From 140c5b3957c26a56225baff0a8bedf0d69a11f5a Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 30 Sep 2024 18:00:47 -0700
Subject: [PATCH 001/376] don't push integration testing docker images (#2584)

* experiment with build and no push

* use slightly more descriptive and consistent tags and names

* name integration test workflow consistently with other workflows

* put the tag back

* try runs-on s3 backend

* try adding runs-on cache

* add with key

* add a dummy path

* forget about multiline

* maybe we don't need runs-on cache immediately

* lower ram slightly, name test with a version bump

* don't need to explicitly include runs-on/cache for docker caching

* comment out flaky portion of knowledge chat test

---------

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 .../{run-it.yml => pr-Integration-tests.yml}  | 56 +++++++++++--------
 .../tests/dev_apis/test_knowledge_chat.py     | 24 ++++----
 2 files changed, 45 insertions(+), 35 deletions(-)
 rename .github/workflows/{run-it.yml => pr-Integration-tests.yml} (67%)

diff --git a/.github/workflows/run-it.yml b/.github/workflows/pr-Integration-tests.yml
similarity index 67%
rename from .github/workflows/run-it.yml
rename to .github/workflows/pr-Integration-tests.yml
index a5bf603e9bc..867817720cb 100644
--- a/.github/workflows/run-it.yml
+++ b/.github/workflows/pr-Integration-tests.yml
@@ -1,4 +1,4 @@
-name: Run Integration Tests
+name: Run Integration Tests v2
 concurrency:
   group: Run-Integration-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }}
   cancel-in-progress: true
@@ -14,7 +14,7 @@ env:
 jobs:
   integration-tests:
     # See https://runs-on.com/runners/linux/
-    runs-on: [runs-on,runner=8cpu-linux-x64,ram=32,"run-id=${{ github.run_id }}"]
+    runs-on: [runs-on,runner=8cpu-linux-x64,ram=16,"run-id=${{ github.run_id }}"]
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -28,25 +28,35 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_TOKEN }}
 
-      # NOTE: we don't need to build the Web Docker image since it's not used
-      # during the IT for now. We have a separate action to verify it builds 
-      # succesfully
+      # tag every docker image with "test" so that we can spin up the correct set
+      # of images during testing
+      
+      # We don't need to build the Web Docker image since it's not yet used
+      # in the integration tests. We have a separate action to verify that it builds 
+      # successfully.
       - name: Pull Web Docker image
         run: |
           docker pull danswer/danswer-web-server:latest
-          docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:it
+          docker tag danswer/danswer-web-server:latest danswer/danswer-web-server:test
 
+      # we use the runs-on cache for docker builds
+      # in conjunction with runs-on runners, it has better speed and unlimited caching
+      # https://runs-on.com/caching/s3-cache-for-github-actions/
+      # https://runs-on.com/caching/docker/
+      # https://github.com/moby/buildkit#s3-cache-experimental
+      
+      # images are built and run locally for testing purposes. Not pushed.
       - name: Build Backend Docker image
         uses: ./.github/actions/custom-build-and-push
         with:
           context: ./backend
           file: ./backend/Dockerfile
           platforms: linux/amd64
-          tags: danswer/danswer-backend:it
-          cache-from: type=registry,ref=danswer/danswer-backend:it
-          cache-to: |
-            type=registry,ref=danswer/danswer-backend:it,mode=max
-            type=inline
+          tags: danswer/danswer-backend:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/backend/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
       - name: Build Model Server Docker image
         uses: ./.github/actions/custom-build-and-push
@@ -54,11 +64,11 @@ jobs:
           context: ./backend
           file: ./backend/Dockerfile.model_server
           platforms: linux/amd64
-          tags: danswer/danswer-model-server:it
-          cache-from: type=registry,ref=danswer/danswer-model-server:it
-          cache-to: |
-            type=registry,ref=danswer/danswer-model-server:it,mode=max
-            type=inline
+          tags: danswer/danswer-model-server:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
       - name: Build integration test Docker image
         uses: ./.github/actions/custom-build-and-push
@@ -66,11 +76,11 @@ jobs:
           context: ./backend
           file: ./backend/tests/integration/Dockerfile
           platforms: linux/amd64
-          tags: danswer/integration-test-runner:it
-          cache-from: type=registry,ref=danswer/integration-test-runner:it
-          cache-to: |
-            type=registry,ref=danswer/integration-test-runner:it,mode=max
-            type=inline
+          tags: danswer/danswer-integration:test
+          push: false
+          load: true
+          cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
+          cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
       - name: Start Docker containers
         run: |
@@ -79,7 +89,7 @@ jobs:
           AUTH_TYPE=basic \
           REQUIRE_EMAIL_VERIFICATION=false \
           DISABLE_TELEMETRY=true \
-          IMAGE_TAG=it \
+          IMAGE_TAG=test \
           docker compose -f docker-compose.dev.yml -p danswer-stack up -d
         id: start_docker
 
@@ -131,7 +141,7 @@ jobs:
             -e API_SERVER_HOST=api_server \
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
             -e TEST_WEB_HOSTNAME=test-runner \
-            danswer/integration-test-runner:it
+            danswer/danswer-integration:test
         continue-on-error: true
         id: run_tests
 
diff --git a/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py b/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
index 2cf6fd399ea..a6bc259c640 100644
--- a/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
+++ b/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
@@ -71,8 +71,8 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     answer_1 = response_json["answer"]
     assert "blue" in answer_1.lower()
 
-    # check that the llm selected a document
-    assert 0 in response_json["llm_selected_doc_indices"]
+    # FLAKY - check that the llm selected a document
+    # assert 0 in response_json["llm_selected_doc_indices"]
 
     # check that the final context documents are correct
     # (it should contain all documents because there arent enough to exclude any)
@@ -80,8 +80,8 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     assert 1 in response_json["final_context_doc_indices"]
     assert 2 in response_json["final_context_doc_indices"]
 
-    # check that the cited documents are correct
-    assert cc_pair_1.documents[0].id in response_json["cited_documents"].values()
+    # FLAKY - check that the cited documents are correct
+    # assert cc_pair_1.documents[0].id in response_json["cited_documents"].values()
 
     # check that the top documents are correct
     assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[0].id
@@ -117,8 +117,8 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     answer_2 = response_json["answer"]
     assert "red" in answer_2.lower()
 
-    # check that the llm selected a document
-    assert 0 in response_json["llm_selected_doc_indices"]
+    # FLAKY - check that the llm selected a document
+    # assert 0 in response_json["llm_selected_doc_indices"]
 
     # check that the final context documents are correct
     # (it should contain all documents because there arent enough to exclude any)
@@ -126,8 +126,8 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     assert 1 in response_json["final_context_doc_indices"]
     assert 2 in response_json["final_context_doc_indices"]
 
-    # check that the cited documents are correct
-    assert cc_pair_1.documents[1].id in response_json["cited_documents"].values()
+    # FLAKY - check that the cited documents are correct
+    # assert cc_pair_1.documents[1].id in response_json["cited_documents"].values()
 
     # check that the top documents are correct
     assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[1].id
@@ -171,8 +171,8 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     answer_3 = response_json["answer"]
     assert "green" in answer_3.lower()
 
-    # check that the llm selected a document
-    assert 0 in response_json["llm_selected_doc_indices"]
+    # FLAKY - check that the llm selected a document
+    # assert 0 in response_json["llm_selected_doc_indices"]
 
     # check that the final context documents are correct
     # (it should contain all documents because there arent enough to exclude any)
@@ -180,8 +180,8 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     assert 1 in response_json["final_context_doc_indices"]
     assert 2 in response_json["final_context_doc_indices"]
 
-    # check that the cited documents are correct
-    assert cc_pair_1.documents[2].id in response_json["cited_documents"].values()
+    # FLAKY - check that the cited documents are correct
+    # assert cc_pair_1.documents[2].id in response_json["cited_documents"].values()
 
     # check that the top documents are correct
     assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[2].id

From e229d27734a4f478a593db97453809a4ef37fa98 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Mon, 30 Sep 2024 21:50:03 -0700
Subject: [PATCH 002/376] Unstructured UI (#2636)

* checkpoint

* k

* k

* need frontend

* add api key check + ui component

* add proper ports + icons + functions

* k

* k

* k

---------

Co-authored-by: pablodanswer <pablo@danswer.ai>
---
 backend/danswer/auth/users.py                 |   1 -
 backend/danswer/configs/app_configs.py        |   1 +
 backend/danswer/configs/constants.py          |   1 +
 backend/danswer/connectors/blob/connector.py  |   2 +-
 .../connectors/confluence/connector.py        |   6 +-
 .../danswer/connectors/dropbox/connector.py   |   2 +-
 backend/danswer/connectors/file/connector.py  |   5 +-
 .../connectors/google_drive/connector.py      |  28 ++--
 .../connectors/sharepoint/connector.py        |   2 +-
 .../file_processing/extract_file_text.py      |  32 ++--
 .../danswer/file_processing/unstructured.py   |  67 +++++++++
 .../danswer/server/manage/search_settings.py  |  28 +++-
 .../server/query_and_chat/chat_backend.py     |   5 +-
 backend/requirements/default.txt              |  12 +-
 .../confluence/test_confluence_basic.py       |  12 +-
 .../document_processing/page.tsx              | 138 ++++++++++++++++++
 web/src/components/admin/ClientLayout.tsx     |  11 +-
 web/src/components/icons/icons.tsx            |  25 ++++
 18 files changed, 340 insertions(+), 38 deletions(-)
 create mode 100644 backend/danswer/file_processing/unstructured.py
 create mode 100644 web/src/app/admin/configuration/document_processing/page.tsx

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index ac02d125850..a583a93235f 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -342,7 +342,6 @@ def get_database_strategy(
     strategy = DatabaseStrategy(
         access_token_db, lifetime_seconds=SESSION_EXPIRE_TIME_SECONDS  # type: ignore
     )
-
     return strategy
 
 
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index c943bc7f603..24ea43717d7 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -338,6 +338,7 @@
 # exception without aborting the attempt.
 INDEXING_EXCEPTION_LIMIT = int(os.environ.get("INDEXING_EXCEPTION_LIMIT", 0))
 
+
 #####
 # Miscellaneous
 #####
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 52314db920c..a2b0f752ffe 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -48,6 +48,7 @@
 # Key-Value store keys
 KV_REINDEX_KEY = "needs_reindexing"
 KV_SEARCH_SETTINGS = "search_settings"
+KV_UNSTRUCTURED_API_KEY = "unstructured_api_key"
 KV_USER_STORE_KEY = "INVITED_USERS"
 KV_NO_AUTH_USER_PREFERENCES_KEY = "no_auth_user_preferences"
 KV_CRED_KEY = "credential_id_{}"
diff --git a/backend/danswer/connectors/blob/connector.py b/backend/danswer/connectors/blob/connector.py
index a664a3d764a..1f030a7564f 100644
--- a/backend/danswer/connectors/blob/connector.py
+++ b/backend/danswer/connectors/blob/connector.py
@@ -194,8 +194,8 @@ def _yield_blob_objects(
 
                 try:
                     text = extract_file_text(
-                        name,
                         BytesIO(downloaded_file),
+                        file_name=name,
                         break_on_unprocessable=False,
                     )
                     batch.append(
diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index d3caf66cc14..f800aa49520 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -519,7 +519,9 @@ def _attachment_to_content(
             return None
 
         extracted_text = extract_file_text(
-            attachment["title"], io.BytesIO(response.content), False
+            io.BytesIO(response.content),
+            file_name=attachment["title"],
+            break_on_unprocessable=False,
         )
         if len(extracted_text) > CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD:
             logger.warning(
@@ -625,7 +627,7 @@ def _get_doc_batch(
             )
             unused_attachments.extend(unused_page_attachments)
 
-            page_text += attachment_text
+            page_text += "\n" + attachment_text if attachment_text else ""
             comments_text = self._fetch_comments(self.confluence_client, page_id)
             page_text += comments_text
             doc_metadata: dict[str, str | list[str]] = {"Wiki Space Name": self.space}
diff --git a/backend/danswer/connectors/dropbox/connector.py b/backend/danswer/connectors/dropbox/connector.py
index b36f0fbd122..7d2eb0166c7 100644
--- a/backend/danswer/connectors/dropbox/connector.py
+++ b/backend/danswer/connectors/dropbox/connector.py
@@ -97,8 +97,8 @@ def _yield_files_recursive(
                     link = self._get_shared_link(entry.path_display)
                     try:
                         text = extract_file_text(
-                            entry.name,
                             BytesIO(downloaded_file),
+                            file_name=entry.name,
                             break_on_unprocessable=False,
                         )
                         batch.append(
diff --git a/backend/danswer/connectors/file/connector.py b/backend/danswer/connectors/file/connector.py
index 83d0af2c12e..8ef98716c91 100644
--- a/backend/danswer/connectors/file/connector.py
+++ b/backend/danswer/connectors/file/connector.py
@@ -74,13 +74,14 @@ def _process_file(
         )
 
     # Using the PDF reader function directly to pass in password cleanly
-    elif extension == ".pdf":
+    elif extension == ".pdf" and pdf_pass is not None:
         file_content_raw, file_metadata = read_pdf_file(file=file, pdf_pass=pdf_pass)
 
     else:
         file_content_raw = extract_file_text(
-            file_name=file_name,
             file=file,
+            file_name=file_name,
+            break_on_unprocessable=True,
         )
 
     all_metadata = {**metadata, **file_metadata} if metadata else file_metadata
diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index bf267ab7786..48b514e80b5 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -36,6 +36,8 @@
 from danswer.file_processing.extract_file_text import docx_to_text
 from danswer.file_processing.extract_file_text import pptx_to_text
 from danswer.file_processing.extract_file_text import read_pdf_file
+from danswer.file_processing.unstructured import get_unstructured_api_key
+from danswer.file_processing.unstructured import unstructured_to_text
 from danswer.utils.batching import batch_generator
 from danswer.utils.logger import setup_logger
 
@@ -327,16 +329,24 @@ def extract_text(file: dict[str, str], service: discovery.Resource) -> str:
         GDriveMimeType.MARKDOWN.value,
     ]:
         return service.files().get_media(fileId=file["id"]).execute().decode("utf-8")
-    elif mime_type == GDriveMimeType.WORD_DOC.value:
-        response = service.files().get_media(fileId=file["id"]).execute()
-        return docx_to_text(file=io.BytesIO(response))
-    elif mime_type == GDriveMimeType.PDF.value:
-        response = service.files().get_media(fileId=file["id"]).execute()
-        text, _ = read_pdf_file(file=io.BytesIO(response))
-        return text
-    elif mime_type == GDriveMimeType.POWERPOINT.value:
+    if mime_type in [
+        GDriveMimeType.WORD_DOC.value,
+        GDriveMimeType.POWERPOINT.value,
+        GDriveMimeType.PDF.value,
+    ]:
         response = service.files().get_media(fileId=file["id"]).execute()
-        return pptx_to_text(file=io.BytesIO(response))
+        if get_unstructured_api_key():
+            return unstructured_to_text(
+                file=io.BytesIO(response), file_name=file.get("name", file["id"])
+            )
+
+        if mime_type == GDriveMimeType.WORD_DOC.value:
+            return docx_to_text(file=io.BytesIO(response))
+        elif mime_type == GDriveMimeType.PDF.value:
+            text, _ = read_pdf_file(file=io.BytesIO(response))
+            return text
+        elif mime_type == GDriveMimeType.POWERPOINT.value:
+            return pptx_to_text(file=io.BytesIO(response))
 
     return UNSUPPORTED_FILE_TYPE_CONTENT
 
diff --git a/backend/danswer/connectors/sharepoint/connector.py b/backend/danswer/connectors/sharepoint/connector.py
index e74dcbf7edd..a32b4ecb1ab 100644
--- a/backend/danswer/connectors/sharepoint/connector.py
+++ b/backend/danswer/connectors/sharepoint/connector.py
@@ -40,8 +40,8 @@ def _convert_driveitem_to_document(
     driveitem: DriveItem,
 ) -> Document:
     file_text = extract_file_text(
-        file_name=driveitem.name,
         file=io.BytesIO(driveitem.get_content().execute_query().value),
+        file_name=driveitem.name,
         break_on_unprocessable=False,
     )
 
diff --git a/backend/danswer/file_processing/extract_file_text.py b/backend/danswer/file_processing/extract_file_text.py
index 36df08ac465..0f8c4e782c6 100644
--- a/backend/danswer/file_processing/extract_file_text.py
+++ b/backend/danswer/file_processing/extract_file_text.py
@@ -20,6 +20,8 @@
 
 from danswer.configs.constants import DANSWER_METADATA_FILENAME
 from danswer.file_processing.html_utils import parse_html_page_basic
+from danswer.file_processing.unstructured import get_unstructured_api_key
+from danswer.file_processing.unstructured import unstructured_to_text
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -331,9 +333,10 @@ def file_io_to_text(file: IO[Any]) -> str:
 
 
 def extract_file_text(
-    file_name: str | None,
     file: IO[Any],
+    file_name: str,
     break_on_unprocessable: bool = True,
+    extension: str | None = None,
 ) -> str:
     extension_to_function: dict[str, Callable[[IO[Any]], str]] = {
         ".pdf": pdf_to_text,
@@ -345,22 +348,29 @@ def extract_file_text(
         ".html": parse_html_page_basic,
     }
 
-    def _process_file() -> str:
-        if file_name:
-            extension = get_file_ext(file_name)
-            if check_file_ext_is_valid(extension):
-                return extension_to_function.get(extension, file_io_to_text)(file)
+    try:
+        if get_unstructured_api_key():
+            return unstructured_to_text(file, file_name)
+
+        if file_name or extension:
+            if extension is not None:
+                final_extension = extension
+            elif file_name is not None:
+                final_extension = get_file_ext(file_name)
 
-        # Either the file somehow has no name or the extension is not one that we are familiar with
+            if check_file_ext_is_valid(final_extension):
+                return extension_to_function.get(final_extension, file_io_to_text)(file)
+
+        # Either the file somehow has no name or the extension is not one that we recognize
         if is_text_file(file):
             return file_io_to_text(file)
 
         raise ValueError("Unknown file extension and unknown text encoding")
 
-    try:
-        return _process_file()
     except Exception as e:
         if break_on_unprocessable:
-            raise RuntimeError(f"Failed to process file: {str(e)}") from e
-        logger.warning(f"Failed to process file: {str(e)}")
+            raise RuntimeError(
+                f"Failed to process file {file_name or 'Unknown'}: {str(e)}"
+            ) from e
+        logger.warning(f"Failed to process file {file_name or 'Unknown'}: {str(e)}")
         return ""
diff --git a/backend/danswer/file_processing/unstructured.py b/backend/danswer/file_processing/unstructured.py
new file mode 100644
index 00000000000..c5a14d87617
--- /dev/null
+++ b/backend/danswer/file_processing/unstructured.py
@@ -0,0 +1,67 @@
+from typing import Any
+from typing import cast
+from typing import IO
+
+from unstructured.staging.base import dict_to_elements
+from unstructured_client import UnstructuredClient  # type: ignore
+from unstructured_client.models import operations  # type: ignore
+from unstructured_client.models import shared
+
+from danswer.configs.constants import KV_UNSTRUCTURED_API_KEY
+from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
+
+
+def get_unstructured_api_key() -> str | None:
+    kv_store = get_dynamic_config_store()
+    try:
+        return cast(str, kv_store.load(KV_UNSTRUCTURED_API_KEY))
+    except ConfigNotFoundError:
+        return None
+
+
+def update_unstructured_api_key(api_key: str) -> None:
+    kv_store = get_dynamic_config_store()
+    kv_store.store(KV_UNSTRUCTURED_API_KEY, api_key)
+
+
+def delete_unstructured_api_key() -> None:
+    kv_store = get_dynamic_config_store()
+    kv_store.delete(KV_UNSTRUCTURED_API_KEY)
+
+
+def _sdk_partition_request(
+    file: IO[Any], file_name: str, **kwargs: Any
+) -> operations.PartitionRequest:
+    try:
+        request = operations.PartitionRequest(
+            partition_parameters=shared.PartitionParameters(
+                files=shared.Files(content=file.read(), file_name=file_name),
+                **kwargs,
+            ),
+        )
+        return request
+    except Exception as e:
+        logger.error(f"Error creating partition request for file {file_name}: {str(e)}")
+        raise
+
+
+def unstructured_to_text(file: IO[Any], file_name: str) -> str:
+    logger.debug(f"Starting to read file: {file_name}")
+    req = _sdk_partition_request(file, file_name, strategy="auto")
+
+    unstructured_client = UnstructuredClient(api_key_auth=get_unstructured_api_key())
+
+    response = unstructured_client.general.partition(req)  # type: ignore
+    elements = dict_to_elements(response.elements)
+
+    if response.status_code != 200:
+        err = f"Received unexpected status code {response.status_code} from Unstructured API."
+        logger.error(err)
+        raise ValueError(err)
+
+    return "\n\n".join(str(el) for el in elements)
diff --git a/backend/danswer/server/manage/search_settings.py b/backend/danswer/server/manage/search_settings.py
index c8433467f6c..6436a0bd8c0 100644
--- a/backend/danswer/server/manage/search_settings.py
+++ b/backend/danswer/server/manage/search_settings.py
@@ -21,6 +21,9 @@
 from danswer.db.search_settings import update_current_search_settings
 from danswer.db.search_settings import update_search_settings_status
 from danswer.document_index.factory import get_default_document_index
+from danswer.file_processing.unstructured import delete_unstructured_api_key
+from danswer.file_processing.unstructured import get_unstructured_api_key
+from danswer.file_processing.unstructured import update_unstructured_api_key
 from danswer.natural_language_processing.search_nlp_models import clean_model_name
 from danswer.search.models import SavedSearchSettings
 from danswer.search.models import SearchSettingsCreationRequest
@@ -30,7 +33,6 @@
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import ALT_INDEX_SUFFIX
 
-
 router = APIRouter(prefix="/search-settings")
 logger = setup_logger()
 
@@ -196,3 +198,27 @@ def update_saved_search_settings(
     update_current_search_settings(
         search_settings=search_settings, db_session=db_session
     )
+
+
+@router.get("/unstructured-api-key-set")
+def unstructured_api_key_set(
+    _: User | None = Depends(current_admin_user),
+) -> bool:
+    api_key = get_unstructured_api_key()
+    print(api_key)
+    return api_key is not None
+
+
+@router.put("/upsert-unstructured-api-key")
+def upsert_unstructured_api_key(
+    unstructured_api_key: str,
+    _: User | None = Depends(current_admin_user),
+) -> None:
+    update_unstructured_api_key(unstructured_api_key)
+
+
+@router.delete("/delete-unstructured-api-key")
+def delete_unstructured_api_key_endpoint(
+    _: User | None = Depends(current_admin_user),
+) -> None:
+    delete_unstructured_api_key()
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index c7f5983417d..36a09afde19 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -588,7 +588,10 @@ def upload_files_for_chat(
         # if the file is a doc, extract text and store that so we don't need
         # to re-extract it every time we send a message
         if file_type == ChatFileType.DOC:
-            extracted_text = extract_file_text(file_name=file.filename, file=file.file)
+            extracted_text = extract_file_text(
+                file=file.file,
+                file_name=file.filename or "",
+            )
             text_file_id = str(uuid.uuid4())
             file_store.save_file(
                 file_name=text_file_id,
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 1a49310088c..9855b8662ff 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -2,7 +2,7 @@ aiohttp==3.10.2
 alembic==1.10.4
 asyncpg==0.27.0
 atlassian-python-api==3.37.0
-beautifulsoup4==4.12.2
+beautifulsoup4==4.12.3
 boto3==1.34.84
 celery==5.3.4
 chardet==5.2.0
@@ -19,9 +19,9 @@ google-auth-oauthlib==1.0.0
 # GPT4All library has issues running on Macs and python:3.11.4-slim-bookworm
 # will reintroduce this when library version catches up
 # gpt4all==2.0.2
-httpcore==0.16.3
-httpx[http2]==0.23.3
-httpx-oauth==0.11.2
+httpcore==1.0.5
+httpx[http2]==0.27.0
+httpx-oauth==0.15.1
 huggingface-hub==0.20.1
 jira==3.5.1
 jsonref==1.1.0
@@ -46,7 +46,7 @@ PyGithub==1.58.2
 python-dateutil==2.8.2
 python-gitlab==3.9.0
 python-pptx==0.6.23
-pypdf==3.17.0
+pypdf==4.3.0
 pytest-mock==3.12.0
 pytest-playwright==0.3.2
 python-docx==1.1.2
@@ -67,6 +67,8 @@ supervisor==4.2.5
 tiktoken==0.7.0
 timeago==1.0.16
 transformers==4.39.2
+unstructured==0.15.1
+unstructured-client==0.25.4
 uvicorn==0.21.1
 zulip==0.8.2
 hubspot-api-client==8.1.0
diff --git a/backend/tests/daily/connectors/confluence/test_confluence_basic.py b/backend/tests/daily/connectors/confluence/test_confluence_basic.py
index 4eb25207814..a791b1eab3b 100644
--- a/backend/tests/daily/connectors/confluence/test_confluence_basic.py
+++ b/backend/tests/daily/connectors/confluence/test_confluence_basic.py
@@ -1,5 +1,7 @@
 import os
 import time
+from unittest.mock import MagicMock
+from unittest.mock import patch
 
 import pytest
 
@@ -24,7 +26,13 @@ def confluence_connector() -> ConfluenceConnector:
     return connector
 
 
-def test_confluence_connector_basic(confluence_connector: ConfluenceConnector) -> None:
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_confluence_connector_basic(
+    mock_get_api_key: MagicMock, confluence_connector: ConfluenceConnector
+) -> None:
     doc_batch_generator = confluence_connector.poll_source(0, time.time())
 
     doc_batch = next(doc_batch_generator)
@@ -41,7 +49,7 @@ def test_confluence_connector_basic(confluence_connector: ConfluenceConnector) -
     assert len(doc.sections) == 1
 
     section = doc.sections[0]
-    assert section.text == "test123small"
+    assert section.text == "test123\nsmall"
     assert (
         section.link
         == "https://danswerai.atlassian.net/wiki/spaces/DailyConne/overview"
diff --git a/web/src/app/admin/configuration/document_processing/page.tsx b/web/src/app/admin/configuration/document_processing/page.tsx
new file mode 100644
index 00000000000..9ccd72b73d1
--- /dev/null
+++ b/web/src/app/admin/configuration/document_processing/page.tsx
@@ -0,0 +1,138 @@
+"use client";
+
+import { useState } from "react";
+import { Button, Card } from "@tremor/react";
+import { DocumentIcon2 } from "@/components/icons/icons";
+import useSWR from "swr";
+import { ThreeDotsLoader } from "@/components/Loading";
+import { AdminPageTitle } from "@/components/admin/Title";
+import { Lock } from "@phosphor-icons/react";
+
+function Main() {
+  const {
+    data: isApiKeySet,
+    error,
+    mutate,
+    isLoading,
+  } = useSWR<{
+    unstructured_api_key: string | null;
+  }>("/api/search-settings/unstructured-api-key-set", (url: string) =>
+    fetch(url).then((res) => res.json())
+  );
+
+  const [apiKey, setApiKey] = useState("");
+
+  const handleSave = async () => {
+    try {
+      await fetch(
+        `/api/search-settings/upsert-unstructured-api-key?unstructured_api_key=${apiKey}`,
+        {
+          method: "PUT",
+        }
+      );
+    } catch (error) {
+      console.error("Failed to save API key:", error);
+    }
+    mutate();
+  };
+
+  const handleDelete = async () => {
+    try {
+      await fetch("/api/search-settings/delete-unstructured-api-key", {
+        method: "DELETE",
+      });
+      setApiKey("");
+    } catch (error) {
+      console.error("Failed to delete API key:", error);
+    }
+    mutate();
+  };
+
+  if (isLoading) {
+    return <ThreeDotsLoader />;
+  }
+  return (
+    <div className="container mx-auto p-4">
+      <Card className="mb-8 max-w-2xl bg-white text-text shadow-lg rounded-lg">
+        <h3 className="text-2xl text-text-800 font-bold mb-4 text-text border-b border-b-border pb-2">
+          Unstructured API Integration
+        </h3>
+
+        <div className="space-y-4">
+          <p className="text-text-600">
+            Unstructured effortlessly extracts and transforms complex data from
+            difficult-to-use formats like HTML, PDF, CSV, PNG, PPTX, and more.
+            Enter an API key to enable this powerful document processing. If not
+            set, standard document processing will be used.
+          </p>
+          <p className="text-text-600">
+            Learn more about Unstructured{" "}
+            <a
+              href="https://unstructured.io/docs"
+              target="_blank"
+              rel="noopener noreferrer"
+              className="text-blue-500 hover:underline font-medium"
+            >
+              here
+            </a>
+            .
+          </p>
+          <div className="mt-4">
+            {isApiKeySet ? (
+              <div className="w-full p-3 border rounded-md bg-background text-text flex items-center">
+                <span className="flex-grow">••••••••••••••••</span>
+                <Lock className="h-5 w-5 text-gray-400" />
+              </div>
+            ) : (
+              <input
+                type="text"
+                placeholder="Enter API Key"
+                value={apiKey}
+                onChange={(e) => setApiKey(e.target.value)}
+                className="w-full p-3 border rounded-md bg-background text-text focus:ring-2 focus:ring-blue-500 transition duration-200"
+              />
+            )}
+          </div>
+          <div className="flex space-x-4 mt-6">
+            {isApiKeySet ? (
+              <>
+                <Button
+                  color="red"
+                  onClick={handleDelete}
+                  variant="secondary"
+                  className="bg-red-100 text-red-600 hover:bg-red-400 transition duration-200"
+                >
+                  Delete API Key
+                </Button>
+                <p className="text-text-600 my-auto">
+                  Delete the current API key before updating.
+                </p>
+              </>
+            ) : (
+              <Button
+                onClick={handleSave}
+                className="bg-blue-500 text-white hover:bg-blue-600 transition duration-200"
+              >
+                Save API Key
+              </Button>
+            )}
+          </div>
+        </div>
+      </Card>
+    </div>
+  );
+}
+
+function Page() {
+  return (
+    <div className="mx-auto container">
+      <AdminPageTitle
+        title="Document Processing"
+        icon={<DocumentIcon2 size={32} className="my-auto" />}
+      />
+      <Main />
+    </div>
+  );
+}
+
+export default Page;
diff --git a/web/src/components/admin/ClientLayout.tsx b/web/src/components/admin/ClientLayout.tsx
index 961f5a9c21b..e0415f84544 100644
--- a/web/src/components/admin/ClientLayout.tsx
+++ b/web/src/components/admin/ClientLayout.tsx
@@ -21,6 +21,7 @@ import {
   AssistantsIconSkeleton,
   ClosedBookIcon,
   SearchIcon,
+  DocumentIcon2,
 } from "@/components/icons/icons";
 import { UserRole } from "@/lib/types";
 import { FiActivity, FiBarChart2 } from "react-icons/fi";
@@ -29,7 +30,6 @@ import { User } from "@/lib/types";
 import { usePathname } from "next/navigation";
 import { SettingsContext } from "../settings/SettingsProvider";
 import { useContext } from "react";
-import { CustomTooltip } from "../tooltip/CustomTooltip";
 
 export function ClientLayout({
   user,
@@ -246,6 +246,15 @@ export function ClientLayout({
                           ),
                           link: "/admin/configuration/search",
                         },
+                        {
+                          name: (
+                            <div className="flex">
+                              <DocumentIcon2 className="text-icon-settings-sidebar" />
+                              <div className="ml-1">Document Processing</div>
+                            </div>
+                          ),
+                          link: "/admin/configuration/document-processing",
+                        },
                       ],
                     },
                     {
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 1f6f1f2e8d1..186e4473cce 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -2791,6 +2791,31 @@ export const MacIcon = ({
   );
 };
 
+export const DocumentIcon2 = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <svg
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+      xmlns="http://www.w3.org/2000/svg"
+      width="200"
+      height="200"
+      viewBox="0 0 24 24"
+    >
+      <path
+        fill="none"
+        stroke="currentColor"
+        strokeLinecap="round"
+        strokeLinejoin="round"
+        strokeWidth="1.5"
+        d="M19.5 14.25v-2.625a3.375 3.375 0 0 0-3.375-3.375h-1.5A1.125 1.125 0 0 1 13.5 7.125v-1.5a3.375 3.375 0 0 0-3.375-3.375H8.25m0 12.75h7.5m-7.5 3H12M10.5 2.25H5.625c-.621 0-1.125.504-1.125 1.125v17.25c0 .621.504 1.125 1.125 1.125h12.75c.621 0 1.125-.504 1.125-1.125V11.25a9 9 0 0 0-9-9Z"
+      />
+    </svg>
+  );
+};
+
 export const WindowsIcon = ({
   size = 16,
   className = "my-auto flex flex-shrink-0 ",

From c9bdf4c443c9f70a4a80e5b7bc5a34bb3469becb Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Tue, 1 Oct 2024 08:46:25 -0700
Subject: [PATCH 003/376] Update CONTRIBUTING.md

---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 3e4415188a1..779cabbb491 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -22,7 +22,7 @@ Your input is vital to making sure that Danswer moves in the right direction.
 Before starting on implementation, please raise a GitHub issue.
 
 And always feel free to message us (Chris Weaver / Yuhong Sun) on 
-[Slack](https://join.slack.com/t/danswer/shared_invite/zt-2afut44lv-Rw3kSWu6_OmdAXRpCv80DQ) / 
+[Slack](https://join.slack.com/t/danswer/shared_invite/zt-2lcmqw703-071hBuZBfNEOGUsLa5PXvQ) / 
 [Discord](https://discord.gg/TDJ59cGV2X) directly about anything at all. 
 
 

From 3fa1b18306d9cfdc994ea475e21d68f1de3d850c Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 1 Oct 2024 09:34:30 -0700
Subject: [PATCH 004/376] update nav link name (#2643)

* update nav link name

* underscore -> dash
---
 .../{document_processing => document-processing}/page.tsx         | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename web/src/app/admin/configuration/{document_processing => document-processing}/page.tsx (100%)

diff --git a/web/src/app/admin/configuration/document_processing/page.tsx b/web/src/app/admin/configuration/document-processing/page.tsx
similarity index 100%
rename from web/src/app/admin/configuration/document_processing/page.tsx
rename to web/src/app/admin/configuration/document-processing/page.tsx

From ec02665ffa26d53a2f46a2222eb7cd81cb2dfd49 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 1 Oct 2024 09:36:40 -0700
Subject: [PATCH 005/376] run the nightly tag overnight relative to pacific
 time (#2637)

---
 .github/workflows/tag-nightly.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tag-nightly.yml b/.github/workflows/tag-nightly.yml
index bf2699d9fd4..50bb20808a3 100644
--- a/.github/workflows/tag-nightly.yml
+++ b/.github/workflows/tag-nightly.yml
@@ -2,7 +2,7 @@ name: Nightly Tag Push
 
 on:
   schedule:
-    - cron: '0 0 * * *' # Runs every day at midnight UTC
+    - cron: '0 10 * * *' # Runs every day at 2 AM PST / 3 AM PDT / 10 AM UTC
 
 permissions:
   contents: write  # Allows pushing tags to the repository

From 834c76e30a93bee90ddceab160d6457acf8bb38e Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Tue, 1 Oct 2024 10:32:41 -0700
Subject: [PATCH 006/376] Added quotes to project name to handle reserved words
 (#2639)

---
 .../workflows/pr-python-connector-tests.yml   |  3 ++
 .../connectors/danswer_jira/connector.py      |  8 +++-
 .../daily/connectors/jira/test_jira_basic.py  | 48 +++++++++++++++++++
 3 files changed, 57 insertions(+), 2 deletions(-)
 create mode 100644 backend/tests/daily/connectors/jira/test_jira_basic.py

diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml
index e5dd79c016d..642618000d2 100644
--- a/.github/workflows/pr-python-connector-tests.yml
+++ b/.github/workflows/pr-python-connector-tests.yml
@@ -15,6 +15,9 @@ env:
   CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
   CONFLUENCE_USER_NAME: ${{ secrets.CONFLUENCE_USER_NAME }}
   CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
+  # Jira
+  JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
+  JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
 
 jobs:
   connectors-check:
diff --git a/backend/danswer/connectors/danswer_jira/connector.py b/backend/danswer/connectors/danswer_jira/connector.py
index 05fa2e1e24d..097aa41c372 100644
--- a/backend/danswer/connectors/danswer_jira/connector.py
+++ b/backend/danswer/connectors/danswer_jira/connector.py
@@ -245,10 +245,12 @@ def load_from_state(self) -> GenerateDocumentsOutput:
         if self.jira_client is None:
             raise ConnectorMissingCredentialError("Jira")
 
+        # Quote the project name to handle reserved words
+        quoted_project = f'"{self.jira_project}"'
         start_ind = 0
         while True:
             doc_batch, fetched_batch_size = fetch_jira_issues_batch(
-                jql=f"project = {self.jira_project}",
+                jql=f"project = {quoted_project}",
                 start_index=start_ind,
                 jira_client=self.jira_client,
                 batch_size=self.batch_size,
@@ -276,8 +278,10 @@ def poll_source(
             "%Y-%m-%d %H:%M"
         )
 
+        # Quote the project name to handle reserved words
+        quoted_project = f'"{self.jira_project}"'
         jql = (
-            f"project = {self.jira_project} AND "
+            f"project = {quoted_project} AND "
             f"updated >= '{start_date_str}' AND "
             f"updated <= '{end_date_str}'"
         )
diff --git a/backend/tests/daily/connectors/jira/test_jira_basic.py b/backend/tests/daily/connectors/jira/test_jira_basic.py
new file mode 100644
index 00000000000..19d69dfadcf
--- /dev/null
+++ b/backend/tests/daily/connectors/jira/test_jira_basic.py
@@ -0,0 +1,48 @@
+import os
+import time
+
+import pytest
+
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.danswer_jira.connector import JiraConnector
+
+
+@pytest.fixture
+def jira_connector() -> JiraConnector:
+    connector = JiraConnector(
+        "https://danswerai.atlassian.net/jira/software/c/projects/AS/boards/6",
+        comment_email_blacklist=[],
+    )
+    connector.load_credentials(
+        {
+            "jira_user_email": os.environ["JIRA_USER_EMAIL"],
+            "jira_api_token": os.environ["JIRA_API_TOKEN"],
+        }
+    )
+    return connector
+
+
+def test_jira_connector_basic(jira_connector: JiraConnector) -> None:
+    doc_batch_generator = jira_connector.poll_source(0, time.time())
+
+    doc_batch = next(doc_batch_generator)
+    with pytest.raises(StopIteration):
+        next(doc_batch_generator)
+
+    assert len(doc_batch) == 1
+
+    doc = doc_batch[0]
+
+    assert doc.id == "https://danswerai.atlassian.net/browse/AS-2"
+    assert doc.semantic_identifier == "test123small"
+    assert doc.source == DocumentSource.JIRA
+    assert doc.metadata == {"priority": "Medium", "status": "Backlog"}
+    assert doc.secondary_owners is None
+    assert doc.title is None
+    assert doc.from_ingestion_api is False
+    assert doc.additional_info is None
+
+    assert len(doc.sections) == 1
+    section = doc.sections[0]
+    assert section.text == "example_text\n"
+    assert section.link == "https://danswerai.atlassian.net/browse/AS-2"

From c68c6fdc446f8114f3d37d2b4c7ba52b2e30f391 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 1 Oct 2024 10:23:18 -0700
Subject: [PATCH 007/376] welcome flow

---
 web/src/components/initialSetup/welcome/WelcomeModal.tsx | 7 +------
 web/src/lib/chat/fetchChatData.ts                        | 1 +
 2 files changed, 2 insertions(+), 6 deletions(-)

diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index 0cf74e31357..ea59cfa00f9 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -2,13 +2,10 @@
 
 import { Button, Divider, Text } from "@tremor/react";
 import { Modal } from "../../Modal";
-import Link from "next/link";
 import Cookies from "js-cookie";
 import { useRouter } from "next/navigation";
 import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants";
-import { FiCheckCircle, FiMessageSquare, FiShare2 } from "react-icons/fi";
 import { useEffect, useState } from "react";
-import { BackButton } from "@/components/BackButton";
 import { ApiKeyForm } from "@/components/llm/ApiKeyForm";
 import { WellKnownLLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { checkLlmProvider } from "./lib";
@@ -26,14 +23,12 @@ export function _CompletedWelcomeFlowDummyComponent() {
 
 export function _WelcomeModal({ user }: { user: User | null }) {
   const router = useRouter();
-  const [selectedFlow, setSelectedFlow] = useState<null | "search" | "chat">(
-    null
-  );
   const [canBegin, setCanBegin] = useState(false);
   const [apiKeyVerified, setApiKeyVerified] = useState<boolean>(false);
   const [providerOptions, setProviderOptions] = useState<
     WellKnownLLMProviderDescriptor[]
   >([]);
+
   const { refreshProviderInfo } = useProviderStatus();
   const clientSetWelcomeFlowComplete = async () => {
     setWelcomeFlowComplete();
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index fe6a5d9d717..d17c3da01b8 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -184,6 +184,7 @@ export async function fetchChatData(searchParams: {
 
   const hasAnyConnectors = ccPairs.length > 0;
   const shouldShowWelcomeModal =
+    !llmProviders.length &&
     !hasCompletedWelcomeFlowSS() &&
     !hasAnyConnectors &&
     (!user || user.role === "admin");

From 2f2fc08553ec331687771fb3fe64823e0beb4453 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 1 Oct 2024 10:27:17 -0700
Subject: [PATCH 008/376] =?UTF-8?q?raise=20redis=20connections=20and=20usi?=
 =?UTF-8?q?ng=20blocking=20connection=20pool=20for=20more=20d=E2=80=A6=20(?=
 =?UTF-8?q?#2635)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* raise redis connections and using blocking connection pool for more deterministic behavior

* improve comment
---
 backend/danswer/redis/redis_pool.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py
index 45ab39d8a02..9664019eb6d 100644
--- a/backend/danswer/redis/redis_pool.py
+++ b/backend/danswer/redis/redis_pool.py
@@ -3,7 +3,6 @@
 
 import redis
 from redis.client import Redis
-from redis.connection import ConnectionPool
 
 from danswer.configs.app_configs import REDIS_DB_NUMBER
 from danswer.configs.app_configs import REDIS_HOST
@@ -13,13 +12,13 @@
 from danswer.configs.app_configs import REDIS_SSL_CA_CERTS
 from danswer.configs.app_configs import REDIS_SSL_CERT_REQS
 
-REDIS_POOL_MAX_CONNECTIONS = 10
+REDIS_POOL_MAX_CONNECTIONS = 128
 
 
 class RedisPool:
     _instance: Optional["RedisPool"] = None
     _lock: threading.Lock = threading.Lock()
-    _pool: ConnectionPool
+    _pool: redis.BlockingConnectionPool
 
     def __new__(cls) -> "RedisPool":
         if not cls._instance:
@@ -45,27 +44,33 @@ def create_pool(
         ssl_ca_certs: str | None = REDIS_SSL_CA_CERTS,
         ssl_cert_reqs: str = REDIS_SSL_CERT_REQS,
         ssl: bool = False,
-    ) -> redis.ConnectionPool:
+    ) -> redis.BlockingConnectionPool:
+        """We use BlockingConnectionPool because it will block and wait for a connection
+        rather than error if max_connections is reached. This is far more deterministic
+        behavior and aligned with how we want to use Redis."""
+
         # Using ConnectionPool is not well documented.
         # Useful examples: https://github.com/redis/redis-py/issues/780
         if ssl:
-            return redis.ConnectionPool(
+            return redis.BlockingConnectionPool(
                 host=host,
                 port=port,
                 db=db,
                 password=password,
                 max_connections=max_connections,
+                timeout=None,
                 connection_class=redis.SSLConnection,
                 ssl_ca_certs=ssl_ca_certs,
                 ssl_cert_reqs=ssl_cert_reqs,
             )
 
-        return redis.ConnectionPool(
+        return redis.BlockingConnectionPool(
             host=host,
             port=port,
             db=db,
             password=password,
             max_connections=max_connections,
+            timeout=None,
         )
 
 

From 9a4e51a18ecaa5074bdf2edc63d986cfe3f55f8c Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 1 Oct 2024 10:43:43 -0700
Subject: [PATCH 009/376] add default model + minor fixes (#2638)

* add default model + minor fixes

* fix build

* minor additional fix

* build fix
---
 .../slack/handlers/handle_regular_answer.py        |  2 +-
 web/src/app/admin/assistants/PersonaTable.tsx      |  2 +-
 web/src/app/chat/ChatPage.tsx                      |  3 ++-
 web/src/app/chat/modal/SetDefaultModelModal.tsx    |  6 +++---
 web/src/lib/hooks.ts                               | 14 +++++++++++++-
 5 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
index f1c9bd077cf..5bd920c4b6c 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
@@ -160,7 +160,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
                         detail="Slack bot does not support persona config",
                     )
 
-                elif new_message_request.persona_id:
+                elif new_message_request.persona_id is not None:
                     persona = cast(
                         Persona,
                         fetch_persona_by_id(
diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index e30e858f13c..e1e87137285 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -119,7 +119,7 @@ export function PersonasTable({
             id: persona.id.toString(),
             cells: [
               <div key="name" className="flex">
-                {!persona.is_default_persona && (
+                {!persona.builtin_persona && (
                   <FiEdit2
                     className="mr-1 my-auto cursor-pointer"
                     onClick={() =>
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 0d13b8607d2..e9676d28246 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -208,7 +208,7 @@ export function ChatPage({
   };
 
   const llmOverrideManager = useLlmOverride(
-    user?.preferences.default_model,
+    user?.preferences.default_model ?? null,
     selectedChatSession,
     defaultTemperature
   );
@@ -1779,6 +1779,7 @@ export function ChatPage({
 
       {settingsToggled && (
         <SetDefaultModelModal
+          setPopup={setPopup}
           setLlmOverride={llmOverrideManager.setGlobalDefault}
           defaultModel={user?.preferences.default_model!}
           refreshUser={refreshUser}
diff --git a/web/src/app/chat/modal/SetDefaultModelModal.tsx b/web/src/app/chat/modal/SetDefaultModelModal.tsx
index e5536f19c61..97e05cf59b0 100644
--- a/web/src/app/chat/modal/SetDefaultModelModal.tsx
+++ b/web/src/app/chat/modal/SetDefaultModelModal.tsx
@@ -7,22 +7,23 @@ import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces"
 import { destructureValue, structureValue } from "@/lib/llm/utils";
 import { setUserDefaultModel } from "@/lib/users/UserSettings";
 import { useRouter } from "next/navigation";
-import { usePopup } from "@/components/admin/connectors/Popup";
+import { PopupSpec } from "@/components/admin/connectors/Popup";
 
 export function SetDefaultModelModal({
+  setPopup,
   llmProviders,
   onClose,
   setLlmOverride,
   defaultModel,
   refreshUser,
 }: {
+  setPopup: (popupSpec: PopupSpec | null) => void;
   llmProviders: LLMProviderDescriptor[];
   setLlmOverride: Dispatch<SetStateAction<LlmOverride>>;
   onClose: () => void;
   defaultModel: string | null;
   refreshUser: () => void;
 }) {
-  const { popup, setPopup } = usePopup();
   const containerRef = useRef<HTMLDivElement>(null);
   const messageRef = useRef<HTMLDivElement>(null);
 
@@ -127,7 +128,6 @@ export function SetDefaultModelModal({
       modalClassName="rounded-lg  bg-white max-w-xl"
     >
       <>
-        {popup}
         <div className="flex mb-4">
           <h2 className="text-2xl text-emphasis font-bold flex my-auto">
             Set Default Model
diff --git a/web/src/lib/hooks.ts b/web/src/lib/hooks.ts
index 39613e388fd..a760c082471 100644
--- a/web/src/lib/hooks.ts
+++ b/web/src/lib/hooks.ts
@@ -151,7 +151,7 @@ export function useLlmOverride(
   defaultTemperature?: number
 ): LlmOverrideManager {
   const [globalDefault, setGlobalDefault] = useState<LlmOverride>(
-    globalModel
+    globalModel != null
       ? destructureValue(globalModel)
       : {
           name: "",
@@ -182,6 +182,18 @@ export function useLlmOverride(
     defaultTemperature != undefined ? defaultTemperature : 0
   );
 
+  useEffect(() => {
+    setGlobalDefault(
+      globalModel != null
+        ? destructureValue(globalModel)
+        : {
+            name: "",
+            provider: "",
+            modelName: "",
+          }
+    );
+  }, [globalModel]);
+
   useEffect(() => {
     setTemperature(defaultTemperature !== undefined ? defaultTemperature : 0);
   }, [defaultTemperature]);

From f513c5bbede4f7d29257d5bb31da6367412eef0b Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 1 Oct 2024 10:59:10 -0700
Subject: [PATCH 010/376] sync up when checks run with branch protection
 required checks (#2628)

---
 .github/workflows/pr-Integration-tests.yml | 4 +++-
 .github/workflows/pr-python-checks.yml     | 4 +++-
 .github/workflows/pr-python-tests.yml      | 4 +++-
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-Integration-tests.yml b/.github/workflows/pr-Integration-tests.yml
index 867817720cb..ae7cf0be78e 100644
--- a/.github/workflows/pr-Integration-tests.yml
+++ b/.github/workflows/pr-Integration-tests.yml
@@ -6,7 +6,9 @@ concurrency:
 on:
   merge_group:
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
+      - 'release/**'
 
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
diff --git a/.github/workflows/pr-python-checks.yml b/.github/workflows/pr-python-checks.yml
index 0af0603667e..0a9e9f96a63 100644
--- a/.github/workflows/pr-python-checks.yml
+++ b/.github/workflows/pr-python-checks.yml
@@ -3,7 +3,9 @@ name: Python Checks
 on:
   merge_group:
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
+      - 'release/**'
 
 jobs:
   mypy-check:
diff --git a/.github/workflows/pr-python-tests.yml b/.github/workflows/pr-python-tests.yml
index 2a318ebbbf4..ce57a7a5814 100644
--- a/.github/workflows/pr-python-tests.yml
+++ b/.github/workflows/pr-python-tests.yml
@@ -3,7 +3,9 @@ name: Python Unit Tests
 on:
   merge_group:
   pull_request:
-    branches: [ main ]
+    branches:
+      - main
+      - 'release/**'
 
 jobs:
   backend-check:

From fffb9c155a26d27064ef93a9fcb3d7189ea5963a Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Tue, 1 Oct 2024 11:31:18 -0700
Subject: [PATCH 011/376] Redis Cache for KV Store (#2603)

* k

* k

* k

* k
---
 .../703313b75876_add_tokenratelimit_tables.py |   8 +-
 backend/danswer/auth/invited_users.py         |  12 +--
 backend/danswer/auth/noauth_user.py           |  12 +--
 .../danswer/background/celery/celery_app.py   |  16 ++-
 .../danswer/background/celery/celery_utils.py |   6 +-
 .../celery/tasks/connector_deletion/tasks.py  |   5 +-
 .../background/celery/tasks/vespa/tasks.py    |   7 +-
 backend/danswer/configs/app_configs.py        |   3 -
 .../connectors/gmail/connector_auth.py        |  30 +++---
 .../connectors/google_drive/connector_auth.py |  24 ++---
 backend/danswer/danswerbot/slack/listener.py  |   4 +-
 backend/danswer/danswerbot/slack/tokens.py    |   6 +-
 backend/danswer/db/models.py                  |   2 +-
 backend/danswer/db/swap_index.py              |   4 +-
 backend/danswer/document_index/vespa/index.py |   4 +-
 backend/danswer/dynamic_configs/factory.py    |  15 ---
 backend/danswer/dynamic_configs/store.py      | 102 ------------------
 .../danswer/file_processing/unstructured.py   |  12 +--
 .../__init__.py                               |   0
 backend/danswer/key_value_store/factory.py    |   7 ++
 .../interface.py                              |   4 +-
 backend/danswer/key_value_store/store.py      |  99 +++++++++++++++++
 backend/danswer/main.py                       |  12 +--
 backend/danswer/redis/redis_pool.py           |   7 ++
 backend/danswer/search/search_settings.py     |   8 +-
 backend/danswer/server/documents/connector.py |  22 ++--
 .../danswer/server/manage/administrative.py   |   8 +-
 backend/danswer/server/manage/slack_bot.py    |   4 +-
 backend/danswer/server/manage/users.py        |  10 +-
 backend/danswer/server/settings/api.py        |  12 +--
 backend/danswer/server/settings/store.py      |  10 +-
 backend/danswer/tools/custom/custom_tool.py   |   2 +-
 .../tools/images/image_generation_tool.py     |   2 +-
 .../internet_search/internet_search_tool.py   |   2 +-
 backend/danswer/tools/search/search_tool.py   |   2 +-
 backend/danswer/tools/tool.py                 |   2 +-
 backend/danswer/utils/telemetry.py            |  12 +--
 .../server/enterprise_settings/store.py       |  18 ++--
 backend/pyproject.toml                        |   4 +
 39 files changed, 250 insertions(+), 269 deletions(-)
 delete mode 100644 backend/danswer/dynamic_configs/factory.py
 delete mode 100644 backend/danswer/dynamic_configs/store.py
 rename backend/danswer/{dynamic_configs => key_value_store}/__init__.py (100%)
 create mode 100644 backend/danswer/key_value_store/factory.py
 rename backend/danswer/{dynamic_configs => key_value_store}/interface.py (89%)
 create mode 100644 backend/danswer/key_value_store/store.py

diff --git a/backend/alembic/versions/703313b75876_add_tokenratelimit_tables.py b/backend/alembic/versions/703313b75876_add_tokenratelimit_tables.py
index ed1993efed3..9e1fdf3cb9e 100644
--- a/backend/alembic/versions/703313b75876_add_tokenratelimit_tables.py
+++ b/backend/alembic/versions/703313b75876_add_tokenratelimit_tables.py
@@ -9,7 +9,7 @@
 from typing import cast
 from alembic import op
 import sqlalchemy as sa
-from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.key_value_store.factory import get_kv_store
 
 # revision identifiers, used by Alembic.
 revision = "703313b75876"
@@ -54,9 +54,7 @@ def upgrade() -> None:
     )
 
     try:
-        settings_json = cast(
-            str, get_dynamic_config_store().load("token_budget_settings")
-        )
+        settings_json = cast(str, get_kv_store().load("token_budget_settings"))
         settings = json.loads(settings_json)
 
         is_enabled = settings.get("enable_token_budget", False)
@@ -71,7 +69,7 @@ def upgrade() -> None:
             )
 
         # Delete the dynamic config
-        get_dynamic_config_store().delete("token_budget_settings")
+        get_kv_store().delete("token_budget_settings")
 
     except Exception:
         # Ignore if the dynamic config is not found
diff --git a/backend/danswer/auth/invited_users.py b/backend/danswer/auth/invited_users.py
index efce858f265..2bbf79050ca 100644
--- a/backend/danswer/auth/invited_users.py
+++ b/backend/danswer/auth/invited_users.py
@@ -1,20 +1,20 @@
 from typing import cast
 
 from danswer.configs.constants import KV_USER_STORE_KEY
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import JSON_ro
+from danswer.key_value_store.interface import KvKeyNotFoundError
 
 
 def get_invited_users() -> list[str]:
     try:
-        store = get_dynamic_config_store()
+        store = get_kv_store()
         return cast(list, store.load(KV_USER_STORE_KEY))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         return list()
 
 
 def write_invited_users(emails: list[str]) -> int:
-    store = get_dynamic_config_store()
+    store = get_kv_store()
     store.store(KV_USER_STORE_KEY, cast(JSON_ro, emails))
     return len(emails)
diff --git a/backend/danswer/auth/noauth_user.py b/backend/danswer/auth/noauth_user.py
index 9520ef41c23..9eb589dbb25 100644
--- a/backend/danswer/auth/noauth_user.py
+++ b/backend/danswer/auth/noauth_user.py
@@ -4,29 +4,29 @@
 
 from danswer.auth.schemas import UserRole
 from danswer.configs.constants import KV_NO_AUTH_USER_PREFERENCES_KEY
-from danswer.dynamic_configs.store import ConfigNotFoundError
-from danswer.dynamic_configs.store import DynamicConfigStore
+from danswer.key_value_store.store import KeyValueStore
+from danswer.key_value_store.store import KvKeyNotFoundError
 from danswer.server.manage.models import UserInfo
 from danswer.server.manage.models import UserPreferences
 
 
 def set_no_auth_user_preferences(
-    store: DynamicConfigStore, preferences: UserPreferences
+    store: KeyValueStore, preferences: UserPreferences
 ) -> None:
     store.store(KV_NO_AUTH_USER_PREFERENCES_KEY, preferences.model_dump())
 
 
-def load_no_auth_user_preferences(store: DynamicConfigStore) -> UserPreferences:
+def load_no_auth_user_preferences(store: KeyValueStore) -> UserPreferences:
     try:
         preferences_data = cast(
             Mapping[str, Any], store.load(KV_NO_AUTH_USER_PREFERENCES_KEY)
         )
         return UserPreferences(**preferences_data)
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         return UserPreferences(chosen_assistants=None, default_model=None)
 
 
-def fetch_no_auth_user(store: DynamicConfigStore) -> UserInfo:
+def fetch_no_auth_user(store: KeyValueStore) -> UserInfo:
     return UserInfo(
         id="__no_auth_user__",
         email="anonymous@danswer.ai",
diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 0440f275c36..5244d9b94da 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -30,7 +30,7 @@
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_LIGHT_APP_NAME
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
 from danswer.db.engine import SqlEngine
-from danswer.redis.redis_pool import RedisPool
+from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import ColoredFormatter
 from danswer.utils.logger import PlainFormatter
 from danswer.utils.logger import setup_logger
@@ -40,8 +40,6 @@
 # use this within celery tasks to get celery task specific logging
 task_logger = get_task_logger(__name__)
 
-redis_pool = RedisPool()
-
 celery_app = Celery(__name__)
 celery_app.config_from_object(
     "danswer.background.celery.celeryconfig"
@@ -79,13 +77,13 @@ def celery_task_postrun(
     if not task_id:
         return
 
+    r = get_redis_client()
+
     if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
-        r = redis_pool.get_client()
         r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)
         return
 
     if task_id.startswith(RedisDocumentSet.PREFIX):
-        r = redis_pool.get_client()
         document_set_id = RedisDocumentSet.get_id_from_task_id(task_id)
         if document_set_id is not None:
             rds = RedisDocumentSet(document_set_id)
@@ -93,7 +91,6 @@ def celery_task_postrun(
         return
 
     if task_id.startswith(RedisUserGroup.PREFIX):
-        r = redis_pool.get_client()
         usergroup_id = RedisUserGroup.get_id_from_task_id(task_id)
         if usergroup_id is not None:
             rug = RedisUserGroup(usergroup_id)
@@ -101,7 +98,6 @@ def celery_task_postrun(
         return
 
     if task_id.startswith(RedisConnectorDeletion.PREFIX):
-        r = redis_pool.get_client()
         cc_pair_id = RedisConnectorDeletion.get_id_from_task_id(task_id)
         if cc_pair_id is not None:
             rcd = RedisConnectorDeletion(cc_pair_id)
@@ -130,7 +126,7 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
         SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
         SqlEngine.init_engine(pool_size=8, max_overflow=0)
 
-    r = redis_pool.get_client()
+    r = get_redis_client()
 
     WAIT_INTERVAL = 5
     WAIT_LIMIT = 60
@@ -190,7 +186,7 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
 
     # This is singleton work that should be done on startup exactly once
     # by the primary worker
-    r = redis_pool.get_client()
+    r = get_redis_client()
 
     # For the moment, we're assuming that we are the only primary worker
     # that should be running.
@@ -364,7 +360,7 @@ def run_periodic_task(self, worker: Any) -> None:
             if not hasattr(worker, "primary_worker_lock"):
                 return
 
-            r = redis_pool.get_client()
+            r = get_redis_client()
 
             lock: redis.lock.Lock = worker.primary_worker_lock
 
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 9ee282e1af3..344ef5291b9 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -25,12 +25,12 @@
 from danswer.db.tasks import check_task_is_live_and_not_timed_out
 from danswer.db.tasks import get_latest_task
 from danswer.db.tasks import get_latest_task_by_type
-from danswer.redis.redis_pool import RedisPool
+from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import DeletionAttemptSnapshot
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
-redis_pool = RedisPool()
 
 
 def _get_deletion_status(
@@ -47,7 +47,7 @@ def _get_deletion_status(
 
     rcd = RedisConnectorDeletion(cc_pair.id)
 
-    r = redis_pool.get_client()
+    r = get_redis_client()
     if not r.exists(rcd.fence_key):
         return None
 
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index 655487f7168..a16b9fda34a 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -18,9 +18,8 @@
 from danswer.db.index_attempt import get_last_attempt
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.search_settings import get_current_search_settings
-from danswer.redis.redis_pool import RedisPool
+from danswer.redis.redis_pool import get_redis_client
 
-redis_pool = RedisPool()
 
 # use this within celery tasks to get celery task specific logging
 task_logger = get_task_logger(__name__)
@@ -32,7 +31,7 @@
     trail=False,
 )
 def check_for_connector_deletion_task() -> None:
-    r = redis_pool.get_client()
+    r = get_redis_client()
 
     lock_beat = r.lock(
         DanswerRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index d11d317d0b1..0ae214ca470 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -41,14 +41,13 @@
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import UpdateRequest
-from danswer.redis.redis_pool import RedisPool
+from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from danswer.utils.variable_functionality import (
     fetch_versioned_implementation_with_fallback,
 )
 from danswer.utils.variable_functionality import noop_fallback
 
-redis_pool = RedisPool()
 
 # use this within celery tasks to get celery task specific logging
 task_logger = get_task_logger(__name__)
@@ -65,7 +64,7 @@ def check_for_vespa_sync_task() -> None:
     """Runs periodically to check if any document needs syncing.
     Generates sets of tasks for Celery if syncing is needed."""
 
-    r = redis_pool.get_client()
+    r = get_redis_client()
 
     lock_beat = r.lock(
         DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK,
@@ -426,7 +425,7 @@ def monitor_vespa_sync() -> None:
     This task lock timeout is CELERY_METADATA_SYNC_BEAT_LOCK_TIMEOUT seconds, so don't
     do anything too expensive in this function!
     """
-    r = redis_pool.get_client()
+    r = get_redis_client()
 
     lock_beat = r.lock(
         DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK,
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 24ea43717d7..460e15bd1f4 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -342,9 +342,6 @@
 #####
 # Miscellaneous
 #####
-# File based Key Value store no longer used
-DYNAMIC_CONFIG_STORE = "PostgresBackedDynamicConfigStore"
-
 JOB_TIMEOUT = 60 * 60 * 6  # 6 hours default
 # used to allow the background indexing jobs to use a different embedding
 # model server than the API server
diff --git a/backend/danswer/connectors/gmail/connector_auth.py b/backend/danswer/connectors/gmail/connector_auth.py
index ad80d1e1eb1..7d996c5e687 100644
--- a/backend/danswer/connectors/gmail/connector_auth.py
+++ b/backend/danswer/connectors/gmail/connector_auth.py
@@ -25,7 +25,7 @@
 from danswer.connectors.gmail.constants import SCOPES
 from danswer.db.credentials import update_credential_json
 from danswer.db.models import User
-from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.key_value_store.factory import get_kv_store
 from danswer.server.documents.models import CredentialBase
 from danswer.server.documents.models import GoogleAppCredentials
 from danswer.server.documents.models import GoogleServiceAccountKey
@@ -72,7 +72,7 @@ def get_gmail_creds_for_service_account(
 
 
 def verify_csrf(credential_id: int, state: str) -> None:
-    csrf = get_dynamic_config_store().load(KV_CRED_KEY.format(str(credential_id)))
+    csrf = get_kv_store().load(KV_CRED_KEY.format(str(credential_id)))
     if csrf != state:
         raise PermissionError(
             "State from Gmail Connector callback does not match expected"
@@ -80,7 +80,7 @@ def verify_csrf(credential_id: int, state: str) -> None:
 
 
 def get_gmail_auth_url(credential_id: int) -> str:
-    creds_str = str(get_dynamic_config_store().load(KV_GMAIL_CRED_KEY))
+    creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
     credential_json = json.loads(creds_str)
     flow = InstalledAppFlow.from_client_config(
         credential_json,
@@ -92,14 +92,14 @@ def get_gmail_auth_url(credential_id: int) -> str:
     parsed_url = cast(ParseResult, urlparse(auth_url))
     params = parse_qs(parsed_url.query)
 
-    get_dynamic_config_store().store(
+    get_kv_store().store(
         KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
     )  # type: ignore
     return str(auth_url)
 
 
 def get_auth_url(credential_id: int) -> str:
-    creds_str = str(get_dynamic_config_store().load(KV_GMAIL_CRED_KEY))
+    creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
     credential_json = json.loads(creds_str)
     flow = InstalledAppFlow.from_client_config(
         credential_json,
@@ -111,7 +111,7 @@ def get_auth_url(credential_id: int) -> str:
     parsed_url = cast(ParseResult, urlparse(auth_url))
     params = parse_qs(parsed_url.query)
 
-    get_dynamic_config_store().store(
+    get_kv_store().store(
         KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
     )  # type: ignore
     return str(auth_url)
@@ -158,42 +158,40 @@ def build_service_account_creds(
 
 
 def get_google_app_gmail_cred() -> GoogleAppCredentials:
-    creds_str = str(get_dynamic_config_store().load(KV_GMAIL_CRED_KEY))
+    creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
     return GoogleAppCredentials(**json.loads(creds_str))
 
 
 def upsert_google_app_gmail_cred(app_credentials: GoogleAppCredentials) -> None:
-    get_dynamic_config_store().store(
-        KV_GMAIL_CRED_KEY, app_credentials.json(), encrypt=True
-    )
+    get_kv_store().store(KV_GMAIL_CRED_KEY, app_credentials.json(), encrypt=True)
 
 
 def delete_google_app_gmail_cred() -> None:
-    get_dynamic_config_store().delete(KV_GMAIL_CRED_KEY)
+    get_kv_store().delete(KV_GMAIL_CRED_KEY)
 
 
 def get_gmail_service_account_key() -> GoogleServiceAccountKey:
-    creds_str = str(get_dynamic_config_store().load(KV_GMAIL_SERVICE_ACCOUNT_KEY))
+    creds_str = str(get_kv_store().load(KV_GMAIL_SERVICE_ACCOUNT_KEY))
     return GoogleServiceAccountKey(**json.loads(creds_str))
 
 
 def upsert_gmail_service_account_key(
     service_account_key: GoogleServiceAccountKey,
 ) -> None:
-    get_dynamic_config_store().store(
+    get_kv_store().store(
         KV_GMAIL_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
     )
 
 
 def upsert_service_account_key(service_account_key: GoogleServiceAccountKey) -> None:
-    get_dynamic_config_store().store(
+    get_kv_store().store(
         KV_GMAIL_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
     )
 
 
 def delete_gmail_service_account_key() -> None:
-    get_dynamic_config_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
+    get_kv_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
 
 
 def delete_service_account_key() -> None:
-    get_dynamic_config_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
+    get_kv_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
diff --git a/backend/danswer/connectors/google_drive/connector_auth.py b/backend/danswer/connectors/google_drive/connector_auth.py
index cc68fec54ea..777deae990a 100644
--- a/backend/danswer/connectors/google_drive/connector_auth.py
+++ b/backend/danswer/connectors/google_drive/connector_auth.py
@@ -28,7 +28,7 @@
 from danswer.connectors.google_drive.constants import FETCH_PERMISSIONS_SCOPES
 from danswer.db.credentials import update_credential_json
 from danswer.db.models import User
-from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.key_value_store.factory import get_kv_store
 from danswer.server.documents.models import CredentialBase
 from danswer.server.documents.models import GoogleAppCredentials
 from danswer.server.documents.models import GoogleServiceAccountKey
@@ -134,7 +134,7 @@ def get_google_drive_creds(
 
 
 def verify_csrf(credential_id: int, state: str) -> None:
-    csrf = get_dynamic_config_store().load(KV_CRED_KEY.format(str(credential_id)))
+    csrf = get_kv_store().load(KV_CRED_KEY.format(str(credential_id)))
     if csrf != state:
         raise PermissionError(
             "State from Google Drive Connector callback does not match expected"
@@ -142,7 +142,7 @@ def verify_csrf(credential_id: int, state: str) -> None:
 
 
 def get_auth_url(credential_id: int) -> str:
-    creds_str = str(get_dynamic_config_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
+    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
     credential_json = json.loads(creds_str)
     flow = InstalledAppFlow.from_client_config(
         credential_json,
@@ -154,7 +154,7 @@ def get_auth_url(credential_id: int) -> str:
     parsed_url = cast(ParseResult, urlparse(auth_url))
     params = parse_qs(parsed_url.query)
 
-    get_dynamic_config_store().store(
+    get_kv_store().store(
         KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
     )  # type: ignore
     return str(auth_url)
@@ -202,32 +202,28 @@ def build_service_account_creds(
 
 
 def get_google_app_cred() -> GoogleAppCredentials:
-    creds_str = str(get_dynamic_config_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
+    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
     return GoogleAppCredentials(**json.loads(creds_str))
 
 
 def upsert_google_app_cred(app_credentials: GoogleAppCredentials) -> None:
-    get_dynamic_config_store().store(
-        KV_GOOGLE_DRIVE_CRED_KEY, app_credentials.json(), encrypt=True
-    )
+    get_kv_store().store(KV_GOOGLE_DRIVE_CRED_KEY, app_credentials.json(), encrypt=True)
 
 
 def delete_google_app_cred() -> None:
-    get_dynamic_config_store().delete(KV_GOOGLE_DRIVE_CRED_KEY)
+    get_kv_store().delete(KV_GOOGLE_DRIVE_CRED_KEY)
 
 
 def get_service_account_key() -> GoogleServiceAccountKey:
-    creds_str = str(
-        get_dynamic_config_store().load(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY)
-    )
+    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY))
     return GoogleServiceAccountKey(**json.loads(creds_str))
 
 
 def upsert_service_account_key(service_account_key: GoogleServiceAccountKey) -> None:
-    get_dynamic_config_store().store(
+    get_kv_store().store(
         KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
     )
 
 
 def delete_service_account_key() -> None:
-    get_dynamic_config_store().delete(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY)
+    get_kv_store().delete(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY)
diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index c430f1b31b7..553e9979189 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -49,7 +49,7 @@
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.search_settings import get_current_search_settings
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
 from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.one_shot_answer.models import ThreadMessage
@@ -522,7 +522,7 @@ def _initialize_socket_client(socket_client: SocketModeClient) -> None:
 
             # Let the handlers run in the background + re-check for token updates every 60 seconds
             Event().wait(timeout=60)
-        except ConfigNotFoundError:
+        except KvKeyNotFoundError:
             # try again every 30 seconds. This is needed since the user may add tokens
             # via the UI at any point in the programs lifecycle - if we just allow it to
             # fail, then the user will need to restart the containers after adding tokens
diff --git a/backend/danswer/danswerbot/slack/tokens.py b/backend/danswer/danswerbot/slack/tokens.py
index 5de3a6a0135..3f67e4649fc 100644
--- a/backend/danswer/danswerbot/slack/tokens.py
+++ b/backend/danswer/danswerbot/slack/tokens.py
@@ -2,7 +2,7 @@
 from typing import cast
 
 from danswer.configs.constants import KV_SLACK_BOT_TOKENS_CONFIG_KEY
-from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.key_value_store.factory import get_kv_store
 from danswer.server.manage.models import SlackBotTokens
 
 
@@ -13,7 +13,7 @@ def fetch_tokens() -> SlackBotTokens:
     if app_token and bot_token:
         return SlackBotTokens(app_token=app_token, bot_token=bot_token)
 
-    dynamic_config_store = get_dynamic_config_store()
+    dynamic_config_store = get_kv_store()
     return SlackBotTokens(
         **cast(dict, dynamic_config_store.load(key=KV_SLACK_BOT_TOKENS_CONFIG_KEY))
     )
@@ -22,7 +22,7 @@ def fetch_tokens() -> SlackBotTokens:
 def save_tokens(
     tokens: SlackBotTokens,
 ) -> None:
-    dynamic_config_store = get_dynamic_config_store()
+    dynamic_config_store = get_kv_store()
     dynamic_config_store.store(
         key=KV_SLACK_BOT_TOKENS_CONFIG_KEY, val=dict(tokens), encrypt=True
     )
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index fff6b12336d..f3d25e5e9d5 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -50,7 +50,7 @@
 from danswer.db.enums import IndexModelStatus
 from danswer.db.enums import TaskStatus
 from danswer.db.pydantic_type import PydanticType
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.interface import JSON_ro
 from danswer.file_store.models import FileDescriptor
 from danswer.llm.override_models import LLMOverride
 from danswer.llm.override_models import PromptOverride
diff --git a/backend/danswer/db/swap_index.py b/backend/danswer/db/swap_index.py
index 8f6d1718924..a11db4dd693 100644
--- a/backend/danswer/db/swap_index.py
+++ b/backend/danswer/db/swap_index.py
@@ -11,7 +11,7 @@
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.search_settings import get_secondary_search_settings
 from danswer.db.search_settings import update_search_settings_status
-from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.key_value_store.factory import get_kv_store
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -54,7 +54,7 @@ def check_index_swap(db_session: Session) -> None:
         )
 
         if cc_pair_count > 0:
-            kv_store = get_dynamic_config_store()
+            kv_store = get_kv_store()
             kv_store.store(KV_REINDEX_KEY, False)
 
             # Expire jobs for the now past index/embedding model
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 5d5d63d39eb..700f8860fb5 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -58,8 +58,8 @@
 from danswer.document_index.vespa_constants import VESPA_DIM_REPLACEMENT_PAT
 from danswer.document_index.vespa_constants import VESPA_TIMEOUT
 from danswer.document_index.vespa_constants import YQL_BASE
-from danswer.dynamic_configs.factory import get_dynamic_config_store
 from danswer.indexing.models import DocMetadataAwareIndexChunk
+from danswer.key_value_store.factory import get_kv_store
 from danswer.search.models import IndexFilters
 from danswer.search.models import InferenceChunkUncleaned
 from danswer.utils.batching import batch_generator
@@ -140,7 +140,7 @@ def ensure_indices_exist(
             SEARCH_THREAD_NUMBER_PAT, str(VESPA_SEARCHER_THREADS)
         )
 
-        kv_store = get_dynamic_config_store()
+        kv_store = get_kv_store()
 
         needs_reindexing = False
         try:
diff --git a/backend/danswer/dynamic_configs/factory.py b/backend/danswer/dynamic_configs/factory.py
deleted file mode 100644
index 44b6e096b6d..00000000000
--- a/backend/danswer/dynamic_configs/factory.py
+++ /dev/null
@@ -1,15 +0,0 @@
-from danswer.configs.app_configs import DYNAMIC_CONFIG_STORE
-from danswer.dynamic_configs.interface import DynamicConfigStore
-from danswer.dynamic_configs.store import FileSystemBackedDynamicConfigStore
-from danswer.dynamic_configs.store import PostgresBackedDynamicConfigStore
-
-
-def get_dynamic_config_store() -> DynamicConfigStore:
-    dynamic_config_store_type = DYNAMIC_CONFIG_STORE
-    if dynamic_config_store_type == FileSystemBackedDynamicConfigStore.__name__:
-        raise NotImplementedError("File based config store no longer supported")
-    if dynamic_config_store_type == PostgresBackedDynamicConfigStore.__name__:
-        return PostgresBackedDynamicConfigStore()
-
-    # TODO: change exception type
-    raise Exception("Unknown dynamic config store type")
diff --git a/backend/danswer/dynamic_configs/store.py b/backend/danswer/dynamic_configs/store.py
deleted file mode 100644
index cc53da938ad..00000000000
--- a/backend/danswer/dynamic_configs/store.py
+++ /dev/null
@@ -1,102 +0,0 @@
-import json
-import os
-from collections.abc import Iterator
-from contextlib import contextmanager
-from pathlib import Path
-from typing import cast
-
-from filelock import FileLock
-from sqlalchemy.orm import Session
-
-from danswer.db.engine import get_session_factory
-from danswer.db.models import KVStore
-from danswer.dynamic_configs.interface import ConfigNotFoundError
-from danswer.dynamic_configs.interface import DynamicConfigStore
-from danswer.dynamic_configs.interface import JSON_ro
-
-
-FILE_LOCK_TIMEOUT = 10
-
-
-def _get_file_lock(file_name: Path) -> FileLock:
-    return FileLock(file_name.with_suffix(".lock"))
-
-
-class FileSystemBackedDynamicConfigStore(DynamicConfigStore):
-    def __init__(self, dir_path: str) -> None:
-        # TODO (chris): maybe require all possible keys to be passed in
-        # at app start somehow to prevent key overlaps
-        self.dir_path = Path(dir_path)
-
-    def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
-        file_path = self.dir_path / key
-        lock = _get_file_lock(file_path)
-        with lock.acquire(timeout=FILE_LOCK_TIMEOUT):
-            with open(file_path, "w+") as f:
-                json.dump(val, f)
-
-    def load(self, key: str) -> JSON_ro:
-        file_path = self.dir_path / key
-        if not file_path.exists():
-            raise ConfigNotFoundError
-        lock = _get_file_lock(file_path)
-        with lock.acquire(timeout=FILE_LOCK_TIMEOUT):
-            with open(self.dir_path / key) as f:
-                return cast(JSON_ro, json.load(f))
-
-    def delete(self, key: str) -> None:
-        file_path = self.dir_path / key
-        if not file_path.exists():
-            raise ConfigNotFoundError
-        lock = _get_file_lock(file_path)
-        with lock.acquire(timeout=FILE_LOCK_TIMEOUT):
-            os.remove(file_path)
-
-
-class PostgresBackedDynamicConfigStore(DynamicConfigStore):
-    @contextmanager
-    def get_session(self) -> Iterator[Session]:
-        factory = get_session_factory()
-        session: Session = factory()
-        try:
-            yield session
-        finally:
-            session.close()
-
-    def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
-        # The actual encryption/decryption is done in Postgres, we just need to choose
-        # which field to set
-        encrypted_val = val if encrypt else None
-        plain_val = val if not encrypt else None
-        with self.get_session() as session:
-            obj = session.query(KVStore).filter_by(key=key).first()
-            if obj:
-                obj.value = plain_val
-                obj.encrypted_value = encrypted_val
-            else:
-                obj = KVStore(
-                    key=key, value=plain_val, encrypted_value=encrypted_val
-                )  # type: ignore
-                session.query(KVStore).filter_by(key=key).delete()  # just in case
-                session.add(obj)
-            session.commit()
-
-    def load(self, key: str) -> JSON_ro:
-        with self.get_session() as session:
-            obj = session.query(KVStore).filter_by(key=key).first()
-            if not obj:
-                raise ConfigNotFoundError
-
-            if obj.value is not None:
-                return cast(JSON_ro, obj.value)
-            if obj.encrypted_value is not None:
-                return cast(JSON_ro, obj.encrypted_value)
-
-            return None
-
-    def delete(self, key: str) -> None:
-        with self.get_session() as session:
-            result = session.query(KVStore).filter_by(key=key).delete()  # type: ignore
-            if result == 0:
-                raise ConfigNotFoundError
-            session.commit()
diff --git a/backend/danswer/file_processing/unstructured.py b/backend/danswer/file_processing/unstructured.py
index c5a14d87617..dc61869ee9c 100644
--- a/backend/danswer/file_processing/unstructured.py
+++ b/backend/danswer/file_processing/unstructured.py
@@ -8,8 +8,8 @@
 from unstructured_client.models import shared
 
 from danswer.configs.constants import KV_UNSTRUCTURED_API_KEY
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.utils.logger import setup_logger
 
 
@@ -17,20 +17,20 @@
 
 
 def get_unstructured_api_key() -> str | None:
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     try:
         return cast(str, kv_store.load(KV_UNSTRUCTURED_API_KEY))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         return None
 
 
 def update_unstructured_api_key(api_key: str) -> None:
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     kv_store.store(KV_UNSTRUCTURED_API_KEY, api_key)
 
 
 def delete_unstructured_api_key() -> None:
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     kv_store.delete(KV_UNSTRUCTURED_API_KEY)
 
 
diff --git a/backend/danswer/dynamic_configs/__init__.py b/backend/danswer/key_value_store/__init__.py
similarity index 100%
rename from backend/danswer/dynamic_configs/__init__.py
rename to backend/danswer/key_value_store/__init__.py
diff --git a/backend/danswer/key_value_store/factory.py b/backend/danswer/key_value_store/factory.py
new file mode 100644
index 00000000000..7b52fdabef8
--- /dev/null
+++ b/backend/danswer/key_value_store/factory.py
@@ -0,0 +1,7 @@
+from danswer.key_value_store.interface import KeyValueStore
+from danswer.key_value_store.store import PgRedisKVStore
+
+
+def get_kv_store() -> KeyValueStore:
+    # this is the only one supported currently
+    return PgRedisKVStore()
diff --git a/backend/danswer/dynamic_configs/interface.py b/backend/danswer/key_value_store/interface.py
similarity index 89%
rename from backend/danswer/dynamic_configs/interface.py
rename to backend/danswer/key_value_store/interface.py
index 999ad939615..190c53189dd 100644
--- a/backend/danswer/dynamic_configs/interface.py
+++ b/backend/danswer/key_value_store/interface.py
@@ -9,11 +9,11 @@
 )
 
 
-class ConfigNotFoundError(Exception):
+class KvKeyNotFoundError(Exception):
     pass
 
 
-class DynamicConfigStore:
+class KeyValueStore:
     @abc.abstractmethod
     def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
         raise NotImplementedError
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
new file mode 100644
index 00000000000..450056c40b1
--- /dev/null
+++ b/backend/danswer/key_value_store/store.py
@@ -0,0 +1,99 @@
+import json
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import cast
+
+from sqlalchemy.orm import Session
+
+from danswer.db.engine import get_session_factory
+from danswer.db.models import KVStore
+from danswer.key_value_store.interface import JSON_ro
+from danswer.key_value_store.interface import KeyValueStore
+from danswer.key_value_store.interface import KvKeyNotFoundError
+from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+REDIS_KEY_PREFIX = "danswer_kv_store:"
+KV_REDIS_KEY_EXPIRATION = 60 * 60 * 24  # 1 Day
+
+
+class PgRedisKVStore(KeyValueStore):
+    def __init__(self) -> None:
+        self.redis_client = get_redis_client()
+
+    @contextmanager
+    def get_session(self) -> Iterator[Session]:
+        factory = get_session_factory()
+        session: Session = factory()
+        try:
+            yield session
+        finally:
+            session.close()
+
+    def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
+        # Not encrypted in Redis, but encrypted in Postgres
+        try:
+            self.redis_client.set(
+                REDIS_KEY_PREFIX + key, json.dumps(val), ex=KV_REDIS_KEY_EXPIRATION
+            )
+        except Exception as e:
+            # Fallback gracefully to Postgres if Redis fails
+            logger.error(f"Failed to set value in Redis for key '{key}': {str(e)}")
+
+        encrypted_val = val if encrypt else None
+        plain_val = val if not encrypt else None
+        with self.get_session() as session:
+            obj = session.query(KVStore).filter_by(key=key).first()
+            if obj:
+                obj.value = plain_val
+                obj.encrypted_value = encrypted_val
+            else:
+                obj = KVStore(
+                    key=key, value=plain_val, encrypted_value=encrypted_val
+                )  # type: ignore
+                session.query(KVStore).filter_by(key=key).delete()  # just in case
+                session.add(obj)
+            session.commit()
+
+    def load(self, key: str) -> JSON_ro:
+        try:
+            redis_value = self.redis_client.get(REDIS_KEY_PREFIX + key)
+            if redis_value:
+                assert isinstance(redis_value, bytes)
+                return json.loads(redis_value.decode("utf-8"))
+        except Exception as e:
+            logger.error(f"Failed to get value from Redis for key '{key}': {str(e)}")
+
+        with self.get_session() as session:
+            obj = session.query(KVStore).filter_by(key=key).first()
+            if not obj:
+                raise KvKeyNotFoundError
+
+            if obj.value is not None:
+                value = obj.value
+            elif obj.encrypted_value is not None:
+                value = obj.encrypted_value
+            else:
+                value = None
+
+            try:
+                self.redis_client.set(REDIS_KEY_PREFIX + key, json.dumps(value))
+            except Exception as e:
+                logger.error(f"Failed to set value in Redis for key '{key}': {str(e)}")
+
+            return cast(JSON_ro, value)
+
+    def delete(self, key: str) -> None:
+        try:
+            self.redis_client.delete(REDIS_KEY_PREFIX + key)
+        except Exception as e:
+            logger.error(f"Failed to delete value from Redis for key '{key}': {str(e)}")
+
+        with self.get_session() as session:
+            result = session.query(KVStore).filter_by(key=key).delete()  # type: ignore
+            if result == 0:
+                raise KvKeyNotFoundError
+            session.commit()
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index a5abb8f28c2..de150cd5823 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -65,9 +65,9 @@
 from danswer.db.swap_index import check_index_swap
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import DocumentIndex
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
 from danswer.indexing.models import IndexingSetting
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
 from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder
@@ -256,7 +256,7 @@ def update_default_multipass_indexing(db_session: Session) -> None:
 
 
 def translate_saved_search_settings(db_session: Session) -> None:
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
 
     try:
         search_settings_dict = kv_store.load(KV_SEARCH_SETTINGS)
@@ -294,17 +294,17 @@ def translate_saved_search_settings(db_session: Session) -> None:
             logger.notice("Search settings updated and KV store entry deleted.")
         else:
             logger.notice("KV store search settings is empty.")
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         logger.notice("No search config found in KV store.")
 
 
 def mark_reindex_flag(db_session: Session) -> None:
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     try:
         value = kv_store.load(KV_REINDEX_KEY)
         logger.debug(f"Re-indexing flag has value {value}")
         return
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         # Only need to update the flag if it hasn't been set
         pass
 
diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py
index 9664019eb6d..233a51a849f 100644
--- a/backend/danswer/redis/redis_pool.py
+++ b/backend/danswer/redis/redis_pool.py
@@ -74,6 +74,13 @@ def create_pool(
         )
 
 
+redis_pool = RedisPool()
+
+
+def get_redis_client() -> Redis:
+    return redis_pool.get_client()
+
+
 # # Usage example
 # redis_pool = RedisPool()
 # redis_client = redis_pool.get_client()
diff --git a/backend/danswer/search/search_settings.py b/backend/danswer/search/search_settings.py
index d502205dfe7..f5870de83f1 100644
--- a/backend/danswer/search/search_settings.py
+++ b/backend/danswer/search/search_settings.py
@@ -1,8 +1,8 @@
 from typing import cast
 
 from danswer.configs.constants import KV_SEARCH_SETTINGS
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.search.models import SavedSearchSettings
 from danswer.utils.logger import setup_logger
 
@@ -17,10 +17,10 @@ def get_kv_search_settings() -> SavedSearchSettings | None:
     if the value is updated by another process/instance of the API server. If this reads from an in memory cache like
     reddis then it will be ok. Until then this has some performance implications (though minor)
     """
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     try:
         return SavedSearchSettings(**cast(dict, kv_store.load(KV_SEARCH_SETTINGS)))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         return None
     except Exception as e:
         logger.error(f"Error loading search settings: {e}")
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index 58dcf7e7691..c6076dee744 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -74,8 +74,8 @@
 from danswer.db.models import User
 from danswer.db.models import UserRole
 from danswer.db.search_settings import get_current_search_settings
-from danswer.dynamic_configs.interface import ConfigNotFoundError
 from danswer.file_store.file_store import get_default_file_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.server.documents.models import AuthStatus
 from danswer.server.documents.models import AuthUrl
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
@@ -116,7 +116,7 @@ def check_google_app_gmail_credentials_exist(
 ) -> dict[str, str]:
     try:
         return {"client_id": get_google_app_gmail_cred().web.client_id}
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         raise HTTPException(status_code=404, detail="Google App Credentials not found")
 
 
@@ -140,7 +140,7 @@ def delete_google_app_gmail_credentials(
 ) -> StatusResponse:
     try:
         delete_google_app_gmail_cred()
-    except ConfigNotFoundError as e:
+    except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
     return StatusResponse(
@@ -154,7 +154,7 @@ def check_google_app_credentials_exist(
 ) -> dict[str, str]:
     try:
         return {"client_id": get_google_app_cred().web.client_id}
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         raise HTTPException(status_code=404, detail="Google App Credentials not found")
 
 
@@ -178,7 +178,7 @@ def delete_google_app_credentials(
 ) -> StatusResponse:
     try:
         delete_google_app_cred()
-    except ConfigNotFoundError as e:
+    except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
     return StatusResponse(
@@ -192,7 +192,7 @@ def check_google_service_gmail_account_key_exist(
 ) -> dict[str, str]:
     try:
         return {"service_account_email": get_gmail_service_account_key().client_email}
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         raise HTTPException(
             status_code=404, detail="Google Service Account Key not found"
         )
@@ -218,7 +218,7 @@ def delete_google_service_gmail_account_key(
 ) -> StatusResponse:
     try:
         delete_gmail_service_account_key()
-    except ConfigNotFoundError as e:
+    except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
     return StatusResponse(
@@ -232,7 +232,7 @@ def check_google_service_account_key_exist(
 ) -> dict[str, str]:
     try:
         return {"service_account_email": get_service_account_key().client_email}
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         raise HTTPException(
             status_code=404, detail="Google Service Account Key not found"
         )
@@ -258,7 +258,7 @@ def delete_google_service_account_key(
 ) -> StatusResponse:
     try:
         delete_service_account_key()
-    except ConfigNotFoundError as e:
+    except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
     return StatusResponse(
@@ -280,7 +280,7 @@ def upsert_service_account_credential(
             DocumentSource.GOOGLE_DRIVE,
             delegated_user_email=service_account_credential_request.google_drive_delegated_user,
         )
-    except ConfigNotFoundError as e:
+    except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
     # first delete all existing service account credentials
@@ -306,7 +306,7 @@ def upsert_gmail_service_account_credential(
             DocumentSource.GMAIL,
             delegated_user_email=service_account_credential_request.gmail_delegated_user,
         )
-    except ConfigNotFoundError as e:
+    except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
     # first delete all existing service account credentials
diff --git a/backend/danswer/server/manage/administrative.py b/backend/danswer/server/manage/administrative.py
index 1ebe5bd0691..9c87e60b18a 100644
--- a/backend/danswer/server/manage/administrative.py
+++ b/backend/danswer/server/manage/administrative.py
@@ -29,9 +29,9 @@
 from danswer.db.models import User
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
 from danswer.file_store.file_store import get_default_file_store
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.llm.factory import get_default_llms
 from danswer.llm.utils import test_llm
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
@@ -114,7 +114,7 @@ def validate_existing_genai_api_key(
     _: User = Depends(current_admin_user),
 ) -> None:
     # Only validate every so often
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     curr_time = datetime.now(tz=timezone.utc)
     try:
         last_check = datetime.fromtimestamp(
@@ -123,7 +123,7 @@ def validate_existing_genai_api_key(
         check_freq_sec = timedelta(seconds=GENERATIVE_MODEL_ACCESS_CHECK_FREQ)
         if curr_time - last_check < check_freq_sec:
             return
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         # First time checking the key, nothing unusual
         pass
 
diff --git a/backend/danswer/server/manage/slack_bot.py b/backend/danswer/server/manage/slack_bot.py
index 9a06b225cce..abee8b8644e 100644
--- a/backend/danswer/server/manage/slack_bot.py
+++ b/backend/danswer/server/manage/slack_bot.py
@@ -18,7 +18,7 @@
 from danswer.db.slack_bot_config import insert_slack_bot_config
 from danswer.db.slack_bot_config import remove_slack_bot_config
 from danswer.db.slack_bot_config import update_slack_bot_config
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.server.manage.models import SlackBotConfig
 from danswer.server.manage.models import SlackBotConfigCreationRequest
 from danswer.server.manage.models import SlackBotTokens
@@ -212,5 +212,5 @@ def put_tokens(
 def get_tokens(_: User | None = Depends(current_admin_user)) -> SlackBotTokens:
     try:
         return fetch_tokens()
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         raise HTTPException(status_code=404, detail="No tokens found")
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index e72b85dedad..2a43542460a 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -38,7 +38,7 @@
 from danswer.db.models import User__UserGroup
 from danswer.db.users import get_user_by_email
 from danswer.db.users import list_users
-from danswer.dynamic_configs.factory import get_dynamic_config_store
+from danswer.key_value_store.factory import get_kv_store
 from danswer.server.manage.models import AllUsersResponse
 from danswer.server.manage.models import UserByEmail
 from danswer.server.manage.models import UserInfo
@@ -367,7 +367,7 @@ def verify_user_logged_in(
         # if auth type is disabled, return a dummy user with preferences from
         # the key-value store
         if AUTH_TYPE == AuthType.DISABLED:
-            store = get_dynamic_config_store()
+            store = get_kv_store()
             return fetch_no_auth_user(store)
 
         raise HTTPException(
@@ -405,7 +405,7 @@ def update_user_default_model(
 ) -> None:
     if user is None:
         if AUTH_TYPE == AuthType.DISABLED:
-            store = get_dynamic_config_store()
+            store = get_kv_store()
             no_auth_user = fetch_no_auth_user(store)
             no_auth_user.preferences.default_model = request.default_model
             set_no_auth_user_preferences(store, no_auth_user.preferences)
@@ -433,7 +433,7 @@ def update_user_assistant_list(
 ) -> None:
     if user is None:
         if AUTH_TYPE == AuthType.DISABLED:
-            store = get_dynamic_config_store()
+            store = get_kv_store()
 
             no_auth_user = fetch_no_auth_user(store)
             no_auth_user.preferences.chosen_assistants = request.chosen_assistants
@@ -487,7 +487,7 @@ def update_user_assistant_visibility(
 ) -> None:
     if user is None:
         if AUTH_TYPE == AuthType.DISABLED:
-            store = get_dynamic_config_store()
+            store = get_kv_store()
             no_auth_user = fetch_no_auth_user(store)
             preferences = no_auth_user.preferences
             updated_preferences = update_assistant_list(preferences, assistant_id, show)
diff --git a/backend/danswer/server/settings/api.py b/backend/danswer/server/settings/api.py
index 5b8564c3d3a..48157253c9a 100644
--- a/backend/danswer/server/settings/api.py
+++ b/backend/danswer/server/settings/api.py
@@ -19,8 +19,8 @@
 from danswer.db.notification import get_notification_by_id
 from danswer.db.notification import get_notifications
 from danswer.db.notification import update_notification_last_shown
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.server.settings.models import Notification
 from danswer.server.settings.models import Settings
 from danswer.server.settings.models import UserSettings
@@ -58,9 +58,9 @@ def fetch_settings(
     user_notifications = get_user_notifications(user, db_session)
 
     try:
-        kv_store = get_dynamic_config_store()
+        kv_store = get_kv_store()
         needs_reindexing = cast(bool, kv_store.load(KV_REINDEX_KEY))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         needs_reindexing = False
 
     return UserSettings(
@@ -97,7 +97,7 @@ def get_user_notifications(
         # Reindexing flag should only be shown to admins, basic users can't trigger it anyway
         return []
 
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
     try:
         needs_index = cast(bool, kv_store.load(KV_REINDEX_KEY))
         if not needs_index:
@@ -105,7 +105,7 @@ def get_user_notifications(
                 notif_type=NotificationType.REINDEX, db_session=db_session
             )
             return []
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         # If something goes wrong and the flag is gone, better to not start a reindexing
         # it's a heavyweight long running job and maybe this flag is cleaned up later
         logger.warning("Could not find reindex flag")
diff --git a/backend/danswer/server/settings/store.py b/backend/danswer/server/settings/store.py
index 6f2872f40f9..c3875c6aecb 100644
--- a/backend/danswer/server/settings/store.py
+++ b/backend/danswer/server/settings/store.py
@@ -1,16 +1,16 @@
 from typing import cast
 
 from danswer.configs.constants import KV_SETTINGS_KEY
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.server.settings.models import Settings
 
 
 def load_settings() -> Settings:
-    dynamic_config_store = get_dynamic_config_store()
+    dynamic_config_store = get_kv_store()
     try:
         settings = Settings(**cast(dict, dynamic_config_store.load(KV_SETTINGS_KEY)))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         settings = Settings()
         dynamic_config_store.store(KV_SETTINGS_KEY, settings.model_dump())
 
@@ -18,4 +18,4 @@ def load_settings() -> Settings:
 
 
 def store_settings(settings: Settings) -> None:
-    get_dynamic_config_store().store(KV_SETTINGS_KEY, settings.model_dump())
+    get_kv_store().store(KV_SETTINGS_KEY, settings.model_dump())
diff --git a/backend/danswer/tools/custom/custom_tool.py b/backend/danswer/tools/custom/custom_tool.py
index 3d36d7bb055..85830f1ca30 100644
--- a/backend/danswer/tools/custom/custom_tool.py
+++ b/backend/danswer/tools/custom/custom_tool.py
@@ -8,7 +8,7 @@
 from langchain_core.messages import SystemMessage
 from pydantic import BaseModel
 
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
 from danswer.tools.custom.base_tool_types import ToolResultType
diff --git a/backend/danswer/tools/images/image_generation_tool.py b/backend/danswer/tools/images/image_generation_tool.py
index 6e2515a8e9f..3c1fa75c742 100644
--- a/backend/danswer/tools/images/image_generation_tool.py
+++ b/backend/danswer/tools/images/image_generation_tool.py
@@ -9,7 +9,7 @@
 
 from danswer.chat.chat_utils import combine_message_chain
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.headers import build_llm_extra_headers
 from danswer.llm.interfaces import LLM
diff --git a/backend/danswer/tools/internet_search/internet_search_tool.py b/backend/danswer/tools/internet_search/internet_search_tool.py
index 3012eb465f4..70b4483b996 100644
--- a/backend/danswer/tools/internet_search/internet_search_tool.py
+++ b/backend/danswer/tools/internet_search/internet_search_tool.py
@@ -10,7 +10,7 @@
 from danswer.chat.models import LlmDoc
 from danswer.configs.constants import DocumentSource
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
 from danswer.llm.utils import message_to_string
diff --git a/backend/danswer/tools/search/search_tool.py b/backend/danswer/tools/search/search_tool.py
index cbfaf4f3d92..96ab7b843f6 100644
--- a/backend/danswer/tools/search/search_tool.py
+++ b/backend/danswer/tools/search/search_tool.py
@@ -16,7 +16,7 @@
 from danswer.configs.model_configs import GEN_AI_MODEL_FALLBACK_MAX_TOKENS
 from danswer.db.models import Persona
 from danswer.db.models import User
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import ContextualPruningConfig
 from danswer.llm.answering.models import DocumentPruningConfig
 from danswer.llm.answering.models import PreviousMessage
diff --git a/backend/danswer/tools/tool.py b/backend/danswer/tools/tool.py
index 81b9b457178..29e5311fc15 100644
--- a/backend/danswer/tools/tool.py
+++ b/backend/danswer/tools/tool.py
@@ -2,7 +2,7 @@
 from collections.abc import Generator
 from typing import Any
 
-from danswer.dynamic_configs.interface import JSON_ro
+from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
 from danswer.tools.models import ToolResponse
diff --git a/backend/danswer/utils/telemetry.py b/backend/danswer/utils/telemetry.py
index d8a021877e6..f5fb23ef86f 100644
--- a/backend/danswer/utils/telemetry.py
+++ b/backend/danswer/utils/telemetry.py
@@ -12,8 +12,8 @@
 from danswer.configs.constants import KV_INSTANCE_DOMAIN_KEY
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.models import User
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 
 _DANSWER_TELEMETRY_ENDPOINT = "https://telemetry.danswer.ai/anonymous_telemetry"
 _CACHED_UUID: str | None = None
@@ -34,11 +34,11 @@ def get_or_generate_uuid() -> str:
     if _CACHED_UUID is not None:
         return _CACHED_UUID
 
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
 
     try:
         _CACHED_UUID = cast(str, kv_store.load(KV_CUSTOMER_UUID_KEY))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         _CACHED_UUID = str(uuid.uuid4())
         kv_store.store(KV_CUSTOMER_UUID_KEY, _CACHED_UUID, encrypt=True)
 
@@ -51,11 +51,11 @@ def _get_or_generate_instance_domain() -> str | None:
     if _CACHED_INSTANCE_DOMAIN is not None:
         return _CACHED_INSTANCE_DOMAIN
 
-    kv_store = get_dynamic_config_store()
+    kv_store = get_kv_store()
 
     try:
         _CACHED_INSTANCE_DOMAIN = cast(str, kv_store.load(KV_INSTANCE_DOMAIN_KEY))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         with Session(get_sqlalchemy_engine()) as db_session:
             first_user = db_session.query(User).first()
             if first_user:
diff --git a/backend/ee/danswer/server/enterprise_settings/store.py b/backend/ee/danswer/server/enterprise_settings/store.py
index 30b72d5d2e8..74706e0f769 100644
--- a/backend/ee/danswer/server/enterprise_settings/store.py
+++ b/backend/ee/danswer/server/enterprise_settings/store.py
@@ -11,9 +11,9 @@
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import KV_CUSTOM_ANALYTICS_SCRIPT_KEY
 from danswer.configs.constants import KV_ENTERPRISE_SETTINGS_KEY
-from danswer.dynamic_configs.factory import get_dynamic_config_store
-from danswer.dynamic_configs.interface import ConfigNotFoundError
 from danswer.file_store.file_store import get_default_file_store
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.utils.logger import setup_logger
 from ee.danswer.server.enterprise_settings.models import AnalyticsScriptUpload
 from ee.danswer.server.enterprise_settings.models import EnterpriseSettings
@@ -23,12 +23,12 @@
 
 
 def load_settings() -> EnterpriseSettings:
-    dynamic_config_store = get_dynamic_config_store()
+    dynamic_config_store = get_kv_store()
     try:
         settings = EnterpriseSettings(
             **cast(dict, dynamic_config_store.load(KV_ENTERPRISE_SETTINGS_KEY))
         )
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         settings = EnterpriseSettings()
         dynamic_config_store.store(KV_ENTERPRISE_SETTINGS_KEY, settings.model_dump())
 
@@ -36,17 +36,17 @@ def load_settings() -> EnterpriseSettings:
 
 
 def store_settings(settings: EnterpriseSettings) -> None:
-    get_dynamic_config_store().store(KV_ENTERPRISE_SETTINGS_KEY, settings.model_dump())
+    get_kv_store().store(KV_ENTERPRISE_SETTINGS_KEY, settings.model_dump())
 
 
 _CUSTOM_ANALYTICS_SECRET_KEY = os.environ.get("CUSTOM_ANALYTICS_SECRET_KEY")
 
 
 def load_analytics_script() -> str | None:
-    dynamic_config_store = get_dynamic_config_store()
+    dynamic_config_store = get_kv_store()
     try:
         return cast(str, dynamic_config_store.load(KV_CUSTOM_ANALYTICS_SCRIPT_KEY))
-    except ConfigNotFoundError:
+    except KvKeyNotFoundError:
         return None
 
 
@@ -57,9 +57,7 @@ def store_analytics_script(analytics_script_upload: AnalyticsScriptUpload) -> No
     ):
         raise ValueError("Invalid secret key")
 
-    get_dynamic_config_store().store(
-        KV_CUSTOM_ANALYTICS_SCRIPT_KEY, analytics_script_upload.script
-    )
+    get_kv_store().store(KV_CUSTOM_ANALYTICS_SCRIPT_KEY, analytics_script_upload.script)
 
 
 _LOGO_FILENAME = "__logo__"
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index a9cf3650e13..993b46e2c30 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -4,6 +4,10 @@ mypy_path = "$MYPY_CONFIG_FILE_DIR"
 explicit_package_bases = true
 disallow_untyped_defs = true
 
+[[tool.mypy.overrides]]
+module = "alembic.versions.*"
+disable_error_code = ["var-annotated"]
+
 [tool.ruff]
 ignore = []
 line-length = 130

From b8232e06814288b3b67b593b1df1f8f7be0581eb Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Tue, 1 Oct 2024 20:09:57 -0700
Subject: [PATCH 012/376] Update litellm to fix bedrock models (#2649)

---
 .github/workflows/pr-python-model-tests.yml | 58 +++++++++++++++
 backend/danswer/llm/chat_llm.py             | 10 ++-
 backend/requirements/default.txt            |  2 +-
 backend/tests/daily/conftest.py             | 24 ++++++
 backend/tests/daily/llm/test_bedrock.py     | 81 +++++++++++++++++++++
 5 files changed, 170 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/pr-python-model-tests.yml
 create mode 100644 backend/tests/daily/conftest.py
 create mode 100644 backend/tests/daily/llm/test_bedrock.py

diff --git a/.github/workflows/pr-python-model-tests.yml b/.github/workflows/pr-python-model-tests.yml
new file mode 100644
index 00000000000..f55178281a4
--- /dev/null
+++ b/.github/workflows/pr-python-model-tests.yml
@@ -0,0 +1,58 @@
+name: Connector Tests
+
+on:
+  schedule:
+    # This cron expression runs the job daily at 16:00 UTC (9am PT)
+    - cron: "0 16 * * *"
+
+env:
+  # Bedrock
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION_NAME: ${{ secrets.AWS_REGION_NAME }}
+
+  # OpenAI
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+jobs:
+  connectors-check:
+    # See https://runs-on.com/runners/linux/
+    runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
+
+    env:
+      PYTHONPATH: ./backend
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: "pip"
+          cache-dependency-path: |
+            backend/requirements/default.txt
+            backend/requirements/dev.txt
+
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+
+      - name: Run Tests
+        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
+        run: |
+          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/llm
+          py.test -o junit_family=xunit2 -xv --ff backend/tests/daily/embedding
+
+      - name: Alert on Failure
+        if: failure() && github.event_name == 'schedule'
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          curl -X POST \
+            -H 'Content-type: application/json' \
+            --data '{"text":"Scheduled Model Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
+            $SLACK_WEBHOOK
diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
index 08131f581a4..274e0006cd8 100644
--- a/backend/danswer/llm/chat_llm.py
+++ b/backend/danswer/llm/chat_llm.py
@@ -290,10 +290,12 @@ def _completion(
             return litellm.completion(
                 # model choice
                 model=f"{self.config.model_provider}/{self.config.model_name}",
-                api_key=self._api_key,
-                base_url=self._api_base,
-                api_version=self._api_version,
-                custom_llm_provider=self._custom_llm_provider,
+                # NOTE: have to pass in None instead of empty string for these
+                # otherwise litellm can have some issues with bedrock
+                api_key=self._api_key or None,
+                base_url=self._api_base or None,
+                api_version=self._api_version or None,
+                custom_llm_provider=self._custom_llm_provider or None,
                 # actual input
                 messages=prompt,
                 tools=tools,
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 9855b8662ff..30e0e300bb2 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -28,7 +28,7 @@ jsonref==1.1.0
 langchain==0.1.17
 langchain-core==0.1.50
 langchain-text-splitters==0.0.1
-litellm==1.47.1
+litellm==1.48.7
 llama-index==0.9.45
 Mako==1.2.4
 msal==1.28.0
diff --git a/backend/tests/daily/conftest.py b/backend/tests/daily/conftest.py
new file mode 100644
index 00000000000..88a74c7b4ce
--- /dev/null
+++ b/backend/tests/daily/conftest.py
@@ -0,0 +1,24 @@
+import os
+from collections.abc import Generator
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+
+from danswer.main import fetch_versioned_implementation
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+@pytest.fixture(scope="function")
+def client() -> Generator[TestClient, Any, None]:
+    # Set environment variables
+    os.environ["ENABLE_PAID_ENTERPRISE_EDITION_FEATURES"] = "True"
+
+    # Initialize TestClient with the FastAPI app
+    app = fetch_versioned_implementation(
+        module="danswer.main", attribute="get_application"
+    )()
+    client = TestClient(app)
+    yield client
diff --git a/backend/tests/daily/llm/test_bedrock.py b/backend/tests/daily/llm/test_bedrock.py
new file mode 100644
index 00000000000..1d5022abf99
--- /dev/null
+++ b/backend/tests/daily/llm/test_bedrock.py
@@ -0,0 +1,81 @@
+import os
+from typing import Any
+
+import pytest
+from fastapi.testclient import TestClient
+
+from danswer.llm.llm_provider_options import BEDROCK_PROVIDER_NAME
+from danswer.llm.llm_provider_options import fetch_available_well_known_llms
+from danswer.llm.llm_provider_options import WellKnownLLMProviderDescriptor
+
+
+@pytest.fixture
+def bedrock_provider() -> WellKnownLLMProviderDescriptor:
+    provider = next(
+        (
+            provider
+            for provider in fetch_available_well_known_llms()
+            if provider.name == BEDROCK_PROVIDER_NAME
+        ),
+        None,
+    )
+    assert provider is not None, "Bedrock provider not found"
+    return provider
+
+
+def test_bedrock_llm_configuration(
+    client: TestClient, bedrock_provider: WellKnownLLMProviderDescriptor
+) -> None:
+    # Prepare the test request payload
+    test_request: dict[str, Any] = {
+        "provider": BEDROCK_PROVIDER_NAME,
+        "default_model_name": bedrock_provider.default_model,
+        "fast_default_model_name": bedrock_provider.default_fast_model,
+        "api_key": None,
+        "api_base": None,
+        "api_version": None,
+        "custom_config": {
+            "AWS_REGION_NAME": os.environ.get("AWS_REGION_NAME", "us-east-1"),
+            "AWS_ACCESS_KEY_ID": os.environ.get("AWS_ACCESS_KEY_ID"),
+            "AWS_SECRET_ACCESS_KEY": os.environ.get("AWS_SECRET_ACCESS_KEY"),
+        },
+    }
+
+    # Send the test request
+    response = client.post("/admin/llm/test", json=test_request)
+
+    # Assert the response
+    assert (
+        response.status_code == 200
+    ), f"Expected status code 200, but got {response.status_code}. Response: {response.text}"
+
+
+def test_bedrock_llm_configuration_invalid_key(
+    client: TestClient, bedrock_provider: WellKnownLLMProviderDescriptor
+) -> None:
+    # Prepare the test request payload with invalid credentials
+    test_request: dict[str, Any] = {
+        "provider": BEDROCK_PROVIDER_NAME,
+        "default_model_name": bedrock_provider.default_model,
+        "fast_default_model_name": bedrock_provider.default_fast_model,
+        "api_key": None,
+        "api_base": None,
+        "api_version": None,
+        "custom_config": {
+            "AWS_REGION_NAME": "us-east-1",
+            "AWS_ACCESS_KEY_ID": "invalid_access_key_id",
+            "AWS_SECRET_ACCESS_KEY": "invalid_secret_access_key",
+        },
+    }
+
+    # Send the test request
+    response = client.post("/admin/llm/test", json=test_request)
+
+    # Assert the response
+    assert (
+        response.status_code == 400
+    ), f"Expected status code 400, but got {response.status_code}. Response: {response.text}"
+    assert (
+        "Invalid credentials" in response.text
+        or "Invalid Authentication" in response.text
+    ), f"Expected error message about invalid credentials, but got: {response.text}"

From bd40328a73d93ea3d5b0d02900ebbbe64b256d15 Mon Sep 17 00:00:00 2001
From: Evan Lohn <evan@danswer.ai>
Date: Tue, 1 Oct 2024 17:21:39 -0400
Subject: [PATCH 013/376] fix typo

---
 web/src/lib/connectors/connectors.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/lib/connectors/connectors.ts b/web/src/lib/connectors/connectors.ts
index 19db22d5180..61ae2b076cf 100644
--- a/web/src/lib/connectors/connectors.ts
+++ b/web/src/lib/connectors/connectors.ts
@@ -616,7 +616,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
     ],
   },
   linear: {
-    description: "Configure Dropbox connector",
+    description: "Configure Linear connector",
     values: [],
   },
   dropbox: {

From 07aeea69e7a02caa82f31705d81a60c9f115e327 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 1 Oct 2024 20:11:39 -0700
Subject: [PATCH 014/376] Dupe welcome modal logic (#2656)

---
 web/src/app/prompts/page.tsx |  2 --
 web/src/app/search/page.tsx  | 24 +++++++++---------------
 2 files changed, 9 insertions(+), 17 deletions(-)

diff --git a/web/src/app/prompts/page.tsx b/web/src/app/prompts/page.tsx
index 8bf104d5b7f..c3933e8ab10 100644
--- a/web/src/app/prompts/page.tsx
+++ b/web/src/app/prompts/page.tsx
@@ -1,4 +1,3 @@
-import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 import { fetchChatData } from "@/lib/chat/fetchChatData";
 import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
@@ -23,7 +22,6 @@ export default async function GalleryPage({
     assistants,
     folders,
     openedFolders,
-    shouldShowWelcomeModal,
     toggleSidebar,
   } = data;
 
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index 72f52a40994..c490e1971cf 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -34,7 +34,8 @@ import {
 } from "@/lib/constants";
 import WrappedSearch from "./WrappedSearch";
 import { SearchProvider } from "@/components/context/SearchContext";
-import { ProviderContextProvider } from "@/components/chat_search/ProviderContext";
+import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
+import { LLMProviderDescriptor } from "../admin/configuration/llm/interfaces";
 
 export default async function Home() {
   // Disable caching so we always get the up to date connector / document set / persona info
@@ -49,8 +50,8 @@ export default async function Home() {
     fetchSS("/manage/document-set"),
     fetchAssistantsSS(),
     fetchSS("/query/valid-tags"),
-    fetchSS("/search-settings/get-all-search-settings"),
     fetchSS("/query/user-searches"),
+    fetchLLMProvidersSS(),
   ];
 
   // catch cases where the backend is completely unreachable here
@@ -62,8 +63,9 @@ export default async function Home() {
     | AuthTypeMetadata
     | FullEmbeddingModelResponse
     | FetchAssistantsResponse
+    | LLMProviderDescriptor[]
     | null
-  )[] = [null, null, null, null, null, null];
+  )[] = [null, null, null, null, null, null, null, null];
   try {
     results = await Promise.all(tasks);
   } catch (e) {
@@ -76,8 +78,8 @@ export default async function Home() {
   const [initialAssistantsList, assistantsFetchError] =
     results[4] as FetchAssistantsResponse;
   const tagsResponse = results[5] as Response | null;
-  const embeddingModelResponse = results[6] as Response | null;
-  const queryResponse = results[7] as Response | null;
+  const queryResponse = results[6] as Response | null;
+  const llmProviders = (results[7] || []) as LLMProviderDescriptor[];
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
@@ -130,16 +132,6 @@ export default async function Home() {
     console.log(`Failed to fetch tags - ${tagsResponse?.status}`);
   }
 
-  const embeddingModelVersionInfo =
-    embeddingModelResponse && embeddingModelResponse.ok
-      ? ((await embeddingModelResponse.json()) as FullEmbeddingModelResponse)
-      : null;
-
-  const currentEmbeddingModelName =
-    embeddingModelVersionInfo?.current_model_name;
-  const nextEmbeddingModelName =
-    embeddingModelVersionInfo?.secondary_model_name;
-
   // needs to be done in a non-client side component due to nextjs
   const storedSearchType = cookies().get("searchType")?.value as
     | string
@@ -151,7 +143,9 @@ export default async function Home() {
       : SearchType.SEMANTIC; // default to semantic
 
   const hasAnyConnectors = ccPairs.length > 0;
+
   const shouldShowWelcomeModal =
+    !llmProviders.length &&
     !hasCompletedWelcomeFlowSS() &&
     !hasAnyConnectors &&
     (!user || user.role === "admin");

From a30de693cb6949c97405e550c1d1acd9d93c4da2 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 2 Oct 2024 09:15:54 -0700
Subject: [PATCH 015/376] Clean, memoized assistant ordering (#2655)

* updated refresh

* memoization and so on

* nit

* build issue
---
 web/src/app/chat/ChatPage.tsx                 | 21 +++++++++-----
 web/src/app/chat/input/ChatInputBar.tsx       |  3 ++
 .../modal/configuration/AssistantsTab.tsx     | 29 +++++++++----------
 web/src/components/user/UserProvider.tsx      |  1 -
 4 files changed, 30 insertions(+), 24 deletions(-)

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index e9676d28246..8f9b36b6265 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -50,6 +50,7 @@ import {
   useContext,
   useEffect,
   useLayoutEffect,
+  useMemo,
   useRef,
   useState,
 } from "react";
@@ -157,14 +158,17 @@ export function ChatPage({
   // Useful for determining which session has been loaded (i.e. still on `new, empty session` or `previous session`)
   const loadedIdSessionRef = useRef<number | null>(existingChatSessionId);
 
-  // Assistants
-  const { visibleAssistants, hiddenAssistants: _ } = classifyAssistants(
-    user,
-    availableAssistants
-  );
-  const finalAssistants = user
-    ? orderAssistantsForUser(visibleAssistants, user)
-    : visibleAssistants;
+  // Assistants in order
+  const { finalAssistants } = useMemo(() => {
+    const { visibleAssistants, hiddenAssistants: _ } = classifyAssistants(
+      user,
+      availableAssistants
+    );
+    const finalAssistants = user
+      ? orderAssistantsForUser(visibleAssistants, user)
+      : visibleAssistants;
+    return { finalAssistants };
+  }, [user, availableAssistants]);
 
   const existingChatSessionAssistantId = selectedChatSession?.persona_id;
   const [selectedAssistant, setSelectedAssistant] = useState<
@@ -2410,6 +2414,7 @@ export function ChatPage({
                               handleFileUpload={handleImageUpload}
                               textAreaRef={textAreaRef}
                               chatSessionId={chatSessionIdRef.current!}
+                              refreshUser={refreshUser}
                             />
 
                             {enterpriseSettings &&
diff --git a/web/src/app/chat/input/ChatInputBar.tsx b/web/src/app/chat/input/ChatInputBar.tsx
index 64535d82b20..cef0bc49f50 100644
--- a/web/src/app/chat/input/ChatInputBar.tsx
+++ b/web/src/app/chat/input/ChatInputBar.tsx
@@ -63,6 +63,7 @@ export function ChatInputBar({
   alternativeAssistant,
   chatSessionId,
   inputPrompts,
+  refreshUser,
 }: {
   showConfigureAPIKey: () => void;
   openModelSettings: () => void;
@@ -86,6 +87,7 @@ export function ChatInputBar({
   handleFileUpload: (files: File[]) => void;
   textAreaRef: React.RefObject<HTMLTextAreaElement>;
   chatSessionId?: number;
+  refreshUser: () => void;
 }) {
   useEffect(() => {
     const textarea = textAreaRef.current;
@@ -532,6 +534,7 @@ export function ChatInputBar({
                       setSelectedAssistant(assistant);
                       close();
                     }}
+                    refreshUser={refreshUser}
                   />
                 )}
                 flexPriority="shrink"
diff --git a/web/src/app/chat/modal/configuration/AssistantsTab.tsx b/web/src/app/chat/modal/configuration/AssistantsTab.tsx
index dcf31138ccf..1db03cd605d 100644
--- a/web/src/app/chat/modal/configuration/AssistantsTab.tsx
+++ b/web/src/app/chat/modal/configuration/AssistantsTab.tsx
@@ -13,25 +13,26 @@ import {
   sortableKeyboardCoordinates,
   verticalListSortingStrategy,
 } from "@dnd-kit/sortable";
-import { CSS } from "@dnd-kit/utilities";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { getFinalLLM } from "@/lib/llm/utils";
 import React, { useState } from "react";
 import { updateUserAssistantList } from "@/lib/assistants/updateAssistantPreferences";
 import { DraggableAssistantCard } from "@/components/assistants/AssistantCards";
-import { orderAssistantsForUser } from "@/lib/assistants/utils";
+import { useRouter } from "next/navigation";
 
 export function AssistantsTab({
   selectedAssistant,
   availableAssistants,
   llmProviders,
   onSelect,
+  refreshUser,
 }: {
   selectedAssistant: Persona;
   availableAssistants: Persona[];
   llmProviders: LLMProviderDescriptor[];
   onSelect: (assistant: Persona) => void;
+  refreshUser: () => void;
 }) {
   const [_, llmName] = getFinalLLM(llmProviders, null, null);
   const [assistants, setAssistants] = useState(availableAssistants);
@@ -43,23 +44,21 @@ export function AssistantsTab({
     })
   );
 
-  function handleDragEnd(event: DragEndEvent) {
+  async function handleDragEnd(event: DragEndEvent) {
     const { active, over } = event;
 
     if (over && active.id !== over.id) {
-      setAssistants((items) => {
-        const oldIndex = items.findIndex(
-          (item) => item.id.toString() === active.id
-        );
-        const newIndex = items.findIndex(
-          (item) => item.id.toString() === over.id
-        );
-        const updatedAssistants = arrayMove(items, oldIndex, newIndex);
+      const oldIndex = assistants.findIndex(
+        (item) => item.id.toString() === active.id
+      );
+      const newIndex = assistants.findIndex(
+        (item) => item.id.toString() === over.id
+      );
+      const updatedAssistants = arrayMove(assistants, oldIndex, newIndex);
 
-        updateUserAssistantList(updatedAssistants.map((a) => a.id));
-
-        return updatedAssistants;
-      });
+      setAssistants(updatedAssistants);
+      await updateUserAssistantList(updatedAssistants.map((a) => a.id));
+      refreshUser();
     }
   }
 
diff --git a/web/src/components/user/UserProvider.tsx b/web/src/components/user/UserProvider.tsx
index 67777277c27..5e7bb753520 100644
--- a/web/src/components/user/UserProvider.tsx
+++ b/web/src/components/user/UserProvider.tsx
@@ -40,7 +40,6 @@ export function UserProvider({ children }: { children: React.ReactNode }) {
   }, []);
 
   const refreshUser = async () => {
-    setIsLoadingUser(true);
     await fetchUser();
   };
 

From a0235b7b7be2c03587b2d1f7b803a2ee5bf09387 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 2 Oct 2024 10:13:19 -0700
Subject: [PATCH 016/376] =?UTF-8?q?replace=20trivy=20download=20endpoint?=
 =?UTF-8?q?=20due=20to=20db=20download=20flakiness=20on=20their=20en?=
 =?UTF-8?q?=E2=80=A6=20(#2661)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* disable trivy for the moment due to db download flakiness on their end causing the action to fail

* try hardcoding to amazon registry as others have suggested
---
 .../docker-build-push-backend-container-on-tag.yml        | 8 ++++++++
 .../docker-build-push-model-server-container-on-tag.yml   | 8 ++++++++
 .../workflows/docker-build-push-web-container-on-tag.yml  | 8 ++++++++
 3 files changed, 24 insertions(+)

diff --git a/.github/workflows/docker-build-push-backend-container-on-tag.yml b/.github/workflows/docker-build-push-backend-container-on-tag.yml
index cee4d5d6568..ef07e051db3 100644
--- a/.github/workflows/docker-build-push-backend-container-on-tag.yml
+++ b/.github/workflows/docker-build-push-backend-container-on-tag.yml
@@ -46,8 +46,16 @@ jobs:
         build-args: |
           DANSWER_VERSION=${{ github.ref_name }}
 
+    # trivy has their own rate limiting issues causing this action to flake
+    # we worked around it by hardcoding to different db repos in env
+    # can re-enable when they figure it out
+    # https://github.com/aquasecurity/trivy/discussions/7538
+    # https://github.com/aquasecurity/trivy-action/issues/389
     - name: Run Trivy vulnerability scanner
       uses: aquasecurity/trivy-action@master
+      env:
+        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
       with:
         # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
         image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
diff --git a/.github/workflows/docker-build-push-model-server-container-on-tag.yml b/.github/workflows/docker-build-push-model-server-container-on-tag.yml
index 7767be9c358..c05d233d1e9 100644
--- a/.github/workflows/docker-build-push-model-server-container-on-tag.yml
+++ b/.github/workflows/docker-build-push-model-server-container-on-tag.yml
@@ -40,8 +40,16 @@ jobs:
         build-args: |
           DANSWER_VERSION=${{ github.ref_name }}
 
+    # trivy has their own rate limiting issues causing this action to flake
+    # we worked around it by hardcoding to different db repos in env
+    # can re-enable when they figure it out
+    # https://github.com/aquasecurity/trivy/discussions/7538
+    # https://github.com/aquasecurity/trivy-action/issues/389
     - name: Run Trivy vulnerability scanner
       uses: aquasecurity/trivy-action@master
+      env:
+        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
       with:
         image-ref: docker.io/danswer/danswer-model-server:${{ github.ref_name }}
         severity: 'CRITICAL,HIGH'
diff --git a/.github/workflows/docker-build-push-web-container-on-tag.yml b/.github/workflows/docker-build-push-web-container-on-tag.yml
index 591071da62f..1c901613563 100644
--- a/.github/workflows/docker-build-push-web-container-on-tag.yml
+++ b/.github/workflows/docker-build-push-web-container-on-tag.yml
@@ -113,8 +113,16 @@ jobs:
         run: |
           docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
 
+    # trivy has their own rate limiting issues causing this action to flake
+    # we worked around it by hardcoding to different db repos in env
+    # can re-enable when they figure it out
+    # https://github.com/aquasecurity/trivy/discussions/7538
+    # https://github.com/aquasecurity/trivy-action/issues/389
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+          TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
         with:
           image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
           severity: 'CRITICAL,HIGH'

From af187c6cfe73494a7db424d0e0177b093fc9aac5 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 2 Oct 2024 11:14:59 -0700
Subject: [PATCH 017/376] Better virtualization (#2653)

---
 web/src/app/chat/ChatPage.tsx                 |  14 +-
 web/src/app/chat/message/CodeBlock.tsx        | 211 ++++++++----------
 .../chat/message/MemoizedTextComponents.tsx   |   6 +-
 web/src/app/chat/message/Messages.tsx         |  57 +++--
 web/src/app/chat/message/codeUtils.ts         |  47 ++++
 .../chat_search/MinimalMarkdown.tsx           |  17 +-
 .../search/results/AnswerSection.tsx          |  13 +-
 7 files changed, 200 insertions(+), 165 deletions(-)
 create mode 100644 web/src/app/chat/message/codeUtils.ts

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 8f9b36b6265..3513b2dc3c6 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -759,7 +759,15 @@ export function ChatPage({
     setAboveHorizon(scrollDist.current > 500);
   };
 
-  scrollableDivRef?.current?.addEventListener("scroll", updateScrollTracking);
+  useEffect(() => {
+    const scrollableDiv = scrollableDivRef.current;
+    if (scrollableDiv) {
+      scrollableDiv.addEventListener("scroll", updateScrollTracking);
+      return () => {
+        scrollableDiv.removeEventListener("scroll", updateScrollTracking);
+      };
+    }
+  }, []);
 
   const handleInputResize = () => {
     setTimeout(() => {
@@ -1137,7 +1145,9 @@ export function ChatPage({
 
       await delay(50);
       while (!stack.isComplete || !stack.isEmpty()) {
-        await delay(0.5);
+        if (stack.isEmpty()) {
+          await delay(0.5);
+        }
 
         if (!stack.isEmpty() && !controller.signal.aborted) {
           const packet = stack.nextPacket();
diff --git a/web/src/app/chat/message/CodeBlock.tsx b/web/src/app/chat/message/CodeBlock.tsx
index 55a6ea7be32..66cc82a6e73 100644
--- a/web/src/app/chat/message/CodeBlock.tsx
+++ b/web/src/app/chat/message/CodeBlock.tsx
@@ -1,20 +1,22 @@
 import React, { useState, ReactNode, useCallback, useMemo, memo } from "react";
 import { FiCheck, FiCopy } from "react-icons/fi";
 
-const CODE_BLOCK_PADDING_TYPE = { padding: "1rem" };
+const CODE_BLOCK_PADDING = { padding: "1rem" };
 
 interface CodeBlockProps {
-  className?: string | undefined;
+  className?: string;
   children?: ReactNode;
-  content: string;
-  [key: string]: any;
+  codeText: string;
 }
 
+const MemoizedCodeLine = memo(({ content }: { content: ReactNode }) => (
+  <>{content}</>
+));
+
 export const CodeBlock = memo(function CodeBlock({
   className = "",
   children,
-  content,
-  ...props
+  codeText,
 }: CodeBlockProps) {
   const [copied, setCopied] = useState(false);
 
@@ -26,132 +28,99 @@ export const CodeBlock = memo(function CodeBlock({
       .join(" ");
   }, [className]);
 
-  const codeText = useMemo(() => {
-    let codeText: string | null = null;
-    if (
-      props.node?.position?.start?.offset &&
-      props.node?.position?.end?.offset
-    ) {
-      codeText = content.slice(
-        props.node.position.start.offset,
-        props.node.position.end.offset
-      );
-      codeText = codeText.trim();
-
-      // Find the last occurrence of closing backticks
-      const lastBackticksIndex = codeText.lastIndexOf("```");
-      if (lastBackticksIndex !== -1) {
-        codeText = codeText.slice(0, lastBackticksIndex + 3);
-      }
-
-      // Remove the language declaration and trailing backticks
-      const codeLines = codeText.split("\n");
-      if (
-        codeLines.length > 1 &&
-        (codeLines[0].startsWith("```") ||
-          codeLines[0].trim().startsWith("```"))
-      ) {
-        codeLines.shift(); // Remove the first line with the language declaration
-        if (
-          codeLines[codeLines.length - 1] === "```" ||
-          codeLines[codeLines.length - 1]?.trim() === "```"
-        ) {
-          codeLines.pop(); // Remove the last line with the trailing backticks
-        }
-
-        const minIndent = codeLines
-          .filter((line) => line.trim().length > 0)
-          .reduce((min, line) => {
-            const match = line.match(/^\s*/);
-            return Math.min(min, match ? match[0].length : 0);
-          }, Infinity);
-
-        const formattedCodeLines = codeLines.map((line) =>
-          line.slice(minIndent)
+  const handleCopy = useCallback(() => {
+    if (!codeText) return;
+    navigator.clipboard.writeText(codeText).then(() => {
+      setCopied(true);
+      setTimeout(() => setCopied(false), 2000);
+    });
+  }, [codeText]);
+
+  const CopyButton = memo(() => (
+    <div
+      className="ml-auto cursor-pointer select-none"
+      onMouseDown={handleCopy}
+    >
+      {copied ? (
+        <div className="flex items-center space-x-2">
+          <FiCheck size={16} />
+          <span>Copied!</span>
+        </div>
+      ) : (
+        <div className="flex items-center space-x-2">
+          <FiCopy size={16} />
+          <span>Copy code</span>
+        </div>
+      )}
+    </div>
+  ));
+  CopyButton.displayName = "CopyButton";
+
+  const CodeContent = memo(() => {
+    if (!language) {
+      if (typeof children === "string") {
+        return (
+          <code
+            className={`
+            font-mono 
+            text-gray-800 
+            bg-gray-50 
+            border 
+            border-gray-300 
+            rounded 
+            px-1
+            py-[3px]
+            text-xs 
+            whitespace-pre-wrap 
+            break-words 
+            overflow-hidden
+            mb-1
+            ${className}
+          `}
+          >
+            {children}
+          </code>
         );
-        codeText = formattedCodeLines.join("\n");
       }
-    }
-
-    // handle unknown languages. They won't have a `node.position.start.offset`
-    if (!codeText) {
-      const findTextNode = (node: any): string | null => {
-        if (node.type === "text") {
-          return node.value;
-        }
-        let finalResult = "";
-        if (node.children) {
-          for (const child of node.children) {
-            const result = findTextNode(child);
-            if (result) {
-              finalResult += result;
-            }
-          }
-        }
-        return finalResult;
-      };
-
-      codeText = findTextNode(props.node);
-    }
-
-    return codeText;
-  }, [content, props.node]);
-
-  const handleCopy = useCallback(
-    (event: React.MouseEvent) => {
-      event.preventDefault();
-      if (!codeText) {
-        return;
-      }
-
-      navigator.clipboard.writeText(codeText).then(() => {
-        setCopied(true);
-        setTimeout(() => setCopied(false), 2000);
-      });
-    },
-    [codeText]
-  );
-
-  if (!language) {
-    if (typeof children === "string") {
-      return <code className={className}>{children}</code>;
+      return (
+        <pre style={CODE_BLOCK_PADDING}>
+          <code className={`text-sm ${className}`}>
+            {Array.isArray(children)
+              ? children.map((child, index) => (
+                  <MemoizedCodeLine key={index} content={child} />
+                ))
+              : children}
+          </code>
+        </pre>
+      );
     }
 
     return (
-      <pre style={CODE_BLOCK_PADDING_TYPE}>
-        <code {...props} className={`text-sm ${className}`}>
-          {children}
+      <pre className="overflow-x-scroll" style={CODE_BLOCK_PADDING}>
+        <code className="text-xs overflow-x-auto">
+          {Array.isArray(children)
+            ? children.map((child, index) => (
+                <MemoizedCodeLine key={index} content={child} />
+              ))
+            : children}
         </code>
       </pre>
     );
-  }
+  });
+  CodeContent.displayName = "CodeContent";
 
   return (
     <div className="overflow-x-hidden">
-      <div className="flex mx-3 py-2 text-xs">
-        {language}
-        {codeText && (
-          <div
-            className="ml-auto cursor-pointer select-none"
-            onMouseDown={handleCopy}
-          >
-            {copied ? (
-              <div className="flex items-center space-x-2">
-                <FiCheck size={16} />
-                <span>Copied!</span>
-              </div>
-            ) : (
-              <div className="flex items-center space-x-2">
-                <FiCopy size={16} />
-                <span>Copy code</span>
-              </div>
-            )}
-          </div>
-        )}
-      </div>
-      <pre {...props} className="overflow-x-scroll" style={{ padding: "1rem" }}>
-        <code className={`text-xs overflow-x-auto `}>{children}</code>
-      </pre>
+      {language && (
+        <div className="flex mx-3 py-2 text-xs">
+          {language}
+          {codeText && <CopyButton />}
+        </div>
+      )}
+      <CodeContent />
     </div>
   );
 });
+
+CodeBlock.displayName = "CodeBlock";
+MemoizedCodeLine.displayName = "MemoizedCodeLine";
diff --git a/web/src/app/chat/message/MemoizedTextComponents.tsx b/web/src/app/chat/message/MemoizedTextComponents.tsx
index 4ab8bc810b2..9ab0e28e3ca 100644
--- a/web/src/app/chat/message/MemoizedTextComponents.tsx
+++ b/web/src/app/chat/message/MemoizedTextComponents.tsx
@@ -25,9 +25,9 @@ export const MemoizedLink = memo((props: any) => {
   }
 });
 
-export const MemoizedParagraph = memo(({ node, ...props }: any) => (
-  <p {...props} className="text-default" />
-));
+export const MemoizedParagraph = memo(({ ...props }: any) => {
+  return <p {...props} className="text-default" />;
+});
 
 MemoizedLink.displayName = "MemoizedLink";
 MemoizedParagraph.displayName = "MemoizedParagraph";
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index edb18138c79..e10f5cea03c 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -54,6 +54,7 @@ import RegenerateOption from "../RegenerateOption";
 import { LlmOverride } from "@/lib/hooks";
 import { ContinueGenerating } from "./ContinueMessage";
 import { MemoizedLink, MemoizedParagraph } from "./MemoizedTextComponents";
+import { extractCodeText } from "./codeUtils";
 
 const TOOLS_WITH_CUSTOM_HANDLING = [
   SEARCH_TOOL_NAME,
@@ -253,6 +254,40 @@ export const AIMessage = ({
     new Set((docs || []).map((doc) => doc.source_type))
   ).slice(0, 3);
 
+  const markdownComponents = useMemo(
+    () => ({
+      a: MemoizedLink,
+      p: MemoizedParagraph,
+      code: ({ node, inline, className, children, ...props }: any) => {
+        const codeText = extractCodeText(
+          node,
+          finalContent as string,
+          children
+        );
+
+        return (
+          <CodeBlock className={className} codeText={codeText}>
+            {children}
+          </CodeBlock>
+        );
+      },
+    }),
+    [messageId, content]
+  );
+
+  const renderedMarkdown = useMemo(() => {
+    return (
+      <ReactMarkdown
+        className="prose max-w-full text-base"
+        components={markdownComponents}
+        remarkPlugins={[remarkGfm]}
+        rehypePlugins={[[rehypePrism, { ignoreMissing: true }]]}
+      >
+        {finalContent as string}
+      </ReactMarkdown>
+    );
+  }, [finalContent]);
+
   const includeMessageSwitcher =
     currentMessageInd !== undefined &&
     onMessageSelection &&
@@ -352,27 +387,7 @@ export const AIMessage = ({
 
                         {typeof content === "string" ? (
                           <div className="overflow-x-visible max-w-content-max">
-                            <ReactMarkdown
-                              key={messageId}
-                              className="prose max-w-full text-base"
-                              components={{
-                                a: MemoizedLink,
-                                p: MemoizedParagraph,
-                                code: (props) => (
-                                  <CodeBlock
-                                    className="w-full"
-                                    {...props}
-                                    content={content as string}
-                                  />
-                                ),
-                              }}
-                              remarkPlugins={[remarkGfm]}
-                              rehypePlugins={[
-                                [rehypePrism, { ignoreMissing: true }],
-                              ]}
-                            >
-                              {finalContent as string}
-                            </ReactMarkdown>
+                            {renderedMarkdown}
                           </div>
                         ) : (
                           content
diff --git a/web/src/app/chat/message/codeUtils.ts b/web/src/app/chat/message/codeUtils.ts
new file mode 100644
index 00000000000..2aaae71bc82
--- /dev/null
+++ b/web/src/app/chat/message/codeUtils.ts
@@ -0,0 +1,47 @@
+export function extractCodeText(
+  node: any,
+  content: string,
+  children: React.ReactNode
+): string {
+  let codeText: string | null = null;
+  if (
+    node?.position?.start?.offset != null &&
+    node?.position?.end?.offset != null
+  ) {
+    codeText = content.slice(
+      node.position.start.offset,
+      node.position.end.offset
+    );
+    codeText = codeText.trim();
+
+    // Find the last occurrence of closing backticks
+    const lastBackticksIndex = codeText.lastIndexOf("```");
+    if (lastBackticksIndex !== -1) {
+      codeText = codeText.slice(0, lastBackticksIndex + 3);
+    }
+
+    // Remove the language declaration and trailing backticks
+    const codeLines = codeText.split("\n");
+    if (codeLines.length > 1 && codeLines[0].trim().startsWith("```")) {
+      codeLines.shift(); // Remove the first line with the language declaration
+      if (codeLines[codeLines.length - 1]?.trim() === "```") {
+        codeLines.pop(); // Remove the last line with the trailing backticks
+      }
+
+      const minIndent = codeLines
+        .filter((line) => line.trim().length > 0)
+        .reduce((min, line) => {
+          const match = line.match(/^\s*/);
+          return Math.min(min, match ? match[0].length : 0);
+        }, Infinity);
+
+      const formattedCodeLines = codeLines.map((line) => line.slice(minIndent));
+      codeText = formattedCodeLines.join("\n");
+    }
+  } else {
+    // Fallback if position offsets are not available
+    codeText = children?.toString() || null;
+  }
+
+  return codeText || "";
+}
diff --git a/web/src/components/chat_search/MinimalMarkdown.tsx b/web/src/components/chat_search/MinimalMarkdown.tsx
index 3516749d1bf..4731e2de9c3 100644
--- a/web/src/components/chat_search/MinimalMarkdown.tsx
+++ b/web/src/components/chat_search/MinimalMarkdown.tsx
@@ -1,4 +1,5 @@
 import { CodeBlock } from "@/app/chat/message/CodeBlock";
+import { extractCodeText } from "@/app/chat/message/codeUtils";
 import {
   MemoizedLink,
   MemoizedParagraph,
@@ -10,13 +11,11 @@ import remarkGfm from "remark-gfm";
 interface MinimalMarkdownProps {
   content: string;
   className?: string;
-  useCodeBlock?: boolean;
 }
 
 export const MinimalMarkdown: React.FC<MinimalMarkdownProps> = ({
   content,
   className = "",
-  useCodeBlock = false,
 }) => {
   return (
     <ReactMarkdown
@@ -24,11 +23,15 @@ export const MinimalMarkdown: React.FC<MinimalMarkdownProps> = ({
       components={{
         a: MemoizedLink,
         p: MemoizedParagraph,
-        code: useCodeBlock
-          ? (props) => (
-              <CodeBlock className="w-full" {...props} content={content} />
-            )
-          : (props) => <code {...props} />,
+        code: ({ node, inline, className, children, ...props }: any) => {
+          const codeText = extractCodeText(node, content, children);
+
+          return (
+            <CodeBlock className={className} codeText={codeText}>
+              {children}
+            </CodeBlock>
+          );
+        },
       }}
       remarkPlugins={[remarkGfm]}
     >
diff --git a/web/src/components/search/results/AnswerSection.tsx b/web/src/components/search/results/AnswerSection.tsx
index 225623b0085..324e41e0a84 100644
--- a/web/src/components/search/results/AnswerSection.tsx
+++ b/web/src/components/search/results/AnswerSection.tsx
@@ -1,7 +1,5 @@
 import { Quote } from "@/lib/search/interfaces";
 import { ResponseSection, StatusOptions } from "./ResponseSection";
-import ReactMarkdown from "react-markdown";
-import remarkGfm from "remark-gfm";
 import { MinimalMarkdown } from "@/components/chat_search/MinimalMarkdown";
 
 const TEMP_STRING = "__$%^TEMP$%^__";
@@ -40,12 +38,7 @@ export const AnswerSection = (props: AnswerSectionProps) => {
     status = "success";
     header = <></>;
 
-    body = (
-      <MinimalMarkdown
-        useCodeBlock
-        content={replaceNewlines(props.answer || "")}
-      />
-    );
+    body = <MinimalMarkdown content={replaceNewlines(props.answer || "")} />;
 
     // error while building answer (NOTE: if error occurs during quote generation
     // the above if statement will hit and the error will not be displayed)
@@ -61,9 +54,7 @@ export const AnswerSection = (props: AnswerSectionProps) => {
   } else if (props.answer) {
     status = "success";
     header = <></>;
-    body = (
-      <MinimalMarkdown useCodeBlock content={replaceNewlines(props.answer)} />
-    );
+    body = <MinimalMarkdown content={replaceNewlines(props.answer)} />;
   }
 
   return (

From 457d32fef07d8d0bbdef92388f8aa507515edbd0 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 2 Oct 2024 11:00:06 -0700
Subject: [PATCH 018/376] add clarity around assistants and names (#2663)

---
 web/src/app/admin/assistants/PersonaTable.tsx | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index e1e87137285..dac260a1888 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -20,12 +20,16 @@ import { UserRole, User } from "@/lib/types";
 import { useUser } from "@/components/user/UserProvider";
 
 function PersonaTypeDisplay({ persona }: { persona: Persona }) {
-  if (persona.is_default_persona) {
+  if (persona.builtin_persona) {
     return <Text>Built-In</Text>;
   }
 
+  if (persona.is_default_persona) {
+    return <Text>Default</Text>;
+  }
+
   if (persona.is_public) {
-    return <Text>Global</Text>;
+    return <Text>Public</Text>;
   }
 
   if (persona.groups.length > 0 || persona.users.length > 0) {

From b3c367d09cd10e9066b0fe9d7e90791b331871e0 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Wed, 2 Oct 2024 11:01:40 -0700
Subject: [PATCH 019/376] [tiny] adjust user group sync log (#2664)

---
 backend/ee/danswer/background/celery/tasks/vespa/tasks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
index d194b2ef9a9..805083efcfc 100644
--- a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
@@ -34,7 +34,7 @@ def monitor_usergroup_taskset(key_bytes: bytes, r: Redis, db_session: Session) -
 
     count = cast(int, r.scard(rug.taskset_key))
     task_logger.info(
-        f"User group sync: usergroup_id={usergroup_id} remaining={count} initial={initial_count}"
+        f"User group sync progress: usergroup_id={usergroup_id} remaining={count} initial={initial_count}"
     )
     if count > 0:
         return

From c2088602e193888f95fac21a7565acddeda1e31e Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Wed, 2 Oct 2024 16:16:07 -0700
Subject: [PATCH 020/376] Implement source testing framework + Slack (#2650)

* Added permission sync tests for Slack

* moved folders

* prune test + mypy

* added wait for indexing to cc_pair creation

* commented out check

* should fix other tests

* added slack channel pool

* fixed everything and mypy

* reduced flake
---
 .github/workflows/pr-Integration-tests.yml    |   2 +
 backend/danswer/connectors/slack/connector.py |  11 +-
 backend/danswer/server/documents/cc_pair.py   |  99 +++++-
 backend/danswer/server/documents/connector.py |   5 +-
 backend/danswer/server/documents/models.py    |   2 +-
 backend/ee/danswer/background/celery_utils.py |  33 +-
 .../external_permissions/permission_sync.py   |  29 --
 .../integration/common_utils/constants.py     |   2 +-
 .../common_utils/managers/cc_pair.py          | 116 ++++++-
 .../common_utils/managers/llm_provider.py     |   2 -
 .../integration/common_utils/managers/user.py |  15 +-
 .../connector_job_tests/slack/conftest.py     |  28 ++
 .../slack/slack_api_utils.py                  | 311 ++++++++++++++++++
 .../slack/test_permission_sync.py             | 251 ++++++++++++++
 .../connector_job_tests/slack/test_prune.py   | 255 ++++++++++++++
 15 files changed, 1098 insertions(+), 63 deletions(-)
 create mode 100644 backend/tests/integration/connector_job_tests/slack/conftest.py
 create mode 100644 backend/tests/integration/connector_job_tests/slack/slack_api_utils.py
 create mode 100644 backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
 create mode 100644 backend/tests/integration/connector_job_tests/slack/test_prune.py

diff --git a/.github/workflows/pr-Integration-tests.yml b/.github/workflows/pr-Integration-tests.yml
index ae7cf0be78e..98fa8be16e6 100644
--- a/.github/workflows/pr-Integration-tests.yml
+++ b/.github/workflows/pr-Integration-tests.yml
@@ -12,6 +12,7 @@ on:
 
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
 jobs:
   integration-tests:
@@ -142,6 +143,7 @@ jobs:
             -e REDIS_HOST=cache \
             -e API_SERVER_HOST=api_server \
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
             -e TEST_WEB_HOSTNAME=test-runner \
             danswer/danswer-integration:test
         continue-on-error: true
diff --git a/backend/danswer/connectors/slack/connector.py b/backend/danswer/connectors/slack/connector.py
index d7a23714a37..4a9249e5157 100644
--- a/backend/danswer/connectors/slack/connector.py
+++ b/backend/danswer/connectors/slack/connector.py
@@ -205,12 +205,17 @@ def thread_to_doc(
     "group_leave",
     "group_archive",
     "group_unarchive",
+    "channel_leave",
+    "channel_name",
+    "channel_join",
 }
 
 
-def _default_msg_filter(message: MessageType) -> bool:
+def default_msg_filter(message: MessageType) -> bool:
     # Don't keep messages from bots
     if message.get("bot_id") or message.get("app_id"):
+        if message.get("bot_profile", {}).get("name") == "DanswerConnector":
+            return False
         return True
 
     # Uninformative
@@ -261,7 +266,7 @@ def _get_all_docs(
     channel_name_regex_enabled: bool = False,
     oldest: str | None = None,
     latest: str | None = None,
-    msg_filter_func: Callable[[MessageType], bool] = _default_msg_filter,
+    msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
 ) -> Generator[Document, None, None]:
     """Get all documents in the workspace, channel by channel"""
     slack_cleaner = SlackTextCleaner(client=client)
@@ -320,7 +325,7 @@ def _get_all_doc_ids(
     client: WebClient,
     channels: list[str] | None = None,
     channel_name_regex_enabled: bool = False,
-    msg_filter_func: Callable[[MessageType], bool] = _default_msg_filter,
+    msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
 ) -> set[str]:
     """
     Get all document ids in the workspace, channel by channel
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index 428666751a4..a9ee03c0577 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -29,15 +29,19 @@
 from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
 from danswer.db.index_attempt import get_paginated_index_attempts_for_cc_pair_id
 from danswer.db.models import User
+from danswer.db.tasks import check_task_is_live_and_not_timed_out
 from danswer.db.tasks import get_latest_task
 from danswer.server.documents.models import CCPairFullInfo
-from danswer.server.documents.models import CCPairPruningTask
 from danswer.server.documents.models import CCStatusUpdateRequest
+from danswer.server.documents.models import CeleryTaskStatus
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.server.documents.models import ConnectorCredentialPairMetadata
 from danswer.server.documents.models import PaginatedIndexAttempts
 from danswer.server.models import StatusResponse
 from danswer.utils.logger import setup_logger
+from ee.danswer.background.task_name_builders import (
+    name_sync_external_doc_permissions_task,
+)
 from ee.danswer.db.user_group import validate_user_creation_permissions
 
 logger = setup_logger()
@@ -199,7 +203,7 @@ def get_cc_pair_latest_prune(
     cc_pair_id: int,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-) -> CCPairPruningTask:
+) -> CeleryTaskStatus:
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id=cc_pair_id,
         db_session=db_session,
@@ -223,7 +227,7 @@ def get_cc_pair_latest_prune(
             detail="No pruning task found.",
         )
 
-    return CCPairPruningTask(
+    return CeleryTaskStatus(
         id=last_pruning_task.task_id,
         name=last_pruning_task.task_name,
         status=last_pruning_task.status,
@@ -280,6 +284,95 @@ def prune_cc_pair(
     )
 
 
+@router.get("/admin/cc-pair/{cc_pair_id}/sync")
+def get_cc_pair_latest_sync(
+    cc_pair_id: int,
+    user: User = Depends(current_curator_or_admin_user),
+    db_session: Session = Depends(get_session),
+) -> CeleryTaskStatus:
+    cc_pair = get_connector_credential_pair_from_id(
+        cc_pair_id=cc_pair_id,
+        db_session=db_session,
+        user=user,
+        get_editable=False,
+    )
+    if not cc_pair:
+        raise HTTPException(
+            status_code=400,
+            detail="Connection not found for current user's permissions",
+        )
+
+    # look up the last sync task for this connector (if it exists)
+    sync_task_name = name_sync_external_doc_permissions_task(cc_pair_id=cc_pair_id)
+    last_sync_task = get_latest_task(sync_task_name, db_session)
+    if not last_sync_task:
+        raise HTTPException(
+            status_code=HTTPStatus.NOT_FOUND,
+            detail="No sync task found.",
+        )
+
+    return CeleryTaskStatus(
+        id=last_sync_task.task_id,
+        name=last_sync_task.task_name,
+        status=last_sync_task.status,
+        start_time=last_sync_task.start_time,
+        register_time=last_sync_task.register_time,
+    )
+
+
+@router.post("/admin/cc-pair/{cc_pair_id}/sync")
+def sync_cc_pair(
+    cc_pair_id: int,
+    user: User = Depends(current_curator_or_admin_user),
+    db_session: Session = Depends(get_session),
+) -> StatusResponse[list[int]]:
+    # avoiding circular refs
+    from ee.danswer.background.celery.celery_app import (
+        sync_external_doc_permissions_task,
+    )
+
+    cc_pair = get_connector_credential_pair_from_id(
+        cc_pair_id=cc_pair_id,
+        db_session=db_session,
+        user=user,
+        get_editable=False,
+    )
+    if not cc_pair:
+        raise HTTPException(
+            status_code=400,
+            detail="Connection not found for current user's permissions",
+        )
+
+    sync_task_name = name_sync_external_doc_permissions_task(cc_pair_id=cc_pair_id)
+    last_sync_task = get_latest_task(sync_task_name, db_session)
+
+    if last_sync_task and check_task_is_live_and_not_timed_out(
+        last_sync_task, db_session
+    ):
+        raise HTTPException(
+            status_code=HTTPStatus.CONFLICT,
+            detail="Sync task already in progress.",
+        )
+    if skip_cc_pair_pruning_by_task(
+        last_sync_task,
+        db_session=db_session,
+    ):
+        raise HTTPException(
+            status_code=HTTPStatus.CONFLICT,
+            detail="Sync task already in progress.",
+        )
+
+    logger.info(f"Syncing the {cc_pair.connector.name} connector.")
+    sync_external_doc_permissions_task.apply_async(
+        kwargs=dict(cc_pair_id=cc_pair_id),
+    )
+
+    return StatusResponse(
+        success=True,
+        message="Successfully created the sync task.",
+    )
+
+
 @router.put("/connector/{connector_id}/credential/{credential_id}")
 def associate_credential_to_connector(
     connector_id: int,
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index c6076dee744..732d2511976 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -781,6 +781,7 @@ def connector_run_once(
             detail="Connector has no valid credentials, cannot create index attempts.",
         )
 
+    # Prevents index attempts for cc pairs that already have an index attempt currently running
     skipped_credentials = [
         credential_id
         for credential_id in credential_ids
@@ -790,15 +791,15 @@ def connector_run_once(
                 credential_id=credential_id,
             ),
             only_current=True,
-            disinclude_finished=True,
             db_session=db_session,
+            disinclude_finished=True,
         )
     ]
 
     search_settings = get_current_search_settings(db_session)
 
     connector_credential_pairs = [
-        get_connector_credential_pair(run_info.connector_id, credential_id, db_session)
+        get_connector_credential_pair(connector_id, credential_id, db_session)
         for credential_id in credential_ids
         if credential_id not in skipped_credentials
     ]
diff --git a/backend/danswer/server/documents/models.py b/backend/danswer/server/documents/models.py
index ee266eca8b8..15354a8f381 100644
--- a/backend/danswer/server/documents/models.py
+++ b/backend/danswer/server/documents/models.py
@@ -268,7 +268,7 @@ def from_models(
         )
 
 
-class CCPairPruningTask(BaseModel):
+class CeleryTaskStatus(BaseModel):
     id: str
     name: str
     status: TaskStatus
diff --git a/backend/ee/danswer/background/celery_utils.py b/backend/ee/danswer/background/celery_utils.py
index c42812f81c3..80278d8c433 100644
--- a/backend/ee/danswer/background/celery_utils.py
+++ b/backend/ee/danswer/background/celery_utils.py
@@ -1,3 +1,6 @@
+from datetime import datetime
+from datetime import timezone
+
 from sqlalchemy.orm import Session
 
 from danswer.db.enums import AccessType
@@ -12,10 +15,32 @@
 from ee.danswer.background.task_name_builders import (
     name_sync_external_group_permissions_task,
 )
+from ee.danswer.external_permissions.sync_params import PERMISSION_SYNC_PERIODS
 
 logger = setup_logger()
 
 
+def _is_time_to_run_sync(cc_pair: ConnectorCredentialPair) -> bool:
+    source_sync_period = PERMISSION_SYNC_PERIODS.get(cc_pair.connector.source)
+
+    # If RESTRICTED_FETCH_PERIOD[source] is None, we always run the sync.
+    if not source_sync_period:
+        return True
+
+    # If the last sync is None, it has never been run so we run the sync
+    if cc_pair.last_time_perm_sync is None:
+        return True
+
+    last_sync = cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc)
+    current_time = datetime.now(timezone.utc)
+
+    # If the last sync is greater than the full fetch period, we run the sync
+    if (current_time - last_sync).total_seconds() > source_sync_period:
+        return True
+
+    return False
+
+
 def should_perform_chat_ttl_check(
     retention_limit_days: int | None, db_session: Session
 ) -> bool:
@@ -28,7 +53,7 @@ def should_perform_chat_ttl_check(
     if not latest_task:
         return True
 
-    if latest_task and check_task_is_live_and_not_timed_out(latest_task, db_session):
+    if check_task_is_live_and_not_timed_out(latest_task, db_session):
         logger.debug(f"{task_name} is already being performed. Skipping.")
         return False
     return True
@@ -50,6 +75,9 @@ def should_perform_external_doc_permissions_check(
         logger.debug(f"{task_name} is already being performed. Skipping.")
         return False
 
+    if not _is_time_to_run_sync(cc_pair):
+        return False
+
     return True
 
 
@@ -69,4 +97,7 @@ def should_perform_external_group_permissions_check(
         logger.debug(f"{task_name} is already being performed. Skipping.")
         return False
 
+    if not _is_time_to_run_sync(cc_pair):
+        return False
+
     return True
diff --git a/backend/ee/danswer/external_permissions/permission_sync.py b/backend/ee/danswer/external_permissions/permission_sync.py
index 3a4357f7c10..0f07411da52 100644
--- a/backend/ee/danswer/external_permissions/permission_sync.py
+++ b/backend/ee/danswer/external_permissions/permission_sync.py
@@ -6,38 +6,15 @@
 from danswer.access.access import get_access_for_documents
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.document import get_document_ids_for_connector_credential_pair
-from danswer.db.models import ConnectorCredentialPair
 from danswer.document_index.factory import get_current_primary_default_document_index
 from danswer.document_index.interfaces import UpdateRequest
 from danswer.utils.logger import setup_logger
 from ee.danswer.external_permissions.sync_params import DOC_PERMISSIONS_FUNC_MAP
 from ee.danswer.external_permissions.sync_params import GROUP_PERMISSIONS_FUNC_MAP
-from ee.danswer.external_permissions.sync_params import PERMISSION_SYNC_PERIODS
 
 logger = setup_logger()
 
 
-def _is_time_to_run_sync(cc_pair: ConnectorCredentialPair) -> bool:
-    source_sync_period = PERMISSION_SYNC_PERIODS.get(cc_pair.connector.source)
-
-    # If RESTRICTED_FETCH_PERIOD[source] is None, we always run the sync.
-    if not source_sync_period:
-        return True
-
-    # If the last sync is None, it has never been run so we run the sync
-    if cc_pair.last_time_perm_sync is None:
-        return True
-
-    last_sync = cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc)
-    current_time = datetime.now(timezone.utc)
-
-    # If the last sync is greater than the full fetch period, we run the sync
-    if (current_time - last_sync).total_seconds() > source_sync_period:
-        return True
-
-    return False
-
-
 def run_external_group_permission_sync(
     db_session: Session,
     cc_pair_id: int,
@@ -53,9 +30,6 @@ def run_external_group_permission_sync(
         # Not all sync connectors support group permissions so this is fine
         return
 
-    if not _is_time_to_run_sync(cc_pair):
-        return
-
     try:
         # This function updates:
         # - the user_email <-> external_user_group_id mapping
@@ -91,9 +65,6 @@ def run_external_doc_permission_sync(
             f"No permission sync function found for source type: {source_type}"
         )
 
-    if not _is_time_to_run_sync(cc_pair):
-        return
-
     try:
         # This function updates:
         # - the user_email <-> document mapping
diff --git a/backend/tests/integration/common_utils/constants.py b/backend/tests/integration/common_utils/constants.py
index 7d729191cf6..57db1ad9a32 100644
--- a/backend/tests/integration/common_utils/constants.py
+++ b/backend/tests/integration/common_utils/constants.py
@@ -4,7 +4,7 @@
 API_SERVER_HOST = os.getenv("API_SERVER_HOST") or "localhost"
 API_SERVER_PORT = os.getenv("API_SERVER_PORT") or "8080"
 API_SERVER_URL = f"{API_SERVER_PROTOCOL}://{API_SERVER_HOST}:{API_SERVER_PORT}"
-MAX_DELAY = 30
+MAX_DELAY = 45
 
 GENERAL_HEADERS = {"Content-Type": "application/json"}
 
diff --git a/backend/tests/integration/common_utils/managers/cc_pair.py b/backend/tests/integration/common_utils/managers/cc_pair.py
index 000bbac59d0..6bcbc34e9bd 100644
--- a/backend/tests/integration/common_utils/managers/cc_pair.py
+++ b/backend/tests/integration/common_utils/managers/cc_pair.py
@@ -9,7 +9,7 @@
 from danswer.db.enums import AccessType
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.enums import TaskStatus
-from danswer.server.documents.models import CCPairPruningTask
+from danswer.server.documents.models import CeleryTaskStatus
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.server.documents.models import ConnectorIndexingStatus
 from danswer.server.documents.models import DocumentSource
@@ -85,7 +85,7 @@ def create_from_scratch(
             groups=groups,
             user_performing_action=user_performing_action,
         )
-        return _cc_pair_creator(
+        cc_pair = _cc_pair_creator(
             connector_id=connector.id,
             credential_id=credential.id,
             name=name,
@@ -93,6 +93,7 @@ def create_from_scratch(
             groups=groups,
             user_performing_action=user_performing_action,
         )
+        return cc_pair
 
     @staticmethod
     def create(
@@ -103,7 +104,7 @@ def create(
         groups: list[int] | None = None,
         user_performing_action: DATestUser | None = None,
     ) -> DATestCCPair:
-        return _cc_pair_creator(
+        cc_pair = _cc_pair_creator(
             connector_id=connector_id,
             credential_id=credential_id,
             name=name,
@@ -111,6 +112,7 @@ def create(
             groups=groups,
             user_performing_action=user_performing_action,
         )
+        return cc_pair
 
     @staticmethod
     def pause_cc_pair(
@@ -203,9 +205,28 @@ def verify(
         if not verify_deleted:
             raise ValueError(f"CC pair {cc_pair.id} not found")
 
+    @staticmethod
+    def run_once(
+        cc_pair: DATestCCPair,
+        user_performing_action: DATestUser | None = None,
+    ) -> None:
+        body = {
+            "connector_id": cc_pair.connector_id,
+            "credential_ids": [cc_pair.credential_id],
+            "from_beginning": True,
+        }
+        result = requests.post(
+            url=f"{API_SERVER_URL}/manage/admin/connector/run-once",
+            json=body,
+            headers=user_performing_action.headers
+            if user_performing_action
+            else GENERAL_HEADERS,
+        )
+        result.raise_for_status()
+
     @staticmethod
     def wait_for_indexing(
-        cc_pair_test: DATestCCPair,
+        cc_pair: DATestCCPair,
         after: datetime,
         timeout: float = MAX_DELAY,
         user_performing_action: DATestUser | None = None,
@@ -213,14 +234,20 @@ def wait_for_indexing(
         """after: Wait for an indexing success time after this time"""
         start = time.monotonic()
         while True:
-            cc_pairs = CCPairManager.get_all(user_performing_action)
-            for cc_pair in cc_pairs:
-                if cc_pair.cc_pair_id != cc_pair_test.id:
+            fetched_cc_pairs = CCPairManager.get_all(user_performing_action)
+            for fetched_cc_pair in fetched_cc_pairs:
+                if fetched_cc_pair.cc_pair_id != cc_pair.id:
                     continue
 
-                if cc_pair.last_success and cc_pair.last_success > after:
-                    print(f"cc_pair {cc_pair_test.id} indexing complete.")
+                if (
+                    fetched_cc_pair.last_success
+                    and fetched_cc_pair.last_success > after
+                ):
+                    print(f"cc_pair {cc_pair.id} indexing complete.")
                     return
+                else:
+                    print("cc_pair found but not finished:")
+                    # print(fetched_cc_pair.__dict__)
 
             elapsed = time.monotonic() - start
             if elapsed > timeout:
@@ -250,7 +277,7 @@ def prune(
     def get_prune_task(
         cc_pair: DATestCCPair,
         user_performing_action: DATestUser | None = None,
-    ) -> CCPairPruningTask:
+    ) -> CeleryTaskStatus:
         response = requests.get(
             url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair.id}/prune",
             headers=user_performing_action.headers
@@ -258,11 +285,11 @@ def get_prune_task(
             else GENERAL_HEADERS,
         )
         response.raise_for_status()
-        return CCPairPruningTask(**response.json())
+        return CeleryTaskStatus(**response.json())
 
     @staticmethod
     def wait_for_prune(
-        cc_pair_test: DATestCCPair,
+        cc_pair: DATestCCPair,
         after: datetime,
         timeout: float = MAX_DELAY,
         user_performing_action: DATestUser | None = None,
@@ -270,7 +297,7 @@ def wait_for_prune(
         """after: The task register time must be after this time."""
         start = time.monotonic()
         while True:
-            task = CCPairManager.get_prune_task(cc_pair_test, user_performing_action)
+            task = CCPairManager.get_prune_task(cc_pair, user_performing_action)
             if not task:
                 raise ValueError("Prune task not found.")
 
@@ -292,16 +319,75 @@ def wait_for_prune(
             )
             time.sleep(5)
 
+    @staticmethod
+    def sync(
+        cc_pair: DATestCCPair,
+        user_performing_action: DATestUser | None = None,
+    ) -> None:
+        result = requests.post(
+            url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair.id}/sync",
+            headers=user_performing_action.headers
+            if user_performing_action
+            else GENERAL_HEADERS,
+        )
+        result.raise_for_status()
+
+    @staticmethod
+    def get_sync_task(
+        cc_pair: DATestCCPair,
+        user_performing_action: DATestUser | None = None,
+    ) -> CeleryTaskStatus:
+        response = requests.get(
+            url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair.id}/sync",
+            headers=user_performing_action.headers
+            if user_performing_action
+            else GENERAL_HEADERS,
+        )
+        response.raise_for_status()
+        return CeleryTaskStatus(**response.json())
+
+    @staticmethod
+    def wait_for_sync(
+        cc_pair: DATestCCPair,
+        after: datetime,
+        timeout: float = MAX_DELAY,
+        user_performing_action: DATestUser | None = None,
+    ) -> None:
+        """after: The task register time must be after this time."""
+        start = time.monotonic()
+        while True:
+            task = CCPairManager.get_sync_task(cc_pair, user_performing_action)
+            if not task:
+                raise ValueError("Sync task not found.")
+
+            if not task.register_time or task.register_time < after:
+                raise ValueError("Sync task register time is too early.")
+
+            if task.status == TaskStatus.SUCCESS:
+                # Sync succeeded
+                return
+
+            elapsed = time.monotonic() - start
+            if elapsed > timeout:
+                raise TimeoutError(
+                    f"CC pair syncing was not completed within {timeout} seconds"
+                )
+
+            print(
+                f"Waiting for CC syncing to complete. elapsed={elapsed:.2f} timeout={timeout}"
+            )
+            time.sleep(5)
+
     @staticmethod
     def wait_for_deletion_completion(
         user_performing_action: DATestUser | None = None,
     ) -> None:
         start = time.monotonic()
         while True:
-            cc_pairs = CCPairManager.get_all(user_performing_action)
+            fetched_cc_pairs = CCPairManager.get_all(user_performing_action)
             if all(
                 cc_pair.cc_pair_status != ConnectorCredentialPairStatus.DELETING
-                for cc_pair in cc_pairs
+                for cc_pair in fetched_cc_pairs
             ):
                 return
 
diff --git a/backend/tests/integration/common_utils/managers/llm_provider.py b/backend/tests/integration/common_utils/managers/llm_provider.py
index cde75284ca8..6ac4693496d 100644
--- a/backend/tests/integration/common_utils/managers/llm_provider.py
+++ b/backend/tests/integration/common_utils/managers/llm_provider.py
@@ -50,9 +50,7 @@ def create(
         )
         llm_response.raise_for_status()
         response_data = llm_response.json()
-        import json
 
-        print(json.dumps(response_data, indent=4))
         result_llm = DATestLLMProvider(
             id=response_data["id"],
             name=response_data["name"],
diff --git a/backend/tests/integration/common_utils/managers/user.py b/backend/tests/integration/common_utils/managers/user.py
index c299a5eb38a..4dcafc23cf9 100644
--- a/backend/tests/integration/common_utils/managers/user.py
+++ b/backend/tests/integration/common_utils/managers/user.py
@@ -17,11 +17,14 @@ class UserManager:
     @staticmethod
     def create(
         name: str | None = None,
+        email: str | None = None,
     ) -> DATestUser:
         if name is None:
             name = f"test{str(uuid4())}"
 
-        email = f"{name}@test.com"
+        if email is None:
+            email = f"{name}@test.com"
+
         password = "test"
 
         body = {
@@ -44,12 +47,10 @@ def create(
         )
         print(f"Created user {test_user.email}")
 
-        test_user.headers["Cookie"] = UserManager.login_as_user(test_user)
-
-        return test_user
+        return UserManager.login_as_user(test_user)
 
     @staticmethod
-    def login_as_user(test_user: DATestUser) -> str:
+    def login_as_user(test_user: DATestUser) -> DATestUser:
         data = urlencode(
             {
                 "username": test_user.email,
@@ -71,7 +72,9 @@ def login_as_user(test_user: DATestUser) -> str:
             raise Exception("Failed to login")
 
         print(f"Logged in as {test_user.email}")
-        return f"{result_cookie.name}={result_cookie.value}"
+        cookie = f"{result_cookie.name}={result_cookie.value}"
+        test_user.headers["Cookie"] = cookie
+        return test_user
 
     @staticmethod
     def verify_role(user_to_verify: DATestUser, target_role: UserRole) -> bool:
diff --git a/backend/tests/integration/connector_job_tests/slack/conftest.py b/backend/tests/integration/connector_job_tests/slack/conftest.py
new file mode 100644
index 00000000000..03d99737ce7
--- /dev/null
+++ b/backend/tests/integration/connector_job_tests/slack/conftest.py
@@ -0,0 +1,28 @@
+import os
+from collections.abc import Generator
+from typing import Any
+
+import pytest
+
+from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
+
+
+@pytest.fixture()
+def slack_test_setup() -> Generator[tuple[dict[str, Any], dict[str, Any]], None, None]:
+    slack_client = SlackManager.get_slack_client(os.environ["SLACK_BOT_TOKEN"])
+    admin_user_id = SlackManager.build_slack_user_email_id_map(slack_client)[
+        "admin@onyx-test.com"
+    ]
+
+    (
+        public_channel,
+        private_channel,
+        run_id,
+    ) = SlackManager.get_and_provision_available_slack_channels(
+        slack_client=slack_client, admin_user_id=admin_user_id
+    )
+
+    yield public_channel, private_channel
+
+    # This part will always run after the test, even if it fails
+    SlackManager.cleanup_after_test(slack_client=slack_client, test_id=run_id)
diff --git a/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py b/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py
new file mode 100644
index 00000000000..db399328917
--- /dev/null
+++ b/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py
@@ -0,0 +1,311 @@
+"""
+Assumptions:
+- The test users have already been created
+- General is empty of messages
+- In addition to the normal slack oauth permissions, the following scopes are needed:
+    - channels:manage
+    - groups:write
+    - chat:write
+    - chat:write.public
+"""
+from typing import Any
+from uuid import uuid4
+
+from slack_sdk import WebClient
+from slack_sdk.errors import SlackApiError
+
+from danswer.connectors.slack.connector import default_msg_filter
+from danswer.connectors.slack.connector import get_channel_messages
+from danswer.connectors.slack.utils import make_paginated_slack_api_call_w_retries
+from danswer.connectors.slack.utils import make_slack_api_call_w_retries
+
+
+def _get_slack_channel_id(channel: dict[str, Any]) -> str:
+    if not (channel_id := channel.get("id")):
+        raise ValueError("Channel ID is missing")
+    return channel_id
+
+
+def _get_non_general_channels(
+    slack_client: WebClient,
+    get_private: bool,
+    get_public: bool,
+    only_get_done: bool = False,
+) -> list[dict[str, Any]]:
+    channel_types = []
+    if get_private:
+        channel_types.append("private_channel")
+    if get_public:
+        channel_types.append("public_channel")
+
+    conversations: list[dict[str, Any]] = []
+    for result in make_paginated_slack_api_call_w_retries(
+        slack_client.conversations_list,
+        exclude_archived=False,
+        types=channel_types,
+    ):
+        conversations.extend(result["channels"])
+
+    filtered_conversations = []
+    for conversation in conversations:
+        if conversation.get("is_general", False):
+            continue
+        if only_get_done and "done" not in conversation.get("name", ""):
+            continue
+        filtered_conversations.append(conversation)
+    return filtered_conversations
+
+
+def _clear_slack_conversation_members(
+    slack_client: WebClient,
+    admin_user_id: str,
+    channel: dict[str, Any],
+) -> None:
+    channel_id = _get_slack_channel_id(channel)
+    member_ids: list[str] = []
+    for result in make_paginated_slack_api_call_w_retries(
+        slack_client.conversations_members,
+        channel=channel_id,
+    ):
+        member_ids.extend(result["members"])
+
+    for member_id in member_ids:
+        if member_id == admin_user_id:
+            continue
+        try:
+            make_slack_api_call_w_retries(
+                slack_client.conversations_kick, channel=channel_id, user=member_id
+            )
+            print(f"Kicked member: {member_id}")
+        except Exception as e:
+            if "cant_kick_self" in str(e):
+                continue
+            print(f"Error kicking member: {e}")
+            print(member_id)
+    try:
+        make_slack_api_call_w_retries(
+            slack_client.conversations_unarchive, channel=channel_id
+        )
+        channel["is_archived"] = False
+    except Exception:
+        # Channel is already unarchived
+        pass
+
+
+def _add_slack_conversation_members(
+    slack_client: WebClient, channel: dict[str, Any], member_ids: list[str]
+) -> None:
+    channel_id = _get_slack_channel_id(channel)
+    for user_id in member_ids:
+        try:
+            make_slack_api_call_w_retries(
+                slack_client.conversations_invite,
+                channel=channel_id,
+                users=user_id,
+            )
+        except Exception as e:
+            if "already_in_channel" in str(e):
+                continue
+            print(f"Error inviting member: {e}")
+            print(user_id)
+
+
+def _delete_slack_conversation_messages(
+    slack_client: WebClient,
+    channel: dict[str, Any],
+    message_to_delete: str | None = None,
+) -> None:
+    """deletes all messages from a channel if message_to_delete is None"""
+    channel_id = _get_slack_channel_id(channel)
+    for message_batch in get_channel_messages(slack_client, channel):
+        for message in message_batch:
+            if default_msg_filter(message):
+                continue
+
+            if message_to_delete and message.get("text") != message_to_delete:
+                continue
+            print(" removing message: ", message.get("text"))
+
+            try:
+                if not (ts := message.get("ts")):
+                    raise ValueError("Message timestamp is missing")
+                make_slack_api_call_w_retries(
+                    slack_client.chat_delete,
+                    channel=channel_id,
+                    ts=ts,
+                )
+            except Exception as e:
+                print(f"Error deleting message: {e}")
+                print(message)
+
+
+def _build_slack_channel_from_name(
+    slack_client: WebClient,
+    admin_user_id: str,
+    suffix: str,
+    is_private: bool,
+    channel: dict[str, Any] | None,
+) -> dict[str, Any]:
+    base = "public_channel" if not is_private else "private_channel"
+    channel_name = f"{base}-{suffix}"
+    if channel:
+        # If channel is provided, we rename it
+        channel_id = _get_slack_channel_id(channel)
+        channel_response = make_slack_api_call_w_retries(
+            slack_client.conversations_rename,
+            channel=channel_id,
+            name=channel_name,
+        )
+    else:
+        # Otherwise, we create a new channel
+        channel_response = make_slack_api_call_w_retries(
+            slack_client.conversations_create,
+            name=channel_name,
+            is_private=is_private,
+        )
+
+    try:
+        channel_response = make_slack_api_call_w_retries(
+            slack_client.conversations_unarchive,
+            channel=channel_response["channel"]["id"],
+        )
+    except Exception:
+        # Channel is already unarchived
+        pass
+    try:
+        channel_response = make_slack_api_call_w_retries(
+            slack_client.conversations_invite,
+            channel=channel_response["channel"]["id"],
+            users=[admin_user_id],
+        )
+    except Exception:
+        pass
+
+    final_channel = channel_response["channel"] if channel_response else {}
+    return final_channel
+
+
+class SlackManager:
+    @staticmethod
+    def get_slack_client(token: str) -> WebClient:
+        return WebClient(token=token)
+
+    @staticmethod
+    def get_and_provision_available_slack_channels(
+        slack_client: WebClient, admin_user_id: str
+    ) -> tuple[dict[str, Any], dict[str, Any], str]:
+        run_id = str(uuid4())
+        public_channels = _get_non_general_channels(
+            slack_client, get_private=False, get_public=True, only_get_done=True
+        )
+
+        first_available_channel = (
+            None if len(public_channels) < 1 else public_channels[0]
+        )
+        public_channel = _build_slack_channel_from_name(
+            slack_client=slack_client,
+            admin_user_id=admin_user_id,
+            suffix=run_id,
+            is_private=False,
+            channel=first_available_channel,
+        )
+        _delete_slack_conversation_messages(
+            slack_client=slack_client, channel=public_channel
+        )
+
+        private_channels = _get_non_general_channels(
+            slack_client, get_private=True, get_public=False, only_get_done=True
+        )
+        second_available_channel = (
+            None if len(private_channels) < 1 else private_channels[0]
+        )
+        private_channel = _build_slack_channel_from_name(
+            slack_client=slack_client,
+            admin_user_id=admin_user_id,
+            suffix=run_id,
+            is_private=True,
+            channel=second_available_channel,
+        )
+        _delete_slack_conversation_messages(
+            slack_client=slack_client, channel=private_channel
+        )
+
+        return public_channel, private_channel, run_id
+
+    @staticmethod
+    def build_slack_user_email_id_map(slack_client: WebClient) -> dict[str, str]:
+        users_results = make_slack_api_call_w_retries(
+            slack_client.users_list,
+        )
+        users: list[dict[str, Any]] = users_results.get("members", [])
+        user_email_id_map = {}
+        for user in users:
+            if not (email := user.get("profile", {}).get("email")):
+                continue
+            if not (user_id := user.get("id")):
+                raise ValueError("User ID is missing")
+            user_email_id_map[email] = user_id
+        return user_email_id_map
+
+    @staticmethod
+    def set_channel_members(
+        slack_client: WebClient,
+        admin_user_id: str,
+        channel: dict[str, Any],
+        user_ids: list[str],
+    ) -> None:
+        _clear_slack_conversation_members(
+            slack_client=slack_client,
+            channel=channel,
+            admin_user_id=admin_user_id,
+        )
+        _add_slack_conversation_members(
+            slack_client=slack_client, channel=channel, member_ids=user_ids
+        )
+
+    @staticmethod
+    def add_message_to_channel(
+        slack_client: WebClient, channel: dict[str, Any], message: str
+    ) -> None:
+        channel_id = _get_slack_channel_id(channel)
+        make_slack_api_call_w_retries(
+            slack_client.chat_postMessage,
+            channel=channel_id,
+            text=message,
+        )
+
+    @staticmethod
+    def remove_message_from_channel(
+        slack_client: WebClient, channel: dict[str, Any], message: str
+    ) -> None:
+        _delete_slack_conversation_messages(
+            slack_client=slack_client, channel=channel, message_to_delete=message
+        )
+
+    @staticmethod
+    def cleanup_after_test(
+        slack_client: WebClient,
+        test_id: str,
+    ) -> None:
+        channel_types = ["private_channel", "public_channel"]
+        channels: list[dict[str, Any]] = []
+        for result in make_paginated_slack_api_call_w_retries(
+            slack_client.conversations_list,
+            exclude_archived=False,
+            types=channel_types,
+        ):
+            channels.extend(result["channels"])
+
+        for channel in channels:
+            if test_id not in channel.get("name", ""):
+                continue
+            # "done" in the channel name indicates that this channel is free to be used for a new test
+            new_name = f"done_{str(uuid4())}"
+            try:
+                make_slack_api_call_w_retries(
+                    slack_client.conversations_rename,
+                    channel=channel["id"],
+                    name=new_name,
+                )
+            except SlackApiError as e:
+                print(f"Error renaming channel {channel['id']}: {e}")
diff --git a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
new file mode 100644
index 00000000000..20f2c1913d6
--- /dev/null
+++ b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
@@ -0,0 +1,251 @@
+import os
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+
+import requests
+
+from danswer.connectors.models import InputType
+from danswer.db.enums import AccessType
+from danswer.search.enums import LLMEvaluationType
+from danswer.search.enums import SearchType
+from danswer.search.models import RetrievalDetails
+from danswer.server.documents.models import DocumentSource
+from ee.danswer.server.query_and_chat.models import DocumentSearchRequest
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.managers.cc_pair import CCPairManager
+from tests.integration.common_utils.managers.connector import ConnectorManager
+from tests.integration.common_utils.managers.credential import CredentialManager
+from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
+from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.test_models import DATestCCPair
+from tests.integration.common_utils.test_models import DATestConnector
+from tests.integration.common_utils.test_models import DATestCredential
+from tests.integration.common_utils.test_models import DATestUser
+from tests.integration.common_utils.vespa import vespa_fixture
+from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
+
+
+def test_slack_permission_sync(
+    reset: None,
+    vespa_client: vespa_fixture,
+    slack_test_setup: tuple[dict[str, Any], dict[str, Any]],
+) -> None:
+    public_channel, private_channel = slack_test_setup
+
+    # Creating an admin user (first user created is automatically an admin)
+    admin_user: DATestUser = UserManager.create(
+        email="admin@onyx-test.com",
+    )
+
+    # Creating a non-admin user
+    test_user_1: DATestUser = UserManager.create(
+        email="test_user_1@onyx-test.com",
+    )
+
+    # Creating a non-admin user
+    test_user_2: DATestUser = UserManager.create(
+        email="test_user_2@onyx-test.com",
+    )
+
+    slack_client = SlackManager.get_slack_client(os.environ["SLACK_BOT_TOKEN"])
+    email_id_map = SlackManager.build_slack_user_email_id_map(slack_client)
+    admin_user_id = email_id_map[admin_user.email]
+
+    LLMProviderManager.create(user_performing_action=admin_user)
+
+    before = datetime.now(timezone.utc)
+    credential: DATestCredential = CredentialManager.create(
+        source=DocumentSource.SLACK,
+        credential_json={
+            "slack_bot_token": os.environ["SLACK_BOT_TOKEN"],
+        },
+        user_performing_action=admin_user,
+    )
+    connector: DATestConnector = ConnectorManager.create(
+        name="Slack",
+        input_type=InputType.POLL,
+        source=DocumentSource.SLACK,
+        connector_specific_config={
+            "workspace": "onyx-test-workspace",
+            "channels": [public_channel["name"], private_channel["name"]],
+        },
+        is_public=True,
+        groups=[],
+        user_performing_action=admin_user,
+    )
+    cc_pair: DATestCCPair = CCPairManager.create(
+        credential_id=credential.id,
+        connector_id=connector.id,
+        access_type=AccessType.SYNC,
+        user_performing_action=admin_user,
+    )
+    CCPairManager.wait_for_indexing(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # Add test_user_1 and admin_user to the private channel
+    desired_channel_members = [admin_user, test_user_1]
+    SlackManager.set_channel_members(
+        slack_client=slack_client,
+        admin_user_id=admin_user_id,
+        channel=private_channel,
+        user_ids=[email_id_map[user.email] for user in desired_channel_members],
+    )
+
+    public_message = "Steve's favorite number is 809752"
+    private_message = "Sara's favorite number is 346794"
+
+    SlackManager.add_message_to_channel(
+        slack_client=slack_client,
+        channel=public_channel,
+        message=public_message,
+    )
+    SlackManager.add_message_to_channel(
+        slack_client=slack_client,
+        channel=private_channel,
+        message=private_message,
+    )
+
+    # Run indexing
+    before = datetime.now(timezone.utc)
+    CCPairManager.run_once(cc_pair, admin_user)
+    CCPairManager.wait_for_indexing(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # Run permission sync
+    before = datetime.now(timezone.utc)
+    CCPairManager.sync(
+        cc_pair=cc_pair,
+        user_performing_action=admin_user,
+    )
+    CCPairManager.wait_for_sync(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # Search as admin with access to both channels
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=admin_user.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+
+    # Ensure admin user can see messages from both channels
+    assert public_message in danswer_doc_message_strings
+    assert private_message in danswer_doc_message_strings
+
+    # Search as test_user_2 with access to only the public channel
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=test_user_2.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content before removing from private channel for test_user_2: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure test_user_2 can only see messages from the public channel
+    assert public_message in danswer_doc_message_strings
+    assert private_message not in danswer_doc_message_strings
+
+    # Search as test_user_1 with access to both channels
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=test_user_1.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content before removing from private channel for test_user_1: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure test_user_1 can see messages from both channels
+    assert public_message in danswer_doc_message_strings
+    assert private_message in danswer_doc_message_strings
+
+    # ----------------------MAKE THE CHANGES--------------------------
+    print("\nRemoving test_user_1 from the private channel")
+    # Remove test_user_1 from the private channel
+    desired_channel_members = [admin_user]
+    SlackManager.set_channel_members(
+        slack_client=slack_client,
+        admin_user_id=admin_user_id,
+        channel=private_channel,
+        user_ids=[email_id_map[user.email] for user in desired_channel_members],
+    )
+
+    # Run permission sync
+    CCPairManager.sync(
+        cc_pair=cc_pair,
+        user_performing_action=admin_user,
+    )
+    CCPairManager.wait_for_sync(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # ----------------------------VERIFY THE CHANGES---------------------------
+    # Ensure test_user_1 can no longer see messages from the private channel
+    # Search as test_user_1 with access to only the public channel
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=test_user_1.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content after removing from private channel for test_user_1: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure test_user_1 can only see messages from the public channel
+    assert public_message in danswer_doc_message_strings
+    assert private_message not in danswer_doc_message_strings
diff --git a/backend/tests/integration/connector_job_tests/slack/test_prune.py b/backend/tests/integration/connector_job_tests/slack/test_prune.py
new file mode 100644
index 00000000000..fcac7db1384
--- /dev/null
+++ b/backend/tests/integration/connector_job_tests/slack/test_prune.py
@@ -0,0 +1,255 @@
+import os
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+
+import requests
+
+from danswer.connectors.models import InputType
+from danswer.db.enums import AccessType
+from danswer.search.enums import LLMEvaluationType
+from danswer.search.enums import SearchType
+from danswer.search.models import RetrievalDetails
+from danswer.server.documents.models import DocumentSource
+from ee.danswer.server.query_and_chat.models import DocumentSearchRequest
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.managers.cc_pair import CCPairManager
+from tests.integration.common_utils.managers.connector import ConnectorManager
+from tests.integration.common_utils.managers.credential import CredentialManager
+from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
+from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.test_models import DATestCCPair
+from tests.integration.common_utils.test_models import DATestConnector
+from tests.integration.common_utils.test_models import DATestCredential
+from tests.integration.common_utils.test_models import DATestUser
+from tests.integration.common_utils.vespa import vespa_fixture
+from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
+
+
+def test_slack_prune(
+    reset: None,
+    vespa_client: vespa_fixture,
+    slack_test_setup: tuple[dict[str, Any], dict[str, Any]],
+) -> None:
+    public_channel, private_channel = slack_test_setup
+
+    # Creating an admin user (first user created is automatically an admin)
+    admin_user: DATestUser = UserManager.create(
+        email="admin@onyx-test.com",
+    )
+
+    # Creating a non-admin user
+    test_user_1: DATestUser = UserManager.create(
+        email="test_user_1@onyx-test.com",
+    )
+
+    slack_client = SlackManager.get_slack_client(os.environ["SLACK_BOT_TOKEN"])
+    email_id_map = SlackManager.build_slack_user_email_id_map(slack_client)
+    admin_user_id = email_id_map[admin_user.email]
+
+    LLMProviderManager.create(user_performing_action=admin_user)
+
+    before = datetime.now(timezone.utc)
+    credential: DATestCredential = CredentialManager.create(
+        source=DocumentSource.SLACK,
+        credential_json={
+            "slack_bot_token": os.environ["SLACK_BOT_TOKEN"],
+        },
+        user_performing_action=admin_user,
+    )
+    connector: DATestConnector = ConnectorManager.create(
+        name="Slack",
+        input_type=InputType.POLL,
+        source=DocumentSource.SLACK,
+        connector_specific_config={
+            "workspace": "onyx-test-workspace",
+            "channels": [public_channel["name"], private_channel["name"]],
+        },
+        is_public=True,
+        groups=[],
+        user_performing_action=admin_user,
+    )
+    cc_pair: DATestCCPair = CCPairManager.create(
+        credential_id=credential.id,
+        connector_id=connector.id,
+        access_type=AccessType.SYNC,
+        user_performing_action=admin_user,
+    )
+    CCPairManager.wait_for_indexing(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # ----------------------SETUP INITIAL SLACK STATE--------------------------
+    # Add test_user_1 and admin_user to the private channel
+    desired_channel_members = [admin_user, test_user_1]
+    SlackManager.set_channel_members(
+        slack_client=slack_client,
+        admin_user_id=admin_user_id,
+        channel=private_channel,
+        user_ids=[email_id_map[user.email] for user in desired_channel_members],
+    )
+
+    public_message = "Steve's favorite number is 809752"
+    private_message = "Sara's favorite number is 346794"
+    message_to_delete = "Rebecca's favorite number is 753468"
+
+    SlackManager.add_message_to_channel(
+        slack_client=slack_client,
+        channel=public_channel,
+        message=public_message,
+    )
+    SlackManager.add_message_to_channel(
+        slack_client=slack_client,
+        channel=private_channel,
+        message=private_message,
+    )
+    SlackManager.add_message_to_channel(
+        slack_client=slack_client,
+        channel=private_channel,
+        message=message_to_delete,
+    )
+
+    # Run indexing
+    before = datetime.now(timezone.utc)
+    CCPairManager.run_once(cc_pair, admin_user)
+    CCPairManager.wait_for_indexing(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # Run permission sync
+    before = datetime.now(timezone.utc)
+    CCPairManager.sync(
+        cc_pair=cc_pair,
+        user_performing_action=admin_user,
+    )
+    CCPairManager.wait_for_sync(
+        cc_pair=cc_pair,
+        after=before,
+        user_performing_action=admin_user,
+    )
+
+    # ----------------------TEST THE SETUP--------------------------
+    # Search as admin with access to both channels
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=admin_user.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content before deleting for admin: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure admin user can see all messages
+    assert public_message in danswer_doc_message_strings
+    assert private_message in danswer_doc_message_strings
+    assert message_to_delete in danswer_doc_message_strings
+
+    # Search as test_user_1 with access to both channels
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=test_user_1.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content before deleting for test_user_1: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure test_user_1 can see all messages
+    assert public_message in danswer_doc_message_strings
+    assert private_message in danswer_doc_message_strings
+    assert message_to_delete in danswer_doc_message_strings
+
+    # ----------------------MAKE THE CHANGES--------------------------
+    # Delete messages
+    print("\nDeleting message: ", message_to_delete)
+    SlackManager.remove_message_from_channel(
+        slack_client=slack_client,
+        channel=private_channel,
+        message=message_to_delete,
+    )
+
+    # Prune the cc_pair
+    before = datetime.now(timezone.utc)
+    CCPairManager.prune(cc_pair, user_performing_action=admin_user)
+    CCPairManager.wait_for_prune(cc_pair, before, user_performing_action=admin_user)
+
+    # ----------------------------VERIFY THE CHANGES---------------------------
+    # Ensure admin user can't see deleted messages
+    # Search as admin user with access to only the public channel
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=admin_user.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content after deleting for admin: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure admin can't see deleted messages
+    assert public_message in danswer_doc_message_strings
+    assert private_message in danswer_doc_message_strings
+    assert message_to_delete not in danswer_doc_message_strings
+
+    # Ensure test_user_1 can't see deleted messages
+    # Search as test_user_1 with access to only the public channel
+    search_request = DocumentSearchRequest(
+        message="favorite number",
+        search_type=SearchType.KEYWORD,
+        retrieval_options=RetrievalDetails(),
+        evaluation_type=LLMEvaluationType.SKIP,
+    )
+    search_request_body = search_request.model_dump()
+    result = requests.post(
+        url=f"{API_SERVER_URL}/query/document-search",
+        json=search_request_body,
+        headers=test_user_1.headers,
+    )
+    result.raise_for_status()
+    found_docs = result.json()["top_documents"]
+    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    print(
+        "\ntop_documents content after prune for test_user_1: ",
+        danswer_doc_message_strings,
+    )
+
+    # Ensure test_user_1 can't see deleted messages
+    assert public_message in danswer_doc_message_strings
+    assert private_message in danswer_doc_message_strings
+    assert message_to_delete not in danswer_doc_message_strings

From 0c54d9d57dc95866259e89617d09af8ea8d7cd17 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Wed, 2 Oct 2024 17:48:11 -0700
Subject: [PATCH 021/376] Unstructured Update Copy (#2668)

---
 .../configuration/document-processing/page.tsx      | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/web/src/app/admin/configuration/document-processing/page.tsx b/web/src/app/admin/configuration/document-processing/page.tsx
index 9ccd72b73d1..37934181e17 100644
--- a/web/src/app/admin/configuration/document-processing/page.tsx
+++ b/web/src/app/admin/configuration/document-processing/page.tsx
@@ -55,15 +55,18 @@ function Main() {
     <div className="container mx-auto p-4">
       <Card className="mb-8 max-w-2xl bg-white text-text shadow-lg rounded-lg">
         <h3 className="text-2xl text-text-800 font-bold mb-4 text-text border-b border-b-border pb-2">
-          Unstructured API Integration
+          Process with Unstructured API
         </h3>
 
         <div className="space-y-4">
           <p className="text-text-600">
-            Unstructured effortlessly extracts and transforms complex data from
-            difficult-to-use formats like HTML, PDF, CSV, PNG, PPTX, and more.
-            Enter an API key to enable this powerful document processing. If not
-            set, standard document processing will be used.
+            Unstructured extracts and transforms complex data from formats like
+            .pdf, .docx, .png, .pptx, etc. into clean text for Danswer to
+            ingest. Provide an API key to enable Unstructured document
+            processing.
+            <br />
+            <br /> <strong>Note:</strong> this will send documents to
+            Unstructured servers for processing.
           </p>
           <p className="text-text-600">
             Learn more about Unstructured{" "}

From 3fdd233e84521270865f3a34bc579452c87c7ba6 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 2 Oct 2024 18:57:25 -0700
Subject: [PATCH 022/376] delete directly via selection instead of making
 multiple calls to get chunk ids and delete each one (#2666)

---
 .../danswer/background/connector_deletion.py  |  2 +-
 backend/danswer/document_index/interfaces.py  | 10 +++
 backend/danswer/document_index/vespa/index.py | 61 +++++++++++++++++++
 3 files changed, 72 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/background/connector_deletion.py b/backend/danswer/background/connector_deletion.py
index 983a3c129ba..84b696dd8e4 100644
--- a/backend/danswer/background/connector_deletion.py
+++ b/backend/danswer/background/connector_deletion.py
@@ -148,7 +148,7 @@ def document_by_cc_pair_cleanup_task(
             if count == 1:
                 # count == 1 means this is the only remaining cc_pair reference to the doc
                 # delete it from vespa and the db
-                document_index.delete(doc_ids=[document_id])
+                document_index.delete_single(doc_id=document_id)
                 delete_documents_complete__no_commit(
                     db_session=db_session,
                     document_ids=[document_id],
diff --git a/backend/danswer/document_index/interfaces.py b/backend/danswer/document_index/interfaces.py
index eaa34b37752..b499d696743 100644
--- a/backend/danswer/document_index/interfaces.py
+++ b/backend/danswer/document_index/interfaces.py
@@ -156,6 +156,16 @@ class Deletable(abc.ABC):
     Class must implement the ability to delete document by their unique document ids.
     """
 
+    @abc.abstractmethod
+    def delete_single(self, doc_id: str) -> None:
+        """
+        Given a single document id, hard delete it from the document index
+
+        Parameters:
+        - doc_id: document id as specified by the connector
+        """
+        raise NotImplementedError
+
     @abc.abstractmethod
     def delete(self, doc_ids: list[str]) -> None:
         """
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 700f8860fb5..467260ed619 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -13,6 +13,7 @@
 import httpx
 import requests
 
+from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
 from danswer.configs.chat_configs import DOC_TIME_DECAY
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
 from danswer.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -479,6 +480,66 @@ def delete(self, doc_ids: list[str]) -> None:
                     document_ids=doc_ids, index_name=index_name, http_client=http_client
                 )
 
+    def delete_single(self, doc_id: str) -> None:
+        """Possibly faster overall than the delete method due to using a single
+        delete call with a selection query."""
+
+        # Vespa deletion is poorly documented ... luckily we found this
+        # https://docs.vespa.ai/en/operations/batch-delete.html#example
+
+        doc_id = replace_invalid_doc_id_characters(doc_id)
+
+        # NOTE: using `httpx` here since `requests` doesn't support HTTP2. This is beneficial for
+        # indexing / updates / deletes since we have to make a large volume of requests.
+        index_names = [self.index_name]
+        if self.secondary_index_name:
+            index_names.append(self.secondary_index_name)
+
+        with httpx.Client(http2=True) as http_client:
+            for index_name in index_names:
+                params = httpx.QueryParams(
+                    {
+                        "selection": f"{index_name}.document_id=='{doc_id}'",
+                        "cluster": DOCUMENT_INDEX_NAME,
+                    }
+                )
+
+                total_chunks_deleted = 0
+                while True:
+                    try:
+                        resp = http_client.delete(
+                            f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}",
+                            params=params,
+                        )
+                        resp.raise_for_status()
+                    except httpx.HTTPStatusError as e:
+                        logger.error(
+                            f"Failed to delete chunk, details: {e.response.text}"
+                        )
+                        raise
+
+                    resp_data = resp.json()
+
+                    if "documentCount" in resp_data:
+                        chunks_deleted = resp_data["documentCount"]
+                        total_chunks_deleted += chunks_deleted
+
+                    # Check for continuation token to handle pagination
+                    if "continuation" not in resp_data:
+                        break  # Exit loop if no continuation token
+
+                    if not resp_data["continuation"]:
+                        break  # Exit loop if continuation token is empty
+
+                    params = params.set("continuation", resp_data["continuation"])
+
+                logger.debug(
+                    f"VespaIndex.delete_single: "
+                    f"index={index_name} "
+                    f"doc={doc_id} "
+                    f"chunks_deleted={total_chunks_deleted}"
+                )
+
     def id_based_retrieval(
         self,
         chunk_requests: list[VespaChunkRequest],

From 4f47004d4788bc4ecb1840bad043f5caed2467a9 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 3 Oct 2024 17:25:46 -0700
Subject: [PATCH 023/376] disable another flaky assert (#2678)

---
 .../tests/integration/tests/dev_apis/test_knowledge_chat.py  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py b/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
index a6bc259c640..2911c0f11f1 100644
--- a/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
+++ b/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
@@ -183,6 +183,7 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     # FLAKY - check that the cited documents are correct
     # assert cc_pair_1.documents[2].id in response_json["cited_documents"].values()
 
-    # check that the top documents are correct
-    assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[2].id
+    # flakiness likely due to non-deterministic rephrasing
+    # FLAKY - check that the top documents are correct
+    # assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[2].id
     print("response 3/3 passed")

From 1362d4b58315167f8ac0a4d87b6c5333581edb86 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Thu, 3 Oct 2024 17:55:28 -0700
Subject: [PATCH 024/376] Allow config of background concurrency (#2648)

* Allow config of background concurrency

* Add comment

* Fix light worker

* use backslashes to continue lines in supervisord with bash

---------

Co-authored-by: Richard Kuo (Danswer) <rkuo@danswer.ai>
---
 backend/supervisord.conf                      | 19 +++++++++++--------
 .../docker_compose/docker-compose.dev.yml     |  6 +++++-
 .../docker_compose/docker-compose.gpu-dev.yml |  6 +++++-
 .../docker-compose.prod-no-letsencrypt.yml    |  2 +-
 .../docker_compose/docker-compose.prod.yml    |  2 +-
 .../docker-compose.search-testing.yml         |  2 +-
 .../postgres-service-deployment.yaml          |  2 +-
 7 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/backend/supervisord.conf b/backend/supervisord.conf
index 5b9dca95b4c..2c73545904f 100644
--- a/backend/supervisord.conf
+++ b/backend/supervisord.conf
@@ -39,15 +39,18 @@ autorestart=true
 startsecs=10
 stopasgroup=true
 
+# NOTE: only allowing configuration here and not in the other celery workers, 
+# since this is often the bottleneck for "sync" jobs (e.g. document set syncing, 
+# user group syncing, deletion, etc.) 
 [program:celery_worker_light]
-command=celery -A danswer.background.celery.celery_run:celery_app worker 
-    --pool=threads
-    --concurrency=16
-    --prefetch-multiplier=8 
-    --loglevel=INFO 
-    --logfile=/var/log/celery_worker_light_supervisor.log 
-    --hostname=light@%%n 
-    -Q vespa_metadata_sync,connector_deletion
+command=bash -c "celery -A danswer.background.celery.celery_run:celery_app worker \
+    --pool=threads \
+    --concurrency=${CELERY_WORKER_LIGHT_CONCURRENCY:-24} \
+    --prefetch-multiplier=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-8} \
+    --loglevel=INFO \
+    --logfile=/var/log/celery_worker_light_supervisor.log \
+    --hostname=light@%%n \
+    -Q vespa_metadata_sync,connector_deletion"
 environment=LOG_FILE_NAME=celery_worker_light
 redirect_stderr=true
 autorestart=true
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 5893fc2960f..639db923a9f 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -169,6 +169,10 @@ services:
       - GONG_CONNECTOR_START_TIME=${GONG_CONNECTOR_START_TIME:-}
       - NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP=${NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP:-}
       - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
+      # Celery Configs (defaults are set in the supervisord.conf file, prefer doing that to have on source
+      # of defaults)
+      - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
+      - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
 
       # Danswer SlackBot Configs
       - DANSWER_BOT_SLACK_APP_TOKEN=${DANSWER_BOT_SLACK_APP_TOKEN:-}
@@ -287,7 +291,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index bc8bf10dffc..9a1d62c03f0 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -182,6 +182,10 @@ services:
       # Log all of Danswer prompts and interactions with the LLM
       - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
+      # Celery Configs (defaults are set in the supervisord.conf file, prefer doing that to have on source
+      # of defaults)
+      - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
+      - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
 
       # Enterprise Edition only
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
@@ -297,7 +301,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}
diff --git a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
index 7a56346f074..8ec6d437646 100644
--- a/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
+++ b/deployment/docker_compose/docker-compose.prod-no-letsencrypt.yml
@@ -147,7 +147,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
     env_file:
diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml
index 983881fff7f..eaeaa7646b7 100644
--- a/deployment/docker_compose/docker-compose.prod.yml
+++ b/deployment/docker_compose/docker-compose.prod.yml
@@ -89,7 +89,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
     env_file:
diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index a64b30f09d7..f9be3360d52 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -148,7 +148,7 @@ services:
 
   relational_db:
     image: postgres:15.2-alpine
-    command: -c 'max_connections=150'
+    command: -c 'max_connections=250'
     restart: always
     environment:
       - POSTGRES_USER=${POSTGRES_USER:-postgres}
diff --git a/deployment/kubernetes/postgres-service-deployment.yaml b/deployment/kubernetes/postgres-service-deployment.yaml
index 4a0b2bbdbcc..e89e625589d 100644
--- a/deployment/kubernetes/postgres-service-deployment.yaml
+++ b/deployment/kubernetes/postgres-service-deployment.yaml
@@ -40,7 +40,7 @@ spec:
             secretKeyRef:
               name: danswer-secrets
               key: postgres_password
-        args: ["-c", "max_connections=150"]
+        args: ["-c", "max_connections=250"]
         ports:
         - containerPort: 5432
         volumeMounts:

From 7f788e4b1e3fa69baec4240cc5791f6036bfae26 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 3 Oct 2024 22:54:32 -0700
Subject: [PATCH 025/376] bump celery to 5.5.0b4 (#2681)

---
 backend/requirements/default.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 30e0e300bb2..20305c1a3d0 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -4,7 +4,7 @@ asyncpg==0.27.0
 atlassian-python-api==3.37.0
 beautifulsoup4==4.12.3
 boto3==1.34.84
-celery==5.3.4
+celery==5.5.0b4
 chardet==5.2.0
 dask==2023.8.1
 ddtrace==2.6.5

From 63655cfbedafdf912f6b15b97752f17ee9890bc4 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Fri, 4 Oct 2024 08:43:04 -0700
Subject: [PATCH 026/376] update_single should be optimized for a single call
 now (#2671)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 .../danswer/background/celery/celery_app.py   |   2 -
 .../danswer/background/connector_deletion.py  |   6 +-
 .../background/indexing/run_indexing.py       |  18 ++-
 backend/danswer/background/update.py          |  10 +-
 backend/danswer/document_index/interfaces.py  |  29 +++--
 backend/danswer/document_index/vespa/index.py | 123 +++++++++---------
 6 files changed, 106 insertions(+), 82 deletions(-)

diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 5244d9b94da..d20c49831a9 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -364,8 +364,6 @@ def run_periodic_task(self, worker: Any) -> None:
 
             lock: redis.lock.Lock = worker.primary_worker_lock
 
-            task_logger.info("Reacquiring primary worker lock.")
-
             if lock.owned():
                 task_logger.debug("Reacquiring primary worker lock.")
                 lock.reacquire()
diff --git a/backend/danswer/background/connector_deletion.py b/backend/danswer/background/connector_deletion.py
index 84b696dd8e4..962183f71ae 100644
--- a/backend/danswer/background/connector_deletion.py
+++ b/backend/danswer/background/connector_deletion.py
@@ -33,6 +33,7 @@
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import DocumentIndex
 from danswer.document_index.interfaces import UpdateRequest
+from danswer.document_index.interfaces import VespaDocumentFields
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.utils.logger import setup_logger
 
@@ -168,8 +169,7 @@ def document_by_cc_pair_cleanup_task(
                 doc_sets = fetch_document_sets_for_document(document_id, db_session)
                 update_doc_sets: set[str] = set(doc_sets)
 
-                update_request = UpdateRequest(
-                    document_ids=[document_id],
+                fields = VespaDocumentFields(
                     document_sets=update_doc_sets,
                     access=doc_access,
                     boost=doc.boost,
@@ -177,7 +177,7 @@ def document_by_cc_pair_cleanup_task(
                 )
 
                 # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-                document_index.update_single(update_request=update_request)
+                document_index.update_single(document_id, fields=fields)
 
                 # there are still other cc_pair references to the doc, so just resync to Vespa
                 delete_document_by_connector_credential_pair__no_commit(
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index 499899ac225..b3d011a422b 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -14,6 +14,7 @@
 from danswer.connectors.connector_runner import ConnectorRunner
 from danswer.connectors.factory import instantiate_connector
 from danswer.connectors.models import IndexAttemptMetadata
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.connector_credential_pair import get_last_successful_attempt_time
 from danswer.db.connector_credential_pair import update_connector_credential_pair
 from danswer.db.engine import get_sqlalchemy_engine
@@ -49,7 +50,7 @@ def _get_connector_runner(
     """
     NOTE: `start_time` and `end_time` are only used for poll connectors
 
-    Returns an interator of document batches and whether the returned documents
+    Returns an iterator of document batches and whether the returned documents
     are the complete list of existing documents of the connector. If the task
     of type LOAD_STATE, the list will be considered complete and otherwise incomplete.
     """
@@ -67,12 +68,17 @@ def _get_connector_runner(
         logger.exception(f"Unable to instantiate connector due to {e}")
         # since we failed to even instantiate the connector, we pause the CCPair since
         # it will never succeed
-        update_connector_credential_pair(
-            db_session=db_session,
-            connector_id=attempt.connector_credential_pair.connector.id,
-            credential_id=attempt.connector_credential_pair.credential.id,
-            status=ConnectorCredentialPairStatus.PAUSED,
+
+        cc_pair = get_connector_credential_pair_from_id(
+            attempt.connector_credential_pair.id, db_session
         )
+        if cc_pair and cc_pair.status == ConnectorCredentialPairStatus.ACTIVE:
+            update_connector_credential_pair(
+                db_session=db_session,
+                connector_id=attempt.connector_credential_pair.connector.id,
+                credential_id=attempt.connector_credential_pair.credential.id,
+                status=ConnectorCredentialPairStatus.PAUSED,
+            )
         raise e
 
     return ConnectorRunner(
diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index 94e703635ee..c88b349e4fe 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -96,14 +96,20 @@ def _should_create_new_indexing(
             if last_index.status == IndexingStatus.IN_PROGRESS:
                 return False
         else:
-            if connector.id == 0:  # Ingestion API
+            if (
+                connector.id == 0 or connector.source == DocumentSource.INGESTION_API
+            ):  # Ingestion API
                 return False
         return True
 
     # If the connector is paused or is the ingestion API, don't index
     # NOTE: during an embedding model switch over, the following logic
     # is bypassed by the above check for a future model
-    if not cc_pair.status.is_active() or connector.id == 0:
+    if (
+        not cc_pair.status.is_active()
+        or connector.id == 0
+        or connector.source == DocumentSource.INGESTION_API
+    ):
         return False
 
     if not last_index:
diff --git a/backend/danswer/document_index/interfaces.py b/backend/danswer/document_index/interfaces.py
index b499d696743..e4ff90a40cf 100644
--- a/backend/danswer/document_index/interfaces.py
+++ b/backend/danswer/document_index/interfaces.py
@@ -55,6 +55,21 @@ class DocumentMetadata:
     from_ingestion_api: bool = False
 
 
+@dataclass
+class VespaDocumentFields:
+    """
+    Specifies fields in Vespa for a document.  Fields set to None will be ignored.
+    Perhaps we should name this in an implementation agnostic fashion, but it's more
+    understandable like this for now.
+    """
+
+    # all other fields except these 4 will always be left alone by the update request
+    access: DocumentAccess | None = None
+    document_sets: set[str] | None = None
+    boost: float | None = None
+    hidden: bool | None = None
+
+
 @dataclass
 class UpdateRequest:
     """
@@ -188,11 +203,9 @@ class Updatable(abc.ABC):
     """
 
     @abc.abstractmethod
-    def update_single(self, update_request: UpdateRequest) -> None:
+    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
         """
-        Updates some set of chunks for a document. The document and fields to update
-        are specified in the update request. Each update request in the list applies
-        its changes to a list of document ids.
+        Updates all chunks for a document with the specified fields.
         None values mean that the field does not need an update.
 
         The rationale for a single update function is that it allows retries and parallelism
@@ -200,14 +213,10 @@ def update_single(self, update_request: UpdateRequest) -> None:
         us to individually handle error conditions per document.
 
         Parameters:
-        - update_request: for a list of document ids in the update request, apply the same updates
-                to all of the documents with those ids.
+        - fields: the fields to update in the document. Any field set to None will not be changed.
 
         Return:
-        - an HTTPStatus code. The code can used to decide whether to fail immediately,
-        retry, etc.  Although this method likely hits an HTTP API behind the
-        scenes, the usage of HTTPStatus is a convenience and the interface is not
-        actually HTTP specific.
+            None
         """
         raise NotImplementedError
 
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 467260ed619..512eb932156 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -1,5 +1,6 @@
 import concurrent.futures
 import io
+import logging
 import os
 import re
 import time
@@ -23,6 +24,7 @@
 from danswer.document_index.interfaces import DocumentInsertionRecord
 from danswer.document_index.interfaces import UpdateRequest
 from danswer.document_index.interfaces import VespaChunkRequest
+from danswer.document_index.interfaces import VespaDocumentFields
 from danswer.document_index.vespa.chunk_retrieval import batch_search_api_retrieval
 from danswer.document_index.vespa.chunk_retrieval import (
     get_all_vespa_ids_for_document_id,
@@ -69,6 +71,10 @@
 
 logger = setup_logger()
 
+# Set the logging level to WARNING to ignore INFO and DEBUG logs
+httpx_logger = logging.getLogger("httpx")
+httpx_logger.setLevel(logging.WARNING)
+
 
 @dataclass
 class _VespaUpdateRequest:
@@ -378,89 +384,86 @@ def update(self, update_requests: list[UpdateRequest]) -> None:
             time.monotonic() - update_start,
         )
 
-    def update_single(self, update_request: UpdateRequest) -> None:
+    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
         """Note: if the document id does not exist, the update will be a no-op and the
         function will complete with no errors or exceptions.
         Handle other exceptions if you wish to implement retry behavior
         """
-        if len(update_request.document_ids) != 1:
-            raise ValueError("update_request must contain a single document id")
 
         # Handle Vespa character limitations
         # Mutating update_request but it's not used later anyway
-        update_request.document_ids = [
-            replace_invalid_doc_id_characters(doc_id)
-            for doc_id in update_request.document_ids
-        ]
-
-        # update_start = time.monotonic()
-
-        # Fetch all chunks for each document ahead of time
-        index_names = [self.index_name]
-        if self.secondary_index_name:
-            index_names.append(self.secondary_index_name)
-
-        chunk_id_start_time = time.monotonic()
-        all_doc_chunk_ids: list[str] = []
-        for index_name in index_names:
-            for document_id in update_request.document_ids:
-                # this calls vespa and can raise http exceptions
-                doc_chunk_ids = get_all_vespa_ids_for_document_id(
-                    document_id=document_id,
-                    index_name=index_name,
-                    filters=None,
-                    get_large_chunks=True,
-                )
-                all_doc_chunk_ids.extend(doc_chunk_ids)
-        logger.debug(
-            f"Took {time.monotonic() - chunk_id_start_time:.2f} seconds to fetch all Vespa chunk IDs"
-        )
+        normalized_doc_id = replace_invalid_doc_id_characters(doc_id)
 
         # Build the _VespaUpdateRequest objects
         update_dict: dict[str, dict] = {"fields": {}}
-        if update_request.boost is not None:
-            update_dict["fields"][BOOST] = {"assign": update_request.boost}
-        if update_request.document_sets is not None:
+        if fields.boost is not None:
+            update_dict["fields"][BOOST] = {"assign": fields.boost}
+        if fields.document_sets is not None:
             update_dict["fields"][DOCUMENT_SETS] = {
-                "assign": {
-                    document_set: 1 for document_set in update_request.document_sets
-                }
+                "assign": {document_set: 1 for document_set in fields.document_sets}
             }
-        if update_request.access is not None:
+        if fields.access is not None:
             update_dict["fields"][ACCESS_CONTROL_LIST] = {
-                "assign": {acl_entry: 1 for acl_entry in update_request.access.to_acl()}
+                "assign": {acl_entry: 1 for acl_entry in fields.access.to_acl()}
             }
-        if update_request.hidden is not None:
-            update_dict["fields"][HIDDEN] = {"assign": update_request.hidden}
+        if fields.hidden is not None:
+            update_dict["fields"][HIDDEN] = {"assign": fields.hidden}
 
         if not update_dict["fields"]:
             logger.error("Update request received but nothing to update")
             return
 
-        processed_update_requests: list[_VespaUpdateRequest] = []
-        for document_id in update_request.document_ids:
-            for doc_chunk_id in all_doc_chunk_ids:
-                processed_update_requests.append(
-                    _VespaUpdateRequest(
-                        document_id=document_id,
-                        url=f"{DOCUMENT_ID_ENDPOINT.format(index_name=self.index_name)}/{doc_chunk_id}",
-                        update_request=update_dict,
-                    )
-                )
+        index_names = [self.index_name]
+        if self.secondary_index_name:
+            index_names.append(self.secondary_index_name)
 
         with httpx.Client(http2=True) as http_client:
-            for update in processed_update_requests:
-                http_client.put(
-                    update.url,
-                    headers={"Content-Type": "application/json"},
-                    json=update.update_request,
+            for index_name in index_names:
+                params = httpx.QueryParams(
+                    {
+                        "selection": f"{index_name}.document_id=='{normalized_doc_id}'",
+                        "cluster": DOCUMENT_INDEX_NAME,
+                    }
                 )
 
-        # logger.debug(
-        #     "Finished updating Vespa documents in %.2f seconds",
-        #     time.monotonic() - update_start,
-        # )
+                total_chunks_updated = 0
+                while True:
+                    try:
+                        resp = http_client.put(
+                            f"{DOCUMENT_ID_ENDPOINT.format(index_name=self.index_name)}",
+                            params=params,
+                            headers={"Content-Type": "application/json"},
+                            json=update_dict,
+                        )
+
+                        resp.raise_for_status()
+                    except httpx.HTTPStatusError as e:
+                        logger.error(
+                            f"Failed to update chunks, details: {e.response.text}"
+                        )
+                        raise
+
+                    resp_data = resp.json()
+
+                    if "documentCount" in resp_data:
+                        chunks_updated = resp_data["documentCount"]
+                        total_chunks_updated += chunks_updated
 
+                    # Check for continuation token to handle pagination
+                    if "continuation" not in resp_data:
+                        break  # Exit loop if no continuation token
+
+                    if not resp_data["continuation"]:
+                        break  # Exit loop if continuation token is empty
+
+                    params = params.set("continuation", resp_data["continuation"])
+
+                logger.debug(
+                    f"VespaIndex.update_single: "
+                    f"index={index_name} "
+                    f"doc={normalized_doc_id} "
+                    f"chunks_deleted={total_chunks_updated}"
+                )
         return
 
     def delete(self, doc_ids: list[str]) -> None:
@@ -479,6 +482,7 @@ def delete(self, doc_ids: list[str]) -> None:
                 delete_vespa_docs(
                     document_ids=doc_ids, index_name=index_name, http_client=http_client
                 )
+        return
 
     def delete_single(self, doc_id: str) -> None:
         """Possibly faster overall than the delete method due to using a single
@@ -539,6 +543,7 @@ def delete_single(self, doc_id: str) -> None:
                     f"doc={doc_id} "
                     f"chunks_deleted={total_chunks_deleted}"
                 )
+        return
 
     def id_based_retrieval(
         self,

From 3755e575a57b9294bbfbd3f06e84a24f09743d5a Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Fri, 4 Oct 2024 09:00:48 -0700
Subject: [PATCH 027/376] harden connections to redis (#2677)

* set broker_connection_retry_on_startup to silence deprecation warning (we're OK with retrying on startup)

* env var for CELERY_BROKER_POOL_LIMIT

* add redis retry on timeout and health check interval

* set socket_keepalive = True

* remove shadow declaration of REDIS_HEALTH_CHECK_INTERVAL, add socket_keepalive_options where possible

* fix mypy complaint

* pass through vars in docker compose

* remove extra '='

* wrap in a try
---
 .../danswer/background/celery/celeryconfig.py | 21 +++++++++++++++++++
 backend/danswer/configs/app_configs.py        | 16 ++++++++++++++
 backend/danswer/configs/constants.py          | 12 +++++++++++
 backend/danswer/redis/redis_pool.py           | 11 ++++++++--
 .../docker_compose/docker-compose.dev.yml     |  1 +
 .../docker_compose/docker-compose.gpu-dev.yml |  3 ++-
 6 files changed, 61 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/background/celery/celeryconfig.py b/backend/danswer/background/celery/celeryconfig.py
index d0314adf865..1b1aa092d17 100644
--- a/backend/danswer/background/celery/celeryconfig.py
+++ b/backend/danswer/background/celery/celeryconfig.py
@@ -1,7 +1,9 @@
 # docs: https://docs.celeryq.dev/en/stable/userguide/configuration.html
+from danswer.configs.app_configs import CELERY_BROKER_POOL_LIMIT
 from danswer.configs.app_configs import CELERY_RESULT_EXPIRES
 from danswer.configs.app_configs import REDIS_DB_NUMBER_CELERY
 from danswer.configs.app_configs import REDIS_DB_NUMBER_CELERY_RESULT_BACKEND
+from danswer.configs.app_configs import REDIS_HEALTH_CHECK_INTERVAL
 from danswer.configs.app_configs import REDIS_HOST
 from danswer.configs.app_configs import REDIS_PASSWORD
 from danswer.configs.app_configs import REDIS_PORT
@@ -9,6 +11,7 @@
 from danswer.configs.app_configs import REDIS_SSL_CA_CERTS
 from danswer.configs.app_configs import REDIS_SSL_CERT_REQS
 from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
 
 CELERY_SEPARATOR = ":"
 
@@ -36,12 +39,30 @@
 # can stall other tasks.
 worker_prefetch_multiplier = 4
 
+broker_connection_retry_on_startup = True
+broker_pool_limit = CELERY_BROKER_POOL_LIMIT
+
+# redis broker settings
+# https://docs.celeryq.dev/projects/kombu/en/stable/reference/kombu.transport.redis.html
 broker_transport_options = {
     "priority_steps": list(range(len(DanswerCeleryPriority))),
     "sep": CELERY_SEPARATOR,
     "queue_order_strategy": "priority",
+    "retry_on_timeout": True,
+    "health_check_interval": REDIS_HEALTH_CHECK_INTERVAL,
+    "socket_keepalive": True,
+    "socket_keepalive_options": REDIS_SOCKET_KEEPALIVE_OPTIONS,
 }
 
+# redis backend settings
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-backend-settings
+
+# there doesn't appear to be a way to set socket_keepalive_options on the redis result backend
+redis_socket_keepalive = True
+redis_retry_on_timeout = True
+redis_backend_health_check_interval = REDIS_HEALTH_CHECK_INTERVAL
+
+
 task_default_priority = DanswerCeleryPriority.MEDIUM
 task_acks_late = True
 
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 460e15bd1f4..6096561b5a5 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -164,6 +164,12 @@
 )
 REDIS_DB_NUMBER_CELERY = int(os.environ.get("REDIS_DB_NUMBER_CELERY", 15))  # broker
 
+# will propagate to both our redis client as well as celery's redis client
+REDIS_HEALTH_CHECK_INTERVAL = int(os.environ.get("REDIS_HEALTH_CHECK_INTERVAL", 60))
+
+# our redis client only, not celery's
+REDIS_POOL_MAX_CONNECTIONS = int(os.environ.get("REDIS_POOL_MAX_CONNECTIONS", 128))
+
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-backend-settings
 # should be one of "required", "optional", or "none"
 REDIS_SSL_CERT_REQS = os.getenv("REDIS_SSL_CERT_REQS", "none")
@@ -171,6 +177,16 @@
 
 CELERY_RESULT_EXPIRES = int(os.environ.get("CELERY_RESULT_EXPIRES", 86400))  # seconds
 
+# https://docs.celeryq.dev/en/stable/userguide/configuration.html#broker-pool-limit
+# Setting to None may help when there is a proxy in the way closing idle connections
+CELERY_BROKER_POOL_LIMIT_DEFAULT = 10
+try:
+    CELERY_BROKER_POOL_LIMIT = int(
+        os.environ.get("CELERY_BROKER_POOL_LIMIT", CELERY_BROKER_POOL_LIMIT_DEFAULT)
+    )
+except ValueError:
+    CELERY_BROKER_POOL_LIMIT = CELERY_BROKER_POOL_LIMIT_DEFAULT
+
 #####
 # Connector Configs
 #####
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index a2b0f752ffe..d8470d17195 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -1,3 +1,5 @@
+import platform
+import socket
 from enum import auto
 from enum import Enum
 
@@ -204,3 +206,13 @@ class DanswerCeleryPriority(int, Enum):
     MEDIUM = auto()
     LOW = auto()
     LOWEST = auto()
+
+
+REDIS_SOCKET_KEEPALIVE_OPTIONS = {}
+REDIS_SOCKET_KEEPALIVE_OPTIONS[socket.TCP_KEEPINTVL] = 15
+REDIS_SOCKET_KEEPALIVE_OPTIONS[socket.TCP_KEEPCNT] = 3
+
+if platform.system() == "Darwin":
+    REDIS_SOCKET_KEEPALIVE_OPTIONS[socket.TCP_KEEPALIVE] = 60  # type: ignore
+else:
+    REDIS_SOCKET_KEEPALIVE_OPTIONS[socket.TCP_KEEPIDLE] = 60  # type: ignore
diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py
index 233a51a849f..fd08b9157bd 100644
--- a/backend/danswer/redis/redis_pool.py
+++ b/backend/danswer/redis/redis_pool.py
@@ -5,14 +5,15 @@
 from redis.client import Redis
 
 from danswer.configs.app_configs import REDIS_DB_NUMBER
+from danswer.configs.app_configs import REDIS_HEALTH_CHECK_INTERVAL
 from danswer.configs.app_configs import REDIS_HOST
 from danswer.configs.app_configs import REDIS_PASSWORD
+from danswer.configs.app_configs import REDIS_POOL_MAX_CONNECTIONS
 from danswer.configs.app_configs import REDIS_PORT
 from danswer.configs.app_configs import REDIS_SSL
 from danswer.configs.app_configs import REDIS_SSL_CA_CERTS
 from danswer.configs.app_configs import REDIS_SSL_CERT_REQS
-
-REDIS_POOL_MAX_CONNECTIONS = 128
+from danswer.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
 
 
 class RedisPool:
@@ -59,6 +60,9 @@ def create_pool(
                 password=password,
                 max_connections=max_connections,
                 timeout=None,
+                health_check_interval=REDIS_HEALTH_CHECK_INTERVAL,
+                socket_keepalive=True,
+                socket_keepalive_options=REDIS_SOCKET_KEEPALIVE_OPTIONS,
                 connection_class=redis.SSLConnection,
                 ssl_ca_certs=ssl_ca_certs,
                 ssl_cert_reqs=ssl_cert_reqs,
@@ -71,6 +75,9 @@ def create_pool(
             password=password,
             max_connections=max_connections,
             timeout=None,
+            health_check_interval=REDIS_HEALTH_CHECK_INTERVAL,
+            socket_keepalive=True,
+            socket_keepalive_options=REDIS_SOCKET_KEEPALIVE_OPTIONS,
         )
 
 
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 639db923a9f..86d988e7d90 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -87,6 +87,7 @@ services:
       - LOG_ENDPOINT_LATENCY=${LOG_ENDPOINT_LATENCY:-}
       - LOG_POSTGRES_LATENCY=${LOG_POSTGRES_LATENCY:-}
       - LOG_POSTGRES_CONN_COUNTS=${LOG_POSTGRES_CONN_COUNTS:-}
+      - CELERY_BROKER_POOL_LIMIT=${CELERY_BROKER_POOL_LIMIT:-}
 
       # Chat Configs
       - HARD_DELETE_CHATS=${HARD_DELETE_CHATS:-}
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index 9a1d62c03f0..ebce01eadb2 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -80,7 +80,8 @@ services:
       # If set to `true` will enable additional logs about Vespa query performance
       # (time spent on finding the right docs + time spent fetching summaries from disk)
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
-
+      - CELERY_BROKER_POOL_LIMIT=${CELERY_BROKER_POOL_LIMIT:-}
+      
       # Chat Configs
       - HARD_DELETE_CHATS=${HARD_DELETE_CHATS:-}
 

From b04e9e9b67b07d3a26c514959a01f9ab5f6300d9 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 4 Oct 2024 19:29:45 -0700
Subject: [PATCH 028/376] Improved api key forms + fix non-submittable azure
 (#2654)

---
 .../llm/CustomLLMProviderUpdateForm.tsx       |  5 --
 .../llm/LLMProviderUpdateForm.tsx             | 17 +-----
 web/src/app/chat/ChatPage.tsx                 |  5 +-
 .../initialSetup/welcome/WelcomeModal.tsx     | 60 ++++++++++---------
 web/src/components/llm/ApiKeyForm.tsx         | 15 +----
 web/src/components/llm/ApiKeyModal.tsx        | 10 +++-
 web/src/components/search/SearchSection.tsx   |  5 +-
 7 files changed, 55 insertions(+), 62 deletions(-)

diff --git a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
index 72fe281d288..66b306d7792 100644
--- a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
@@ -16,16 +16,11 @@ import {
   SubLabel,
   TextArrayField,
   TextFormField,
-  BooleanFormField,
 } from "@/components/admin/connectors/Field";
 import { useState } from "react";
-import { Bubble } from "@/components/Bubble";
-import { GroupsIcon } from "@/components/icons/icons";
 import { useSWRConfig } from "swr";
-import { useUserGroups } from "@/lib/hooks";
 import { FullLLMProvider } from "./interfaces";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import * as Yup from "yup";
 import isEqual from "lodash/isEqual";
 import { IsPublicGroupSelector } from "@/components/IsPublicGroupSelector";
diff --git a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
index 11c84825232..70a3ce7ff99 100644
--- a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
@@ -7,21 +7,13 @@ import { LLM_PROVIDERS_ADMIN_URL } from "./constants";
 import {
   SelectorFormField,
   TextFormField,
-  BooleanFormField,
   MultiSelectField,
 } from "@/components/admin/connectors/Field";
 import { useState } from "react";
-import { Bubble } from "@/components/Bubble";
-import { GroupsIcon } from "@/components/icons/icons";
 import { useSWRConfig } from "swr";
-import {
-  defaultModelsByProvider,
-  getDisplayNameForModel,
-  useUserGroups,
-} from "@/lib/hooks";
+import { defaultModelsByProvider, getDisplayNameForModel } from "@/lib/hooks";
 import { FullLLMProvider, WellKnownLLMProviderDescriptor } from "./interfaces";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import * as Yup from "yup";
 import isEqual from "lodash/isEqual";
 import { IsPublicGroupSelector } from "@/components/IsPublicGroupSelector";
@@ -43,11 +35,6 @@ export function LLMProviderUpdateForm({
 }) {
   const { mutate } = useSWRConfig();
 
-  const isPaidEnterpriseFeaturesEnabled = usePaidEnterpriseFeaturesEnabled();
-
-  // EE only
-  const { data: userGroups, isLoading: userGroupsIsLoading } = useUserGroups();
-
   const [isTesting, setIsTesting] = useState(false);
   const [testError, setTestError] = useState<string>("");
 
@@ -278,7 +265,7 @@ export function LLMProviderUpdateForm({
             </div>
           ))}
 
-          {!hideAdvanced && (
+          {!(hideAdvanced && llmProviderDescriptor.name != "azure") && (
             <>
               <Divider />
 
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 3513b2dc3c6..4f052e7388c 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -1767,7 +1767,10 @@ export function ChatPage({
       <HealthCheckBanner />
 
       {showApiKeyModal && !shouldShowWelcomeModal && (
-        <ApiKeyModal hide={() => setShowApiKeyModal(false)} />
+        <ApiKeyModal
+          hide={() => setShowApiKeyModal(false)}
+          setPopup={setPopup}
+        />
       )}
 
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 
diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index ea59cfa00f9..d2229ab9008 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -12,6 +12,8 @@ import { checkLlmProvider } from "./lib";
 import { User } from "@/lib/types";
 import { useProviderStatus } from "@/components/chat_search/ProviderContext";
 
+import { usePopup } from "@/components/admin/connectors/Popup";
+
 function setWelcomeFlowComplete() {
   Cookies.set(COMPLETED_WELCOME_FLOW_COOKIE, "true", { expires: 365 });
 }
@@ -28,6 +30,7 @@ export function _WelcomeModal({ user }: { user: User | null }) {
   const [providerOptions, setProviderOptions] = useState<
     WellKnownLLMProviderDescriptor[]
   >([]);
+  const { popup, setPopup } = usePopup();
 
   const { refreshProviderInfo } = useProviderStatus();
   const clientSetWelcomeFlowComplete = async () => {
@@ -48,34 +51,37 @@ export function _WelcomeModal({ user }: { user: User | null }) {
   }, []);
 
   return (
-    <Modal title={"Welcome to Danswer!"} width="w-full max-w-3xl">
-      <div>
-        <Text className="mb-4">
-          Danswer brings all your company&apos;s knowledge to your fingertips,
-          ready to be accessed instantly.
-        </Text>
-        <Text className="mb-4">
-          To get started, we need to set up an API key for the Language Model
-          (LLM) provider. This key allows Danswer to interact with the AI model,
-          enabling intelligent responses to your queries.
-        </Text>
+    <>
+      {popup}
+      <Modal title={"Welcome to Danswer!"} width="w-full max-w-3xl">
+        <div>
+          <Text className="mb-4">
+            Danswer brings all your company&apos;s knowledge to your fingertips,
+            ready to be accessed instantly.
+          </Text>
+          <Text className="mb-4">
+            To get started, we need to set up an API key for the Language Model
+            (LLM) provider. This key allows Danswer to interact with the AI
+            model, enabling intelligent responses to your queries.
+          </Text>
 
-        <div className="max-h-[900px] overflow-y-scroll">
-          <ApiKeyForm
-            hidePopup
-            onSuccess={() => {
-              router.refresh();
-              refreshProviderInfo();
-              setCanBegin(true);
-            }}
-            providerOptions={providerOptions}
-          />
+          <div className="max-h-[900px] overflow-y-scroll">
+            <ApiKeyForm
+              setPopup={setPopup}
+              onSuccess={() => {
+                router.refresh();
+                refreshProviderInfo();
+                setCanBegin(true);
+              }}
+              providerOptions={providerOptions}
+            />
+          </div>
+          <Divider />
+          <Button disabled={!canBegin} onClick={clientSetWelcomeFlowComplete}>
+            Get Started
+          </Button>
         </div>
-        <Divider />
-        <Button disabled={!canBegin} onClick={clientSetWelcomeFlowComplete}>
-          Get Started
-        </Button>
-      </div>
-    </Modal>
+      </Modal>
+    </>
   );
 }
diff --git a/web/src/components/llm/ApiKeyForm.tsx b/web/src/components/llm/ApiKeyForm.tsx
index f307e0c74e7..189d1fc5738 100644
--- a/web/src/components/llm/ApiKeyForm.tsx
+++ b/web/src/components/llm/ApiKeyForm.tsx
@@ -1,4 +1,4 @@
-import { Popup } from "../admin/connectors/Popup";
+import { PopupSpec } from "../admin/connectors/Popup";
 import { useState } from "react";
 import { TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react";
 import { WellKnownLLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
@@ -8,17 +8,12 @@ import { CustomLLMProviderUpdateForm } from "@/app/admin/configuration/llm/Custo
 export const ApiKeyForm = ({
   onSuccess,
   providerOptions,
-  hidePopup,
+  setPopup,
 }: {
   onSuccess: () => void;
   providerOptions: WellKnownLLMProviderDescriptor[];
-  hidePopup?: boolean;
+  setPopup: (popup: PopupSpec) => void;
 }) => {
-  const [popup, setPopup] = useState<{
-    message: string;
-    type: "success" | "error";
-  } | null>(null);
-
   const defaultProvider = providerOptions[0]?.name;
   const providerNameToIndexMap = new Map<string, number>();
   providerOptions.forEach((provider, index) => {
@@ -35,10 +30,6 @@ export const ApiKeyForm = ({
 
   return (
     <div>
-      {!hidePopup && popup && (
-        <Popup message={popup.message} type={popup.type} />
-      )}
-
       <TabGroup
         index={providerNameToIndexMap.get(providerName) || 0}
         onIndexChange={(index) =>
diff --git a/web/src/components/llm/ApiKeyModal.tsx b/web/src/components/llm/ApiKeyModal.tsx
index 8a6342b4c0c..d1301311c1c 100644
--- a/web/src/components/llm/ApiKeyModal.tsx
+++ b/web/src/components/llm/ApiKeyModal.tsx
@@ -4,8 +4,15 @@ import { ApiKeyForm } from "./ApiKeyForm";
 import { Modal } from "../Modal";
 import { useRouter } from "next/navigation";
 import { useProviderStatus } from "../chat_search/ProviderContext";
+import { PopupSpec } from "../admin/connectors/Popup";
 
-export const ApiKeyModal = ({ hide }: { hide: () => void }) => {
+export const ApiKeyModal = ({
+  hide,
+  setPopup,
+}: {
+  hide: () => void;
+  setPopup: (popup: PopupSpec) => void;
+}) => {
   const router = useRouter();
 
   const {
@@ -39,6 +46,7 @@ export const ApiKeyModal = ({ hide }: { hide: () => void }) => {
           </div>
 
           <ApiKeyForm
+            setPopup={setPopup}
             onSuccess={() => {
               router.refresh();
               refreshProviderInfo();
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 14c3487850a..b171b2bcb05 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -597,7 +597,10 @@ export const SearchSection = ({
         {!shouldDisplayNoSources &&
           showApiKeyModal &&
           !shouldShowWelcomeModal && (
-            <ApiKeyModal hide={() => setShowApiKeyModal(false)} />
+            <ApiKeyModal
+              setPopup={setPopup}
+              hide={() => setShowApiKeyModal(false)}
+            />
           )}
 
         {deletingChatSession && (

From 493c3d73143686894d3e6228482b7cf0ddc031dd Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 5 Oct 2024 14:08:41 -0700
Subject: [PATCH 029/376] Add only multi tenant dependency injection (#2588)

* add only dependency injection

* quick typing fix

* additional non-dependency context

* update nits
---
 backend/danswer/configs/app_configs.py |   4 +
 backend/danswer/configs/constants.py   |   1 +
 backend/danswer/db/engine.py           | 157 ++++++++++++++++++++-----
 3 files changed, 132 insertions(+), 30 deletions(-)

diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 6096561b5a5..4857e2aa96e 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -406,3 +406,7 @@
 ENTERPRISE_EDITION_ENABLED = (
     os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() == "true"
 )
+
+
+MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
+SECRET_JWT_KEY = os.environ.get("SECRET_JWT_KEY", "")
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index d8470d17195..c26c2fbd602 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -41,6 +41,7 @@
 POSTGRES_CELERY_WORKER_HEAVY_APP_NAME = "celery_worker_heavy"
 POSTGRES_PERMISSIONS_APP_NAME = "permissions"
 POSTGRES_UNKNOWN_APP_NAME = "unknown"
+POSTGRES_DEFAULT_SCHEMA = "public"
 
 # API Keys
 DANSWER_API_KEY_PREFIX = "API_KEY__"
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index af44498be24..559c2dec0cc 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -1,4 +1,6 @@
 import contextlib
+import contextvars
+import re
 import threading
 import time
 from collections.abc import AsyncGenerator
@@ -7,6 +9,10 @@
 from typing import Any
 from typing import ContextManager
 
+import jwt
+from fastapi import Depends
+from fastapi import HTTPException
+from fastapi import Request
 from sqlalchemy import event
 from sqlalchemy import text
 from sqlalchemy.engine import create_engine
@@ -19,6 +25,7 @@
 
 from danswer.configs.app_configs import LOG_POSTGRES_CONN_COUNTS
 from danswer.configs.app_configs import LOG_POSTGRES_LATENCY
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import POSTGRES_DB
 from danswer.configs.app_configs import POSTGRES_HOST
 from danswer.configs.app_configs import POSTGRES_PASSWORD
@@ -26,9 +33,12 @@
 from danswer.configs.app_configs import POSTGRES_POOL_RECYCLE
 from danswer.configs.app_configs import POSTGRES_PORT
 from danswer.configs.app_configs import POSTGRES_USER
+from danswer.configs.app_configs import SECRET_JWT_KEY
+from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 SYNC_DB_API = "psycopg2"
@@ -37,11 +47,10 @@
 # global so we don't create more than one engine per process
 # outside of being best practice, this is needed so we can properly pool
 # connections and not create a new pool on every request
-_ASYNC_ENGINE: AsyncEngine | None = None
 
+_ASYNC_ENGINE: AsyncEngine | None = None
 SessionFactory: sessionmaker[Session] | None = None
 
-
 if LOG_POSTGRES_LATENCY:
     # Function to log before query execution
     @event.listens_for(Engine, "before_cursor_execute")
@@ -105,10 +114,19 @@ def get_db_current_time(db_session: Session) -> datetime:
     return result
 
 
+# Regular expression to validate schema names to prevent SQL injection
+SCHEMA_NAME_REGEX = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+
+
+def is_valid_schema_name(name: str) -> bool:
+    return SCHEMA_NAME_REGEX.match(name) is not None
+
+
 class SqlEngine:
-    """Class to manage a global sql alchemy engine (needed for proper resource control)
+    """Class to manage a global SQLAlchemy engine (needed for proper resource control).
     Will eventually subsume most of the standalone functions in this file.
-    Sync only for now"""
+    Sync only for now.
+    """
 
     _engine: Engine | None = None
     _lock: threading.Lock = threading.Lock()
@@ -137,16 +155,18 @@ def _init_engine(cls, **engine_kwargs: Any) -> Engine:
     @classmethod
     def init_engine(cls, **engine_kwargs: Any) -> None:
         """Allow the caller to init the engine with extra params. Different clients
-        such as the API server and different celery workers and tasks
-        need different settings."""
+        such as the API server and different Celery workers and tasks
+        need different settings.
+        """
         with cls._lock:
             if not cls._engine:
                 cls._engine = cls._init_engine(**engine_kwargs)
 
     @classmethod
     def get_engine(cls) -> Engine:
-        """Gets the sql alchemy engine. Will init a default engine if init hasn't
-        already been called. You probably want to init first!"""
+        """Gets the SQLAlchemy engine. Will init a default engine if init hasn't
+        already been called. You probably want to init first!
+        """
         if not cls._engine:
             with cls._lock:
                 if not cls._engine:
@@ -178,7 +198,6 @@ def build_connection_string(
 ) -> str:
     if app_name:
         return f"postgresql+{db_api}://{user}:{password}@{host}:{port}/{db}?application_name={app_name}"
-
     return f"postgresql+{db_api}://{user}:{password}@{host}:{port}/{db}"
 
 
@@ -193,7 +212,7 @@ def get_sqlalchemy_engine() -> Engine:
 def get_sqlalchemy_async_engine() -> AsyncEngine:
     global _ASYNC_ENGINE
     if _ASYNC_ENGINE is None:
-        # underlying asyncpg cannot accept application_name directly in the connection string
+        # Underlying asyncpg cannot accept application_name directly in the connection string
         # https://github.com/MagicStack/asyncpg/issues/798
         connection_string = build_connection_string()
         _ASYNC_ENGINE = create_async_engine(
@@ -211,25 +230,110 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
     return _ASYNC_ENGINE
 
 
-def get_session_context_manager() -> ContextManager[Session]:
-    return contextlib.contextmanager(get_session)()
-
-
-def get_session() -> Generator[Session, None, None]:
-    # The line below was added to monitor the latency caused by Postgres connections
-    # during API calls.
-    # with tracer.trace("db.get_session"):
-    with Session(get_sqlalchemy_engine(), expire_on_commit=False) as session:
+# Context variable to store the current tenant ID
+# This allows us to maintain tenant-specific context throughout the request lifecycle
+# The default value is set to POSTGRES_DEFAULT_SCHEMA for non-multi-tenant setups
+# This context variable works in both synchronous and asynchronous contexts
+# In async code, it's automatically carried across coroutines
+# In sync code, it's managed per thread
+current_tenant_id = contextvars.ContextVar(
+    "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
+)
+
+
+# Dependency to get the current tenant ID and set the context variable
+def get_current_tenant_id(request: Request) -> str:
+    """Dependency that extracts the tenant ID from the JWT token in the request and sets the context variable."""
+    if not MULTI_TENANT:
+        tenant_id = POSTGRES_DEFAULT_SCHEMA
+        current_tenant_id.set(tenant_id)
+        return tenant_id
+
+    token = request.cookies.get("tenant_details")
+    if not token:
+        # If no token is present, use the default schema or handle accordingly
+        tenant_id = POSTGRES_DEFAULT_SCHEMA
+        current_tenant_id.set(tenant_id)
+        return tenant_id
+
+    try:
+        payload = jwt.decode(token, SECRET_JWT_KEY, algorithms=["HS256"])
+        tenant_id = payload.get("tenant_id")
+        if not tenant_id:
+            raise HTTPException(
+                status_code=400, detail="Invalid token: tenant_id missing"
+            )
+        if not is_valid_schema_name(tenant_id):
+            raise ValueError("Invalid tenant ID format")
+        current_tenant_id.set(tenant_id)
+        return tenant_id
+    except jwt.InvalidTokenError:
+        raise HTTPException(status_code=401, detail="Invalid token format")
+    except ValueError as e:
+        # Let the 400 error bubble up
+        raise HTTPException(status_code=400, detail=str(e))
+    except Exception:
+        raise HTTPException(status_code=500, detail="Internal server error")
+
+
+def get_session_with_tenant(tenant_id: str | None = None) -> Session:
+    if tenant_id is None:
+        tenant_id = current_tenant_id.get()
+
+    if not is_valid_schema_name(tenant_id):
+        raise Exception("Invalid tenant ID")
+
+    engine = SqlEngine.get_engine()
+    session = Session(engine, expire_on_commit=False)
+
+    @event.listens_for(session, "after_begin")
+    def set_search_path(session: Session, transaction: Any, connection: Any) -> None:
+        connection.execute(text("SET search_path TO :schema"), {"schema": tenant_id})
+
+    return session
+
+
+def get_session(
+    tenant_id: str = Depends(get_current_tenant_id),
+) -> Generator[Session, None, None]:
+    """Generate a database session with the appropriate tenant schema set."""
+    engine = get_sqlalchemy_engine()
+    with Session(engine, expire_on_commit=False) as session:
+        if MULTI_TENANT:
+            if not is_valid_schema_name(tenant_id):
+                raise HTTPException(status_code=400, detail="Invalid tenant ID")
+            # Set the search_path to the tenant's schema
+            session.execute(text(f'SET search_path = "{tenant_id}"'))
         yield session
 
 
-async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
-    async with AsyncSession(
-        get_sqlalchemy_async_engine(), expire_on_commit=False
-    ) as async_session:
+async def get_async_session(
+    tenant_id: str = Depends(get_current_tenant_id),
+) -> AsyncGenerator[AsyncSession, None]:
+    """Generate an async database session with the appropriate tenant schema set."""
+    engine = get_sqlalchemy_async_engine()
+    async with AsyncSession(engine, expire_on_commit=False) as async_session:
+        if MULTI_TENANT:
+            if not is_valid_schema_name(tenant_id):
+                raise HTTPException(status_code=400, detail="Invalid tenant ID")
+            # Set the search_path to the tenant's schema
+            await async_session.execute(text(f'SET search_path = "{tenant_id}"'))
         yield async_session
 
 
+def get_session_context_manager() -> ContextManager[Session]:
+    """Context manager for database sessions."""
+    return contextlib.contextmanager(get_session)()
+
+
+def get_session_factory() -> sessionmaker[Session]:
+    """Get a session factory."""
+    global SessionFactory
+    if SessionFactory is None:
+        SessionFactory = sessionmaker(bind=get_sqlalchemy_engine())
+    return SessionFactory
+
+
 async def warm_up_connections(
     sync_connections_to_warm_up: int = 20, async_connections_to_warm_up: int = 20
 ) -> None:
@@ -251,10 +355,3 @@ async def warm_up_connections(
         await async_conn.execute(text("SELECT 1"))
     for async_conn in async_connections:
         await async_conn.close()
-
-
-def get_session_factory() -> sessionmaker[Session]:
-    global SessionFactory
-    if SessionFactory is None:
-        SessionFactory = sessionmaker(bind=get_sqlalchemy_engine())
-    return SessionFactory

From 28e65669b440133ad908a10f5418d7828933a54e Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 5 Oct 2024 14:59:15 -0700
Subject: [PATCH 030/376] add multi tenant alembic (#2589)

---
 backend/alembic/env.py | 52 ++++++++++++++++++++++++++++--------------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index 154d6ff3d66..d7ac37af562 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -9,9 +9,9 @@
 from sqlalchemy.ext.asyncio import create_async_engine
 from celery.backends.database.session import ResultModelBase  # type: ignore
 from sqlalchemy.schema import SchemaItem
+from sqlalchemy.sql import text
 
-# this is the Alembic Config object, which provides
-# access to the values within the .ini file in use.
+# Alembic Config object
 config = context.config
 
 # Interpret the config file for Python logging.
@@ -21,16 +21,26 @@
 ):
     fileConfig(config.config_file_name)
 
-# add your model's MetaData object here
+# Add your model's MetaData object here
 # for 'autogenerate' support
 # from myapp import mymodel
 # target_metadata = mymodel.Base.metadata
 target_metadata = [Base.metadata, ResultModelBase.metadata]
 
-# other values from the config, defined by the needs of env.py,
-# can be acquired:
-# my_important_option = config.get_main_option("my_important_option")
-# ... etc.
+
+def get_schema_options() -> tuple[str, bool]:
+    x_args_raw = context.get_x_argument()
+    x_args = {}
+    for arg in x_args_raw:
+        for pair in arg.split(","):
+            if "=" in pair:
+                key, value = pair.split("=", 1)
+                x_args[key] = value
+
+    schema_name = x_args.get("schema", "public")
+    create_schema = x_args.get("create_schema", "true").lower() == "true"
+    return schema_name, create_schema
+
 
 EXCLUDE_TABLES = {"kombu_queue", "kombu_message"}
 
@@ -54,17 +64,20 @@ def run_migrations_offline() -> None:
     and not an Engine, though an Engine is acceptable
     here as well.  By skipping the Engine creation
     we don't even need a DBAPI to be available.
-
     Calls to context.execute() here emit the given string to the
     script output.
-
     """
     url = build_connection_string()
+    schema, _ = get_schema_options()
+
     context.configure(
         url=url,
         target_metadata=target_metadata,  # type: ignore
         literal_binds=True,
+        include_object=include_object,
         dialect_opts={"paramstyle": "named"},
+        version_table_schema=schema,
+        include_schemas=True,
     )
 
     with context.begin_transaction():
@@ -72,22 +85,28 @@ def run_migrations_offline() -> None:
 
 
 def do_run_migrations(connection: Connection) -> None:
+    schema, create_schema = get_schema_options()
+    if create_schema:
+        connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
+        connection.execute(text("COMMIT"))
+
+    connection.execute(text(f'SET search_path TO "{schema}"'))
+
     context.configure(
         connection=connection,
         target_metadata=target_metadata,  # type: ignore
-        include_object=include_object,
-    )  # type: ignore
+        version_table_schema=schema,
+        include_schemas=True,
+        compare_type=True,
+        compare_server_default=True,
+    )
 
     with context.begin_transaction():
         context.run_migrations()
 
 
 async def run_async_migrations() -> None:
-    """In this scenario we need to create an Engine
-    and associate a connection with the context.
-
-    """
-
+    """Run migrations in 'online' mode."""
     connectable = create_async_engine(
         build_connection_string(),
         poolclass=pool.NullPool,
@@ -101,7 +120,6 @@ async def run_async_migrations() -> None:
 
 def run_migrations_online() -> None:
     """Run migrations in 'online' mode."""
-
     asyncio.run(run_async_migrations())
 
 

From e56fd43ba6d9a2d9ccf792879f76064004e3a226 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 5 Oct 2024 16:08:28 -0700
Subject: [PATCH 031/376] cors update (#2686)

---
 backend/shared_configs/configs.py | 31 ++++++++++++++++++++++++-------
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index ea37b031c7a..e8b599b7795 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -1,4 +1,5 @@
 import os
+from typing import List
 from urllib.parse import urlparse
 
 # Used for logging
@@ -76,16 +77,32 @@
 ]
 
 
-# CORS
 def validate_cors_origin(origin: str) -> None:
     parsed = urlparse(origin)
     if parsed.scheme not in ["http", "https"] or not parsed.netloc:
         raise ValueError(f"Invalid CORS origin: '{origin}'")
 
 
-CORS_ALLOWED_ORIGIN = os.environ.get("CORS_ALLOWED_ORIGIN", "*").split(",") or ["*"]
-
-# Validate non-wildcard origins
-for origin in CORS_ALLOWED_ORIGIN:
-    if origin != "*" and (stripped_origin := origin.strip()):
-        validate_cors_origin(stripped_origin)
+# Examples of valid values for the environment variable:
+# - "" (allow all origins)
+# - "http://example.com" (single origin)
+# - "http://example.com,https://example.org" (multiple origins)
+# - "*" (allow all origins)
+CORS_ALLOWED_ORIGIN_ENV = os.environ.get("CORS_ALLOWED_ORIGIN", "")
+
+# Explicitly declare the type of CORS_ALLOWED_ORIGIN
+CORS_ALLOWED_ORIGIN: List[str]
+
+if CORS_ALLOWED_ORIGIN_ENV:
+    # Split the environment variable into a list of origins
+    CORS_ALLOWED_ORIGIN = [
+        origin.strip()
+        for origin in CORS_ALLOWED_ORIGIN_ENV.split(",")
+        if origin.strip()
+    ]
+    # Validate each origin in the list
+    for origin in CORS_ALLOWED_ORIGIN:
+        validate_cors_origin(origin)
+else:
+    # If the environment variable is empty, allow all origins
+    CORS_ALLOWED_ORIGIN = ["*"]

From e00f4678df00444a60f0225cdde95d0a35da8263 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Sat, 5 Oct 2024 16:37:48 -0700
Subject: [PATCH 032/376] Add option to adjust pool size (#2695)

---
 backend/danswer/background/update.py   |  6 ++++--
 backend/danswer/configs/app_configs.py |  6 ++++++
 backend/danswer/db/engine.py           | 16 ++++++++--------
 backend/danswer/main.py                | 13 +++++++++----
 4 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index c88b349e4fe..57d05513ac2 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -23,7 +23,7 @@
 from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
 from danswer.db.engine import get_db_current_time
 from danswer.db.engine import get_sqlalchemy_engine
-from danswer.db.engine import init_sqlalchemy_engine
+from danswer.db.engine import SqlEngine
 from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import get_inprogress_index_attempts
@@ -483,7 +483,9 @@ def update_loop(
 
 def update__main() -> None:
     set_is_ee_based_on_env_variable()
-    init_sqlalchemy_engine(POSTGRES_INDEXER_APP_NAME)
+
+    # initialize the Postgres connection pool
+    SqlEngine.set_app_name(POSTGRES_INDEXER_APP_NAME)
 
     logger.notice("Starting indexing service")
     update_loop()
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 4857e2aa96e..0ac3d6e76e1 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -138,6 +138,12 @@
 POSTGRES_PORT = os.environ.get("POSTGRES_PORT") or "5432"
 POSTGRES_DB = os.environ.get("POSTGRES_DB") or "postgres"
 
+POSTGRES_API_SERVER_POOL_SIZE = int(
+    os.environ.get("POSTGRES_API_SERVER_POOL_SIZE") or 40
+)
+POSTGRES_API_SERVER_POOL_OVERFLOW = int(
+    os.environ.get("POSTGRES_API_SERVER_POOL_OVERFLOW") or 10
+)
 # defaults to False
 POSTGRES_POOL_PRE_PING = os.environ.get("POSTGRES_POOL_PRE_PING", "").lower() == "true"
 
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 559c2dec0cc..dcae5ae1f7a 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -26,6 +26,8 @@
 from danswer.configs.app_configs import LOG_POSTGRES_CONN_COUNTS
 from danswer.configs.app_configs import LOG_POSTGRES_LATENCY
 from danswer.configs.app_configs import MULTI_TENANT
+from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
+from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_SIZE
 from danswer.configs.app_configs import POSTGRES_DB
 from danswer.configs.app_configs import POSTGRES_HOST
 from danswer.configs.app_configs import POSTGRES_PASSWORD
@@ -134,8 +136,8 @@ class SqlEngine:
 
     # Default parameters for engine creation
     DEFAULT_ENGINE_KWARGS = {
-        "pool_size": 40,
-        "max_overflow": 10,
+        "pool_size": 20,
+        "max_overflow": 5,
         "pool_pre_ping": POSTGRES_POOL_PRE_PING,
         "pool_recycle": POSTGRES_POOL_RECYCLE,
     }
@@ -201,10 +203,6 @@ def build_connection_string(
     return f"postgresql+{db_api}://{user}:{password}@{host}:{port}/{db}"
 
 
-def init_sqlalchemy_engine(app_name: str) -> None:
-    SqlEngine.set_app_name(app_name)
-
-
 def get_sqlalchemy_engine() -> Engine:
     return SqlEngine.get_engine()
 
@@ -222,8 +220,10 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
                     "application_name": SqlEngine.get_app_name() + "_async"
                 }
             },
-            pool_size=40,
-            max_overflow=10,
+            # async engine is only used by API server, so we can use those values
+            # here as well
+            pool_size=POSTGRES_API_SERVER_POOL_SIZE,
+            max_overflow=POSTGRES_API_SERVER_POOL_OVERFLOW,
             pool_pre_ping=POSTGRES_POOL_PRE_PING,
             pool_recycle=POSTGRES_POOL_RECYCLE,
         )
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index de150cd5823..7727f25ccd6 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -33,6 +33,8 @@
 from danswer.configs.app_configs import LOG_ENDPOINT_LATENCY
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
+from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
+from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_SIZE
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import AuthType
@@ -49,8 +51,7 @@
 from danswer.db.connector_credential_pair import resync_cc_pair
 from danswer.db.credentials import create_initial_public_credential
 from danswer.db.document import check_docs_exist
-from danswer.db.engine import get_sqlalchemy_engine
-from danswer.db.engine import init_sqlalchemy_engine
+from danswer.db.engine import SqlEngine
 from danswer.db.engine import warm_up_connections
 from danswer.db.index_attempt import cancel_indexing_attempts_past_model
 from danswer.db.index_attempt import expire_index_attempts
@@ -353,8 +354,12 @@ def setup_vespa(
 
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncGenerator:
-    init_sqlalchemy_engine(POSTGRES_WEB_APP_NAME)
-    engine = get_sqlalchemy_engine()
+    SqlEngine.set_app_name(POSTGRES_WEB_APP_NAME)
+    SqlEngine.init_engine(
+        pool_size=POSTGRES_API_SERVER_POOL_SIZE,
+        max_overflow=POSTGRES_API_SERVER_POOL_OVERFLOW,
+    )
+    engine = SqlEngine.get_engine()
 
     verify_auth = fetch_versioned_implementation(
         "danswer.auth.users", "verify_auth_setting"

From 0da736bed945da61e6e0d4dd6f653be29e2c58bf Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 5 Oct 2024 21:08:35 -0700
Subject: [PATCH 033/376] Tenant provisioning in the dataplane (#2694)

* add tenant provisioning to data plane

* minor typing update

* ensure tenant router included

* proper auth check

* update disabling logic

* validated basic provisioning

* use new kv store
---
 backend/danswer/auth/users.py                 |  28 ++
 backend/danswer/chat/load_yamls.py            | 237 +++++++-------
 backend/danswer/configs/app_configs.py        |   4 +
 backend/danswer/db/engine.py                  |   3 +-
 backend/danswer/key_value_store/store.py      |   9 +-
 backend/danswer/main.py                       | 300 +----------------
 backend/danswer/server/auth_check.py          |   2 +
 backend/danswer/setup.py                      | 303 ++++++++++++++++++
 backend/ee/danswer/main.py                    |   3 +
 backend/ee/danswer/server/tenants/__init__.py |   0
 backend/ee/danswer/server/tenants/access.py   |   0
 backend/ee/danswer/server/tenants/api.py      |  46 +++
 backend/ee/danswer/server/tenants/models.py   |   6 +
 .../ee/danswer/server/tenants/provisioning.py |  63 ++++
 .../tests/integration/common_utils/reset.py   |   4 +-
 web/src/components/search/SearchSection.tsx   |  40 ++-
 16 files changed, 620 insertions(+), 428 deletions(-)
 create mode 100644 backend/danswer/setup.py
 create mode 100644 backend/ee/danswer/server/tenants/__init__.py
 create mode 100644 backend/ee/danswer/server/tenants/access.py
 create mode 100644 backend/ee/danswer/server/tenants/api.py
 create mode 100644 backend/ee/danswer/server/tenants/models.py
 create mode 100644 backend/ee/danswer/server/tenants/provisioning.py

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index a583a93235f..81607aab884 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -8,6 +8,7 @@
 from typing import Optional
 from typing import Tuple
 
+import jwt
 from email_validator import EmailNotValidError
 from email_validator import validate_email
 from fastapi import APIRouter
@@ -37,8 +38,10 @@
 from danswer.auth.schemas import UserRole
 from danswer.auth.schemas import UserUpdate
 from danswer.configs.app_configs import AUTH_TYPE
+from danswer.configs.app_configs import DATA_PLANE_SECRET
 from danswer.configs.app_configs import DISABLE_AUTH
 from danswer.configs.app_configs import EMAIL_FROM
+from danswer.configs.app_configs import EXPECTED_API_KEY
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import SMTP_PASS
@@ -504,3 +507,28 @@ async def current_admin_user(user: User | None = Depends(current_user)) -> User
 def get_default_admin_user_emails_() -> list[str]:
     # No default seeding available for Danswer MIT
     return []
+
+
+async def control_plane_dep(request: Request) -> None:
+    api_key = request.headers.get("X-API-KEY")
+    if api_key != EXPECTED_API_KEY:
+        logger.warning("Invalid API key")
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    auth_header = request.headers.get("Authorization")
+    if not auth_header or not auth_header.startswith("Bearer "):
+        logger.warning("Invalid authorization header")
+        raise HTTPException(status_code=401, detail="Invalid authorization header")
+
+    token = auth_header.split(" ")[1]
+    try:
+        payload = jwt.decode(token, DATA_PLANE_SECRET, algorithms=["HS256"])
+        if payload.get("scope") != "tenant:create":
+            logger.warning("Insufficient permissions")
+            raise HTTPException(status_code=403, detail="Insufficient permissions")
+    except jwt.ExpiredSignatureError:
+        logger.warning("Token has expired")
+        raise HTTPException(status_code=401, detail="Token has expired")
+    except jwt.InvalidTokenError:
+        logger.warning("Invalid token")
+        raise HTTPException(status_code=401, detail="Invalid token")
diff --git a/backend/danswer/chat/load_yamls.py b/backend/danswer/chat/load_yamls.py
index 8d0fd34d8da..e8a19c158b2 100644
--- a/backend/danswer/chat/load_yamls.py
+++ b/backend/danswer/chat/load_yamls.py
@@ -6,7 +6,6 @@
 from danswer.configs.chat_configs import PERSONAS_YAML
 from danswer.configs.chat_configs import PROMPTS_YAML
 from danswer.db.document_set import get_or_create_document_set_by_name
-from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.input_prompt import insert_input_prompt_if_not_exists
 from danswer.db.models import DocumentSet as DocumentSetDBModel
 from danswer.db.models import Persona
@@ -18,30 +17,32 @@
 from danswer.search.enums import RecencyBiasSetting
 
 
-def load_prompts_from_yaml(prompts_yaml: str = PROMPTS_YAML) -> None:
+def load_prompts_from_yaml(
+    db_session: Session, prompts_yaml: str = PROMPTS_YAML
+) -> None:
     with open(prompts_yaml, "r") as file:
         data = yaml.safe_load(file)
 
     all_prompts = data.get("prompts", [])
-    with Session(get_sqlalchemy_engine()) as db_session:
-        for prompt in all_prompts:
-            upsert_prompt(
-                user=None,
-                prompt_id=prompt.get("id"),
-                name=prompt["name"],
-                description=prompt["description"].strip(),
-                system_prompt=prompt["system"].strip(),
-                task_prompt=prompt["task"].strip(),
-                include_citations=prompt["include_citations"],
-                datetime_aware=prompt.get("datetime_aware", True),
-                default_prompt=True,
-                personas=None,
-                db_session=db_session,
-                commit=True,
-            )
+    for prompt in all_prompts:
+        upsert_prompt(
+            user=None,
+            prompt_id=prompt.get("id"),
+            name=prompt["name"],
+            description=prompt["description"].strip(),
+            system_prompt=prompt["system"].strip(),
+            task_prompt=prompt["task"].strip(),
+            include_citations=prompt["include_citations"],
+            datetime_aware=prompt.get("datetime_aware", True),
+            default_prompt=True,
+            personas=None,
+            db_session=db_session,
+            commit=True,
+        )
 
 
 def load_personas_from_yaml(
+    db_session: Session,
     personas_yaml: str = PERSONAS_YAML,
     default_chunks: float = MAX_CHUNKS_FED_TO_CHAT,
 ) -> None:
@@ -49,117 +50,117 @@ def load_personas_from_yaml(
         data = yaml.safe_load(file)
 
     all_personas = data.get("personas", [])
-    with Session(get_sqlalchemy_engine()) as db_session:
-        for persona in all_personas:
-            doc_set_names = persona["document_sets"]
-            doc_sets: list[DocumentSetDBModel] = [
-                get_or_create_document_set_by_name(db_session, name)
-                for name in doc_set_names
+    for persona in all_personas:
+        doc_set_names = persona["document_sets"]
+        doc_sets: list[DocumentSetDBModel] = [
+            get_or_create_document_set_by_name(db_session, name)
+            for name in doc_set_names
+        ]
+
+        # Assume if user hasn't set any document sets for the persona, the user may want
+        # to later attach document sets to the persona manually, therefore, don't overwrite/reset
+        # the document sets for the persona
+        doc_set_ids: list[int] | None = None
+        if doc_sets:
+            doc_set_ids = [doc_set.id for doc_set in doc_sets]
+        else:
+            doc_set_ids = None
+
+        prompt_ids: list[int] | None = None
+        prompt_set_names = persona["prompts"]
+        if prompt_set_names:
+            prompts: list[PromptDBModel | None] = [
+                get_prompt_by_name(prompt_name, user=None, db_session=db_session)
+                for prompt_name in prompt_set_names
             ]
-
-            # Assume if user hasn't set any document sets for the persona, the user may want
-            # to later attach document sets to the persona manually, therefore, don't overwrite/reset
-            # the document sets for the persona
-            doc_set_ids: list[int] | None = None
-            if doc_sets:
-                doc_set_ids = [doc_set.id for doc_set in doc_sets]
-            else:
-                doc_set_ids = None
-
-            prompt_ids: list[int] | None = None
-            prompt_set_names = persona["prompts"]
-            if prompt_set_names:
-                prompts: list[PromptDBModel | None] = [
-                    get_prompt_by_name(prompt_name, user=None, db_session=db_session)
-                    for prompt_name in prompt_set_names
-                ]
-                if any([prompt is None for prompt in prompts]):
-                    raise ValueError("Invalid Persona configs, not all prompts exist")
-
-                if prompts:
-                    prompt_ids = [prompt.id for prompt in prompts if prompt is not None]
-
-            p_id = persona.get("id")
-            tool_ids = []
-            if persona.get("image_generation"):
-                image_gen_tool = (
-                    db_session.query(ToolDBModel)
-                    .filter(ToolDBModel.name == "ImageGenerationTool")
-                    .first()
-                )
-                if image_gen_tool:
-                    tool_ids.append(image_gen_tool.id)
-
-            llm_model_provider_override = persona.get("llm_model_provider_override")
-            llm_model_version_override = persona.get("llm_model_version_override")
-
-            # Set specific overrides for image generation persona
-            if persona.get("image_generation"):
-                llm_model_version_override = "gpt-4o"
-
-            existing_persona = (
-                db_session.query(Persona)
-                .filter(Persona.name == persona["name"])
+            if any([prompt is None for prompt in prompts]):
+                raise ValueError("Invalid Persona configs, not all prompts exist")
+
+            if prompts:
+                prompt_ids = [prompt.id for prompt in prompts if prompt is not None]
+
+        p_id = persona.get("id")
+        tool_ids = []
+        if persona.get("image_generation"):
+            image_gen_tool = (
+                db_session.query(ToolDBModel)
+                .filter(ToolDBModel.name == "ImageGenerationTool")
                 .first()
             )
-
-            upsert_persona(
-                user=None,
-                persona_id=(-1 * p_id) if p_id is not None else None,
-                name=persona["name"],
-                description=persona["description"],
-                num_chunks=persona.get("num_chunks")
-                if persona.get("num_chunks") is not None
-                else default_chunks,
-                llm_relevance_filter=persona.get("llm_relevance_filter"),
-                starter_messages=persona.get("starter_messages"),
-                llm_filter_extraction=persona.get("llm_filter_extraction"),
-                icon_shape=persona.get("icon_shape"),
-                icon_color=persona.get("icon_color"),
-                llm_model_provider_override=llm_model_provider_override,
-                llm_model_version_override=llm_model_version_override,
-                recency_bias=RecencyBiasSetting(persona["recency_bias"]),
-                prompt_ids=prompt_ids,
-                document_set_ids=doc_set_ids,
-                tool_ids=tool_ids,
-                builtin_persona=True,
-                is_public=True,
-                display_priority=existing_persona.display_priority
-                if existing_persona is not None
-                else persona.get("display_priority"),
-                is_visible=existing_persona.is_visible
-                if existing_persona is not None
-                else persona.get("is_visible"),
-                db_session=db_session,
-            )
-
-
-def load_input_prompts_from_yaml(input_prompts_yaml: str = INPUT_PROMPT_YAML) -> None:
+            if image_gen_tool:
+                tool_ids.append(image_gen_tool.id)
+
+        llm_model_provider_override = persona.get("llm_model_provider_override")
+        llm_model_version_override = persona.get("llm_model_version_override")
+
+        # Set specific overrides for image generation persona
+        if persona.get("image_generation"):
+            llm_model_version_override = "gpt-4o"
+
+        existing_persona = (
+            db_session.query(Persona).filter(Persona.name == persona["name"]).first()
+        )
+
+        upsert_persona(
+            user=None,
+            persona_id=(-1 * p_id) if p_id is not None else None,
+            name=persona["name"],
+            description=persona["description"],
+            num_chunks=persona.get("num_chunks")
+            if persona.get("num_chunks") is not None
+            else default_chunks,
+            llm_relevance_filter=persona.get("llm_relevance_filter"),
+            starter_messages=persona.get("starter_messages"),
+            llm_filter_extraction=persona.get("llm_filter_extraction"),
+            icon_shape=persona.get("icon_shape"),
+            icon_color=persona.get("icon_color"),
+            llm_model_provider_override=llm_model_provider_override,
+            llm_model_version_override=llm_model_version_override,
+            recency_bias=RecencyBiasSetting(persona["recency_bias"]),
+            prompt_ids=prompt_ids,
+            document_set_ids=doc_set_ids,
+            tool_ids=tool_ids,
+            builtin_persona=True,
+            is_public=True,
+            display_priority=existing_persona.display_priority
+            if existing_persona is not None
+            else persona.get("display_priority"),
+            is_visible=existing_persona.is_visible
+            if existing_persona is not None
+            else persona.get("is_visible"),
+            db_session=db_session,
+        )
+
+
+def load_input_prompts_from_yaml(
+    db_session: Session, input_prompts_yaml: str = INPUT_PROMPT_YAML
+) -> None:
     with open(input_prompts_yaml, "r") as file:
         data = yaml.safe_load(file)
 
     all_input_prompts = data.get("input_prompts", [])
-    with Session(get_sqlalchemy_engine()) as db_session:
-        for input_prompt in all_input_prompts:
-            # If these prompts are deleted (which is a hard delete in the DB), on server startup
-            # they will be recreated, but the user can always just deactivate them, just a light inconvenience
-            insert_input_prompt_if_not_exists(
-                user=None,
-                input_prompt_id=input_prompt.get("id"),
-                prompt=input_prompt["prompt"],
-                content=input_prompt["content"],
-                is_public=input_prompt["is_public"],
-                active=input_prompt.get("active", True),
-                db_session=db_session,
-                commit=True,
-            )
+    for input_prompt in all_input_prompts:
+        # If these prompts are deleted (which is a hard delete in the DB), on server startup
+        # they will be recreated, but the user can always just deactivate them, just a light inconvenience
+
+        insert_input_prompt_if_not_exists(
+            user=None,
+            input_prompt_id=input_prompt.get("id"),
+            prompt=input_prompt["prompt"],
+            content=input_prompt["content"],
+            is_public=input_prompt["is_public"],
+            active=input_prompt.get("active", True),
+            db_session=db_session,
+            commit=True,
+        )
 
 
 def load_chat_yamls(
+    db_session: Session,
     prompt_yaml: str = PROMPTS_YAML,
     personas_yaml: str = PERSONAS_YAML,
     input_prompts_yaml: str = INPUT_PROMPT_YAML,
 ) -> None:
-    load_prompts_from_yaml(prompt_yaml)
-    load_personas_from_yaml(personas_yaml)
-    load_input_prompts_from_yaml(input_prompts_yaml)
+    load_prompts_from_yaml(db_session, prompt_yaml)
+    load_personas_from_yaml(db_session, personas_yaml)
+    load_input_prompts_from_yaml(db_session, input_prompts_yaml)
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 0ac3d6e76e1..4559fed6b87 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -416,3 +416,7 @@
 
 MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
 SECRET_JWT_KEY = os.environ.get("SECRET_JWT_KEY", "")
+
+
+DATA_PLANE_SECRET = os.environ.get("DATA_PLANE_SECRET", "")
+EXPECTED_API_KEY = os.environ.get("EXPECTED_API_KEY", "")
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index dcae5ae1f7a..af7aad23669 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -117,7 +117,7 @@ def get_db_current_time(db_session: Session) -> datetime:
 
 
 # Regular expression to validate schema names to prevent SQL injection
-SCHEMA_NAME_REGEX = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$")
+SCHEMA_NAME_REGEX = re.compile(r"^[a-zA-Z0-9_-]+$")
 
 
 def is_valid_schema_name(name: str) -> bool:
@@ -281,6 +281,7 @@ def get_session_with_tenant(tenant_id: str | None = None) -> Session:
         tenant_id = current_tenant_id.get()
 
     if not is_valid_schema_name(tenant_id):
+        logger.error(f"Invalid tenant ID: {tenant_id}")
         raise Exception("Invalid tenant ID")
 
     engine = SqlEngine.get_engine()
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index 450056c40b1..4306743f875 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -5,7 +5,7 @@
 
 from sqlalchemy.orm import Session
 
-from danswer.db.engine import get_session_factory
+from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.models import KVStore
 from danswer.key_value_store.interface import JSON_ro
 from danswer.key_value_store.interface import KeyValueStore
@@ -26,12 +26,9 @@ def __init__(self) -> None:
 
     @contextmanager
     def get_session(self) -> Iterator[Session]:
-        factory = get_session_factory()
-        session: Session = factory()
-        try:
+        engine = get_sqlalchemy_engine()
+        with Session(engine, expire_on_commit=False) as session:
             yield session
-        finally:
-            session.close()
 
     def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
         # Not encrypted in Redis, but encrypted in Postgres
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index 7727f25ccd6..b9231a9c561 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -1,4 +1,3 @@
-import time
 import traceback
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
@@ -23,13 +22,11 @@
 from danswer.auth.schemas import UserUpdate
 from danswer.auth.users import auth_backend
 from danswer.auth.users import fastapi_users
-from danswer.chat.load_yamls import load_chat_yamls
 from danswer.configs.app_configs import APP_API_PREFIX
 from danswer.configs.app_configs import APP_HOST
 from danswer.configs.app_configs import APP_PORT
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
-from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
 from danswer.configs.app_configs import LOG_ENDPOINT_LATENCY
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
@@ -38,42 +35,9 @@
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import AuthType
-from danswer.configs.constants import KV_REINDEX_KEY
-from danswer.configs.constants import KV_SEARCH_SETTINGS
 from danswer.configs.constants import POSTGRES_WEB_APP_NAME
-from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
-from danswer.configs.model_configs import GEN_AI_API_KEY
-from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
-from danswer.db.connector import check_connectors_exist
-from danswer.db.connector import create_initial_default_connector
-from danswer.db.connector_credential_pair import associate_default_cc_pair
-from danswer.db.connector_credential_pair import get_connector_credential_pairs
-from danswer.db.connector_credential_pair import resync_cc_pair
-from danswer.db.credentials import create_initial_public_credential
-from danswer.db.document import check_docs_exist
 from danswer.db.engine import SqlEngine
 from danswer.db.engine import warm_up_connections
-from danswer.db.index_attempt import cancel_indexing_attempts_past_model
-from danswer.db.index_attempt import expire_index_attempts
-from danswer.db.llm import fetch_default_provider
-from danswer.db.llm import update_default_provider
-from danswer.db.llm import upsert_llm_provider
-from danswer.db.persona import delete_old_default_personas
-from danswer.db.search_settings import get_current_search_settings
-from danswer.db.search_settings import get_secondary_search_settings
-from danswer.db.search_settings import update_current_search_settings
-from danswer.db.search_settings import update_secondary_search_settings
-from danswer.db.swap_index import check_index_swap
-from danswer.document_index.factory import get_default_document_index
-from danswer.document_index.interfaces import DocumentIndex
-from danswer.indexing.models import IndexingSetting
-from danswer.key_value_store.factory import get_kv_store
-from danswer.key_value_store.interface import KvKeyNotFoundError
-from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
-from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
-from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder
-from danswer.search.models import SavedSearchSettings
-from danswer.search.retrieval.search_runner import download_nltk_data
 from danswer.server.auth_check import check_router_auth
 from danswer.server.danswer_api.ingestion import router as danswer_api_router
 from danswer.server.documents.cc_pair import router as cc_pair_router
@@ -99,7 +63,6 @@
 from danswer.server.manage.get_state import router as state_router
 from danswer.server.manage.llm.api import admin_router as llm_admin_router
 from danswer.server.manage.llm.api import basic_router as llm_router
-from danswer.server.manage.llm.models import LLMProviderUpsertRequest
 from danswer.server.manage.search_settings import router as search_settings_router
 from danswer.server.manage.slack_bot import router as slack_bot_management_router
 from danswer.server.manage.users import router as user_router
@@ -111,15 +74,10 @@
 from danswer.server.query_and_chat.query_backend import basic_router as query_router
 from danswer.server.settings.api import admin_router as settings_admin_router
 from danswer.server.settings.api import basic_router as settings_router
-from danswer.server.settings.store import load_settings
-from danswer.server.settings.store import store_settings
 from danswer.server.token_rate_limits.api import (
     router as token_rate_limit_settings_router,
 )
-from danswer.tools.built_in_tools import auto_add_search_tool_to_personas
-from danswer.tools.built_in_tools import load_builtin_tools
-from danswer.tools.built_in_tools import refresh_built_in_tools_cache
-from danswer.utils.gpu_utils import gpu_status_request
+from danswer.setup import setup_danswer
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import get_or_generate_uuid
 from danswer.utils.telemetry import optional_telemetry
@@ -128,8 +86,6 @@
 from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 from shared_configs.configs import CORS_ALLOWED_ORIGIN
-from shared_configs.configs import MODEL_SERVER_HOST
-from shared_configs.configs import MODEL_SERVER_PORT
 
 logger = setup_logger()
 
@@ -182,176 +138,6 @@ def include_router_with_global_prefix_prepended(
     application.include_router(router, **final_kwargs)
 
 
-def setup_postgres(db_session: Session) -> None:
-    logger.notice("Verifying default connector/credential exist.")
-    create_initial_public_credential(db_session)
-    create_initial_default_connector(db_session)
-    associate_default_cc_pair(db_session)
-
-    logger.notice("Loading default Prompts and Personas")
-    delete_old_default_personas(db_session)
-    load_chat_yamls()
-
-    logger.notice("Loading built-in tools")
-    load_builtin_tools(db_session)
-    refresh_built_in_tools_cache(db_session)
-    auto_add_search_tool_to_personas(db_session)
-
-    if GEN_AI_API_KEY and fetch_default_provider(db_session) is None:
-        # Only for dev flows
-        logger.notice("Setting up default OpenAI LLM for dev.")
-        llm_model = GEN_AI_MODEL_VERSION or "gpt-4o-mini"
-        fast_model = FAST_GEN_AI_MODEL_VERSION or "gpt-4o-mini"
-        model_req = LLMProviderUpsertRequest(
-            name="DevEnvPresetOpenAI",
-            provider="openai",
-            api_key=GEN_AI_API_KEY,
-            api_base=None,
-            api_version=None,
-            custom_config=None,
-            default_model_name=llm_model,
-            fast_default_model_name=fast_model,
-            is_public=True,
-            groups=[],
-            display_model_names=[llm_model, fast_model],
-            model_names=[llm_model, fast_model],
-        )
-        new_llm_provider = upsert_llm_provider(
-            llm_provider=model_req, db_session=db_session
-        )
-        update_default_provider(provider_id=new_llm_provider.id, db_session=db_session)
-
-
-def update_default_multipass_indexing(db_session: Session) -> None:
-    docs_exist = check_docs_exist(db_session)
-    connectors_exist = check_connectors_exist(db_session)
-    logger.debug(f"Docs exist: {docs_exist}, Connectors exist: {connectors_exist}")
-
-    if not docs_exist and not connectors_exist:
-        logger.info(
-            "No existing docs or connectors found. Checking GPU availability for multipass indexing."
-        )
-        gpu_available = gpu_status_request()
-        logger.info(f"GPU available: {gpu_available}")
-
-        current_settings = get_current_search_settings(db_session)
-
-        logger.notice(f"Updating multipass indexing setting to: {gpu_available}")
-        updated_settings = SavedSearchSettings.from_db_model(current_settings)
-        # Enable multipass indexing if GPU is available or if using a cloud provider
-        updated_settings.multipass_indexing = (
-            gpu_available or current_settings.cloud_provider is not None
-        )
-        update_current_search_settings(db_session, updated_settings)
-
-        # Update settings with GPU availability
-        settings = load_settings()
-        settings.gpu_enabled = gpu_available
-        store_settings(settings)
-        logger.notice(f"Updated settings with GPU availability: {gpu_available}")
-
-    else:
-        logger.debug(
-            "Existing docs or connectors found. Skipping multipass indexing update."
-        )
-
-
-def translate_saved_search_settings(db_session: Session) -> None:
-    kv_store = get_kv_store()
-
-    try:
-        search_settings_dict = kv_store.load(KV_SEARCH_SETTINGS)
-        if isinstance(search_settings_dict, dict):
-            # Update current search settings
-            current_settings = get_current_search_settings(db_session)
-
-            # Update non-preserved fields
-            if current_settings:
-                current_settings_dict = SavedSearchSettings.from_db_model(
-                    current_settings
-                ).dict()
-
-                new_current_settings = SavedSearchSettings(
-                    **{**current_settings_dict, **search_settings_dict}
-                )
-                update_current_search_settings(db_session, new_current_settings)
-
-            # Update secondary search settings
-            secondary_settings = get_secondary_search_settings(db_session)
-            if secondary_settings:
-                secondary_settings_dict = SavedSearchSettings.from_db_model(
-                    secondary_settings
-                ).dict()
-
-                new_secondary_settings = SavedSearchSettings(
-                    **{**secondary_settings_dict, **search_settings_dict}
-                )
-                update_secondary_search_settings(
-                    db_session,
-                    new_secondary_settings,
-                )
-            # Delete the KV store entry after successful update
-            kv_store.delete(KV_SEARCH_SETTINGS)
-            logger.notice("Search settings updated and KV store entry deleted.")
-        else:
-            logger.notice("KV store search settings is empty.")
-    except KvKeyNotFoundError:
-        logger.notice("No search config found in KV store.")
-
-
-def mark_reindex_flag(db_session: Session) -> None:
-    kv_store = get_kv_store()
-    try:
-        value = kv_store.load(KV_REINDEX_KEY)
-        logger.debug(f"Re-indexing flag has value {value}")
-        return
-    except KvKeyNotFoundError:
-        # Only need to update the flag if it hasn't been set
-        pass
-
-    # If their first deployment is after the changes, it will
-    # enable this when the other changes go in, need to avoid
-    # this being set to False, then the user indexes things on the old version
-    docs_exist = check_docs_exist(db_session)
-    connectors_exist = check_connectors_exist(db_session)
-    if docs_exist or connectors_exist:
-        kv_store.store(KV_REINDEX_KEY, True)
-    else:
-        kv_store.store(KV_REINDEX_KEY, False)
-
-
-def setup_vespa(
-    document_index: DocumentIndex,
-    index_setting: IndexingSetting,
-    secondary_index_setting: IndexingSetting | None,
-) -> bool:
-    # Vespa startup is a bit slow, so give it a few seconds
-    WAIT_SECONDS = 5
-    VESPA_ATTEMPTS = 5
-    for x in range(VESPA_ATTEMPTS):
-        try:
-            logger.notice(f"Setting up Vespa (attempt {x+1}/{VESPA_ATTEMPTS})...")
-            document_index.ensure_indices_exist(
-                index_embedding_dim=index_setting.model_dim,
-                secondary_index_embedding_dim=secondary_index_setting.model_dim
-                if secondary_index_setting
-                else None,
-            )
-
-            logger.notice("Vespa setup complete.")
-            return True
-        except Exception:
-            logger.notice(
-                f"Vespa setup did not succeed. The Vespa service may not be ready yet. Retrying in {WAIT_SECONDS} seconds."
-            )
-            time.sleep(WAIT_SECONDS)
-
-    logger.error(
-        f"Vespa setup did not succeed. Attempt limit reached. ({VESPA_ATTEMPTS})"
-    )
-    return False
-
-
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncGenerator:
     SqlEngine.set_app_name(POSTGRES_WEB_APP_NAME)
@@ -380,89 +166,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
     get_or_generate_uuid()
 
     with Session(engine) as db_session:
-        check_index_swap(db_session=db_session)
-        search_settings = get_current_search_settings(db_session)
-        secondary_search_settings = get_secondary_search_settings(db_session)
-
-        # Break bad state for thrashing indexes
-        if secondary_search_settings and DISABLE_INDEX_UPDATE_ON_SWAP:
-            expire_index_attempts(
-                search_settings_id=search_settings.id, db_session=db_session
-            )
-
-            for cc_pair in get_connector_credential_pairs(db_session):
-                resync_cc_pair(cc_pair, db_session=db_session)
-
-        # Expire all old embedding models indexing attempts, technically redundant
-        cancel_indexing_attempts_past_model(db_session)
-
-        logger.notice(f'Using Embedding model: "{search_settings.model_name}"')
-        if search_settings.query_prefix or search_settings.passage_prefix:
-            logger.notice(f'Query embedding prefix: "{search_settings.query_prefix}"')
-            logger.notice(
-                f'Passage embedding prefix: "{search_settings.passage_prefix}"'
-            )
-
-        if search_settings:
-            if not search_settings.disable_rerank_for_streaming:
-                logger.notice("Reranking is enabled.")
-
-            if search_settings.multilingual_expansion:
-                logger.notice(
-                    f"Multilingual query expansion is enabled with {search_settings.multilingual_expansion}."
-                )
-        if (
-            search_settings.rerank_model_name
-            and not search_settings.provider_type
-            and not search_settings.rerank_provider_type
-        ):
-            warm_up_cross_encoder(search_settings.rerank_model_name)
-
-        logger.notice("Verifying query preprocessing (NLTK) data is downloaded")
-        download_nltk_data()
-
-        # setup Postgres with default credential, llm providers, etc.
-        setup_postgres(db_session)
-
-        translate_saved_search_settings(db_session)
-
-        # Does the user need to trigger a reindexing to bring the document index
-        # into a good state, marked in the kv store
-        mark_reindex_flag(db_session)
-
-        # ensure Vespa is setup correctly
-        logger.notice("Verifying Document Index(s) is/are available.")
-        document_index = get_default_document_index(
-            primary_index_name=search_settings.index_name,
-            secondary_index_name=secondary_search_settings.index_name
-            if secondary_search_settings
-            else None,
-        )
-
-        success = setup_vespa(
-            document_index,
-            IndexingSetting.from_db_model(search_settings),
-            IndexingSetting.from_db_model(secondary_search_settings)
-            if secondary_search_settings
-            else None,
-        )
-        if not success:
-            raise RuntimeError(
-                "Could not connect to Vespa within the specified timeout."
-            )
-
-        logger.notice(f"Model Server: http://{MODEL_SERVER_HOST}:{MODEL_SERVER_PORT}")
-        if search_settings.provider_type is None:
-            warm_up_bi_encoder(
-                embedding_model=EmbeddingModel.from_db_model(
-                    search_settings=search_settings,
-                    server_host=MODEL_SERVER_HOST,
-                    server_port=MODEL_SERVER_PORT,
-                ),
-            )
-
-        # update multipass indexing setting based on GPU availability
-        update_default_multipass_indexing(db_session)
+        setup_danswer(db_session)
 
     optional_telemetry(record_type=RecordType.VERSION, data={"version": __version__})
     yield
diff --git a/backend/danswer/server/auth_check.py b/backend/danswer/server/auth_check.py
index 8a35a560a24..c79b9ad0967 100644
--- a/backend/danswer/server/auth_check.py
+++ b/backend/danswer/server/auth_check.py
@@ -4,6 +4,7 @@
 from fastapi.dependencies.models import Dependant
 from starlette.routing import BaseRoute
 
+from danswer.auth.users import control_plane_dep
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
@@ -98,6 +99,7 @@ def check_router_auth(
                     or depends_fn == current_curator_or_admin_user
                     or depends_fn == api_key_dep
                     or depends_fn == current_user_with_expired_token
+                    or depends_fn == control_plane_dep
                 ):
                     found_auth = True
                     break
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
new file mode 100644
index 00000000000..2baeda4a811
--- /dev/null
+++ b/backend/danswer/setup.py
@@ -0,0 +1,303 @@
+import time
+
+from sqlalchemy.orm import Session
+
+from danswer.chat.load_yamls import load_chat_yamls
+from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+from danswer.configs.constants import KV_REINDEX_KEY
+from danswer.configs.constants import KV_SEARCH_SETTINGS
+from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
+from danswer.configs.model_configs import GEN_AI_API_KEY
+from danswer.configs.model_configs import GEN_AI_MODEL_VERSION
+from danswer.db.connector import check_connectors_exist
+from danswer.db.connector import create_initial_default_connector
+from danswer.db.connector_credential_pair import associate_default_cc_pair
+from danswer.db.connector_credential_pair import get_connector_credential_pairs
+from danswer.db.connector_credential_pair import resync_cc_pair
+from danswer.db.credentials import create_initial_public_credential
+from danswer.db.document import check_docs_exist
+from danswer.db.index_attempt import cancel_indexing_attempts_past_model
+from danswer.db.index_attempt import expire_index_attempts
+from danswer.db.llm import fetch_default_provider
+from danswer.db.llm import update_default_provider
+from danswer.db.llm import upsert_llm_provider
+from danswer.db.persona import delete_old_default_personas
+from danswer.db.search_settings import get_current_search_settings
+from danswer.db.search_settings import get_secondary_search_settings
+from danswer.db.search_settings import update_current_search_settings
+from danswer.db.search_settings import update_secondary_search_settings
+from danswer.db.swap_index import check_index_swap
+from danswer.document_index.factory import get_default_document_index
+from danswer.document_index.interfaces import DocumentIndex
+from danswer.indexing.models import IndexingSetting
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
+from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
+from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
+from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder
+from danswer.search.models import SavedSearchSettings
+from danswer.search.retrieval.search_runner import download_nltk_data
+from danswer.server.manage.llm.models import LLMProviderUpsertRequest
+from danswer.server.settings.store import load_settings
+from danswer.server.settings.store import store_settings
+from danswer.tools.built_in_tools import auto_add_search_tool_to_personas
+from danswer.tools.built_in_tools import load_builtin_tools
+from danswer.tools.built_in_tools import refresh_built_in_tools_cache
+from danswer.utils.gpu_utils import gpu_status_request
+from danswer.utils.logger import setup_logger
+from shared_configs.configs import MODEL_SERVER_HOST
+from shared_configs.configs import MODEL_SERVER_PORT
+
+logger = setup_logger()
+
+
+def setup_danswer(db_session: Session) -> None:
+    check_index_swap(db_session=db_session)
+    search_settings = get_current_search_settings(db_session)
+    secondary_search_settings = get_secondary_search_settings(db_session)
+
+    # Break bad state for thrashing indexes
+    if secondary_search_settings and DISABLE_INDEX_UPDATE_ON_SWAP:
+        expire_index_attempts(
+            search_settings_id=search_settings.id, db_session=db_session
+        )
+
+        for cc_pair in get_connector_credential_pairs(db_session):
+            resync_cc_pair(cc_pair, db_session=db_session)
+
+    # Expire all old embedding models indexing attempts, technically redundant
+    cancel_indexing_attempts_past_model(db_session)
+
+    logger.notice(f'Using Embedding model: "{search_settings.model_name}"')
+    if search_settings.query_prefix or search_settings.passage_prefix:
+        logger.notice(f'Query embedding prefix: "{search_settings.query_prefix}"')
+        logger.notice(f'Passage embedding prefix: "{search_settings.passage_prefix}"')
+
+    if search_settings:
+        if not search_settings.disable_rerank_for_streaming:
+            logger.notice("Reranking is enabled.")
+
+        if search_settings.multilingual_expansion:
+            logger.notice(
+                f"Multilingual query expansion is enabled with {search_settings.multilingual_expansion}."
+            )
+    if (
+        search_settings.rerank_model_name
+        and not search_settings.provider_type
+        and not search_settings.rerank_provider_type
+    ):
+        warm_up_cross_encoder(search_settings.rerank_model_name)
+
+    logger.notice("Verifying query preprocessing (NLTK) data is downloaded")
+    download_nltk_data()
+
+    # setup Postgres with default credential, llm providers, etc.
+    setup_postgres(db_session)
+
+    translate_saved_search_settings(db_session)
+
+    # Does the user need to trigger a reindexing to bring the document index
+    # into a good state, marked in the kv store
+    mark_reindex_flag(db_session)
+
+    # ensure Vespa is setup correctly
+    logger.notice("Verifying Document Index(s) is/are available.")
+    document_index = get_default_document_index(
+        primary_index_name=search_settings.index_name,
+        secondary_index_name=secondary_search_settings.index_name
+        if secondary_search_settings
+        else None,
+    )
+
+    success = setup_vespa(
+        document_index,
+        IndexingSetting.from_db_model(search_settings),
+        IndexingSetting.from_db_model(secondary_search_settings)
+        if secondary_search_settings
+        else None,
+    )
+    if not success:
+        raise RuntimeError("Could not connect to Vespa within the specified timeout.")
+
+    logger.notice(f"Model Server: http://{MODEL_SERVER_HOST}:{MODEL_SERVER_PORT}")
+    if search_settings.provider_type is None:
+        warm_up_bi_encoder(
+            embedding_model=EmbeddingModel.from_db_model(
+                search_settings=search_settings,
+                server_host=MODEL_SERVER_HOST,
+                server_port=MODEL_SERVER_PORT,
+            ),
+        )
+
+    # update multipass indexing setting based on GPU availability
+    update_default_multipass_indexing(db_session)
+
+
+def translate_saved_search_settings(db_session: Session) -> None:
+    kv_store = get_kv_store()
+
+    try:
+        search_settings_dict = kv_store.load(KV_SEARCH_SETTINGS)
+        if isinstance(search_settings_dict, dict):
+            # Update current search settings
+            current_settings = get_current_search_settings(db_session)
+
+            # Update non-preserved fields
+            if current_settings:
+                current_settings_dict = SavedSearchSettings.from_db_model(
+                    current_settings
+                ).dict()
+
+                new_current_settings = SavedSearchSettings(
+                    **{**current_settings_dict, **search_settings_dict}
+                )
+                update_current_search_settings(db_session, new_current_settings)
+
+            # Update secondary search settings
+            secondary_settings = get_secondary_search_settings(db_session)
+            if secondary_settings:
+                secondary_settings_dict = SavedSearchSettings.from_db_model(
+                    secondary_settings
+                ).dict()
+
+                new_secondary_settings = SavedSearchSettings(
+                    **{**secondary_settings_dict, **search_settings_dict}
+                )
+                update_secondary_search_settings(
+                    db_session,
+                    new_secondary_settings,
+                )
+            # Delete the KV store entry after successful update
+            kv_store.delete(KV_SEARCH_SETTINGS)
+            logger.notice("Search settings updated and KV store entry deleted.")
+        else:
+            logger.notice("KV store search settings is empty.")
+    except KvKeyNotFoundError:
+        logger.notice("No search config found in KV store.")
+
+
+def mark_reindex_flag(db_session: Session) -> None:
+    kv_store = get_kv_store()
+    try:
+        value = kv_store.load(KV_REINDEX_KEY)
+        logger.debug(f"Re-indexing flag has value {value}")
+        return
+    except KvKeyNotFoundError:
+        # Only need to update the flag if it hasn't been set
+        pass
+
+    # If their first deployment is after the changes, it will
+    # enable this when the other changes go in, need to avoid
+    # this being set to False, then the user indexes things on the old version
+    docs_exist = check_docs_exist(db_session)
+    connectors_exist = check_connectors_exist(db_session)
+    if docs_exist or connectors_exist:
+        kv_store.store(KV_REINDEX_KEY, True)
+    else:
+        kv_store.store(KV_REINDEX_KEY, False)
+
+
+def setup_vespa(
+    document_index: DocumentIndex,
+    index_setting: IndexingSetting,
+    secondary_index_setting: IndexingSetting | None,
+) -> bool:
+    # Vespa startup is a bit slow, so give it a few seconds
+    WAIT_SECONDS = 5
+    VESPA_ATTEMPTS = 5
+    for x in range(VESPA_ATTEMPTS):
+        try:
+            logger.notice(f"Setting up Vespa (attempt {x+1}/{VESPA_ATTEMPTS})...")
+            document_index.ensure_indices_exist(
+                index_embedding_dim=index_setting.model_dim,
+                secondary_index_embedding_dim=secondary_index_setting.model_dim
+                if secondary_index_setting
+                else None,
+            )
+
+            logger.notice("Vespa setup complete.")
+            return True
+        except Exception:
+            logger.notice(
+                f"Vespa setup did not succeed. The Vespa service may not be ready yet. Retrying in {WAIT_SECONDS} seconds."
+            )
+            time.sleep(WAIT_SECONDS)
+
+    logger.error(
+        f"Vespa setup did not succeed. Attempt limit reached. ({VESPA_ATTEMPTS})"
+    )
+    return False
+
+
+def setup_postgres(db_session: Session) -> None:
+    logger.notice("Verifying default connector/credential exist.")
+    create_initial_public_credential(db_session)
+    create_initial_default_connector(db_session)
+    associate_default_cc_pair(db_session)
+
+    logger.notice("Loading default Prompts and Personas")
+    delete_old_default_personas(db_session)
+    load_chat_yamls(db_session)
+
+    logger.notice("Loading built-in tools")
+    load_builtin_tools(db_session)
+    refresh_built_in_tools_cache(db_session)
+    auto_add_search_tool_to_personas(db_session)
+
+    if GEN_AI_API_KEY and fetch_default_provider(db_session) is None:
+        # Only for dev flows
+        logger.notice("Setting up default OpenAI LLM for dev.")
+        llm_model = GEN_AI_MODEL_VERSION or "gpt-4o-mini"
+        fast_model = FAST_GEN_AI_MODEL_VERSION or "gpt-4o-mini"
+        model_req = LLMProviderUpsertRequest(
+            name="DevEnvPresetOpenAI",
+            provider="openai",
+            api_key=GEN_AI_API_KEY,
+            api_base=None,
+            api_version=None,
+            custom_config=None,
+            default_model_name=llm_model,
+            fast_default_model_name=fast_model,
+            is_public=True,
+            groups=[],
+            display_model_names=[llm_model, fast_model],
+            model_names=[llm_model, fast_model],
+        )
+        new_llm_provider = upsert_llm_provider(
+            llm_provider=model_req, db_session=db_session
+        )
+        update_default_provider(provider_id=new_llm_provider.id, db_session=db_session)
+
+
+def update_default_multipass_indexing(db_session: Session) -> None:
+    docs_exist = check_docs_exist(db_session)
+    connectors_exist = check_connectors_exist(db_session)
+    logger.debug(f"Docs exist: {docs_exist}, Connectors exist: {connectors_exist}")
+
+    if not docs_exist and not connectors_exist:
+        logger.info(
+            "No existing docs or connectors found. Checking GPU availability for multipass indexing."
+        )
+        gpu_available = gpu_status_request()
+        logger.info(f"GPU available: {gpu_available}")
+
+        current_settings = get_current_search_settings(db_session)
+
+        logger.notice(f"Updating multipass indexing setting to: {gpu_available}")
+        updated_settings = SavedSearchSettings.from_db_model(current_settings)
+        # Enable multipass indexing if GPU is available or if using a cloud provider
+        updated_settings.multipass_indexing = (
+            gpu_available or current_settings.cloud_provider is not None
+        )
+        update_current_search_settings(db_session, updated_settings)
+
+        # Update settings with GPU availability
+        settings = load_settings()
+        settings.gpu_enabled = gpu_available
+        store_settings(settings)
+        logger.notice(f"Updated settings with GPU availability: {gpu_available}")
+
+    else:
+        logger.debug(
+            "Existing docs or connectors found. Skipping multipass indexing update."
+        )
diff --git a/backend/ee/danswer/main.py b/backend/ee/danswer/main.py
index 7d150107c75..8422d5494ae 100644
--- a/backend/ee/danswer/main.py
+++ b/backend/ee/danswer/main.py
@@ -34,6 +34,7 @@
 from ee.danswer.server.reporting.usage_export_api import router as usage_export_router
 from ee.danswer.server.saml import router as saml_router
 from ee.danswer.server.seeding import seed_db
+from ee.danswer.server.tenants.api import router as tenants_router
 from ee.danswer.server.token_rate_limits.api import (
     router as token_rate_limit_settings_router,
 )
@@ -79,6 +80,8 @@ def get_application() -> FastAPI:
 
     # RBAC / group access control
     include_router_with_global_prefix_prepended(application, user_group_router)
+    # Tenant management
+    include_router_with_global_prefix_prepended(application, tenants_router)
     # Analytics endpoints
     include_router_with_global_prefix_prepended(application, analytics_router)
     include_router_with_global_prefix_prepended(application, query_history_router)
diff --git a/backend/ee/danswer/server/tenants/__init__.py b/backend/ee/danswer/server/tenants/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/backend/ee/danswer/server/tenants/access.py b/backend/ee/danswer/server/tenants/access.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
new file mode 100644
index 00000000000..ec96351856b
--- /dev/null
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -0,0 +1,46 @@
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+
+from danswer.auth.users import control_plane_dep
+from danswer.configs.app_configs import MULTI_TENANT
+from danswer.db.engine import get_session_with_tenant
+from danswer.setup import setup_danswer
+from danswer.utils.logger import setup_logger
+from ee.danswer.server.tenants.models import CreateTenantRequest
+from ee.danswer.server.tenants.provisioning import ensure_schema_exists
+from ee.danswer.server.tenants.provisioning import run_alembic_migrations
+
+logger = setup_logger()
+router = APIRouter(prefix="/tenants")
+
+
+@router.post("/create")
+def create_tenant(
+    create_tenant_request: CreateTenantRequest, _: None = Depends(control_plane_dep)
+) -> dict[str, str]:
+    try:
+        tenant_id = create_tenant_request.tenant_id
+
+        if not MULTI_TENANT:
+            raise HTTPException(status_code=403, detail="Multi-tenancy is not enabled")
+
+        if not ensure_schema_exists(tenant_id):
+            logger.info(f"Created schema for tenant {tenant_id}")
+        else:
+            logger.info(f"Schema already exists for tenant {tenant_id}")
+
+        run_alembic_migrations(tenant_id)
+        with get_session_with_tenant(tenant_id) as db_session:
+            setup_danswer(db_session)
+
+        logger.info(f"Tenant {tenant_id} created successfully")
+        return {
+            "status": "success",
+            "message": f"Tenant {tenant_id} created successfully",
+        }
+    except Exception as e:
+        logger.exception(f"Failed to create tenant {tenant_id}: {str(e)}")
+        raise HTTPException(
+            status_code=500, detail=f"Failed to create tenant: {str(e)}"
+        )
diff --git a/backend/ee/danswer/server/tenants/models.py b/backend/ee/danswer/server/tenants/models.py
new file mode 100644
index 00000000000..833650c42a6
--- /dev/null
+++ b/backend/ee/danswer/server/tenants/models.py
@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+
+class CreateTenantRequest(BaseModel):
+    tenant_id: str
+    initial_admin_email: str
diff --git a/backend/ee/danswer/server/tenants/provisioning.py b/backend/ee/danswer/server/tenants/provisioning.py
new file mode 100644
index 00000000000..62436c92e17
--- /dev/null
+++ b/backend/ee/danswer/server/tenants/provisioning.py
@@ -0,0 +1,63 @@
+import os
+from types import SimpleNamespace
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+from sqlalchemy.schema import CreateSchema
+
+from alembic import command
+from alembic.config import Config
+from danswer.db.engine import build_connection_string
+from danswer.db.engine import get_sqlalchemy_engine
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def run_alembic_migrations(schema_name: str) -> None:
+    logger.info(f"Starting Alembic migrations for schema: {schema_name}")
+
+    try:
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        root_dir = os.path.abspath(os.path.join(current_dir, "..", "..", "..", ".."))
+        alembic_ini_path = os.path.join(root_dir, "alembic.ini")
+
+        # Configure Alembic
+        alembic_cfg = Config(alembic_ini_path)
+        alembic_cfg.set_main_option("sqlalchemy.url", build_connection_string())
+        alembic_cfg.set_main_option(
+            "script_location", os.path.join(root_dir, "alembic")
+        )
+
+        # Mimic command-line options by adding 'cmd_opts' to the config
+        alembic_cfg.cmd_opts = SimpleNamespace()  # type: ignore
+        alembic_cfg.cmd_opts.x = [f"schema={schema_name}"]  # type: ignore
+
+        # Run migrations programmatically
+        command.upgrade(alembic_cfg, "head")
+
+        # Run migrations programmatically
+        logger.info(
+            f"Alembic migrations completed successfully for schema: {schema_name}"
+        )
+
+    except Exception as e:
+        logger.exception(f"Alembic migration failed for schema {schema_name}: {str(e)}")
+        raise
+
+
+def ensure_schema_exists(tenant_id: str) -> bool:
+    with Session(get_sqlalchemy_engine()) as db_session:
+        with db_session.begin():
+            result = db_session.execute(
+                text(
+                    "SELECT schema_name FROM information_schema.schemata WHERE schema_name = :schema_name"
+                ),
+                {"schema_name": tenant_id},
+            )
+            schema_exists = result.scalar() is not None
+            if not schema_exists:
+                stmt = CreateSchema(tenant_id)
+                db_session.execute(stmt)
+                return True
+            return False
diff --git a/backend/tests/integration/common_utils/reset.py b/backend/tests/integration/common_utils/reset.py
index 95b3f734ed4..a532406c4cd 100644
--- a/backend/tests/integration/common_utils/reset.py
+++ b/backend/tests/integration/common_utils/reset.py
@@ -18,8 +18,8 @@
 from danswer.document_index.vespa.index import DOCUMENT_ID_ENDPOINT
 from danswer.document_index.vespa.index import VespaIndex
 from danswer.indexing.models import IndexingSetting
-from danswer.main import setup_postgres
-from danswer.main import setup_vespa
+from danswer.setup import setup_postgres
+from danswer.setup import setup_vespa
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index b171b2bcb05..4e8ea0abd28 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -14,6 +14,7 @@ import {
   ValidQuestionResponse,
   Relevance,
   SearchDanswerDocument,
+  SourceMetadata,
 } from "@/lib/search/interfaces";
 import { searchRequestStreamed } from "@/lib/search/streamingQa";
 import { CancellationToken, cancellable } from "@/lib/search/cancellable";
@@ -40,6 +41,9 @@ import { ApiKeyModal } from "../llm/ApiKeyModal";
 import { useSearchContext } from "../context/SearchContext";
 import { useUser } from "../user/UserProvider";
 import UnconfiguredProviderText from "../chat_search/UnconfiguredProviderText";
+import { DateRangePickerValue } from "@tremor/react";
+import { Tag } from "@/lib/types";
+import { isEqual } from "lodash";
 
 export type searchState =
   | "input"
@@ -370,8 +374,36 @@ export const SearchSection = ({
     setSearchAnswerExpanded(false);
   };
 
-  const [previousSearch, setPreviousSearch] = useState<string>("");
+  interface SearchDetails {
+    query: string;
+    sources: SourceMetadata[];
+    agentic: boolean;
+    documentSets: string[];
+    timeRange: DateRangePickerValue | null;
+    tags: Tag[];
+    persona: Persona;
+  }
+
+  const [previousSearch, setPreviousSearch] = useState<null | SearchDetails>(
+    null
+  );
   const [agenticResults, setAgenticResults] = useState<boolean | null>(null);
+  const currentSearch = (overrideMessage?: string): SearchDetails => {
+    return {
+      query: overrideMessage || query,
+      sources: filterManager.selectedSources,
+      agentic: agentic!,
+      documentSets: filterManager.selectedDocumentSets,
+      timeRange: filterManager.timeRange,
+      tags: filterManager.selectedTags,
+      persona: assistants.find(
+        (assistant) => assistant.id === selectedPersona
+      ) as Persona,
+    };
+  };
+  const isSearchChanged = () => {
+    return !isEqual(currentSearch(), previousSearch);
+  };
 
   let lastSearchCancellationToken = useRef<CancellationToken | null>(null);
   const onSearch = async ({
@@ -394,7 +426,9 @@ export const SearchSection = ({
 
     setIsFetching(true);
     setSearchResponse(initialSearchResponse);
-    setPreviousSearch(overrideMessage || query);
+
+    setPreviousSearch(currentSearch(overrideMessage));
+
     const searchFnArgs = {
       query: overrideMessage || query,
       sources: filterManager.selectedSources,
@@ -761,7 +795,7 @@ export const SearchSection = ({
                     />
 
                     <FullSearchBar
-                      disabled={previousSearch === query}
+                      disabled={!isSearchChanged()}
                       toggleAgentic={
                         disabledAgentic ? undefined : toggleAgentic
                       }

From 089c734f63143b65543e9c26b31e4adf6741aff1 Mon Sep 17 00:00:00 2001
From: evan-danswer <evan@danswer.ai>
Date: Sun, 6 Oct 2024 14:10:02 -0400
Subject: [PATCH 034/376] disabled llm when skip_gen_ai_answer_question set
 (#2687)

* disabled llm when skip_gen_ai_answer_question set

* added unit test

* typing
---
 backend/danswer/llm/answering/answer.py       |  22 +--
 .../tests/regression/answer_quality/run_qa.py |  38 ++---
 .../danswer/llm/answering/test_skip_gen_ai.py | 132 ++++++++++++++++++
 3 files changed, 165 insertions(+), 27 deletions(-)
 create mode 100644 backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py

diff --git a/backend/danswer/llm/answering/answer.py b/backend/danswer/llm/answering/answer.py
index 922d757d3eb..12c1bc25f4f 100644
--- a/backend/danswer/llm/answering/answer.py
+++ b/backend/danswer/llm/answering/answer.py
@@ -311,13 +311,13 @@ def _raw_output_for_explicit_tool_calling_llms(
                     )
                 )
             yield tool_runner.tool_final_result()
+            if not self.skip_gen_ai_answer_generation:
+                prompt = prompt_builder.build(tool_call_summary=tool_call_summary)
 
-            prompt = prompt_builder.build(tool_call_summary=tool_call_summary)
-
-            yield from self._process_llm_stream(
-                prompt=prompt,
-                tools=[tool.tool_definition() for tool in self.tools],
-            )
+                yield from self._process_llm_stream(
+                    prompt=prompt,
+                    tools=[tool.tool_definition() for tool in self.tools],
+                )
 
             return
 
@@ -413,6 +413,10 @@ def _raw_output_for_non_explicit_tool_calling_llms(
             logger.notice(f"Chosen tool: {chosen_tool_and_args}")
 
         if not chosen_tool_and_args:
+            if self.skip_gen_ai_answer_generation:
+                raise ValueError(
+                    "skip_gen_ai_answer_generation is True, but no tool was chosen; no answer will be generated"
+                )
             prompt_builder.update_system_prompt(
                 default_build_system_message(self.prompt_config)
             )
@@ -477,10 +481,10 @@ def _raw_output_for_non_explicit_tool_calling_llms(
         final = tool_runner.tool_final_result()
 
         yield final
+        if not self.skip_gen_ai_answer_generation:
+            prompt = prompt_builder.build()
 
-        prompt = prompt_builder.build()
-
-        yield from self._process_llm_stream(prompt=prompt, tools=None)
+            yield from self._process_llm_stream(prompt=prompt, tools=None)
 
     @property
     def processed_streamed_output(self) -> AnswerStream:
diff --git a/backend/tests/regression/answer_quality/run_qa.py b/backend/tests/regression/answer_quality/run_qa.py
index 5de034b3740..f6dd0e0b558 100644
--- a/backend/tests/regression/answer_quality/run_qa.py
+++ b/backend/tests/regression/answer_quality/run_qa.py
@@ -77,14 +77,15 @@ def _initialize_files(config: dict) -> tuple[str, list[dict]]:
         "number_of_questions_in_dataset": len(questions),
     }
 
-    env_vars = get_docker_container_env_vars(config["env_name"])
-    if env_vars["ENV_SEED_CONFIGURATION"]:
-        del env_vars["ENV_SEED_CONFIGURATION"]
-    if env_vars["GPG_KEY"]:
-        del env_vars["GPG_KEY"]
-    if metadata["test_config"]["llm"]["api_key"]:
-        del metadata["test_config"]["llm"]["api_key"]
-    metadata.update(env_vars)
+    if config["env_name"]:
+        env_vars = get_docker_container_env_vars(config["env_name"])
+        if env_vars["ENV_SEED_CONFIGURATION"]:
+            del env_vars["ENV_SEED_CONFIGURATION"]
+        if env_vars["GPG_KEY"]:
+            del env_vars["GPG_KEY"]
+        if metadata["test_config"]["llm"]["api_key"]:
+            del metadata["test_config"]["llm"]["api_key"]
+        metadata.update(env_vars)
     metadata_path = os.path.join(test_output_folder, METADATA_FILENAME)
     print("saving metadata to:", metadata_path)
     with open(metadata_path, "w", encoding="utf-8") as yaml_file:
@@ -95,17 +96,18 @@ def _initialize_files(config: dict) -> tuple[str, list[dict]]:
     )
     shutil.copy2(questions_file_path, copied_questions_file_path)
 
-    zipped_files_path = config["zipped_documents_file"]
-    copied_zipped_documents_path = os.path.join(
-        test_output_folder, os.path.basename(zipped_files_path)
-    )
-    shutil.copy2(zipped_files_path, copied_zipped_documents_path)
+    if config["zipped_documents_file"]:
+        zipped_files_path = config["zipped_documents_file"]
+        copied_zipped_documents_path = os.path.join(
+            test_output_folder, os.path.basename(zipped_files_path)
+        )
+        shutil.copy2(zipped_files_path, copied_zipped_documents_path)
 
-    zipped_files_folder = os.path.dirname(zipped_files_path)
-    jsonl_file_path = os.path.join(zipped_files_folder, "target_docs.jsonl")
-    if os.path.exists(jsonl_file_path):
-        copied_jsonl_path = os.path.join(test_output_folder, "target_docs.jsonl")
-        shutil.copy2(jsonl_file_path, copied_jsonl_path)
+        zipped_files_folder = os.path.dirname(zipped_files_path)
+        jsonl_file_path = os.path.join(zipped_files_folder, "target_docs.jsonl")
+        if os.path.exists(jsonl_file_path):
+            copied_jsonl_path = os.path.join(test_output_folder, "target_docs.jsonl")
+            shutil.copy2(jsonl_file_path, copied_jsonl_path)
 
     return test_output_folder, questions
 
diff --git a/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py b/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py
new file mode 100644
index 00000000000..998b2932cbb
--- /dev/null
+++ b/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py
@@ -0,0 +1,132 @@
+from typing import Any
+from typing import cast
+from unittest.mock import Mock
+
+import pytest
+from pytest_mock import MockerFixture
+
+from danswer.llm.answering.answer import Answer
+from danswer.one_shot_answer.answer_question import AnswerObjectIterator
+from danswer.tools.force import ForceUseTool
+from tests.regression.answer_quality.run_qa import _process_and_write_query_results
+
+
+@pytest.mark.parametrize(
+    "config",
+    [
+        {
+            "skip_gen_ai_answer_generation": True,
+            "question": "What is the capital of the moon?",
+        },
+        {
+            "skip_gen_ai_answer_generation": False,
+            "question": "What is the capital of the moon but twice?",
+        },
+    ],
+)
+def test_skip_gen_ai_answer_generation_flag(config: dict[str, Any]) -> None:
+    search_tool = Mock()
+    search_tool.name = "search"
+    search_tool.run = Mock()
+    search_tool.run.return_value = [Mock()]
+    mock_llm = Mock()
+    mock_llm.config = Mock()
+    mock_llm.config.model_name = "gpt-4o-mini"
+    mock_llm.stream = Mock()
+    mock_llm.stream.return_value = [Mock()]
+    answer = Answer(
+        question=config["question"],
+        answer_style_config=Mock(),
+        prompt_config=Mock(),
+        llm=mock_llm,
+        single_message_history="history",
+        tools=[search_tool],
+        force_use_tool=(
+            ForceUseTool(
+                tool_name=search_tool.name,
+                args={"query": config["question"]},
+                force_use=True,
+            )
+        ),
+        skip_explicit_tool_calling=True,
+        return_contexts=True,
+        skip_gen_ai_answer_generation=config["skip_gen_ai_answer_generation"],
+    )
+    count = 0
+    for _ in cast(AnswerObjectIterator, answer.processed_streamed_output):
+        count += 1
+    assert count == 2
+    if not config["skip_gen_ai_answer_generation"]:
+        mock_llm.stream.assert_called_once()
+    else:
+        mock_llm.stream.assert_not_called()
+
+
+##### From here down is the client side test that was not working #####
+
+
+class FinishedTestException(Exception):
+    pass
+
+
+# could not get this to work, it seems like the mock is not being used
+# tests that the main run_qa function passes the skip_gen_ai_answer_generation flag to the Answer object
+@pytest.mark.parametrize(
+    "config, questions",
+    [
+        (
+            {
+                "skip_gen_ai_answer_generation": True,
+                "output_folder": "./test_output_folder",
+                "zipped_documents_file": "./test_docs.jsonl",
+                "questions_file": "./test_questions.jsonl",
+                "commit_sha": None,
+                "launch_web_ui": False,
+                "only_retrieve_docs": True,
+                "use_cloud_gpu": False,
+                "model_server_ip": "PUT_PUBLIC_CLOUD_IP_HERE",
+                "model_server_port": "PUT_PUBLIC_CLOUD_PORT_HERE",
+                "environment_name": "",
+                "env_name": "",
+                "limit": None,
+            },
+            [{"uid": "1", "question": "What is the capital of the moon?"}],
+        ),
+        (
+            {
+                "skip_gen_ai_answer_generation": False,
+                "output_folder": "./test_output_folder",
+                "zipped_documents_file": "./test_docs.jsonl",
+                "questions_file": "./test_questions.jsonl",
+                "commit_sha": None,
+                "launch_web_ui": False,
+                "only_retrieve_docs": True,
+                "use_cloud_gpu": False,
+                "model_server_ip": "PUT_PUBLIC_CLOUD_IP_HERE",
+                "model_server_port": "PUT_PUBLIC_CLOUD_PORT_HERE",
+                "environment_name": "",
+                "env_name": "",
+                "limit": None,
+            },
+            [{"uid": "1", "question": "What is the capital of the moon but twice?"}],
+        ),
+    ],
+)
+@pytest.mark.skip(reason="not working")
+def test_run_qa_skip_gen_ai(
+    config: dict[str, Any], questions: list[dict[str, Any]], mocker: MockerFixture
+) -> None:
+    mocker.patch(
+        "tests.regression.answer_quality.run_qa._initialize_files",
+        return_value=("test", questions),
+    )
+
+    def arg_checker(question_data: dict, config: dict, question_number: int) -> None:
+        assert question_data == questions[0]
+        raise FinishedTestException()
+
+    mocker.patch(
+        "tests.regression.answer_quality.run_qa._process_question", arg_checker
+    )
+    with pytest.raises(FinishedTestException):
+        _process_and_write_query_results(config)

From 0ff5180d7be699f0edb5f3674bd2acf758ad937e Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 6 Oct 2024 11:42:49 -0700
Subject: [PATCH 035/376] Ensure tests don't use LLM (#2702)

---
 deployment/docker_compose/docker-compose.search-testing.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/deployment/docker_compose/docker-compose.search-testing.yml b/deployment/docker_compose/docker-compose.search-testing.yml
index f9be3360d52..fab950c064e 100644
--- a/deployment/docker_compose/docker-compose.search-testing.yml
+++ b/deployment/docker_compose/docker-compose.search-testing.yml
@@ -26,6 +26,9 @@ services:
       - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
       - ENV_SEED_CONFIGURATION=${ENV_SEED_CONFIGURATION:-}
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=True
+      # To enable the LLM for testing, update the value below
+      # NOTE: this is disabled by default since this is a high volume eval that can be costly
+      - DISABLE_GENERATIVE_AI=${DISABLE_GENERATIVE_AI:-true}
     extra_hosts:
       - "host.docker.internal:host-gateway"
     logging:

From 7aaf8224309396fa0620f1f9410a54579a79d943 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 6 Oct 2024 12:05:17 -0700
Subject: [PATCH 036/376] Enable removal of reranking + navigate back to search
 settings (#2674)

* k

* nit
---
 .../admin/embeddings/RerankingFormPage.tsx    |  7 ++++-
 .../embeddings/pages/EmbeddingFormPage.tsx    | 27 ++++++++++---------
 2 files changed, 21 insertions(+), 13 deletions(-)

diff --git a/web/src/app/admin/embeddings/RerankingFormPage.tsx b/web/src/app/admin/embeddings/RerankingFormPage.tsx
index 5425fc89329..1e0aae06594 100644
--- a/web/src/app/admin/embeddings/RerankingFormPage.tsx
+++ b/web/src/app/admin/embeddings/RerankingFormPage.tsx
@@ -79,7 +79,12 @@ const RerankingDetailsForm = forwardRef<
       >
         {({ values, setFieldValue, resetForm }) => {
           const resetRerankingValues = () => {
-            setRerankingDetails(originalRerankingDetails);
+            setRerankingDetails({
+              rerank_api_key: null,
+              rerank_provider_type: null,
+              rerank_model_name: null,
+              rerank_api_url: null,
+            });
             resetForm();
           };
 
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index d6a241400e8..adf486ed5d4 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -13,7 +13,7 @@ import {
 } from "@/components/embedding/interfaces";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import useSWR, { mutate } from "swr";
+import useSWR from "swr";
 import { ThreeDotsLoader } from "@/components/Loading";
 import AdvancedEmbeddingFormPage from "./AdvancedEmbeddingFormPage";
 import {
@@ -173,10 +173,9 @@ export default function EmbeddingForm() {
     const response = await updateSearchSettings(values);
     if (response.ok) {
       setPopup({
-        message: "Updated search settings succesffuly",
+        message: "Updated search settings successfully",
         type: "success",
       });
-      mutate("/api/search-settings/get-current-search-settings");
       return true;
     } else {
       setPopup({ message: "Failed to update search settings", type: "error" });
@@ -184,6 +183,17 @@ export default function EmbeddingForm() {
     }
   };
 
+  const navigateToEmbeddingPage = (changedResource: string) => {
+    setPopup({
+      message: `Changed ${changedResource} successfully. Redirecting to embedding page`,
+      type: "success",
+    });
+
+    setTimeout(() => {
+      window.open("/admin/configuration/search", "_self");
+    }, 2000);
+  };
+
   const onConfirm = async () => {
     if (!selectedProvider) {
       return;
@@ -227,14 +237,7 @@ export default function EmbeddingForm() {
     );
 
     if (response.ok) {
-      setPopup({
-        message: "Changed provider successfully. Redirecting to embedding page",
-        type: "success",
-      });
-      mutate("/api/search-settings/get-secondary-search-settings");
-      setTimeout(() => {
-        window.open("/admin/configuration/search", "_self");
-      }, 2000);
+      navigateToEmbeddingPage("embedding model");
     } else {
       setPopup({ message: "Failed to update embedding model", type: "error" });
 
@@ -286,6 +289,7 @@ export default function EmbeddingForm() {
         className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
         onClick={async () => {
           updateSearch();
+          navigateToEmbeddingPage("search settings");
         }}
       >
         Update Search
@@ -405,7 +409,6 @@ export default function EmbeddingForm() {
               <div className="flex w-full justify-end">
                 <button
                   className={`enabled:cursor-pointer enabled:hover:underline disabled:cursor-not-allowed mt-auto enabled:text-text-600 disabled:text-text-400 ml-auto flex gap-x-1 items-center py-2.5 px-3.5 text-sm font-regular rounded-sm`}
-                  // disabled={!isFormValid}
                   onClick={() => {
                     nextFormStep();
                   }}

From f189eda90430eab03a998f66facf143af901f41b Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 6 Oct 2024 12:05:28 -0700
Subject: [PATCH 037/376] remove left-over `memo` (#2669)

---
 web/src/app/chat/message/CodeBlock.tsx | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/web/src/app/chat/message/CodeBlock.tsx b/web/src/app/chat/message/CodeBlock.tsx
index 66cc82a6e73..ee6b6e6eab6 100644
--- a/web/src/app/chat/message/CodeBlock.tsx
+++ b/web/src/app/chat/message/CodeBlock.tsx
@@ -36,7 +36,7 @@ export const CodeBlock = memo(function CodeBlock({
     });
   }, [codeText]);
 
-  const CopyButton = memo(() => (
+  const CopyButton = () => (
     <div
       className="ml-auto cursor-pointer select-none"
       onMouseDown={handleCopy}
@@ -53,10 +53,9 @@ export const CodeBlock = memo(function CodeBlock({
         </div>
       )}
     </div>
-  ));
-  CopyButton.displayName = "CopyButton";
+  );
 
-  const CodeContent = memo(() => {
+  const CodeContent = () => {
     if (!language) {
       if (typeof children === "string") {
         return (
@@ -106,8 +105,7 @@ export const CodeBlock = memo(function CodeBlock({
         </code>
       </pre>
     );
-  });
-  CodeContent.displayName = "CodeContent";
+  };
 
   return (
     <div className="overflow-x-hidden">

From 3206bb27ce6aed4fe68adf60d478e550238e55e6 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 6 Oct 2024 13:31:19 -0700
Subject: [PATCH 038/376] update disabling logic (#2592)


From 83bc7d46569962655c26d700feedd51d04ce1552 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 6 Oct 2024 14:27:31 -0700
Subject: [PATCH 039/376] DanswerBot Update (#2697)

---
 backend/danswer/danswerbot/slack/listener.py  | 14 ++--
 backend/danswer/danswerbot/slack/utils.py     | 67 +++++++++++++------
 .../llm/answering/prompts/citations_prompt.py | 11 ++-
 backend/danswer/prompts/chat_prompts.py       |  7 +-
 backend/danswer/prompts/direct_qa_prompts.py  |  9 ++-
 5 files changed, 70 insertions(+), 38 deletions(-)

diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index 553e9979189..dbf6eae24cd 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -131,9 +131,8 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool
             )
             return False
 
+        bot_tag_id = get_danswer_bot_app_id(client.web_client)
         if event_type == "message":
-            bot_tag_id = get_danswer_bot_app_id(client.web_client)
-
             is_dm = event.get("channel_type") == "im"
             is_tagged = bot_tag_id and bot_tag_id in msg
             is_danswer_bot_msg = bot_tag_id and bot_tag_id in event.get("user", "")
@@ -159,8 +158,10 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool
                 slack_bot_config = get_slack_bot_config_for_channel(
                     channel_name=channel_name, db_session=db_session
                 )
-            if not slack_bot_config or not slack_bot_config.channel_config.get(
-                "respond_to_bots"
+            # If DanswerBot is not specifically tagged and the channel is not set to respond to bots, ignore the message
+            if (not bot_tag_id or bot_tag_id not in msg) and (
+                not slack_bot_config
+                or not slack_bot_config.channel_config.get("respond_to_bots")
             ):
                 channel_specific_logger.info("Ignoring message from bot")
                 return False
@@ -447,8 +448,9 @@ def process_slack_event(client: SocketModeClient, req: SocketModeRequest) -> Non
                 return view_routing(req, client)
         elif req.type == "events_api" or req.type == "slash_commands":
             return process_message(req, client)
-    except Exception:
-        logger.exception("Failed to process slack event")
+    except Exception as e:
+        logger.exception(f"Failed to process slack event. Error: {e}")
+        logger.error(f"Slack request payload: {req.payload}")
 
 
 def _get_socket_client(slack_bot_tokens: SlackBotTokens) -> SocketModeClient:
diff --git a/backend/danswer/danswerbot/slack/utils.py b/backend/danswer/danswerbot/slack/utils.py
index d762dde7826..81209cf5c17 100644
--- a/backend/danswer/danswerbot/slack/utils.py
+++ b/backend/danswer/danswerbot/slack/utils.py
@@ -430,35 +430,58 @@ def read_slack_thread(
     replies = cast(dict, response.data).get("messages", [])
     for reply in replies:
         if "user" in reply and "bot_id" not in reply:
-            message = remove_danswer_bot_tag(reply["text"], client=client)
-            user_sem_id = fetch_user_semantic_id_from_id(reply["user"], client)
+            message = reply["text"]
+            user_sem_id = (
+                fetch_user_semantic_id_from_id(reply.get("user"), client)
+                or "Unknown User"
+            )
             message_type = MessageType.USER
         else:
             self_app_id = get_danswer_bot_app_id(client)
 
-            # Only include bot messages from Danswer, other bots are not taken in as context
-            if self_app_id != reply.get("user"):
-                continue
-
-            blocks = reply["blocks"]
-            if len(blocks) <= 1:
-                continue
-
-            # For the old flow, the useful block is the second one after the header block that says AI Answer
-            if reply["blocks"][0]["text"]["text"] == "AI Answer":
-                message = reply["blocks"][1]["text"]["text"]
-            else:
-                # for the new flow, the answer is the first block
-                message = reply["blocks"][0]["text"]["text"]
-
-            if message.startswith("_Filters"):
-                if len(blocks) <= 2:
+            if reply.get("user") == self_app_id:
+                # DanswerBot response
+                message_type = MessageType.ASSISTANT
+                user_sem_id = "Assistant"
+
+                # DanswerBot responses have both text and blocks
+                # The useful content is in the blocks, specifically the first block unless there are
+                # auto-detected filters
+                blocks = reply.get("blocks")
+                if not blocks:
+                    logger.warning(f"DanswerBot response has no blocks: {reply}")
                     continue
-                message = reply["blocks"][2]["text"]["text"]
 
-            user_sem_id = "Assistant"
-            message_type = MessageType.ASSISTANT
+                message = blocks[0].get("text", {}).get("text")
+
+                # If auto-detected filters are on, use the second block for the actual answer
+                # The first block is the auto-detected filters
+                if message.startswith("_Filters"):
+                    if len(blocks) < 2:
+                        logger.warning(f"Only filter blocks found: {reply}")
+                        continue
+                    # This is the DanswerBot answer format, if there is a change to how we respond,
+                    # this will need to be updated to get the correct "answer" portion
+                    message = reply["blocks"][1].get("text", {}).get("text")
+            else:
+                # Other bots are not counted as the LLM response which only comes from Danswer
+                message_type = MessageType.USER
+                bot_user_name = fetch_user_semantic_id_from_id(
+                    reply.get("user"), client
+                )
+                user_sem_id = bot_user_name or "Unknown" + " Bot"
+
+                # For other bots, just use the text as we have no way of knowing that the
+                # useful portion is
+                message = reply.get("text")
+                if not message:
+                    message = blocks[0].get("text", {}).get("text")
+
+            if not message:
+                logger.warning("Skipping Slack thread message, no text found")
+                continue
 
+        message = remove_danswer_bot_tag(message, client=client)
         thread_messages.append(
             ThreadMessage(message=message, sender=user_sem_id, role=message_type)
         )
diff --git a/backend/danswer/llm/answering/prompts/citations_prompt.py b/backend/danswer/llm/answering/prompts/citations_prompt.py
index 52345f3e587..a2248da0585 100644
--- a/backend/danswer/llm/answering/prompts/citations_prompt.py
+++ b/backend/danswer/llm/answering/prompts/citations_prompt.py
@@ -18,6 +18,7 @@
 from danswer.prompts.constants import DEFAULT_IGNORE_STATEMENT
 from danswer.prompts.direct_qa_prompts import CITATIONS_PROMPT
 from danswer.prompts.direct_qa_prompts import CITATIONS_PROMPT_FOR_TOOL_CALLING
+from danswer.prompts.direct_qa_prompts import HISTORY_BLOCK
 from danswer.prompts.prompt_utils import add_date_time_to_prompt
 from danswer.prompts.prompt_utils import build_complete_context_str
 from danswer.prompts.prompt_utils import build_task_prompt_reminders
@@ -143,6 +144,12 @@ def build_citations_user_message(
         prompt=prompt_config, use_language_hint=bool(multilingual_expansion)
     )
 
+    history_block = (
+        HISTORY_BLOCK.format(history_str=history_message) + "\n"
+        if history_message
+        else ""
+    )
+
     if context_docs:
         context_docs_str = build_complete_context_str(context_docs)
         optional_ignore = "" if all_doc_useful else DEFAULT_IGNORE_STATEMENT
@@ -152,14 +159,14 @@ def build_citations_user_message(
             context_docs_str=context_docs_str,
             task_prompt=task_prompt_with_reminder,
             user_query=question,
-            history_block=history_message,
+            history_block=history_block,
         )
     else:
         # if no context docs provided, assume we're in the tool calling flow
         user_prompt = CITATIONS_PROMPT_FOR_TOOL_CALLING.format(
             task_prompt=task_prompt_with_reminder,
             user_query=question,
-            history_block=history_message,
+            history_block=history_block,
         )
 
     user_prompt = user_prompt.strip()
diff --git a/backend/danswer/prompts/chat_prompts.py b/backend/danswer/prompts/chat_prompts.py
index a5fa973f37c..a9653254f9a 100644
--- a/backend/danswer/prompts/chat_prompts.py
+++ b/backend/danswer/prompts/chat_prompts.py
@@ -110,8 +110,8 @@
 and additional information or details would provide little or no value.
 - The query is some task that does not require additional information to handle.
 
-{GENERAL_SEP_PAT}
 Conversation History:
+{GENERAL_SEP_PAT}
 {{chat_history}}
 {GENERAL_SEP_PAT}
 
@@ -135,8 +135,8 @@
 Strip out any information that is not relevant for the retrieval task.
 If the follow up message is an error or code snippet, repeat the same input back EXACTLY.
 
-{GENERAL_SEP_PAT}
 Chat History:
+{GENERAL_SEP_PAT}
 {{chat_history}}
 {GENERAL_SEP_PAT}
 
@@ -152,8 +152,8 @@
 If there is a clear change in topic, ensure the query reflects the new topic accurately.
 Strip out any information that is not relevant for the internet search.
 
-{GENERAL_SEP_PAT}
 Chat History:
+{GENERAL_SEP_PAT}
 {{chat_history}}
 {GENERAL_SEP_PAT}
 
@@ -210,6 +210,7 @@
 Focus the name on the important keywords to convey the topic of the conversation.
 
 Chat History:
+{GENERAL_SEP_PAT}
 {{chat_history}}
 {GENERAL_SEP_PAT}
 
diff --git a/backend/danswer/prompts/direct_qa_prompts.py b/backend/danswer/prompts/direct_qa_prompts.py
index 0139da13e88..b1229b896a7 100644
--- a/backend/danswer/prompts/direct_qa_prompts.py
+++ b/backend/danswer/prompts/direct_qa_prompts.py
@@ -72,7 +72,8 @@
 JSON_PROMPT = f"""
 {{system_prompt}}
 {REQUIRE_JSON}
-{{context_block}}{{history_block}}{{task_prompt}}
+{{context_block}}{{history_block}}
+{{task_prompt}}
 
 SAMPLE RESPONSE:
 ```
@@ -91,6 +92,7 @@
 # "conversation history" block
 CITATIONS_PROMPT = f"""
 Refer to the following context documents when responding to me.{DEFAULT_IGNORE_STATEMENT}
+
 CONTEXT:
 {GENERAL_SEP_PAT}
 {{context_docs_str}}
@@ -109,10 +111,7 @@
 Refer to the provided context documents when responding to me.{DEFAULT_IGNORE_STATEMENT} \
 You should always get right to the point, and never use extraneous language.
 
-CHAT HISTORY:
-{{history_block}}
-
-{{task_prompt}}
+{{history_block}}{{task_prompt}}
 
 {QUESTION_PAT.upper()}
 {{user_query}}

From 64909d74f9c88e87aafb2133084150f1eb45d58c Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 7 Oct 2024 10:33:08 -0700
Subject: [PATCH 040/376] UX Cleanup (#2701)

* start

* shared iconlogo class

* clean out of place components

* nit
---
 .../app/admin/assistants/AssistantEditor.tsx  |   2 +-
 web/src/app/admin/assistants/PersonaTable.tsx |   2 +-
 .../prompt-library/modals/EditPromptModal.tsx |   2 +-
 .../admin/prompt-library/promptLibrary.tsx    |  41 +-
 web/src/app/admin/tools/ToolEditor.tsx        |   9 +-
 .../ee/admin/api-key/DanswerApiKeyForm.tsx    |  13 +-
 .../ee/admin/groups/UserGroupCreationForm.tsx |  15 +-
 web/src/components/icons/icons.tsx            | 541 ++++--------------
 .../components/resizable/ResizableSection.tsx |   2 +-
 web/src/components/search/SearchBar.tsx       |   4 +-
 .../search/SearchResultsDisplay.tsx           |   2 +-
 web/src/components/search/SearchSection.tsx   |   5 +-
 .../search/filtering/FilterDropdown.tsx       |   3 +-
 13 files changed, 160 insertions(+), 481 deletions(-)

diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 1f961614dbc..d4287e4c984 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -165,7 +165,7 @@ export function AssistantEditor({
         existingPersona.num_chunks === 0
       );
     }
-  }, []);
+  }, [isUpdate, existingPrompt]);
 
   const defaultProvider = llmProviders.find(
     (llmProvider) => llmProvider.is_default_provider
diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index dac260a1888..7a72d6e2e45 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -176,7 +176,7 @@ export function PersonasTable({
                 </div>
               </div>,
               <div key="edit" className="flex">
-                <div className="mx-auto my-auto">
+                <div className="mr-auto my-auto">
                   {!persona.is_default_persona && isEditable ? (
                     <div
                       className="hover:bg-hover rounded p-1 cursor-pointer"
diff --git a/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx b/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
index a5b44da512d..bd20ce5e453 100644
--- a/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
+++ b/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
@@ -52,7 +52,7 @@ const EditPromptModal = ({
         }}
       >
         {({ isSubmitting, values }) => (
-          <Form>
+          <Form className="items-stretch">
             <h2 className="text-2xl text-emphasis font-bold mb-3 flex items-center">
               <svg
                 className="w-6 h-6 mr-2"
diff --git a/web/src/app/admin/prompt-library/promptLibrary.tsx b/web/src/app/admin/prompt-library/promptLibrary.tsx
index 8a046893dfb..18cb5927b50 100644
--- a/web/src/app/admin/prompt-library/promptLibrary.tsx
+++ b/web/src/app/admin/prompt-library/promptLibrary.tsx
@@ -17,6 +17,7 @@ import { FiTag } from "react-icons/fi";
 import { PageSelector } from "@/components/PageSelector";
 import { InputPrompt } from "./interfaces";
 import { Modal } from "@/components/Modal";
+import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
 
 const CategoryBubble = ({
   name,
@@ -120,6 +121,7 @@ export const PromptLibraryTable = ({
       setPopup({ message: "Failed to delete input prompt", type: "error" });
     }
     refresh();
+    setConfirmDeletionId(null);
   };
 
   const handleStatusSelect = (status: string) => {
@@ -138,35 +140,16 @@ export const PromptLibraryTable = ({
   return (
     <div className="justify-center py-2">
       {confirmDeletionId != null && (
-        <Modal
-          onOutsideClick={() => setConfirmDeletionId(null)}
-          className="max-w-sm"
-        >
-          <>
-            <p className="text-lg mb-2">
-              Are you sure you want to delete this prompt? You will not be able
-              to recover this prompt
-            </p>
-            <div className="mt-6 flex justify-between">
-              <button
-                className="rounded py-1.5 px-2 bg-background-800 text-text-200"
-                onClick={async () => {
-                  await handleDelete(confirmDeletionId);
-                  setConfirmDeletionId(null);
-                }}
-              >
-                Yes
-              </button>
-              <button
-                onClick={() => setConfirmDeletionId(null)}
-                className="rounded py-1.5 px-2 bg-background-150 text-text-800"
-              >
-                {" "}
-                No
-              </button>
-            </div>
-          </>
-        </Modal>
+        <DeleteEntityModal
+          onClose={() => setConfirmDeletionId(null)}
+          onSubmit={() => handleDelete(confirmDeletionId)}
+          entityType="prompt"
+          entityName={
+            paginatedPromptLibrary.find(
+              (prompt) => prompt.id === confirmDeletionId
+            )?.prompt ?? ""
+          }
+        />
       )}
 
       <div className="flex items-center w-full border-2 border-border rounded-lg px-4 py-2 focus-within:border-accent">
diff --git a/web/src/app/admin/tools/ToolEditor.tsx b/web/src/app/admin/tools/ToolEditor.tsx
index b4df98f8623..0ffb8750ef6 100644
--- a/web/src/app/admin/tools/ToolEditor.tsx
+++ b/web/src/app/admin/tools/ToolEditor.tsx
@@ -318,10 +318,11 @@ export function ToolEditor({ tool }: { tool?: ToolSnapshot }) {
       <Formik
         initialValues={{
           definition: prettifiedDefinition,
-          customHeaders: tool?.custom_headers?.map((header) => ({
-            key: header.key,
-            value: header.value,
-          })) ?? [{ key: "test", value: "value" }],
+          customHeaders:
+            tool?.custom_headers?.map((header) => ({
+              key: header.key,
+              value: header.value,
+            })) ?? [],
         }}
         validationSchema={ToolSchema}
         onSubmit={async (values: ToolFormValues) => {
diff --git a/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx b/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
index 6b1c8a860b2..4232929c838 100644
--- a/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
+++ b/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
@@ -28,18 +28,9 @@ export const DanswerApiKeyForm = ({
 
   return (
     <Modal onOutsideClick={onClose} width="w-2/6">
-      <div className="px-8 py-6 bg-background">
+      <>
         <h2 className="text-xl font-bold flex">
           {isUpdate ? "Update API Key" : "Create a new API Key"}
-          <div
-            onClick={onClose}
-            className="ml-auto hover:bg-hover p-1.5 rounded"
-          >
-            <XIcon
-              size={20}
-              className="my-auto flex flex-shrink-0 cursor-pointer"
-            />
-          </div>
         </h2>
 
         <Divider />
@@ -126,7 +117,7 @@ export const DanswerApiKeyForm = ({
             </Form>
           )}
         </Formik>
-      </div>
+      </>
     </Modal>
   );
 };
diff --git a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
index ec1ac52e609..2963eb20b0c 100644
--- a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
+++ b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
@@ -35,18 +35,9 @@ export const UserGroupCreationForm = ({
 
   return (
     <Modal className="w-fit" onOutsideClick={onClose}>
-      <div className="px-8 py-6 bg-background">
+      <>
         <h2 className="text-xl font-bold flex">
           {isUpdate ? "Update a User Group" : "Create a new User Group"}
-          <div
-            onClick={onClose}
-            className="ml-auto hover:bg-hover p-1.5 rounded"
-          >
-            <XIcon
-              size={20}
-              className="my-auto flex flex-shrink-0 cursor-pointer"
-            />
-          </div>
         </h2>
 
         <Divider />
@@ -89,7 +80,7 @@ export const UserGroupCreationForm = ({
         >
           {({ isSubmitting, values, setFieldValue }) => (
             <Form>
-              <div className="p-4">
+              <div className="py-4">
                 <TextFormField
                   name="name"
                   label="Name:"
@@ -150,7 +141,7 @@ export const UserGroupCreationForm = ({
             </Form>
           )}
         </Formik>
-      </div>
+      </>
     </Modal>
   );
 };
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 186e4473cce..9295a08c083 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -43,7 +43,7 @@ import {
   FiSlack,
 } from "react-icons/fi";
 import { SiBookstack } from "react-icons/si";
-import Image from "next/image";
+import Image, { StaticImageData } from "next/image";
 import jiraSVG from "../../../public/Jira.svg";
 import confluenceSVG from "../../../public/Confluence.svg";
 import openAISVG from "../../../public/Openai.svg";
@@ -94,6 +94,23 @@ export interface IconProps {
   className?: string;
 }
 
+export interface LogoIconProps extends IconProps {
+  src: string | StaticImageData;
+}
+
+export const LogoIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+  src,
+}: LogoIconProps) => (
+  <div
+    style={{ width: `${size}px`, height: `${size}px` }}
+    className={`w-[${size}px] h-[${size}px] ` + className}
+  >
+    <Image src={src} alt="Logo" width="96" height="96" />
+  </div>
+);
+
 export const AssistantsIconSkeleton = ({
   size,
   className = defaultTailwindCSS,
@@ -172,36 +189,6 @@ export const AssistantsIcon = ({
   );
 };
 
-// export const AssistantsIcon = ({
-//   size,
-//   className = defaultTailwindCSS,
-// }: IconProps) => {
-//   return (
-//     <svg
-//       style={{ width: `${size}px`, height: `${size}px` }}
-//       className={`w-[${size}px] h-[${size}px] ` + className}
-//       xmlns="http://www.w3.org/2000/svg"
-//       width="200"
-//       height="200"
-//       viewBox="0 0 24 24"
-//     >
-//       <g fill="none" stroke="currentColor" strokeWidth="1.5">
-//         <path
-//           strokeLinecap="round"
-//           d="M21.483 19c-.04.936-.165 1.51-.569 1.914c-.586.586-1.528.586-3.414.586c-1.886 0-2.828 0-3.414-.586c-.586-.586-.586-1.528-.586-3.414v-2c0-1.886 0-2.828.586-3.414c.586-.586 1.528-.586 3.414-.586c1.886 0 2.828 0 3.414.586c.532.531.581 1.357.585 2.914"
-//         />
-//         <path d="M2 8.5c0 1.886 0 2.828.586 3.414c.586.586 1.528.586 3.414.586c1.886 0 2.828 0 3.414-.586C10 11.328 10 10.386 10 8.5v-2c0-1.886 0-2.828-.586-3.414C8.828 2.5 7.886 2.5 6 2.5c-1.886 0-2.828 0-3.414.586C2 3.672 2 4.614 2 6.5v2Z" />
-//         <path
-//           strokeLinecap="round"
-//           d="M15.5 2.513c-.327.017-.562.055-.765.14a2 2 0 0 0-1.083 1.082c-.152.367-.152.833-.152 1.765c0 .932 0 1.398.152 1.765a2 2 0 0 0 1.083 1.083c.367.152.833.152 1.765.152h2c.932 0 1.398 0 1.765-.152a2 2 0 0 0 1.083-1.083c.152-.367.152-.833.152-1.765c0-.932 0-1.398-.152-1.765a2 2 0 0 0-1.083-1.083c-.204-.084-.438-.122-.765-.139"
-//         />
-//         <path d="M2 18.5c0 .932 0 1.398.152 1.765a2 2 0 0 0 1.083 1.083c.367.152.833.152 1.765.152h2c.932 0 1.398 0 1.765-.152a2 2 0 0 0 1.083-1.083C10 19.898 10 19.432 10 18.5c0-.932 0-1.398-.152-1.765a2 2 0 0 0-1.083-1.083C8.398 15.5 7.932 15.5 7 15.5H5c-.932 0-1.398 0-1.765.152a2 2 0 0 0-1.083 1.083C2 17.102 2 17.568 2 18.5Z" />
-//       </g>
-//     </svg>
-
-//   );
-// };
-
 <svg
   xmlns="http://www.w3.org/2000/svg"
   width="200"
@@ -999,103 +986,53 @@ export const LightSettingsIcon = ({
   );
 };
 
-//
 //  COMPANY LOGOS
-//
 
 export const LoopioIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] dark:invert ` + className}
-    >
-      <Image src="/Loopio.png" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon
+    size={size}
+    className={`${className} dark:invert`}
+    src="/Loopio.png"
+  />
+);
 
 export const NewIconTest = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <svg
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-      xmlns="http://www.w3.org/2000/svg"
-      width="200"
-      height="200"
-      viewBox="0 0 24 24"
-    >
-      <g fill="none" stroke="currentColor" strokeWidth="1.5">
-        <path
-          strokeLinecap="round"
-          strokeLinejoin="round"
-          d="M2.75 8.917h18.5"
-        />
-        <rect width="18.5" height="18.5" x="2.75" y="2.75" rx="6" />
-      </g>
-    </svg>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/NewIconTest.svg" />
+);
 
 export const GitlabIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src="/Gitlab.png" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/Gitlab.png" />
+);
+
 export const GithubIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src="/Github.png" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/Github.png" />
+);
 
 export const GmailIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src="/Gmail.png" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/Gmail.png" />
+);
 
 export const GoogleDriveIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src="/GoogleDrive.png" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/GoogleDrive.png" />
+);
 
 export const BookstackIcon = ({
   size = 16,
@@ -1107,440 +1044,240 @@ export const BookstackIcon = ({
 export const ConfluenceIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={confluenceSVG} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon
+    size={size + 4}
+    className={`${className} -m-0.5`}
+    src={confluenceSVG}
+  />
+);
 
 export const OCIStorageIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={OCIStorageSVG} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon
+    size={size + 4}
+    className={`${className} -m-0.5`}
+    src={OCIStorageSVG}
+  />
+);
 
 export const JiraIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  // Jira Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
-  return (
-    <div
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={jiraSVG} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size + 4} className={`${className} -m-0.5`} src={jiraSVG} />
+);
 
 export const ZulipIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={zulipIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => <LogoIcon size={size} className={className} src={zulipIcon} />;
 
 export const OpenAIIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={openAISVG} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => <LogoIcon size={size} className={className} src={openAISVG} />;
 
 export const VoyageIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={voyageIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={voyageIcon} />
+);
 
 export const GoogleIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={googleIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={googleIcon} />
+);
 
 export const CohereIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={cohereIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={cohereIcon} />
+);
 
 export const GoogleStorageIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={googleCloudStorageIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon
+    size={size + 4}
+    className={`${className} -m-0.5`}
+    src={googleCloudStorageIcon}
+  />
+);
 
 export const ProductboardIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src="/Productboard.webp" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/Productboard.webp" />
+);
 
 export const AWSIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      // Linear Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={awsWEBP} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => <LogoIcon size={size} className={className} src={awsWEBP} />;
 
 export const AzureIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      // Linear Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={azureIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => <LogoIcon size={size} className={className} src={azureIcon} />;
 
 export const LinearIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      // Linear Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={linearIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={linearIcon} />
+);
 
 export const SlabIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src="/SlabLogo.png" alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src="/SlabLogo.png" />
 );
 
 export const NotionIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src="/Notion.png" alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src="/Notion.png" />
+);
 
 export const GuruIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={guruIcon} alt="Logo" width="96" height="96" />
-  </div>
-);
+}: IconProps) => <LogoIcon size={size} className={className} src={guruIcon} />;
 
 export const RequestTrackerIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={requestTrackerIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={requestTrackerIcon} />
 );
 
 export const SalesforceIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={salesforceIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={salesforceIcon} />
 );
 
 export const R2Icon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={r2Icon} alt="Logo" width="96" height="96" />
-  </div>
-);
+}: IconProps) => <LogoIcon size={size} className={className} src={r2Icon} />;
 
 export const S3Icon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={s3Icon} alt="Logo" width="96" height="96" />
-  </div>
-);
+}: IconProps) => <LogoIcon size={size} className={className} src={s3Icon} />;
 
 export const SharepointIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={sharepointIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={sharepointIcon} />
 );
 
 export const TeamsIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={teamsIcon} alt="Logo" width="96" height="96" />
-  </div>
-);
+}: IconProps) => <LogoIcon size={size} className={className} src={teamsIcon} />;
 
 export const GongIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={gongIcon} alt="Logo" width="96" height="96" />
-  </div>
-);
+}: IconProps) => <LogoIcon size={size} className={className} src={gongIcon} />;
 
 export const HubSpotIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      // HubSpot Icon has a bit more surrounding whitespace than other icons, which is why we need to adjust it here
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={hubSpotIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={hubSpotIcon} />
+);
 
 export const Document360Icon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={document360Icon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={document360Icon} />
+);
 
 export const GoogleSitesIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={googleSitesIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={googleSitesIcon} />
+);
 
 export const ZendeskIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={zendeskIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={zendeskIcon} />
 );
 
 export const DropboxIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={dropboxIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={dropboxIcon} />
 );
 
 export const DiscourseIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={discourseIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={discourseIcon} />
 );
 
 export const AxeroIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src="/Axero.jpeg" alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src="/Axero.jpeg" />
 );
 
 export const ClickupIcon = ({
   size = 16,
   className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size + 4}px`, height: `${size + 4}px` }}
-      className={`w-[${size + 4}px] h-[${size + 4}px] -m-0.5 ` + className}
-    >
-      <Image src={clickupIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={clickupIcon} />
+);
 
 export const MediaWikiIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={mediawikiIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={mediawikiIcon} />
 );
 
 export const WikipediaIcon = ({
   size = 16,
   className = defaultTailwindCSS,
 }: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={wikipediaIcon} alt="Logo" width="96" height="96" />
-  </div>
+  <LogoIcon size={size} className={className} src={wikipediaIcon} />
+);
+
+export const XenforoIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={xenforoIcon} />
 );
 
+export const AsanaIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => <LogoIcon size={size} className={className} src={asanaIcon} />;
+
 /* 
 EE Icons
 */
@@ -2836,29 +2573,3 @@ export const WindowsIcon = ({
     </svg>
   );
 };
-
-export const XenforoIcon = ({
-  size = 16,
-  className = defaultTailwindCSS,
-}: IconProps) => {
-  return (
-    <div
-      style={{ width: `${size}px`, height: `${size}px` }}
-      className={`w-[${size}px] h-[${size}px] ` + className}
-    >
-      <Image src={xenforoIcon} alt="Logo" width="96" height="96" />
-    </div>
-  );
-};
-
-export const AsanaIcon = ({
-  size = 16,
-  className = defaultTailwindCSS,
-}: IconProps) => (
-  <div
-    style={{ width: `${size}px`, height: `${size}px` }}
-    className={`w-[${size}px] h-[${size}px] ` + className}
-  >
-    <Image src={asanaIcon} alt="Logo" width="96" height="96" />
-  </div>
-);
diff --git a/web/src/components/resizable/ResizableSection.tsx b/web/src/components/resizable/ResizableSection.tsx
index 98a877792b6..0591555d8df 100644
--- a/web/src/components/resizable/ResizableSection.tsx
+++ b/web/src/components/resizable/ResizableSection.tsx
@@ -41,7 +41,7 @@ export function ResizableSection({
     Cookies.set(DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME, newWidth.toString(), {
       path: "/",
     });
-  }, [minWidth, maxWidth]);
+  }, [minWidth, maxWidth, width]);
 
   const startResizing = (mouseDownEvent: React.MouseEvent<HTMLDivElement>) => {
     setIsResizing(true);
diff --git a/web/src/components/search/SearchBar.tsx b/web/src/components/search/SearchBar.tsx
index d268367bd33..df6f45eacdb 100644
--- a/web/src/components/search/SearchBar.tsx
+++ b/web/src/components/search/SearchBar.tsx
@@ -178,10 +178,10 @@ export const FullSearchBar = ({
         suppressContentEditableWarning={true}
       />
       <div
-        className={`flex flex-nowrap overflow-y-hidden ${showingSidebar ? " 2xl:justify-between" : "2xl:justify-end"} justify-between 4xl:justify-end w-full max-w-full items-center space-x-3 py-3 px-4`}
+        className={`flex flex-nowrap ${showingSidebar ? " 2xl:justify-between" : "2xl:justify-end"} justify-between 4xl:justify-end w-full max-w-full items-center space-x-3 py-3 px-4`}
       >
         <div
-          className={`-my-1 overflow-x-scroll flex-grow 4xl:hidden ${!showingSidebar && "2xl:hidden"}`}
+          className={`-my-1 flex-grow 4xl:hidden ${!showingSidebar && "2xl:hidden"}`}
         >
           {(ccPairs.length > 0 || documentSets.length > 0) && (
             <HorizontalSourceSelector
diff --git a/web/src/components/search/SearchResultsDisplay.tsx b/web/src/components/search/SearchResultsDisplay.tsx
index 4be28c9e8a8..7f1036050ec 100644
--- a/web/src/components/search/SearchResultsDisplay.tsx
+++ b/web/src/components/search/SearchResultsDisplay.tsx
@@ -73,7 +73,7 @@ export const SearchResultsDisplay = ({
     return () => {
       window.removeEventListener("keydown", handleKeyDown);
     };
-  }, []);
+  }, [performSweep, agenticResults]);
 
   if (!searchResponse) {
     return null;
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 4e8ea0abd28..a9641511af2 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -140,7 +140,8 @@ export const SearchSection = ({
     return () => {
       window.removeEventListener("keydown", handleKeyDown);
     };
-  }, []);
+  }, [toggleAgentic]);
+
   const [isFetching, setIsFetching] = useState(false);
 
   // Search Type
@@ -522,7 +523,7 @@ export const SearchSection = ({
     return () => {
       window.removeEventListener("keydown", handleKeyDown);
     };
-  }, [router]);
+  }, [router, toggleSidebar]);
 
   useEffect(() => {
     if (settings?.isMobile) {
diff --git a/web/src/components/search/filtering/FilterDropdown.tsx b/web/src/components/search/filtering/FilterDropdown.tsx
index 71fcf84b36d..444382fc73c 100644
--- a/web/src/components/search/filtering/FilterDropdown.tsx
+++ b/web/src/components/search/filtering/FilterDropdown.tsx
@@ -41,7 +41,8 @@ export function FilterDropdown({
               flex-col 
               ${dropdownWidth || width}
               max-h-96 
-              overflow-y-scroll 
+              overflow-y-scroll
+              overscroll-contain
               `}
           >
             {options.map((option, ind) => {

From 3404c7eb1d5149cf9b2d600e43475ee3ce20a74f Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 7 Oct 2024 11:16:17 -0700
Subject: [PATCH 041/376] Feature/background prune 2 (#2583)

* first cut at redis

* some new helper functions for the db

* ignore kombu tables in alembic migrations (used by celery)

* multiline commands for readability, add vespa_metadata_sync queue to worker

* typo fix

* fix returning tuple fields

* add constants

* fix _get_access_for_document

* docstrings!

* fix double function declaration and typing

* fix type hinting

* add a global redis pool

* Add get_document function

* use task_logger in various celery tasks

* add celeryconfig.py to simplify configuration. Will be used in a subsequent commit

* Add celery redis helper. used in a subsequent PR

* kombu warning getting spammy since celery is not self managing its queue in Postgres any more

* add last_modified and last_synced to documents

* fix task naming convention

* use celeryconfig.py

* the big one. adds queues and tasks, updates functions to use the queues with priorities, etc

* change vespa index log line to debug

* mypy fixes

* update alembic migration

* fix fence ordering, rename to "monitor", fix fetch_versioned_implementation call

* mypy

* switch to monotonic time

* fix startup dependencies on redis

* rebase alembic migration

* kombu cleanup - fail silently

* mypy

* add redis_host environment override

* update REDIS_HOST env var in docker-compose.dev.yml

* update the rest of the docker files

* in flight

* harden indexing-status endpoint against db changes happening in the background.  Needs further improvement but OK for now.

* allow no task syncs to run because we create certain objects with no entries but initially marked as out of date

* add back writing to vespa on indexing

* actually working connector deletion

* update contributing guide

* backporting fixes from background_deletion

* renaming cache to cache_volume

* add redis password to various deployments

* try setting up pr testing for helm

* fix indent

* hopefully this release version actually exists

* fix command line option to --chart-dirs

* fetch-depth 0

* edit values.yaml

* try setting ct working directory

* bypass testing only on change for now

* move files and lint them

* update helm testing

* some issues suggest using --config works

* add vespa repo

* add postgresql repo

* increase timeout

* try amd64 runner

* fix redis password reference

* add comment to helm chart testing workflow

* rename helm testing workflow to disable it

* adding clarifying comments

* address code review

* missed a file

* remove commented warning ... just not needed

* fix imports

* refactor to use update_single

* mypy fixes

* add vespa test

* multiple celery workers

* update logs as well and set prefetch multipliers appropriate to the worker intent

* add db refresh to connector deletion

* add some preliminary locking

* organize tasks into separate files

* celery auto associates tasks created inside another task, which bloats the result metadata considerably. trail=False prevents this.

* code review fixes

* move monitor_usergroup_taskset to ee, improve logging

* add multi workers to dev_run_background_jobs.py

* update supervisord with some recommended settings for celery

* name celery workers and shorten dev script prefixing

* add configurable sql alchemy engine settings on startup (needed for various intents like API server, different celery workers and tasks, etc)

* fix comments

* autoscale sqlalchemy pool size to celery concurrency (allow override later?)

* supervisord needs the percent symbols escaped

* use name as primary check, some minor refactoring and type hinting too.

* stash merge (may not function yet)

* remove dead code

* more cleanup

* remove dead file

* we shouldn't be checking for deletion attempts in the db any more

* print cc_pair_id

* print status on status mismatch again

* add logging when cc_pair isn't present

* don't indexing any ingestion type connectors, and don't pause any connectors that aren't active

* add more specific check for deletion completion

* remove flaky mediawiki test site

* move is_pruning

* remove unused code

* remove old function

---------

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 ...49f9_add_last_pruned_to_connector_table.py |  27 +++
 .../danswer/background/celery/celery_app.py   |  33 ++-
 .../danswer/background/celery/celery_redis.py | 119 ++++++++++
 .../danswer/background/celery/celery_utils.py |  85 ++-----
 .../celery/tasks/connector_deletion/tasks.py  |  24 +-
 .../background/celery/tasks/periodic/tasks.py |   5 +-
 .../background/celery/tasks/pruning/tasks.py  | 223 ++++++++++++++----
 .../background/celery/tasks/shared/tasks.py   | 113 +++++++++
 .../background/celery/tasks/vespa/tasks.py    |  73 +++++-
 .../danswer/background/connector_deletion.py  | 211 -----------------
 backend/danswer/configs/constants.py          |   5 +-
 backend/danswer/db/connector.py               |  14 ++
 backend/danswer/db/models.py                  |   6 +
 backend/danswer/server/documents/cc_pair.py   |  68 ++----
 .../background/celery/tasks/vespa/tasks.py    |   6 +-
 backend/ee/danswer/db/user_group.py           |   2 +-
 .../common_utils/managers/cc_pair.py          |  48 ++--
 .../connector_job_tests/slack/test_prune.py   |   3 +-
 .../connector/test_connector_deletion.py      |  25 +-
 .../permissions/test_cc_pair_permissions.py   |   4 +-
 .../permissions/test_whole_curator_flow.py    |   8 +-
 .../integration/tests/pruning/test_pruning.py |   5 +-
 .../mediawiki/test_mediawiki_family.py        |   2 +-
 23 files changed, 649 insertions(+), 460 deletions(-)
 create mode 100644 backend/alembic/versions/ac5eaac849f9_add_last_pruned_to_connector_table.py
 create mode 100644 backend/danswer/background/celery/tasks/shared/tasks.py
 delete mode 100644 backend/danswer/background/connector_deletion.py

diff --git a/backend/alembic/versions/ac5eaac849f9_add_last_pruned_to_connector_table.py b/backend/alembic/versions/ac5eaac849f9_add_last_pruned_to_connector_table.py
new file mode 100644
index 00000000000..b2c33e1688d
--- /dev/null
+++ b/backend/alembic/versions/ac5eaac849f9_add_last_pruned_to_connector_table.py
@@ -0,0 +1,27 @@
+"""add last_pruned to the connector_credential_pair table
+
+Revision ID: ac5eaac849f9
+Revises: 52a219fb5233
+Create Date: 2024-09-10 15:04:26.437118
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "ac5eaac849f9"
+down_revision = "46b7a812670f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # last pruned represents the last time the connector was pruned
+    op.add_column(
+        "connector_credential_pair",
+        sa.Column("last_pruned", sa.DateTime(timezone=True), nullable=True),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("connector_credential_pair", "last_pruned")
diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index d20c49831a9..5d5450315b5 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -19,6 +19,7 @@
 
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
@@ -104,6 +105,13 @@ def celery_task_postrun(
             r.srem(rcd.taskset_key, task_id)
         return
 
+    if task_id.startswith(RedisConnectorPruning.SUBTASK_PREFIX):
+        cc_pair_id = RedisConnectorPruning.get_id_from_task_id(task_id)
+        if cc_pair_id is not None:
+            rcp = RedisConnectorPruning(cc_pair_id)
+            r.srem(rcp.taskset_key, task_id)
+        return
+
 
 @beat_init.connect
 def on_beat_init(sender: Any, **kwargs: Any) -> None:
@@ -236,6 +244,18 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     for key in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
         r.delete(key)
 
+    for key in r.scan_iter(RedisConnectorPruning.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_COMPLETE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_PROGRESS_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+        r.delete(key)
+
 
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:
@@ -330,7 +350,11 @@ def on_setup_logging(
 
 class HubPeriodicTask(bootsteps.StartStopStep):
     """Regularly reacquires the primary worker lock outside of the task queue.
-    Use the task_logger in this class to avoid double logging."""
+    Use the task_logger in this class to avoid double logging.
+
+    This cannot be done inside a regular beat task because it must run on schedule and
+    a queue of existing work would starve the task from running.
+    """
 
     # it's unclear to me whether using the hub's timer or the bootstep timer is better
     requires = {"celery.worker.components:Hub"}
@@ -405,6 +429,7 @@ def stop(self, worker: Any) -> None:
         "danswer.background.celery.tasks.connector_deletion",
         "danswer.background.celery.tasks.periodic",
         "danswer.background.celery.tasks.pruning",
+        "danswer.background.celery.tasks.shared",
         "danswer.background.celery.tasks.vespa",
     ]
 )
@@ -425,7 +450,7 @@ def stop(self, worker: Any) -> None:
             "task": "check_for_connector_deletion_task",
             # don't need to check too often, since we kick off a deletion initially
             # during the API call that actually marks the CC pair for deletion
-            "schedule": timedelta(minutes=1),
+            "schedule": timedelta(seconds=60),
             "options": {"priority": DanswerCeleryPriority.HIGH},
         },
     }
@@ -433,8 +458,8 @@ def stop(self, worker: Any) -> None:
 celery_app.conf.beat_schedule.update(
     {
         "check-for-prune": {
-            "task": "check_for_prune_task",
-            "schedule": timedelta(seconds=5),
+            "task": "check_for_prune_task_2",
+            "schedule": timedelta(seconds=60),
             "options": {"priority": DanswerCeleryPriority.HIGH},
         },
     }
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index 1d837bd51e0..d9d61b6c8a6 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -343,6 +343,125 @@ def generate_tasks(
         return len(async_results)
 
 
+class RedisConnectorPruning(RedisObjectHelper):
+    """Celery will kick off a long running generator task to crawl the connector and
+    find any missing docs, which will each then get a new cleanup task. The progress of
+    those tasks will then be monitored to completion.
+
+    Example rough happy path order:
+    Check connectorpruning_fence_1
+    Send generator task with id connectorpruning+generator_1_{uuid}
+
+    generator runs connector with callbacks that increment connectorpruning_generator_progress_1
+    generator creates many subtasks with id connectorpruning+sub_1_{uuid}
+      in taskset connectorpruning_taskset_1
+    on completion, generator sets connectorpruning_generator_complete_1
+
+    celery postrun removes subtasks from taskset
+    monitor beat task cleans up when taskset reaches 0 items
+    """
+
+    PREFIX = "connectorpruning"
+    FENCE_PREFIX = PREFIX + "_fence"  # a fence for the entire pruning process
+    GENERATOR_TASK_PREFIX = PREFIX + "+generator"
+
+    TASKSET_PREFIX = PREFIX + "_taskset"  # stores a list of prune tasks id's
+    SUBTASK_PREFIX = PREFIX + "+sub"
+
+    GENERATOR_PROGRESS_PREFIX = (
+        PREFIX + "_generator_progress"
+    )  # a signal that contains generator progress
+    GENERATOR_COMPLETE_PREFIX = (
+        PREFIX + "_generator_complete"
+    )  # a signal that the generator has finished
+
+    def __init__(self, id: int) -> None:
+        """id: the cc_pair_id of the connector credential pair"""
+
+        super().__init__(id)
+        self.documents_to_prune: set[str] = set()
+
+    @property
+    def generator_task_id_prefix(self) -> str:
+        return f"{self.GENERATOR_TASK_PREFIX}_{self._id}"
+
+    @property
+    def generator_progress_key(self) -> str:
+        # example: connectorpruning_generator_progress_1
+        return f"{self.GENERATOR_PROGRESS_PREFIX}_{self._id}"
+
+    @property
+    def generator_complete_key(self) -> str:
+        # example: connectorpruning_generator_complete_1
+        return f"{self.GENERATOR_COMPLETE_PREFIX}_{self._id}"
+
+    @property
+    def subtask_id_prefix(self) -> str:
+        return f"{self.SUBTASK_PREFIX}_{self._id}"
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock | None,
+    ) -> int | None:
+        last_lock_time = time.monotonic()
+
+        async_results = []
+        cc_pair = get_connector_credential_pair_from_id(self._id, db_session)
+        if not cc_pair:
+            return None
+
+        for doc_id in self.documents_to_prune:
+            current_time = time.monotonic()
+            if lock and current_time - last_lock_time >= (
+                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
+            ):
+                lock.reacquire()
+                last_lock_time = current_time
+
+            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # the actual redis key is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # we prefix the task id so it's easier to keep track of who created the task
+            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            custom_task_id = f"{self.subtask_id_prefix}_{uuid4()}"
+
+            # add to the tracking taskset in redis BEFORE creating the celery task.
+            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
+            redis_client.sadd(self.taskset_key, custom_task_id)
+
+            # Priority on sync's triggered by new indexing should be medium
+            result = celery_app.send_task(
+                "document_by_cc_pair_cleanup_task",
+                kwargs=dict(
+                    document_id=doc_id,
+                    connector_id=cc_pair.connector_id,
+                    credential_id=cc_pair.credential_id,
+                ),
+                queue=DanswerCeleryQueues.CONNECTOR_DELETION,
+                task_id=custom_task_id,
+                priority=DanswerCeleryPriority.MEDIUM,
+            )
+
+            async_results.append(result)
+
+        return len(async_results)
+
+    def is_pruning(self, db_session: Session, redis_client: Redis) -> bool:
+        """A single example of a helper method being refactored into the redis helper"""
+        cc_pair = get_connector_credential_pair_from_id(
+            cc_pair_id=self._id, db_session=db_session
+        )
+        if not cc_pair:
+            raise ValueError(f"cc_pair_id {self._id} does not exist.")
+
+        if redis_client.exists(self.fence_key):
+            return True
+
+        return False
+
+
 def celery_get_queue_length(queue: str, r: Redis) -> int:
     """This is a redis specific way to get the length of a celery queue.
     It is priority aware and knows how to count across the multiple redis lists
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 344ef5291b9..03ca82d500d 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -1,3 +1,4 @@
+from collections.abc import Callable
 from datetime import datetime
 from datetime import timezone
 from typing import Any
@@ -5,8 +6,6 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.task_utils import name_cc_prune_task
-from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
 from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
@@ -17,14 +16,8 @@
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.models import Document
 from danswer.db.connector_credential_pair import get_connector_credential_pair
-from danswer.db.engine import get_db_current_time
 from danswer.db.enums import TaskStatus
-from danswer.db.models import Connector
-from danswer.db.models import Credential
 from danswer.db.models import TaskQueueState
-from danswer.db.tasks import check_task_is_live_and_not_timed_out
-from danswer.db.tasks import get_latest_task
-from danswer.db.tasks import get_latest_task_by_type
 from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import DeletionAttemptSnapshot
 from danswer.utils.logger import setup_logger
@@ -70,72 +63,19 @@ def get_deletion_attempt_snapshot(
     )
 
 
-def skip_cc_pair_pruning_by_task(
-    pruning_task: TaskQueueState | None, db_session: Session
-) -> bool:
-    """task should be the latest prune task for this cc_pair"""
-    if not ALLOW_SIMULTANEOUS_PRUNING:
-        # if only one prune is allowed at any time, then check to see if any prune
-        # is active
-        pruning_type_task_name = name_cc_prune_task()
-        last_pruning_type_task = get_latest_task_by_type(
-            pruning_type_task_name, db_session
-        )
-
-        if last_pruning_type_task and check_task_is_live_and_not_timed_out(
-            last_pruning_type_task, db_session
-        ):
-            return True
-
-    if pruning_task and check_task_is_live_and_not_timed_out(pruning_task, db_session):
-        # if the last task is live right now, we shouldn't start a new one
-        return True
-
-    return False
-
-
-def should_prune_cc_pair(
-    connector: Connector, credential: Credential, db_session: Session
-) -> bool:
-    if not connector.prune_freq:
-        return False
-
-    pruning_task_name = name_cc_prune_task(
-        connector_id=connector.id, credential_id=credential.id
-    )
-    last_pruning_task = get_latest_task(pruning_task_name, db_session)
-
-    if skip_cc_pair_pruning_by_task(last_pruning_task, db_session):
-        return False
-
-    current_db_time = get_db_current_time(db_session)
-
-    if not last_pruning_task:
-        # If the connector has never been pruned, then compare vs when the connector
-        # was created
-        time_since_initialization = current_db_time - connector.time_created
-        if time_since_initialization.total_seconds() >= connector.prune_freq:
-            return True
-        return False
-
-    if not last_pruning_task.start_time:
-        # if the last prune task hasn't started, we shouldn't start a new one
-        return False
-
-    # if the last prune task has a start time, then compare against it to determine
-    # if we should start
-    time_since_last_pruning = current_db_time - last_pruning_task.start_time
-    return time_since_last_pruning.total_seconds() >= connector.prune_freq
-
-
 def document_batch_to_ids(doc_batch: list[Document]) -> set[str]:
     return {doc.id for doc in doc_batch}
 
 
-def extract_ids_from_runnable_connector(runnable_connector: BaseConnector) -> set[str]:
+def extract_ids_from_runnable_connector(
+    runnable_connector: BaseConnector,
+    progress_callback: Callable[[int], None] | None = None,
+) -> set[str]:
     """
     If the PruneConnector hasnt been implemented for the given connector, just pull
-    all docs using the load_from_state and grab out the IDs
+    all docs using the load_from_state and grab out the IDs.
+
+    Optionally, a callback can be passed to handle the length of each document batch.
     """
     all_connector_doc_ids: set[str] = set()
 
@@ -158,6 +98,8 @@ def extract_ids_from_runnable_connector(runnable_connector: BaseConnector) -> se
                 max_calls=MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE, period=60
             )(document_batch_to_ids)
         for doc_batch in doc_batch_generator:
+            if progress_callback:
+                progress_callback(len(doc_batch))
             all_connector_doc_ids.update(doc_batch_processing_func(doc_batch))
 
     return all_connector_doc_ids
@@ -177,9 +119,10 @@ def celery_is_listening_to_queue(worker: Any, name: str) -> bool:
 
 
 def celery_is_worker_primary(worker: Any) -> bool:
-    """There are multiple approaches that could be taken, but the way we do it is to
-    check the hostname set for the celery worker, either in celeryconfig.py or on the
-    command line."""
+    """There are multiple approaches that could be taken to determine if a celery worker
+    is 'primary', as defined by us. But the way we do it is to check the hostname set
+    for the celery worker, which can be done either in celeryconfig.py or on the
+    command line with '--hostname'."""
     hostname = worker.hostname
     if hostname.startswith("light"):
         return False
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index a16b9fda34a..a3223aacc9f 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -1,12 +1,12 @@
 import redis
 from celery import shared_task
 from celery.exceptions import SoftTimeLimitExceeded
-from celery.utils.log import get_task_logger
 from redis import Redis
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import ObjectDeletedError
 
 from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.celery_app import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
@@ -14,17 +14,10 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.enums import ConnectorCredentialPairStatus
-from danswer.db.enums import IndexingStatus
-from danswer.db.index_attempt import get_last_attempt
 from danswer.db.models import ConnectorCredentialPair
-from danswer.db.search_settings import get_current_search_settings
 from danswer.redis.redis_pool import get_redis_client
 
 
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
-
-
 @shared_task(
     name="check_for_connector_deletion_task",
     soft_time_limit=JOB_TIMEOUT,
@@ -90,21 +83,6 @@ def try_generate_document_cc_pair_cleanup_tasks(
     if cc_pair.status != ConnectorCredentialPairStatus.DELETING:
         return None
 
-    search_settings = get_current_search_settings(db_session)
-
-    last_indexing = get_last_attempt(
-        connector_id=cc_pair.connector_id,
-        credential_id=cc_pair.credential_id,
-        search_settings_id=search_settings.id,
-        db_session=db_session,
-    )
-    if last_indexing:
-        if (
-            last_indexing.status == IndexingStatus.IN_PROGRESS
-            or last_indexing.status == IndexingStatus.NOT_STARTED
-        ):
-            return None
-
     # add tasks to celery and build up the task set to monitor in redis
     r.delete(rcd.taskset_key)
 
diff --git a/backend/danswer/background/celery/tasks/periodic/tasks.py b/backend/danswer/background/celery/tasks/periodic/tasks.py
index bd3b082aeb8..99b1cab7e77 100644
--- a/backend/danswer/background/celery/tasks/periodic/tasks.py
+++ b/backend/danswer/background/celery/tasks/periodic/tasks.py
@@ -7,18 +7,15 @@
 from celery import shared_task
 from celery.contrib.abortable import AbortableTask  # type: ignore
 from celery.exceptions import TaskRevokedError
-from celery.utils.log import get_task_logger
 from sqlalchemy import inspect
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from danswer.background.celery.celery_app import task_logger
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import PostgresAdvisoryLocks
 from danswer.db.engine import get_sqlalchemy_engine  # type: ignore
 
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
-
 
 @shared_task(
     name="kombu_message_cleanup_task",
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index 2f840e430ae..f72229b7d8c 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -1,61 +1,165 @@
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from uuid import uuid4
+
+import redis
 from celery import shared_task
-from celery.utils.log import get_task_logger
+from celery.exceptions import SoftTimeLimitExceeded
+from redis import Redis
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector
-from danswer.background.celery.celery_utils import should_prune_cc_pair
-from danswer.background.connector_deletion import delete_connector_credential_pair_batch
-from danswer.background.task_utils import build_celery_task_wrapper
-from danswer.background.task_utils import name_cc_prune_task
+from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
 from danswer.configs.app_configs import JOB_TIMEOUT
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.configs.constants import DanswerRedisLocks
 from danswer.connectors.factory import instantiate_connector
 from danswer.connectors.models import InputType
 from danswer.db.connector_credential_pair import get_connector_credential_pair
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.document import get_documents_for_connector_credential_pair
 from danswer.db.engine import get_sqlalchemy_engine
-from danswer.document_index.document_index_utils import get_both_index_names
-from danswer.document_index.factory import get_default_document_index
-
-
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
+from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.models import ConnectorCredentialPair
+from danswer.redis.redis_pool import get_redis_client
 
 
 @shared_task(
-    name="check_for_prune_task",
+    name="check_for_prune_task_2",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_for_prune_task() -> None:
-    """Runs periodically to check if any prune tasks should be run and adds them
-    to the queue"""
-
-    with Session(get_sqlalchemy_engine()) as db_session:
-        all_cc_pairs = get_connector_credential_pairs(db_session)
-
-        for cc_pair in all_cc_pairs:
-            if should_prune_cc_pair(
-                connector=cc_pair.connector,
-                credential=cc_pair.credential,
-                db_session=db_session,
-            ):
-                task_logger.info(f"Pruning the {cc_pair.connector.name} connector")
-
-                prune_documents_task.apply_async(
-                    kwargs=dict(
-                        connector_id=cc_pair.connector.id,
-                        credential_id=cc_pair.credential.id,
-                    )
+def check_for_prune_task_2() -> None:
+    r = get_redis_client()
+
+    lock_beat = r.lock(
+        DanswerRedisLocks.CHECK_PRUNE_BEAT_LOCK,
+        timeout=CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT,
+    )
+
+    try:
+        # these tasks should never overlap
+        if not lock_beat.acquire(blocking=False):
+            return
+
+        with Session(get_sqlalchemy_engine()) as db_session:
+            cc_pairs = get_connector_credential_pairs(db_session)
+            for cc_pair in cc_pairs:
+                tasks_created = ccpair_pruning_generator_task_creation_helper(
+                    cc_pair, db_session, r, lock_beat
                 )
-
-
-@build_celery_task_wrapper(name_cc_prune_task)
-@celery_app.task(name="prune_documents_task", soft_time_limit=JOB_TIMEOUT)
-def prune_documents_task(connector_id: int, credential_id: int) -> None:
+                if not tasks_created:
+                    continue
+
+                task_logger.info(f"Pruning started: cc_pair_id={cc_pair.id}")
+    except SoftTimeLimitExceeded:
+        task_logger.info(
+            "Soft time limit exceeded, task is being terminated gracefully."
+        )
+    except Exception:
+        task_logger.exception("Unexpected exception")
+    finally:
+        if lock_beat.owned():
+            lock_beat.release()
+
+
+def ccpair_pruning_generator_task_creation_helper(
+    cc_pair: ConnectorCredentialPair,
+    db_session: Session,
+    r: Redis,
+    lock_beat: redis.lock.Lock,
+) -> int | None:
+    """Returns an int if pruning is triggered.
+    The int represents the number of prune tasks generated (in this case, only one
+    because the task is a long running generator task.)
+    Returns None if no pruning is triggered (due to not being needed or
+    other reasons such as simultaneous pruning restrictions.
+
+    Checks for scheduling related conditions, then delegates the rest of the checks to
+    try_creating_prune_generator_task.
+    """
+
+    lock_beat.reacquire()
+
+    # skip pruning if no prune frequency is set
+    # pruning can still be forced via the API which will run a pruning task directly
+    if not cc_pair.connector.prune_freq:
+        return None
+
+    # skip pruning if the next scheduled prune time hasn't been reached yet
+    last_pruned = cc_pair.last_pruned
+    if not last_pruned:
+        # if never pruned, use the connector time created as the last_pruned time
+        last_pruned = cc_pair.connector.time_created
+
+    next_prune = last_pruned + timedelta(seconds=cc_pair.connector.prune_freq)
+    if datetime.now(timezone.utc) < next_prune:
+        return None
+
+    return try_creating_prune_generator_task(cc_pair, db_session, r)
+
+
+def try_creating_prune_generator_task(
+    cc_pair: ConnectorCredentialPair,
+    db_session: Session,
+    r: Redis,
+) -> int | None:
+    """Checks for any conditions that should block the pruning generator task from being
+    created, then creates the task.
+
+    Does not check for scheduling related conditions as this function
+    is used to trigger prunes immediately.
+    """
+
+    if not ALLOW_SIMULTANEOUS_PRUNING:
+        for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+            return None
+
+    rcp = RedisConnectorPruning(cc_pair.id)
+
+    # skip pruning if already pruning
+    if r.exists(rcp.fence_key):
+        return None
+
+    # skip pruning if the cc_pair is deleting
+    db_session.refresh(cc_pair)
+    if cc_pair.status == ConnectorCredentialPairStatus.DELETING:
+        return None
+
+    # add a long running generator task to the queue
+    r.delete(rcp.generator_complete_key)
+    r.delete(rcp.taskset_key)
+
+    custom_task_id = f"{rcp.generator_task_id_prefix}_{uuid4()}"
+
+    celery_app.send_task(
+        "connector_pruning_generator_task",
+        kwargs=dict(
+            connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
+        ),
+        queue=DanswerCeleryQueues.CONNECTOR_PRUNING,
+        task_id=custom_task_id,
+        priority=DanswerCeleryPriority.LOW,
+    )
+
+    # set this only after all tasks have been added
+    r.set(rcp.fence_key, 1)
+    return 1
+
+
+@shared_task(name="connector_pruning_generator_task", soft_time_limit=JOB_TIMEOUT)
+def connector_pruning_generator_task(connector_id: int, credential_id: int) -> None:
     """connector pruning task. For a cc pair, this task pulls all document IDs from the source
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
     from the most recently pulled document ID list"""
+
+    r = get_redis_client()
+
     with Session(get_sqlalchemy_engine()) as db_session:
         try:
             cc_pair = get_connector_credential_pair(
@@ -70,6 +174,12 @@ def prune_documents_task(connector_id: int, credential_id: int) -> None:
                 )
                 return
 
+            rcp = RedisConnectorPruning(cc_pair.id)
+
+            # Define the callback function
+            def redis_increment_callback(amount: int) -> None:
+                r.incrby(rcp.generator_progress_key, amount)
+
             runnable_connector = instantiate_connector(
                 db_session,
                 cc_pair.connector.source,
@@ -78,10 +188,12 @@ def prune_documents_task(connector_id: int, credential_id: int) -> None:
                 cc_pair.credential,
             )
 
+            # a list of docs in the source
             all_connector_doc_ids: set[str] = extract_ids_from_runnable_connector(
-                runnable_connector
+                runnable_connector, redis_increment_callback
             )
 
+            # a list of docs in our local index
             all_indexed_document_ids = {
                 doc.id
                 for doc in get_documents_for_connector_credential_pair(
@@ -91,30 +203,37 @@ def prune_documents_task(connector_id: int, credential_id: int) -> None:
                 )
             }
 
+            # generate list of docs to remove (no longer in the source)
             doc_ids_to_remove = list(all_indexed_document_ids - all_connector_doc_ids)
 
-            curr_ind_name, sec_ind_name = get_both_index_names(db_session)
-            document_index = get_default_document_index(
-                primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
+            task_logger.info(
+                f"Pruning set collected: "
+                f"cc_pair_id={cc_pair.id} "
+                f"docs_to_remove={len(doc_ids_to_remove)} "
+                f"doc_source={cc_pair.connector.source}"
             )
 
-            if len(doc_ids_to_remove) == 0:
-                task_logger.info(
-                    f"No docs to prune from {cc_pair.connector.source} connector"
-                )
-                return
+            rcp.documents_to_prune = set(doc_ids_to_remove)
 
             task_logger.info(
-                f"pruning {len(doc_ids_to_remove)} doc(s) from {cc_pair.connector.source} connector"
+                f"RedisConnectorPruning.generate_tasks starting. cc_pair_id={cc_pair.id}"
             )
-            delete_connector_credential_pair_batch(
-                document_ids=doc_ids_to_remove,
-                connector_id=connector_id,
-                credential_id=credential_id,
-                document_index=document_index,
+            tasks_generated = rcp.generate_tasks(celery_app, db_session, r, None)
+            if tasks_generated is None:
+                return None
+
+            task_logger.info(
+                f"RedisConnectorPruning.generate_tasks finished. "
+                f"cc_pair_id={cc_pair.id} tasks_generated={tasks_generated}"
             )
+
+            r.set(rcp.generator_complete_key, tasks_generated)
         except Exception as e:
             task_logger.exception(
                 f"Failed to run pruning for connector id {connector_id}."
             )
+
+            r.delete(rcp.generator_progress_key)
+            r.delete(rcp.taskset_key)
+            r.delete(rcp.fence_key)
             raise e
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
new file mode 100644
index 00000000000..0ca4d5031b1
--- /dev/null
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -0,0 +1,113 @@
+from celery import shared_task
+from celery import Task
+from celery.exceptions import SoftTimeLimitExceeded
+from sqlalchemy.orm import Session
+
+from danswer.access.access import get_access_for_document
+from danswer.background.celery.celery_app import task_logger
+from danswer.db.document import delete_document_by_connector_credential_pair__no_commit
+from danswer.db.document import delete_documents_complete__no_commit
+from danswer.db.document import get_document
+from danswer.db.document import get_document_connector_count
+from danswer.db.document import mark_document_as_synced
+from danswer.db.document_set import fetch_document_sets_for_document
+from danswer.db.engine import get_sqlalchemy_engine
+from danswer.document_index.document_index_utils import get_both_index_names
+from danswer.document_index.factory import get_default_document_index
+from danswer.document_index.interfaces import VespaDocumentFields
+from danswer.server.documents.models import ConnectorCredentialPairIdentifier
+
+
+@shared_task(
+    name="document_by_cc_pair_cleanup_task",
+    bind=True,
+    soft_time_limit=45,
+    time_limit=60,
+    max_retries=3,
+)
+def document_by_cc_pair_cleanup_task(
+    self: Task, document_id: str, connector_id: int, credential_id: int
+) -> bool:
+    """A lightweight subtask used to clean up document to cc pair relationships.
+    Created by connection deletion and connector pruning parent tasks."""
+
+    """
+    To delete a connector / credential pair:
+    (1) find all documents associated with connector / credential pair where there
+    this the is only connector / credential pair that has indexed it
+    (2) delete all documents from document stores
+    (3) delete all entries from postgres
+    (4) find all documents associated with connector / credential pair where there
+    are multiple connector / credential pairs that have indexed it
+    (5) update document store entries to remove access associated with the
+    connector / credential pair from the access list
+    (6) delete all relevant entries from postgres
+    """
+    task_logger.info(f"document_id={document_id}")
+
+    try:
+        with Session(get_sqlalchemy_engine()) as db_session:
+            curr_ind_name, sec_ind_name = get_both_index_names(db_session)
+            document_index = get_default_document_index(
+                primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
+            )
+
+            count = get_document_connector_count(db_session, document_id)
+            if count == 1:
+                # count == 1 means this is the only remaining cc_pair reference to the doc
+                # delete it from vespa and the db
+                document_index.delete(doc_ids=[document_id])
+                delete_documents_complete__no_commit(
+                    db_session=db_session,
+                    document_ids=[document_id],
+                )
+            elif count > 1:
+                # count > 1 means the document still has cc_pair references
+                doc = get_document(document_id, db_session)
+                if not doc:
+                    return False
+
+                # the below functions do not include cc_pairs being deleted.
+                # i.e. they will correctly omit access for the current cc_pair
+                doc_access = get_access_for_document(
+                    document_id=document_id, db_session=db_session
+                )
+
+                doc_sets = fetch_document_sets_for_document(document_id, db_session)
+                update_doc_sets: set[str] = set(doc_sets)
+
+                fields = VespaDocumentFields(
+                    document_sets=update_doc_sets,
+                    access=doc_access,
+                    boost=doc.boost,
+                    hidden=doc.hidden,
+                )
+
+                # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
+                document_index.update_single(document_id, fields=fields)
+
+                # there are still other cc_pair references to the doc, so just resync to Vespa
+                delete_document_by_connector_credential_pair__no_commit(
+                    db_session=db_session,
+                    document_id=document_id,
+                    connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
+                        connector_id=connector_id,
+                        credential_id=credential_id,
+                    ),
+                )
+
+                mark_document_as_synced(document_id, db_session)
+            else:
+                pass
+
+            db_session.commit()
+    except SoftTimeLimitExceeded:
+        task_logger.info(f"SoftTimeLimitExceeded exception. doc_id={document_id}")
+    except Exception as e:
+        task_logger.exception("Unexpected exception")
+
+        # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
+        countdown = 2 ** (self.request.retries + 4)
+        self.retry(exc=e, countdown=countdown)
+
+    return True
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 0ae214ca470..62806c7b81d 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -5,20 +5,22 @@
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from celery.utils.log import get_task_logger
 from redis import Redis
 from sqlalchemy.orm import Session
 
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.celery_app import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.db.connector import fetch_connector_by_id
+from danswer.db.connector import mark_ccpair_as_pruned
 from danswer.db.connector_credential_pair import add_deletion_failure_message
 from danswer.db.connector_credential_pair import (
     delete_connector_credential_pair__no_commit,
@@ -49,10 +51,6 @@
 from danswer.utils.variable_functionality import noop_fallback
 
 
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
-
-
 # celery auto associates tasks created inside another task,
 # which bloats the result metadata considerably. trail=False prevents this.
 @shared_task(
@@ -279,7 +277,7 @@ def monitor_document_set_taskset(
     fence_key = key_bytes.decode("utf-8")
     document_set_id = RedisDocumentSet.get_id_from_fence_key(fence_key)
     if document_set_id is None:
-        task_logger.warning("could not parse document set id from {key}")
+        task_logger.warning(f"could not parse document set id from {fence_key}")
         return
 
     rds = RedisDocumentSet(document_set_id)
@@ -326,7 +324,7 @@ def monitor_connector_deletion_taskset(key_bytes: bytes, r: Redis) -> None:
     fence_key = key_bytes.decode("utf-8")
     cc_pair_id = RedisConnectorDeletion.get_id_from_fence_key(fence_key)
     if cc_pair_id is None:
-        task_logger.warning("could not parse document set id from {key}")
+        task_logger.warning(f"could not parse cc_pair_id from {fence_key}")
         return
 
     rcd = RedisConnectorDeletion(cc_pair_id)
@@ -351,6 +349,9 @@ def monitor_connector_deletion_taskset(key_bytes: bytes, r: Redis) -> None:
     with Session(get_sqlalchemy_engine()) as db_session:
         cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session)
         if not cc_pair:
+            task_logger.warning(
+                f"monitor_connector_deletion_taskset - cc_pair_id not found: cc_pair_id={cc_pair_id}"
+            )
             return
 
         try:
@@ -402,20 +403,67 @@ def monitor_connector_deletion_taskset(key_bytes: bytes, r: Redis) -> None:
             add_deletion_failure_message(db_session, cc_pair.id, error_message)
             task_logger.exception(
                 f"Failed to run connector_deletion. "
-                f"connector_id={cc_pair.connector_id} credential_id={cc_pair.credential_id}"
+                f"cc_pair_id={cc_pair_id} connector_id={cc_pair.connector_id} credential_id={cc_pair.credential_id}"
             )
             raise e
 
     task_logger.info(
-        f"Successfully deleted connector_credential_pair with connector_id: '{cc_pair.connector_id}' "
-        f"and credential_id: '{cc_pair.credential_id}'. "
-        f"Deleted {initial_count} docs."
+        f"Successfully deleted cc_pair: "
+        f"cc_pair_id={cc_pair_id} "
+        f"connector_id={cc_pair.connector_id} "
+        f"credential_id={cc_pair.credential_id} "
+        f"docs_deleted={initial_count}"
     )
 
     r.delete(rcd.taskset_key)
     r.delete(rcd.fence_key)
 
 
+def monitor_ccpair_pruning_taskset(
+    key_bytes: bytes, r: Redis, db_session: Session
+) -> None:
+    fence_key = key_bytes.decode("utf-8")
+    cc_pair_id = RedisConnectorPruning.get_id_from_fence_key(fence_key)
+    if cc_pair_id is None:
+        task_logger.warning(
+            f"monitor_connector_pruning_taskset: could not parse cc_pair_id from {fence_key}"
+        )
+        return
+
+    rcp = RedisConnectorPruning(cc_pair_id)
+
+    fence_value = r.get(rcp.fence_key)
+    if fence_value is None:
+        return
+
+    generator_value = r.get(rcp.generator_complete_key)
+    if generator_value is None:
+        return
+
+    try:
+        initial_count = int(cast(int, generator_value))
+    except ValueError:
+        task_logger.error("The value is not an integer.")
+        return
+
+    count = cast(int, r.scard(rcp.taskset_key))
+    task_logger.info(
+        f"Connector pruning progress: cc_pair_id={cc_pair_id} remaining={count} initial={initial_count}"
+    )
+    if count > 0:
+        return
+
+    mark_ccpair_as_pruned(cc_pair_id, db_session)
+    task_logger.info(
+        f"Successfully pruned connector credential pair. cc_pair_id={cc_pair_id}"
+    )
+
+    r.delete(rcp.taskset_key)
+    r.delete(rcp.generator_progress_key)
+    r.delete(rcp.generator_complete_key)
+    r.delete(rcp.fence_key)
+
+
 @shared_task(name="monitor_vespa_sync", soft_time_limit=300)
 def monitor_vespa_sync() -> None:
     """This is a celery beat task that monitors and finalizes metadata sync tasksets.
@@ -457,6 +505,9 @@ def monitor_vespa_sync() -> None:
                 )
                 monitor_usergroup_taskset(key_bytes, r, db_session)
 
+            for key_bytes in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+                monitor_ccpair_pruning_taskset(key_bytes, r, db_session)
+
         # uncomment for debugging if needed
         # r_celery = celery_app.broker_connection().channel().client
         # length = celery_get_queue_length(DanswerCeleryQueues.VESPA_METADATA_SYNC, r_celery)
diff --git a/backend/danswer/background/connector_deletion.py b/backend/danswer/background/connector_deletion.py
deleted file mode 100644
index 962183f71ae..00000000000
--- a/backend/danswer/background/connector_deletion.py
+++ /dev/null
@@ -1,211 +0,0 @@
-"""
-To delete a connector / credential pair:
-(1) find all documents associated with connector / credential pair where there
-this the is only connector / credential pair that has indexed it
-(2) delete all documents from document stores
-(3) delete all entries from postgres
-(4) find all documents associated with connector / credential pair where there
-are multiple connector / credential pairs that have indexed it
-(5) update document store entries to remove access associated with the
-connector / credential pair from the access list
-(6) delete all relevant entries from postgres
-"""
-from celery import shared_task
-from celery import Task
-from celery.exceptions import SoftTimeLimitExceeded
-from celery.utils.log import get_task_logger
-from sqlalchemy.orm import Session
-
-from danswer.access.access import get_access_for_document
-from danswer.access.access import get_access_for_documents
-from danswer.db.document import delete_document_by_connector_credential_pair__no_commit
-from danswer.db.document import delete_documents_by_connector_credential_pair__no_commit
-from danswer.db.document import delete_documents_complete__no_commit
-from danswer.db.document import get_document
-from danswer.db.document import get_document_connector_count
-from danswer.db.document import get_document_connector_counts
-from danswer.db.document import mark_document_as_synced
-from danswer.db.document import prepare_to_modify_documents
-from danswer.db.document_set import fetch_document_sets_for_document
-from danswer.db.document_set import fetch_document_sets_for_documents
-from danswer.db.engine import get_sqlalchemy_engine
-from danswer.document_index.document_index_utils import get_both_index_names
-from danswer.document_index.factory import get_default_document_index
-from danswer.document_index.interfaces import DocumentIndex
-from danswer.document_index.interfaces import UpdateRequest
-from danswer.document_index.interfaces import VespaDocumentFields
-from danswer.server.documents.models import ConnectorCredentialPairIdentifier
-from danswer.utils.logger import setup_logger
-
-logger = setup_logger()
-
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
-
-_DELETION_BATCH_SIZE = 1000
-
-
-def delete_connector_credential_pair_batch(
-    document_ids: list[str],
-    connector_id: int,
-    credential_id: int,
-    document_index: DocumentIndex,
-) -> None:
-    """
-    Removes a batch of documents ids from a cc-pair. If no other cc-pair uses a document anymore
-    it gets permanently deleted.
-    """
-    with Session(get_sqlalchemy_engine()) as db_session:
-        # acquire lock for all documents in this batch so that indexing can't
-        # override the deletion
-        with prepare_to_modify_documents(
-            db_session=db_session, document_ids=document_ids
-        ):
-            document_connector_counts = get_document_connector_counts(
-                db_session=db_session, document_ids=document_ids
-            )
-
-            # figure out which docs need to be completely deleted
-            document_ids_to_delete = [
-                document_id
-                for document_id, cnt in document_connector_counts
-                if cnt == 1
-            ]
-            logger.debug(f"Deleting documents: {document_ids_to_delete}")
-
-            document_index.delete(doc_ids=document_ids_to_delete)
-
-            delete_documents_complete__no_commit(
-                db_session=db_session,
-                document_ids=document_ids_to_delete,
-            )
-
-            # figure out which docs need to be updated
-            document_ids_to_update = [
-                document_id for document_id, cnt in document_connector_counts if cnt > 1
-            ]
-
-            # maps document id to list of document set names
-            new_doc_sets_for_documents: dict[str, set[str]] = {
-                document_id_and_document_set_names_tuple[0]: set(
-                    document_id_and_document_set_names_tuple[1]
-                )
-                for document_id_and_document_set_names_tuple in fetch_document_sets_for_documents(
-                    db_session=db_session,
-                    document_ids=document_ids_to_update,
-                )
-            }
-
-            # determine future ACLs for documents in batch
-            access_for_documents = get_access_for_documents(
-                document_ids=document_ids_to_update,
-                db_session=db_session,
-            )
-
-            # update Vespa
-            logger.debug(f"Updating documents: {document_ids_to_update}")
-            update_requests = [
-                UpdateRequest(
-                    document_ids=[document_id],
-                    access=access,
-                    document_sets=new_doc_sets_for_documents[document_id],
-                )
-                for document_id, access in access_for_documents.items()
-            ]
-            document_index.update(update_requests=update_requests)
-
-            # clean up Postgres
-            delete_documents_by_connector_credential_pair__no_commit(
-                db_session=db_session,
-                document_ids=document_ids_to_update,
-                connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
-                    connector_id=connector_id,
-                    credential_id=credential_id,
-                ),
-            )
-            db_session.commit()
-
-
-@shared_task(
-    name="document_by_cc_pair_cleanup_task",
-    bind=True,
-    soft_time_limit=45,
-    time_limit=60,
-    max_retries=3,
-)
-def document_by_cc_pair_cleanup_task(
-    self: Task, document_id: str, connector_id: int, credential_id: int
-) -> bool:
-    task_logger.info(f"document_id={document_id}")
-
-    try:
-        with Session(get_sqlalchemy_engine()) as db_session:
-            curr_ind_name, sec_ind_name = get_both_index_names(db_session)
-            document_index = get_default_document_index(
-                primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
-            )
-
-            count = get_document_connector_count(db_session, document_id)
-            if count == 1:
-                # count == 1 means this is the only remaining cc_pair reference to the doc
-                # delete it from vespa and the db
-                document_index.delete_single(doc_id=document_id)
-                delete_documents_complete__no_commit(
-                    db_session=db_session,
-                    document_ids=[document_id],
-                )
-            elif count > 1:
-                # count > 1 means the document still has cc_pair references
-                doc = get_document(document_id, db_session)
-                if not doc:
-                    return False
-
-                # the below functions do not include cc_pairs being deleted.
-                # i.e. they will correctly omit access for the current cc_pair
-                doc_access = get_access_for_document(
-                    document_id=document_id, db_session=db_session
-                )
-
-                doc_sets = fetch_document_sets_for_document(document_id, db_session)
-                update_doc_sets: set[str] = set(doc_sets)
-
-                fields = VespaDocumentFields(
-                    document_sets=update_doc_sets,
-                    access=doc_access,
-                    boost=doc.boost,
-                    hidden=doc.hidden,
-                )
-
-                # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-                document_index.update_single(document_id, fields=fields)
-
-                # there are still other cc_pair references to the doc, so just resync to Vespa
-                delete_document_by_connector_credential_pair__no_commit(
-                    db_session=db_session,
-                    document_id=document_id,
-                    connector_credential_pair_identifier=ConnectorCredentialPairIdentifier(
-                        connector_id=connector_id,
-                        credential_id=credential_id,
-                    ),
-                )
-
-                mark_document_as_synced(document_id, db_session)
-            else:
-                pass
-
-            # update_docs_last_modified__no_commit(
-            #     db_session=db_session,
-            #     document_ids=[document_id],
-            # )
-
-            db_session.commit()
-    except SoftTimeLimitExceeded:
-        task_logger.info(f"SoftTimeLimitExceeded exception. doc_id={document_id}")
-    except Exception as e:
-        task_logger.exception("Unexpected exception")
-
-        # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
-        countdown = 2 ** (self.request.retries + 4)
-        self.retry(exc=e, countdown=countdown)
-
-    return True
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index c26c2fbd602..4c43dfcf634 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -187,10 +187,9 @@ class PostgresAdvisoryLocks(Enum):
 
 
 class DanswerCeleryQueues:
-    VESPA_DOCSET_SYNC_GENERATOR = "vespa_docset_sync_generator"
-    VESPA_USERGROUP_SYNC_GENERATOR = "vespa_usergroup_sync_generator"
     VESPA_METADATA_SYNC = "vespa_metadata_sync"
     CONNECTOR_DELETION = "connector_deletion"
+    CONNECTOR_PRUNING = "connector_pruning"
 
 
 class DanswerRedisLocks:
@@ -198,7 +197,7 @@ class DanswerRedisLocks:
     CHECK_VESPA_SYNC_BEAT_LOCK = "da_lock:check_vespa_sync_beat"
     MONITOR_VESPA_SYNC_BEAT_LOCK = "da_lock:monitor_vespa_sync_beat"
     CHECK_CONNECTOR_DELETION_BEAT_LOCK = "da_lock:check_connector_deletion_beat"
-    MONITOR_CONNECTOR_DELETION_BEAT_LOCK = "da_lock:monitor_connector_deletion_beat"
+    CHECK_PRUNE_BEAT_LOCK = "da_lock:check_prune_beat"
 
 
 class DanswerCeleryPriority(int, Enum):
diff --git a/backend/danswer/db/connector.py b/backend/danswer/db/connector.py
index 89e6977103e..0f777d30ec9 100644
--- a/backend/danswer/db/connector.py
+++ b/backend/danswer/db/connector.py
@@ -1,3 +1,5 @@
+from datetime import datetime
+from datetime import timezone
 from typing import cast
 
 from sqlalchemy import and_
@@ -268,3 +270,15 @@ def create_initial_default_connector(db_session: Session) -> None:
     )
     db_session.add(connector)
     db_session.commit()
+
+
+def mark_ccpair_as_pruned(cc_pair_id: int, db_session: Session) -> None:
+    stmt = select(ConnectorCredentialPair).where(
+        ConnectorCredentialPair.id == cc_pair_id
+    )
+    cc_pair = db_session.scalar(stmt)
+    if cc_pair is None:
+        raise ValueError(f"No cc_pair with ID: {cc_pair_id}")
+
+    cc_pair.last_pruned = datetime.now(timezone.utc)
+    db_session.commit()
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index f3d25e5e9d5..4777577d0fd 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -414,6 +414,12 @@ class ConnectorCredentialPair(Base):
     last_successful_index_time: Mapped[datetime.datetime | None] = mapped_column(
         DateTime(timezone=True), default=None
     )
+
+    # last successful prune
+    last_pruned: Mapped[datetime.datetime | None] = mapped_column(
+        DateTime(timezone=True), nullable=True, index=True
+    )
+
     total_docs_indexed: Mapped[int] = mapped_column(Integer, default=0)
 
     connector: Mapped["Connector"] = relationship(
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index a9ee03c0577..ce3131d050f 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -10,9 +10,11 @@
 
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
+from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
-from danswer.background.celery.celery_utils import skip_cc_pair_pruning_by_task
-from danswer.background.task_utils import name_cc_prune_task
+from danswer.background.celery.tasks.pruning.tasks import (
+    try_creating_prune_generator_task,
+)
 from danswer.db.connector_credential_pair import add_credential_to_connector
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.connector_credential_pair import remove_credential_from_connector
@@ -31,6 +33,7 @@
 from danswer.db.models import User
 from danswer.db.tasks import check_task_is_live_and_not_timed_out
 from danswer.db.tasks import get_latest_task
+from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import CCPairFullInfo
 from danswer.server.documents.models import CCStatusUpdateRequest
 from danswer.server.documents.models import CeleryTaskStatus
@@ -203,7 +206,7 @@ def get_cc_pair_latest_prune(
     cc_pair_id: int,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-) -> CeleryTaskStatus:
+) -> bool:
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id=cc_pair_id,
         db_session=db_session,
@@ -216,24 +219,8 @@ def get_cc_pair_latest_prune(
             detail="Connection not found for current user's permissions",
         )
 
-    # look up the last prune task for this connector (if it exists)
-    pruning_task_name = name_cc_prune_task(
-        connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
-    )
-    last_pruning_task = get_latest_task(pruning_task_name, db_session)
-    if not last_pruning_task:
-        raise HTTPException(
-            status_code=HTTPStatus.NOT_FOUND,
-            detail="No pruning task found.",
-        )
-
-    return CeleryTaskStatus(
-        id=last_pruning_task.task_id,
-        name=last_pruning_task.task_name,
-        status=last_pruning_task.status,
-        start_time=last_pruning_task.start_time,
-        register_time=last_pruning_task.register_time,
-    )
+    rcp = RedisConnectorPruning(cc_pair.id)
+    return rcp.is_pruning(db_session, get_redis_client())
 
 
 @router.post("/admin/cc-pair/{cc_pair_id}/prune")
@@ -242,8 +229,7 @@ def prune_cc_pair(
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
 ) -> StatusResponse[list[int]]:
-    # avoiding circular refs
-    from danswer.background.celery.tasks.pruning.tasks import prune_documents_task
+    """Triggers pruning on a particular cc_pair immediately"""
 
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id=cc_pair_id,
@@ -257,26 +243,26 @@ def prune_cc_pair(
             detail="Connection not found for current user's permissions",
         )
 
-    pruning_task_name = name_cc_prune_task(
-        connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
-    )
-    last_pruning_task = get_latest_task(pruning_task_name, db_session)
-    if skip_cc_pair_pruning_by_task(
-        last_pruning_task,
-        db_session=db_session,
-    ):
+    r = get_redis_client()
+    rcp = RedisConnectorPruning(cc_pair_id)
+    if rcp.is_pruning(db_session, r):
         raise HTTPException(
             status_code=HTTPStatus.CONFLICT,
             detail="Pruning task already in progress.",
         )
 
-    logger.info(f"Pruning the {cc_pair.connector.name} connector.")
-    prune_documents_task.apply_async(
-        kwargs=dict(
-            connector_id=cc_pair.connector.id,
-            credential_id=cc_pair.credential.id,
-        )
+    logger.info(
+        f"Pruning cc_pair: cc_pair_id={cc_pair_id} "
+        f"connector_id={cc_pair.connector_id} "
+        f"credential_id={cc_pair.credential_id} "
+        f"{cc_pair.connector.name} connector."
     )
+    tasks_created = try_creating_prune_generator_task(cc_pair, db_session, r)
+    if not tasks_created:
+        raise HTTPException(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+            detail="Pruning task creation failed.",
+        )
 
     return StatusResponse(
         success=True,
@@ -353,14 +339,6 @@ def sync_cc_pair(
             status_code=HTTPStatus.CONFLICT,
             detail="Sync task already in progress.",
         )
-    if skip_cc_pair_pruning_by_task(
-        last_sync_task,
-        db_session=db_session,
-    ):
-        raise HTTPException(
-            status_code=HTTPStatus.CONFLICT,
-            detail="Sync task already in progress.",
-        )
 
     logger.info(f"Syncing the {cc_pair.connector.name} connector.")
     sync_external_doc_permissions_task.apply_async(
diff --git a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
index 805083efcfc..d305f723daf 100644
--- a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
@@ -15,10 +15,10 @@
 
 def monitor_usergroup_taskset(key_bytes: bytes, r: Redis, db_session: Session) -> None:
     """This function is likely to move in the worker refactor happening next."""
-    key = key_bytes.decode("utf-8")
-    usergroup_id = RedisUserGroup.get_id_from_fence_key(key)
+    fence_key = key_bytes.decode("utf-8")
+    usergroup_id = RedisUserGroup.get_id_from_fence_key(fence_key)
     if not usergroup_id:
-        task_logger.warning("Could not parse usergroup id from {key}")
+        task_logger.warning(f"Could not parse usergroup id from {fence_key}")
         return
 
     rug = RedisUserGroup(usergroup_id)
diff --git a/backend/ee/danswer/db/user_group.py b/backend/ee/danswer/db/user_group.py
index 863b9170e3f..470a46e688b 100644
--- a/backend/ee/danswer/db/user_group.py
+++ b/backend/ee/danswer/db/user_group.py
@@ -603,7 +603,7 @@ def delete_user_group_cc_pair_relationship__no_commit(
 
     if cc_pair.status != ConnectorCredentialPairStatus.DELETING:
         raise ValueError(
-            f"Connector Credential Pair '{cc_pair_id}' is not in the DELETING state"
+            f"Connector Credential Pair '{cc_pair_id}' is not in the DELETING state. status={cc_pair.status}"
         )
 
     delete_stmt = delete(UserGroup__ConnectorCredentialPair).where(
diff --git a/backend/tests/integration/common_utils/managers/cc_pair.py b/backend/tests/integration/common_utils/managers/cc_pair.py
index 6bcbc34e9bd..48ae805cbb9 100644
--- a/backend/tests/integration/common_utils/managers/cc_pair.py
+++ b/backend/tests/integration/common_utils/managers/cc_pair.py
@@ -274,10 +274,10 @@ def prune(
         result.raise_for_status()
 
     @staticmethod
-    def get_prune_task(
+    def is_pruning(
         cc_pair: DATestCCPair,
         user_performing_action: DATestUser | None = None,
-    ) -> CeleryTaskStatus:
+    ) -> bool:
         response = requests.get(
             url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair.id}/prune",
             headers=user_performing_action.headers
@@ -285,28 +285,21 @@ def get_prune_task(
             else GENERAL_HEADERS,
         )
         response.raise_for_status()
-        return CeleryTaskStatus(**response.json())
+        response_bool = response.json()
+        return response_bool
 
     @staticmethod
     def wait_for_prune(
         cc_pair: DATestCCPair,
-        after: datetime,
         timeout: float = MAX_DELAY,
         user_performing_action: DATestUser | None = None,
     ) -> None:
         """after: The task register time must be after this time."""
         start = time.monotonic()
         while True:
-            task = CCPairManager.get_prune_task(cc_pair, user_performing_action)
-            if not task:
-                raise ValueError("Prune task not found.")
-
-            if not task.register_time or task.register_time < after:
-                raise ValueError("Prune task register time is too early.")
-
-            if task.status == TaskStatus.SUCCESS:
-                # Pruning succeeded
-                return
+            result = CCPairManager.is_pruning(cc_pair, user_performing_action)
+            if not result:
+                break
 
             elapsed = time.monotonic() - start
             if elapsed > timeout:
@@ -380,16 +373,31 @@ def wait_for_sync(
 
     @staticmethod
     def wait_for_deletion_completion(
+        cc_pair_id: int | None = None,
         user_performing_action: DATestUser | None = None,
     ) -> None:
+        """if cc_pair_id is not specified, just waits until no connectors are in the deleting state.
+        if cc_pair_id is specified, checks to ensure the specific cc_pair_id is gone.
+        We had a bug where the connector was paused in the middle of deleting, so specifying the
+        cc_pair_id is good to do."""
         start = time.monotonic()
         while True:
-            fetched_cc_pairs = CCPairManager.get_all(user_performing_action)
-            if all(
-                cc_pair.cc_pair_status != ConnectorCredentialPairStatus.DELETING
-                for cc_pair in fetched_cc_pairs
-            ):
-                return
+            cc_pairs = CCPairManager.get_all(user_performing_action)
+            if cc_pair_id:
+                found = False
+                for cc_pair in cc_pairs:
+                    if cc_pair.cc_pair_id == cc_pair_id:
+                        found = True
+                        break
+
+                if not found:
+                    return
+            else:
+                if all(
+                    cc_pair.cc_pair_status != ConnectorCredentialPairStatus.DELETING
+                    for cc_pair in cc_pairs
+                ):
+                    return
 
             if time.monotonic() - start > MAX_DELAY:
                 raise TimeoutError(
diff --git a/backend/tests/integration/connector_job_tests/slack/test_prune.py b/backend/tests/integration/connector_job_tests/slack/test_prune.py
index fcac7db1384..96638417c4d 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_prune.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_prune.py
@@ -195,9 +195,8 @@ def test_slack_prune(
     )
 
     # Prune the cc_pair
-    before = datetime.now(timezone.utc)
     CCPairManager.prune(cc_pair, user_performing_action=admin_user)
-    CCPairManager.wait_for_prune(cc_pair, before, user_performing_action=admin_user)
+    CCPairManager.wait_for_prune(cc_pair, user_performing_action=admin_user)
 
     # ----------------------------VERIFY THE CHANGES---------------------------
     # Ensure admin user can't see deleted messages
diff --git a/backend/tests/integration/tests/connector/test_connector_deletion.py b/backend/tests/integration/tests/connector/test_connector_deletion.py
index 46a65f768a9..676ee4d9f4b 100644
--- a/backend/tests/integration/tests/connector/test_connector_deletion.py
+++ b/backend/tests/integration/tests/connector/test_connector_deletion.py
@@ -11,6 +11,7 @@
 
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.enums import IndexingStatus
+from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import create_index_attempt_error
 from danswer.db.models import IndexAttempt
 from danswer.db.search_settings import get_current_search_settings
@@ -117,6 +118,22 @@ def test_connector_deletion(reset: None, vespa_client: vespa_fixture) -> None:
         user_performing_action=admin_user,
     )
 
+    # inject an index attempt and index attempt error (exercises foreign key errors)
+    with Session(get_sqlalchemy_engine()) as db_session:
+        attempt_id = create_index_attempt(
+            connector_credential_pair_id=cc_pair_1.id,
+            search_settings_id=1,
+            db_session=db_session,
+        )
+        create_index_attempt_error(
+            index_attempt_id=attempt_id,
+            batch=1,
+            docs=[],
+            exception_msg="",
+            exception_traceback="",
+            db_session=db_session,
+        )
+
     # Update local records to match the database for later comparison
     user_group_1.cc_pair_ids = []
     user_group_2.cc_pair_ids = [cc_pair_2.id]
@@ -125,7 +142,9 @@ def test_connector_deletion(reset: None, vespa_client: vespa_fixture) -> None:
     cc_pair_1.groups = []
     cc_pair_2.groups = [user_group_2.id]
 
-    CCPairManager.wait_for_deletion_completion(user_performing_action=admin_user)
+    CCPairManager.wait_for_deletion_completion(
+        cc_pair_id=cc_pair_1.id, user_performing_action=admin_user
+    )
 
     # validate vespa documents
     DocumentManager.verify(
@@ -303,7 +322,9 @@ def test_connector_deletion_for_overlapping_connectors(
     )
 
     # wait for deletion to finish
-    CCPairManager.wait_for_deletion_completion(user_performing_action=admin_user)
+    CCPairManager.wait_for_deletion_completion(
+        cc_pair_id=cc_pair_1.id, user_performing_action=admin_user
+    )
 
     print("Connector 1 deleted")
 
diff --git a/backend/tests/integration/tests/permissions/test_cc_pair_permissions.py b/backend/tests/integration/tests/permissions/test_cc_pair_permissions.py
index 5fba8ff64fc..001daacd0c2 100644
--- a/backend/tests/integration/tests/permissions/test_cc_pair_permissions.py
+++ b/backend/tests/integration/tests/permissions/test_cc_pair_permissions.py
@@ -171,7 +171,9 @@ def test_cc_pair_permissions(reset: None) -> None:
 
     # Test deleting the cc pair
     CCPairManager.delete(valid_cc_pair, user_performing_action=curator)
-    CCPairManager.wait_for_deletion_completion(user_performing_action=curator)
+    CCPairManager.wait_for_deletion_completion(
+        cc_pair_id=valid_cc_pair.id, user_performing_action=curator
+    )
 
     CCPairManager.verify(
         cc_pair=valid_cc_pair,
diff --git a/backend/tests/integration/tests/permissions/test_whole_curator_flow.py b/backend/tests/integration/tests/permissions/test_whole_curator_flow.py
index 1ce9052c108..751f41413d4 100644
--- a/backend/tests/integration/tests/permissions/test_whole_curator_flow.py
+++ b/backend/tests/integration/tests/permissions/test_whole_curator_flow.py
@@ -77,7 +77,9 @@ def test_whole_curator_flow(reset: None) -> None:
 
     # Verify that the curator can delete the CC pair
     CCPairManager.delete(cc_pair=test_cc_pair, user_performing_action=curator)
-    CCPairManager.wait_for_deletion_completion(user_performing_action=curator)
+    CCPairManager.wait_for_deletion_completion(
+        cc_pair_id=test_cc_pair.id, user_performing_action=curator
+    )
 
     # Verify that the CC pair has been deleted
     CCPairManager.verify(
@@ -158,7 +160,9 @@ def test_global_curator_flow(reset: None) -> None:
 
     # Verify that the curator can delete the CC pair
     CCPairManager.delete(cc_pair=test_cc_pair, user_performing_action=global_curator)
-    CCPairManager.wait_for_deletion_completion(user_performing_action=global_curator)
+    CCPairManager.wait_for_deletion_completion(
+        cc_pair_id=test_cc_pair.id, user_performing_action=global_curator
+    )
 
     # Verify that the CC pair has been deleted
     CCPairManager.verify(
diff --git a/backend/tests/integration/tests/pruning/test_pruning.py b/backend/tests/integration/tests/pruning/test_pruning.py
index 084ad80b357..4c4e82cdfb0 100644
--- a/backend/tests/integration/tests/pruning/test_pruning.py
+++ b/backend/tests/integration/tests/pruning/test_pruning.py
@@ -105,12 +105,9 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
             logger.info("Removing courses.html.")
             os.remove(os.path.join(website_tgt, "courses.html"))
 
-            # store the time again as a reference for the pruning timestamps
-            now = datetime.now(timezone.utc)
-
             CCPairManager.prune(cc_pair_1, user_performing_action=admin_user)
             CCPairManager.wait_for_prune(
-                cc_pair_1, now, timeout=60, user_performing_action=admin_user
+                cc_pair_1, timeout=60, user_performing_action=admin_user
             )
 
             selected_cc_pair = CCPairManager.get_one(
diff --git a/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py b/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py
index 35a189f6dd4..3e62df7a0c9 100644
--- a/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py
+++ b/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py
@@ -10,7 +10,7 @@
 NON_BUILTIN_WIKIS: Final[list[tuple[str, str]]] = [
     ("https://fallout.fandom.com", "falloutwiki"),
     ("https://harrypotter.fandom.com/wiki/", "harrypotterwiki"),
-    ("https://artofproblemsolving.com/wiki", "artofproblemsolving"),
+    # ("https://artofproblemsolving.com/wiki", "artofproblemsolving"),  # FLAKY
     ("https://www.bogleheads.org/wiki/Main_Page", "bogleheadswiki"),
     ("https://bogleheads.org/wiki/Main_Page", "bogleheadswiki"),
     ("https://www.dandwiki.com/wiki/", "dungeonsanddragons"),

From 150dcc2883bf187f343941ff8838027595ac10ce Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 7 Oct 2024 13:10:58 -0700
Subject: [PATCH 042/376] back button + popups (#2707)

* back button + popups

* remove logs
---
 .../app/admin/assistants/AssistantEditor.tsx  |  1 +
 .../app/admin/configuration/search/page.tsx   |  9 ++++++
 .../[connector]/AddConnectorPage.tsx          | 12 +++-----
 .../embeddings/pages/EmbeddingFormPage.tsx    | 15 ++--------
 web/src/app/admin/indexing/status/page.tsx    | 18 +++++++++---
 .../components/context/EmbeddingContext.tsx   |  8 +++++-
 web/src/components/context/FormContext.tsx    |  9 ++++--
 web/src/components/popup/PopupFromQuery.tsx   | 28 +++++++++++++++++++
 8 files changed, 73 insertions(+), 27 deletions(-)
 create mode 100644 web/src/components/popup/PopupFromQuery.tsx

diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index d4287e4c984..731070783e5 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -106,6 +106,7 @@ export function AssistantEditor({
   admin?: boolean;
 }) {
   const router = useRouter();
+
   const { popup, setPopup } = usePopup();
 
   const colorOptions = [
diff --git a/web/src/app/admin/configuration/search/page.tsx b/web/src/app/admin/configuration/search/page.tsx
index 6a0d428d357..a66e45d9023 100644
--- a/web/src/app/admin/configuration/search/page.tsx
+++ b/web/src/app/admin/configuration/search/page.tsx
@@ -19,7 +19,9 @@ export interface EmbeddingDetails {
   default_model_id?: number;
   name: string;
 }
+
 import { EmbeddingIcon } from "@/components/icons/icons";
+import { usePopupFromQuery } from "@/components/popup/PopupFromQuery";
 
 import Link from "next/link";
 import { SavedSearchSettings } from "../../embeddings/interfaces";
@@ -29,6 +31,12 @@ import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 function Main() {
   const settings = useContext(SettingsContext);
+  const { popup: searchSettingsPopup } = usePopupFromQuery({
+    "search-settings": {
+      message: `Changed search settings successfully`,
+      type: "success",
+    },
+  });
   const {
     data: currentEmeddingModel,
     isLoading: isLoadingCurrentModel,
@@ -74,6 +82,7 @@ function Main() {
 
   return (
     <div className="h-screen">
+      {searchSettingsPopup}
       {!futureEmbeddingModel ? (
         <>
           {settings?.settings.needs_reindexing && (
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index 8294b6ca0d0..07b719d8341 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -41,7 +41,7 @@ import { Formik } from "formik";
 import { AccessTypeForm } from "@/components/admin/connectors/AccessTypeForm";
 import { AccessTypeGroupSelector } from "@/components/admin/connectors/AccessTypeGroupSelector";
 import NavigationRow from "./NavigationRow";
-
+import { useRouter } from "next/navigation";
 export interface AdvancedConfig {
   refreshFreq: number;
   pruneFreq: number;
@@ -111,6 +111,8 @@ export default function AddConnector({
 }: {
   connector: ConfigurableSources;
 }) {
+  const router = useRouter();
+
   // State for managing credentials and files
   const [currentCredential, setCurrentCredential] =
     useState<Credential<any> | null>(null);
@@ -201,13 +203,7 @@ export default function AddConnector({
   };
 
   const onSuccess = () => {
-    setPopup({
-      message: "Connector created! Redirecting to connector home page",
-      type: "success",
-    });
-    setTimeout(() => {
-      window.open("/admin/indexing/status", "_self");
-    }, 1000);
+    router.push("/admin/indexing/status?message=connector-created");
   };
 
   return (
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index adf486ed5d4..196c99da0d2 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -25,9 +25,11 @@ import RerankingDetailsForm from "../RerankingFormPage";
 import { useEmbeddingFormContext } from "@/components/context/EmbeddingContext";
 import { Modal } from "@/components/Modal";
 
+import { useRouter } from "next/navigation";
 export default function EmbeddingForm() {
   const { formStep, nextFormStep, prevFormStep } = useEmbeddingFormContext();
   const { popup, setPopup } = usePopup();
+  const router = useRouter();
 
   const [advancedEmbeddingDetails, setAdvancedEmbeddingDetails] =
     useState<AdvancedSearchConfiguration>({
@@ -172,10 +174,6 @@ export default function EmbeddingForm() {
 
     const response = await updateSearchSettings(values);
     if (response.ok) {
-      setPopup({
-        message: "Updated search settings successfully",
-        type: "success",
-      });
       return true;
     } else {
       setPopup({ message: "Failed to update search settings", type: "error" });
@@ -184,14 +182,7 @@ export default function EmbeddingForm() {
   };
 
   const navigateToEmbeddingPage = (changedResource: string) => {
-    setPopup({
-      message: `Changed ${changedResource} successfully. Redirecting to embedding page`,
-      type: "success",
-    });
-
-    setTimeout(() => {
-      window.open("/admin/configuration/search", "_self");
-    }, 2000);
+    router.push("/admin/configuration/search?message=search-settings");
   };
 
   const onConfirm = async () => {
diff --git a/web/src/app/admin/indexing/status/page.tsx b/web/src/app/admin/indexing/status/page.tsx
index f5d64d3ac3a..bb56ceaa74e 100644
--- a/web/src/app/admin/indexing/status/page.tsx
+++ b/web/src/app/admin/indexing/status/page.tsx
@@ -11,8 +11,15 @@ import { AdminPageTitle } from "@/components/admin/Title";
 import Link from "next/link";
 import { Button, Text } from "@tremor/react";
 import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
+import { usePopupFromQuery } from "@/components/popup/PopupFromQuery";
 
 function Main() {
+  const { popup } = usePopupFromQuery({
+    "connector-created": {
+      message: "Connector created successfully",
+      type: "success",
+    },
+  });
   const {
     data: indexAttemptData,
     isLoading: indexAttemptIsLoading,
@@ -67,10 +74,13 @@ function Main() {
   });
 
   return (
-    <CCPairIndexingStatusTable
-      ccPairsIndexingStatuses={indexAttemptData}
-      editableCcPairsIndexingStatuses={editableIndexAttemptData}
-    />
+    <>
+      {popup}
+      <CCPairIndexingStatusTable
+        ccPairsIndexingStatuses={indexAttemptData}
+        editableCcPairsIndexingStatuses={editableIndexAttemptData}
+      />
+    </>
   );
 }
 
diff --git a/web/src/components/context/EmbeddingContext.tsx b/web/src/components/context/EmbeddingContext.tsx
index 2ca18dc955e..3d03b99b1d5 100644
--- a/web/src/components/context/EmbeddingContext.tsx
+++ b/web/src/components/context/EmbeddingContext.tsx
@@ -57,9 +57,15 @@ export const EmbeddingFormProvider: React.FC<{
   useEffect(() => {
     // Update URL when formStep changes
     const updatedSearchParams = new URLSearchParams(searchParams.toString());
+    const existingStep = updatedSearchParams.get("step");
     updatedSearchParams.set("step", formStep.toString());
     const newUrl = `${pathname}?${updatedSearchParams.toString()}`;
-    router.push(newUrl);
+
+    if (!existingStep) {
+      router.replace(newUrl);
+    } else if (newUrl !== pathname) {
+      router.push(newUrl);
+    }
   }, [formStep, router, pathname, searchParams]);
 
   // Update formStep when URL changes
diff --git a/web/src/components/context/FormContext.tsx b/web/src/components/context/FormContext.tsx
index 8755fbe6da1..d445782f718 100644
--- a/web/src/components/context/FormContext.tsx
+++ b/web/src/components/context/FormContext.tsx
@@ -57,12 +57,17 @@ export const FormProvider: React.FC<{
   useEffect(() => {
     // Update URL when formStep changes
     const updatedSearchParams = new URLSearchParams(searchParams.toString());
+    const existingStep = updatedSearchParams.get("step");
     updatedSearchParams.set("step", formStep.toString());
     const newUrl = `${pathname}?${updatedSearchParams.toString()}`;
-    router.push(newUrl);
+
+    if (!existingStep) {
+      router.replace(newUrl);
+    } else if (newUrl !== pathname) {
+      router.push(newUrl);
+    }
   }, [formStep, router, pathname, searchParams]);
 
-  // Update formStep when URL changes
   useEffect(() => {
     const stepFromUrl = parseInt(searchParams.get("step") || "0", 10);
     if (stepFromUrl !== formStep) {
diff --git a/web/src/components/popup/PopupFromQuery.tsx b/web/src/components/popup/PopupFromQuery.tsx
new file mode 100644
index 00000000000..71834f69987
--- /dev/null
+++ b/web/src/components/popup/PopupFromQuery.tsx
@@ -0,0 +1,28 @@
+import { useEffect } from "react";
+
+import { usePopup } from "../admin/connectors/Popup";
+import { PopupSpec } from "../admin/connectors/Popup";
+import { useRouter } from "next/navigation";
+
+interface PopupMessages {
+  [key: string]: PopupSpec;
+}
+
+export const usePopupFromQuery = (messages: PopupMessages) => {
+  const router = useRouter();
+  const { popup, setPopup } = usePopup();
+
+  useEffect(() => {
+    const searchParams = new URLSearchParams(window.location.search);
+    // Get the value for search param with key "message"
+    const messageValue = searchParams.get("message");
+    // Check if any key from messages object is present in search params
+    if (messageValue && messageValue in messages) {
+      const popupMessage = messages[messageValue];
+      router.replace(window.location.pathname);
+      setPopup(popupMessage);
+    }
+  }, []);
+
+  return { popup };
+};

From 1900a390d81f0ffef9b86e7900fada54209c17c6 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 7 Oct 2024 13:21:07 -0700
Subject: [PATCH 043/376] Linting (#2704)

* effect cleanup

* remove unused imports

* remove unne

* remove unnecessary packages

* k

* temp

* minor
---
 backend/danswer/llm/chat_llm.py               |  1 +
 web/.eslintrc.json                            |  5 +-
 web/package.json                              |  2 +-
 web/src/app/admin/add-connector/page.tsx      | 23 ++---
 .../app/admin/assistants/AssistantEditor.tsx  |  6 +-
 .../admin/assistants/CollapsibleSection.tsx   |  1 -
 web/src/app/admin/assistants/PersonaTable.tsx | 10 +--
 web/src/app/admin/assistants/lib.ts           |  2 +-
 web/src/app/admin/assistants/page.tsx         |  2 +-
 .../admin/bot/SlackBotConfigCreationForm.tsx  |  3 +-
 web/src/app/admin/bot/new/page.tsx            |  3 +-
 web/src/app/admin/bot/page.tsx                | 15 +---
 .../llm/CustomLLMProviderUpdateForm.tsx       |  2 +
 web/src/app/admin/configuration/llm/page.tsx  |  1 -
 .../configuration/search/UpgradingPage.tsx    | 17 ++--
 .../[ccPairId]/IndexingAttemptsTable.tsx      | 88 +++++++++++--------
 .../app/admin/connector/[ccPairId]/page.tsx   | 47 +++++-----
 .../[connector]/AddConnectorPage.tsx          |  5 +-
 .../[connector]/ConnectorWrapper.tsx          |  2 +-
 .../admin/connectors/[connector]/Sidebar.tsx  |  2 +-
 .../pages/ConnectorInput/NumberInput.tsx      |  2 +-
 .../pages/ConnectorInput/SelectInput.tsx      |  6 +-
 .../[connector]/pages/gdrive/Credential.tsx   |  5 +-
 .../pages/gdrive/GoogleDrivePage.tsx          |  7 +-
 .../[connector]/pages/gmail/Credential.tsx    |  1 +
 .../[connector]/pages/gmail/GmailPage.tsx     |  9 +-
 .../app/admin/documents/explorer/Explorer.tsx | 32 ++++---
 .../feedback/DocumentFeedbackTable.tsx        |  1 -
 .../sets/DocumentSetCreationForm.tsx          |  4 +-
 web/src/app/admin/documents/sets/new/page.tsx |  1 -
 web/src/app/admin/documents/sets/page.tsx     |  7 +-
 .../EmbeddingModelSelectionForm.tsx           |  1 -
 .../modals/ChangeCredentialsModal.tsx         |  2 +-
 .../embeddings/modals/ModelSelectionModal.tsx |  5 +-
 .../modals/ProviderCreationModal.tsx          |  3 +
 .../pages/AdvancedEmbeddingFormPage.tsx       |  3 +-
 .../embeddings/pages/CloudEmbeddingPage.tsx   |  4 +-
 .../embeddings/pages/EmbeddingFormPage.tsx    |  2 +-
 .../embeddings/pages/OpenEmbeddingPage.tsx    | 11 ++-
 web/src/app/admin/indexing/[id]/page.tsx      |  1 -
 web/src/app/admin/indexing/status/page.tsx    |  4 -
 .../prompt-library/modals/AddPromptModal.tsx  |  4 +-
 .../admin/prompt-library/promptLibrary.tsx    |  1 -
 web/src/app/admin/settings/SettingsForm.tsx   |  6 +-
 web/src/app/admin/token-rate-limits/page.tsx  |  2 +-
 web/src/app/admin/tools/ToolEditor.tsx        | 43 ++++-----
 web/src/app/admin/tools/ToolsTable.tsx        |  1 -
 .../app/admin/tools/edit/[toolId]/page.tsx    |  1 -
 web/src/app/admin/tools/new/page.tsx          |  1 -
 web/src/app/admin/tools/page.tsx              |  2 +-
 web/src/app/admin/users/page.tsx              |  1 -
 web/src/app/assistants/SidebarWrapper.tsx     | 13 ++-
 .../assistants/gallery/AssistantsGallery.tsx  |  1 -
 .../assistants/mine/AssistantSharingModal.tsx |  2 +-
 .../app/assistants/mine/AssistantsList.tsx    | 12 +--
 .../assistants/mine/WrappedInputPrompts.tsx   |  1 -
 web/src/app/auth/login/LoginText.tsx          |  3 +-
 web/src/app/auth/verify-email/Verify.tsx      |  8 +-
 web/src/app/chat/ChatPage.tsx                 | 26 ++++--
 web/src/app/chat/ChatPopup.tsx                |  2 +-
 web/src/app/chat/RegenerateOption.tsx         |  1 -
 .../documentSidebar/ChatDocumentDisplay.tsx   |  1 +
 .../chat/documentSidebar/DocumentSidebar.tsx  |  2 +-
 web/src/app/chat/files/InputBarPreview.tsx    |  2 +-
 .../files/images/InputBarPreviewImage.tsx     |  1 +
 web/src/app/chat/folders/FolderManagement.tsx |  2 -
 web/src/app/chat/input/ChatInputBar.tsx       |  2 +-
 web/src/app/chat/input/ChatInputOption.tsx    |  6 +-
 web/src/app/chat/lib.tsx                      | 15 +---
 web/src/app/chat/message/ContinueMessage.tsx  |  2 +-
 web/src/app/chat/message/Messages.tsx         |  9 +-
 web/src/app/chat/message/SearchSummary.tsx    |  2 +-
 web/src/app/chat/message/SkippedSearch.tsx    |  2 +-
 web/src/app/chat/modal/FeedbackModal.tsx      |  7 +-
 .../app/chat/modal/SetDefaultModelModal.tsx   |  4 +-
 .../app/chat/modal/ShareChatSessionModal.tsx  |  3 +-
 .../modal/configuration/AssistantsTab.tsx     |  1 -
 .../app/chat/modal/configuration/LlmTab.tsx   | 20 ++---
 .../app/chat/modifiers/SearchTypeSelector.tsx |  2 +-
 .../app/chat/modifiers/SelectedDocuments.tsx  |  3 +-
 web/src/app/chat/page.tsx                     |  2 -
 .../sessionSidebar/ChatSessionDisplay.tsx     |  6 +-
 .../chat/sessionSidebar/HistorySidebar.tsx    |  3 +-
 web/src/app/chat/sessionSidebar/PagesTab.tsx  |  2 +-
 .../ee/admin/api-key/DanswerApiKeyForm.tsx    |  1 -
 web/src/app/ee/admin/api-key/page.tsx         |  2 +-
 .../app/ee/admin/groups/ConnectorEditor.tsx   |  2 +-
 web/src/app/ee/admin/groups/UserEditor.tsx    |  1 -
 .../ee/admin/groups/UserGroupCreationForm.tsx |  2 -
 .../groups/[groupId]/AddConnectorForm.tsx     |  1 -
 .../app/ee/admin/groups/[groupId]/page.tsx    |  1 -
 web/src/app/ee/admin/groups/page.tsx          |  4 +-
 .../query-history/QueryHistoryTable.tsx       |  5 +-
 .../performance/query-history/[id]/page.tsx   |  7 +-
 .../admin/performance/usage/UsageReports.tsx  |  4 +-
 .../app/ee/admin/performance/usage/page.tsx   |  1 -
 .../ee/admin/standard-answer/[id]/page.tsx    |  1 -
 .../app/ee/admin/standard-answer/new/page.tsx |  1 -
 web/src/app/ee/admin/standard-answer/page.tsx | 13 +--
 .../ee/admin/whitelabeling/ImageUpload.tsx    |  6 +-
 web/src/app/layout.tsx                        |  3 +-
 web/src/app/search/page.tsx                   |  2 +-
 web/src/components/Dropdown.tsx               | 22 +++--
 web/src/components/IsPublicGroupSelector.tsx  |  7 +-
 web/src/components/SSRAutoRefresh.tsx         |  2 +-
 web/src/components/UserDropdown.tsx           |  8 +-
 .../connectors/AccessTypeGroupSelector.tsx    |  9 +-
 .../components/assistants/AssistantIcon.tsx   |  1 +
 web/src/components/chat_search/Header.tsx     |  2 +-
 .../chat_search/ProviderContext.tsx           | 14 ++-
 web/src/components/chat_search/hooks.ts       |  1 +
 .../components/context/EmbeddingContext.tsx   |  2 +-
 web/src/components/context/FormContext.tsx    |  2 +-
 web/src/components/health/healthcheck.tsx     | 10 +--
 .../search/NoCompleteSourceModal.tsx          |  2 +-
 .../initialSetup/welcome/WelcomeModal.tsx     |  2 +-
 web/src/components/search/SearchSection.tsx   | 10 +--
 web/src/lib/sources.ts                        |  8 --
 web/src/lib/types.ts                          |  2 +-
 119 files changed, 368 insertions(+), 371 deletions(-)

diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
index 274e0006cd8..1021f82abc6 100644
--- a/backend/danswer/llm/chat_llm.py
+++ b/backend/danswer/llm/chat_llm.py
@@ -283,6 +283,7 @@ def _completion(
                 _convert_message_to_dict(msg) if isinstance(msg, BaseMessage) else msg
                 for msg in prompt
             ]
+
         elif isinstance(prompt, str):
             prompt = [_convert_message_to_dict(HumanMessage(content=prompt))]
 
diff --git a/web/.eslintrc.json b/web/.eslintrc.json
index bffb357a712..f0f3abee419 100644
--- a/web/.eslintrc.json
+++ b/web/.eslintrc.json
@@ -1,3 +1,6 @@
 {
-  "extends": "next/core-web-vitals"
+  "extends": "next/core-web-vitals",
+  "rules": {
+    "@next/next/no-img-element": "off"
+  }
 }
diff --git a/web/package.json b/web/package.json
index 190ec9b1aa6..1e55fec590e 100644
--- a/web/package.json
+++ b/web/package.json
@@ -56,4 +56,4 @@
     "eslint-config-next": "^14.1.0",
     "prettier": "2.8.8"
   }
-}
+}
\ No newline at end of file
diff --git a/web/src/app/admin/add-connector/page.tsx b/web/src/app/admin/add-connector/page.tsx
index 8d73131e69a..ecf459d612f 100644
--- a/web/src/app/admin/add-connector/page.tsx
+++ b/web/src/app/admin/add-connector/page.tsx
@@ -6,7 +6,7 @@ import { SourceCategory, SourceMetadata } from "@/lib/search/interfaces";
 import { listSourceMetadata } from "@/lib/sources";
 import { Title, Text, Button } from "@tremor/react";
 import Link from "next/link";
-import { useEffect, useMemo, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 
 function SourceTile({
   sourceMetadata,
@@ -49,15 +49,18 @@ export default function Page() {
       searchInputRef.current.focus();
     }
   }, []);
-  const filterSources = (sources: SourceMetadata[]) => {
-    if (!searchTerm) return sources;
-    const lowerSearchTerm = searchTerm.toLowerCase();
-    return sources.filter(
-      (source) =>
-        source.displayName.toLowerCase().includes(lowerSearchTerm) ||
-        source.category.toLowerCase().includes(lowerSearchTerm)
-    );
-  };
+  const filterSources = useCallback(
+    (sources: SourceMetadata[]) => {
+      if (!searchTerm) return sources;
+      const lowerSearchTerm = searchTerm.toLowerCase();
+      return sources.filter(
+        (source) =>
+          source.displayName.toLowerCase().includes(lowerSearchTerm) ||
+          source.category.toLowerCase().includes(lowerSearchTerm)
+      );
+    },
+    [searchTerm]
+  );
 
   const categorizedSources = useMemo(() => {
     const filtered = filterSources(sources);
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 731070783e5..b9bd5152149 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -132,7 +132,7 @@ export function AssistantEditor({
     if (defaultIconShape === null) {
       setDefaultIconShape(generateRandomIconShape().encodedGrid);
     }
-  }, []);
+  }, [defaultIconShape]);
 
   const [isIconDropdownOpen, setIsIconDropdownOpen] = useState(false);
 
@@ -166,7 +166,7 @@ export function AssistantEditor({
         existingPersona.num_chunks === 0
       );
     }
-  }, [isUpdate, existingPrompt]);
+  }, [isUpdate, existingPrompt, existingPersona?.num_chunks]);
 
   const defaultProvider = llmProviders.find(
     (llmProvider) => llmProvider.is_default_provider
@@ -888,7 +888,7 @@ export function AssistantEditor({
                                               values.document_set_ids.indexOf(
                                                 documentSet.id
                                               );
-                                            let isSelected = ind !== -1;
+                                            const isSelected = ind !== -1;
                                             return (
                                               <DocumentSetSelectable
                                                 key={documentSet.id}
diff --git a/web/src/app/admin/assistants/CollapsibleSection.tsx b/web/src/app/admin/assistants/CollapsibleSection.tsx
index 139f93d26e7..1ff3ce09a7c 100644
--- a/web/src/app/admin/assistants/CollapsibleSection.tsx
+++ b/web/src/app/admin/assistants/CollapsibleSection.tsx
@@ -1,5 +1,4 @@
 "use client";
-import { Button } from "@tremor/react";
 import React, { ReactNode, useState } from "react";
 import { FiSettings } from "react-icons/fi";
 
diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index 7a72d6e2e45..fad5853d88e 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -5,7 +5,7 @@ import { Persona } from "./interfaces";
 import { useRouter } from "next/navigation";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { useState, useMemo, useEffect } from "react";
+import { useState, useMemo } from "react";
 import { UniqueIdentifier } from "@dnd-kit/core";
 import { DraggableTable } from "@/components/table/DraggableTable";
 import {
@@ -15,8 +15,6 @@ import {
 } from "./lib";
 import { FiEdit2 } from "react-icons/fi";
 import { TrashIcon } from "@/components/icons/icons";
-import { getCurrentUser } from "@/lib/user";
-import { UserRole, User } from "@/lib/types";
 import { useUser } from "@/components/user/UserProvider";
 
 function PersonaTypeDisplay({ persona }: { persona: Persona }) {
@@ -51,9 +49,9 @@ export function PersonasTable({
 
   const { isLoadingUser, isAdmin } = useUser();
 
-  const editablePersonaIds = new Set(
-    editablePersonas.map((p) => p.id.toString())
-  );
+  const editablePersonaIds = useMemo(() => {
+    return new Set(editablePersonas.map((p) => p.id.toString()));
+  }, [editablePersonas]);
 
   const sortedPersonas = useMemo(() => {
     const editable = editablePersonas.sort(personaComparator);
diff --git a/web/src/app/admin/assistants/lib.ts b/web/src/app/admin/assistants/lib.ts
index fd5851ac7a3..9b79ea7ef6c 100644
--- a/web/src/app/admin/assistants/lib.ts
+++ b/web/src/app/admin/assistants/lib.ts
@@ -276,7 +276,7 @@ export function buildFinalPrompt(
   taskPrompt: string,
   retrievalDisabled: boolean
 ) {
-  let queryString = Object.entries({
+  const queryString = Object.entries({
     system_prompt: systemPrompt,
     task_prompt: taskPrompt,
     retrieval_disabled: retrievalDisabled,
diff --git a/web/src/app/admin/assistants/page.tsx b/web/src/app/admin/assistants/page.tsx
index 15909470582..7f3922ac4a7 100644
--- a/web/src/app/admin/assistants/page.tsx
+++ b/web/src/app/admin/assistants/page.tsx
@@ -5,7 +5,7 @@ import { Divider, Text, Title } from "@tremor/react";
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { Persona } from "./interfaces";
-import { AssistantsIcon, RobotIcon } from "@/components/icons/icons";
+import { AssistantsIcon } from "@/components/icons/icons";
 import { AdminPageTitle } from "@/components/admin/Title";
 
 export default async function Page() {
diff --git a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
index 4f79c79936a..9fb5aa063d2 100644
--- a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
+++ b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
@@ -20,7 +20,6 @@ import { Button, Card, Divider } from "@tremor/react";
 import { useRouter } from "next/navigation";
 import { Persona } from "../assistants/interfaces";
 import { useState } from "react";
-import MultiSelectDropdown from "@/components/MultiSelectDropdown";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import { DocumentSetSelectable } from "@/components/documentSet/DocumentSetSelectable";
 import CollapsibleSection from "../assistants/CollapsibleSection";
@@ -229,7 +228,7 @@ export const SlackBotCreationForm = ({
                                 const ind = values.document_sets.indexOf(
                                   documentSet.id
                                 );
-                                let isSelected = ind !== -1;
+                                const isSelected = ind !== -1;
 
                                 return (
                                   <DocumentSetSelectable
diff --git a/web/src/app/admin/bot/new/page.tsx b/web/src/app/admin/bot/new/page.tsx
index 094682c3696..79761934023 100644
--- a/web/src/app/admin/bot/new/page.tsx
+++ b/web/src/app/admin/bot/new/page.tsx
@@ -3,9 +3,8 @@ import { CPUIcon } from "@/components/icons/icons";
 import { SlackBotCreationForm } from "../SlackBotConfigCreationForm";
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { DocumentSet, StandardAnswerCategory } from "@/lib/types";
+import { DocumentSet } from "@/lib/types";
 import { BackButton } from "@/components/BackButton";
-import { Text } from "@tremor/react";
 import {
   FetchAssistantsResponse,
   fetchAssistantsSS,
diff --git a/web/src/app/admin/bot/page.tsx b/web/src/app/admin/bot/page.tsx
index c3ef70ccbf6..62f74277d1c 100644
--- a/web/src/app/admin/bot/page.tsx
+++ b/web/src/app/admin/bot/page.tsx
@@ -2,12 +2,7 @@
 
 import { ThreeDotsLoader } from "@/components/Loading";
 import { PageSelector } from "@/components/PageSelector";
-import {
-  CPUIcon,
-  EditIcon,
-  SlackIcon,
-  TrashIcon,
-} from "@/components/icons/icons";
+import { EditIcon, SlackIcon, TrashIcon } from "@/components/icons/icons";
 import { SlackBotConfig } from "@/lib/types";
 import { useState } from "react";
 import { useSlackBotConfigs, useSlackBotTokens } from "./hooks";
@@ -26,12 +21,7 @@ import {
   Text,
   Title,
 } from "@tremor/react";
-import {
-  FiArrowUpRight,
-  FiChevronDown,
-  FiChevronUp,
-  FiSlack,
-} from "react-icons/fi";
+import { FiArrowUpRight, FiChevronDown, FiChevronUp } from "react-icons/fi";
 import Link from "next/link";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import { ErrorCallout } from "@/components/ErrorCallout";
@@ -223,6 +213,7 @@ const Main = () => {
           className="text-blue-500"
           href="https://docs.danswer.dev/slack_bot_setup"
           target="_blank"
+          rel="noreferrer"
         >
           guide{" "}
         </a>
diff --git a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
index 66b306d7792..18ef2202c47 100644
--- a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
@@ -220,6 +220,7 @@ export function CustomLLMProviderUpdateForm({
                     target="_blank"
                     href="https://docs.litellm.ai/docs/providers"
                     className="text-link"
+                    rel="noreferrer"
                   >
                     https://docs.litellm.ai/docs/providers
                   </a>
@@ -373,6 +374,7 @@ export function CustomLLMProviderUpdateForm({
                     target="_blank"
                     href="https://models.litellm.ai/"
                     className="text-link"
+                    rel="noreferrer"
                   >
                     here
                   </a>
diff --git a/web/src/app/admin/configuration/llm/page.tsx b/web/src/app/admin/configuration/llm/page.tsx
index 9771a53c3af..e68b225dfa4 100644
--- a/web/src/app/admin/configuration/llm/page.tsx
+++ b/web/src/app/admin/configuration/llm/page.tsx
@@ -1,7 +1,6 @@
 "use client";
 
 import { AdminPageTitle } from "@/components/admin/Title";
-import { FiCpu } from "react-icons/fi";
 import { LLMConfiguration } from "./LLMConfiguration";
 import { CpuIcon } from "@/components/icons/icons";
 
diff --git a/web/src/app/admin/configuration/search/UpgradingPage.tsx b/web/src/app/admin/configuration/search/UpgradingPage.tsx
index bbbb75797c7..2d9415e10e8 100644
--- a/web/src/app/admin/configuration/search/UpgradingPage.tsx
+++ b/web/src/app/admin/configuration/search/UpgradingPage.tsx
@@ -63,13 +63,16 @@ export default function UpgradingPage({
     }
     setIsCancelling(false);
   };
-  const statusOrder: Record<ValidStatuses, number> = {
-    failed: 0,
-    completed_with_errors: 1,
-    not_started: 2,
-    in_progress: 3,
-    success: 4,
-  };
+  const statusOrder: Record<ValidStatuses, number> = useMemo(
+    () => ({
+      failed: 0,
+      completed_with_errors: 1,
+      not_started: 2,
+      in_progress: 3,
+      success: 4,
+    }),
+    []
+  );
 
   const sortedReindexingProgress = useMemo(() => {
     return [...(ongoingReIndexingStatus || [])].sort((a, b) => {
diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index 37cfac8740c..517fb1585c4 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { useEffect, useRef } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import {
   Table,
   TableHead,
@@ -10,9 +10,8 @@ import {
   TableCell,
   Text,
 } from "@tremor/react";
-import { CCPairFullInfo } from "./types";
+import { CCPairFullInfo, PaginatedIndexAttempts } from "./types";
 import { IndexAttemptStatus } from "@/components/Status";
-import { useState } from "react";
 import { PageSelector } from "@/components/PageSelector";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { buildCCPairInfoUrl } from "./lib";
@@ -22,7 +21,6 @@ import { ErrorCallout } from "@/components/ErrorCallout";
 import { InfoIcon, SearchIcon } from "@/components/icons/icons";
 import Link from "next/link";
 import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
-import { PaginatedIndexAttempts } from "./types";
 import { useRouter } from "next/navigation";
 import { Tooltip } from "@/components/tooltip/Tooltip";
 
@@ -61,47 +59,59 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
   // we use it to avoid duplicate requests
   const ongoingRequestsRef = useRef<Set<number>>(new Set());
 
-  const batchRetrievalUrlBuilder = (batchNum: number) =>
-    `${buildCCPairInfoUrl(ccPair.id)}/index-attempts?page=${batchNum}&page_size=${BATCH_SIZE * NUM_IN_PAGE}`;
+  const batchRetrievalUrlBuilder = useCallback(
+    (batchNum: number) => {
+      return `${buildCCPairInfoUrl(ccPair.id)}/index-attempts?page=${batchNum}&page_size=${BATCH_SIZE * NUM_IN_PAGE}`;
+    },
+    [ccPair.id]
+  );
 
   // This fetches and caches the data for a given batch number
-  const fetchBatchData = async (batchNum: number) => {
-    if (ongoingRequestsRef.current.has(batchNum)) return;
-    ongoingRequestsRef.current.add(batchNum);
+  const fetchBatchData = useCallback(
+    async (batchNum: number) => {
+      if (ongoingRequestsRef.current.has(batchNum)) return;
+      ongoingRequestsRef.current.add(batchNum);
 
-    try {
-      const response = await fetch(batchRetrievalUrlBuilder(batchNum + 1));
-      if (!response.ok) {
-        throw new Error("Failed to fetch data");
-      }
-      const data = await response.json();
+      try {
+        const response = await fetch(batchRetrievalUrlBuilder(batchNum + 1));
+        if (!response.ok) {
+          throw new Error("Failed to fetch data");
+        }
+        const data = await response.json();
 
-      const newBatchData: PaginatedIndexAttempts[] = [];
-      for (let i = 0; i < BATCH_SIZE; i++) {
-        const startIndex = i * NUM_IN_PAGE;
-        const endIndex = startIndex + NUM_IN_PAGE;
-        const pageIndexAttempts = data.index_attempts.slice(
-          startIndex,
-          endIndex
+        const newBatchData: PaginatedIndexAttempts[] = [];
+        for (let i = 0; i < BATCH_SIZE; i++) {
+          const startIndex = i * NUM_IN_PAGE;
+          const endIndex = startIndex + NUM_IN_PAGE;
+          const pageIndexAttempts = data.index_attempts.slice(
+            startIndex,
+            endIndex
+          );
+          newBatchData.push({
+            ...data,
+            index_attempts: pageIndexAttempts,
+          });
+        }
+
+        setCachedBatches((prev) => ({
+          ...prev,
+          [batchNum]: newBatchData,
+        }));
+      } catch (error) {
+        setCurrentPageError(
+          error instanceof Error ? error : new Error("An error occurred")
         );
-        newBatchData.push({
-          ...data,
-          index_attempts: pageIndexAttempts,
-        });
+      } finally {
+        ongoingRequestsRef.current.delete(batchNum);
       }
-
-      setCachedBatches((prev) => ({
-        ...prev,
-        [batchNum]: newBatchData,
-      }));
-    } catch (error) {
-      setCurrentPageError(
-        error instanceof Error ? error : new Error("An error occurred")
-      );
-    } finally {
-      ongoingRequestsRef.current.delete(batchNum);
-    }
-  };
+    },
+    [
+      ongoingRequestsRef,
+      setCachedBatches,
+      setCurrentPageError,
+      batchRetrievalUrlBuilder,
+    ]
+  );
 
   // This fetches and caches the data for the current batch and the next and previous batches
   useEffect(() => {
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index dc5ca7aeafb..91c2e197b4e 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -1,29 +1,29 @@
 "use client";
 
-import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
-import { CCPairStatus } from "@/components/Status";
 import { BackButton } from "@/components/BackButton";
-import { Button, Divider, Title } from "@tremor/react";
-import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
-import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
-import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
-import { DeletionButton } from "./DeletionButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { ReIndexButton } from "./ReIndexButton";
-import { ValidSources } from "@/lib/types";
-import useSWR, { mutate } from "swr";
-import { errorHandlingFetcher } from "@/lib/fetcher";
 import { ThreeDotsLoader } from "@/components/Loading";
-import CredentialSection from "@/components/credentials/CredentialSection";
-import { buildCCPairInfoUrl } from "./lib";
 import { SourceIcon } from "@/components/SourceIcon";
-import { credentialTemplates } from "@/lib/connectors/credentials";
-import { useEffect, useRef, useState } from "react";
-import { CheckmarkIcon, EditIcon, XIcon } from "@/components/icons/icons";
+import { CCPairStatus } from "@/components/Status";
 import { usePopup } from "@/components/admin/connectors/Popup";
+import CredentialSection from "@/components/credentials/CredentialSection";
+import { CheckmarkIcon, EditIcon, XIcon } from "@/components/icons/icons";
 import { updateConnectorCredentialPairName } from "@/lib/connector";
-import DeletionErrorStatus from "./DeletionErrorStatus";
+import { credentialTemplates } from "@/lib/connectors/credentials";
+import { errorHandlingFetcher } from "@/lib/fetcher";
+import { ValidSources } from "@/lib/types";
+import { Button, Divider, Title } from "@tremor/react";
 import { useRouter } from "next/navigation";
+import { useCallback, useEffect, useRef, useState } from "react";
+import useSWR, { mutate } from "swr";
+import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
+import { DeletionButton } from "./DeletionButton";
+import DeletionErrorStatus from "./DeletionErrorStatus";
+import { IndexingAttemptsTable } from "./IndexingAttemptsTable";
+import { ModifyStatusButtonCluster } from "./ModifyStatusButtonCluster";
+import { ReIndexButton } from "./ReIndexButton";
+import { buildCCPairInfoUrl } from "./lib";
+import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
 
 // since the uploaded files are cleaned up after some period of time
 // re-indexing will not work for the file connector. Also, it would not
@@ -49,7 +49,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
 
   const { popup, setPopup } = usePopup();
 
-  const finishConnectorDeletion = () => {
+  const finishConnectorDeletion = useCallback(() => {
     setPopup({
       message: "Connector deleted successfully",
       type: "success",
@@ -57,7 +57,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
     setTimeout(() => {
       router.push("/admin/indexing/status");
     }, 2000);
-  };
+  }, [router, setPopup]);
 
   useEffect(() => {
     if (isEditing && inputRef.current) {
@@ -80,7 +80,14 @@ function Main({ ccPairId }: { ccPairId: number }) {
     ) {
       finishConnectorDeletion();
     }
-  }, [isLoading, ccPair, error, hasLoadedOnce, router]);
+  }, [
+    isLoading,
+    ccPair,
+    error,
+    hasLoadedOnce,
+    router,
+    finishConnectorDeletion,
+  ]);
 
   const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
     setEditableName(e.target.value);
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index 07b719d8341..b01347edab6 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
+import { errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR, { mutate } from "swr";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 
@@ -29,6 +29,8 @@ import {
   defaultPruneFreqDays,
   defaultRefreshFreqMinutes,
   isLoadState,
+  Connector,
+  ConnectorBase,
 } from "@/lib/connectors/connectors";
 import { Modal } from "@/components/Modal";
 import GDriveMain from "./pages/gdrive/GoogleDrivePage";
@@ -47,7 +49,6 @@ export interface AdvancedConfig {
   pruneFreq: number;
   indexingStart: string;
 }
-import { Connector, ConnectorBase } from "@/lib/connectors/connectors";
 
 const BASE_CONNECTOR_URL = "/api/manage/admin/connector";
 
diff --git a/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx b/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx
index c038cdbb4b2..282aafbfbb6 100644
--- a/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx
+++ b/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { ConfigurableSources, ValidSources } from "@/lib/types";
+import { ConfigurableSources } from "@/lib/types";
 import AddConnector from "./AddConnectorPage";
 import { FormProvider } from "@/components/context/FormContext";
 import Sidebar from "./Sidebar";
diff --git a/web/src/app/admin/connectors/[connector]/Sidebar.tsx b/web/src/app/admin/connectors/[connector]/Sidebar.tsx
index 4b7f2970daf..c31e3930c24 100644
--- a/web/src/app/admin/connectors/[connector]/Sidebar.tsx
+++ b/web/src/app/admin/connectors/[connector]/Sidebar.tsx
@@ -1,7 +1,7 @@
 import { useFormContext } from "@/components/context/FormContext";
 import { HeaderTitle } from "@/components/header/HeaderTitle";
 
-import { BackIcon, SettingsIcon } from "@/components/icons/icons";
+import { SettingsIcon } from "@/components/icons/icons";
 import { Logo } from "@/components/Logo";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { credentialTemplates } from "@/lib/connectors/credentials";
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/NumberInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/NumberInput.tsx
index b7fcb49cf1e..0f19c22adb5 100644
--- a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/NumberInput.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/NumberInput.tsx
@@ -1,5 +1,5 @@
 import { SubLabel } from "@/components/admin/connectors/Field";
-import { Field, useFormikContext } from "formik";
+import { Field } from "formik";
 
 export default function NumberInput({
   label,
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx
index a7c14deec20..2f926e64176 100644
--- a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/SelectInput.tsx
@@ -1,9 +1,5 @@
 import CredentialSubText from "@/components/credentials/CredentialFields";
-import {
-  ListOption,
-  SelectOption,
-  StringWithDescription,
-} from "@/lib/connectors/connectors";
+import { StringWithDescription } from "@/lib/connectors/connectors";
 import { Field } from "formik";
 
 export default function SelectInput({
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
index 8fc1fa76787..371bbef6dd1 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
@@ -10,15 +10,13 @@ import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
 import Cookies from "js-cookie";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
-import { Card } from "@tremor/react";
+import { Button as TremorButton } from "@tremor/react";
 import {
   Credential,
   GoogleDriveCredentialJson,
   GoogleDriveServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
 
-import { Button as TremorButton } from "@tremor/react";
-
 type GoogleDriveCredentialJsonTypes = "authorized_user" | "service_account";
 
 export const DriveJsonUpload = ({
@@ -285,6 +283,7 @@ export const DriveJsonUploadSection = ({
           className="text-link"
           target="_blank"
           href="https://docs.danswer.dev/connectors/google_drive#authorization"
+          rel="noreferrer"
         >
           here
         </a>{" "}
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
index 247b64e61b4..d8a14db03a1 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
@@ -1,14 +1,16 @@
 "use client";
 
 import React from "react";
-import { useState, useEffect } from "react";
 import useSWR from "swr";
 import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { LoadingAnimation } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { ConnectorIndexingStatus } from "@/lib/types";
-import { usePublicCredentials } from "@/lib/hooks";
+import {
+  usePublicCredentials,
+  useConnectorCredentialIndexingStatus,
+} from "@/lib/hooks";
 import { Title } from "@tremor/react";
 import { DriveJsonUploadSection, DriveOAuthSection } from "./Credential";
 import {
@@ -18,7 +20,6 @@ import {
 } from "@/lib/connectors/credentials";
 import { GoogleDriveConfig } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
-import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
 
 const GDriveMain = ({}: {}) => {
   const { isLoadingUser, isAdmin } = useUser();
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
index 8b456884f1a..51502bd443f 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
@@ -271,6 +271,7 @@ export const GmailJsonUploadSection = ({
           className="text-link"
           target="_blank"
           href="https://docs.danswer.dev/connectors/gmail#authorization"
+          rel="noreferrer"
         >
           here
         </a>{" "}
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
index 5f52eb31013..1a7fbe5de69 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
@@ -5,20 +5,19 @@ import { errorHandlingFetcher } from "@/lib/fetcher";
 import { LoadingAnimation } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { ConnectorIndexingStatus } from "@/lib/types";
-import { getCurrentUser } from "@/lib/user";
-import { User, UserRole } from "@/lib/types";
 import {
   Credential,
   GmailCredentialJson,
   GmailServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
 import { GmailOAuthSection, GmailJsonUploadSection } from "./Credential";
-import { usePublicCredentials } from "@/lib/hooks";
+import {
+  usePublicCredentials,
+  useConnectorCredentialIndexingStatus,
+} from "@/lib/hooks";
 import { Title } from "@tremor/react";
 import { GmailConfig } from "@/lib/connectors/connectors";
-import { useState, useEffect } from "react";
 import { useUser } from "@/components/user/UserProvider";
-import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
 
 export const GmailMain = () => {
   const { isLoadingUser, isAdmin } = useUser();
diff --git a/web/src/app/admin/documents/explorer/Explorer.tsx b/web/src/app/admin/documents/explorer/Explorer.tsx
index c1722b01edf..4d47d6c7af6 100644
--- a/web/src/app/admin/documents/explorer/Explorer.tsx
+++ b/web/src/app/admin/documents/explorer/Explorer.tsx
@@ -2,7 +2,7 @@
 
 import { adminSearch } from "./lib";
 import { MagnifyingGlass } from "@phosphor-icons/react";
-import { useState, useEffect } from "react";
+import { useState, useEffect, useCallback } from "react";
 import { DanswerDocument } from "@/lib/search/interfaces";
 import { buildDocumentSummaryDisplay } from "@/components/search/DocumentDisplay";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
@@ -121,19 +121,27 @@ export function Explorer({
 
   const filterManager = useFilters();
 
-  const onSearch = async (query: string) => {
-    const filters = buildFilters(
-      filterManager.selectedSources,
+  const onSearch = useCallback(
+    async (query: string) => {
+      const filters = buildFilters(
+        filterManager.selectedSources,
+        filterManager.selectedDocumentSets,
+        filterManager.timeRange,
+        filterManager.selectedTags
+      );
+      const results = await adminSearch(query, filters);
+      if (results.ok) {
+        setResults((await results.json()).documents);
+      }
+      setTimeoutId(null);
+    },
+    [
       filterManager.selectedDocumentSets,
+      filterManager.selectedSources,
       filterManager.timeRange,
-      filterManager.selectedTags
-    );
-    const results = await adminSearch(query, filters);
-    if (results.ok) {
-      setResults((await results.json()).documents);
-    }
-    setTimeoutId(null);
-  };
+      filterManager.selectedTags,
+    ]
+  );
 
   useEffect(() => {
     if (timeoutId !== null) {
diff --git a/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx b/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx
index 936afad1209..09713b104ff 100644
--- a/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx
+++ b/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx
@@ -1,4 +1,3 @@
-import { BasicTable } from "@/components/admin/connectors/BasicTable";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { useState } from "react";
 import {
diff --git a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
index fb7e56cc60c..8da5ee0a8b8 100644
--- a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
+++ b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
@@ -177,7 +177,7 @@ export const DocumentSetCreationForm = ({
                               const ind = props.values.cc_pair_ids.indexOf(
                                 ccPair.cc_pair_id
                               );
-                              let isSelected = ind !== -1;
+                              const isSelected = ind !== -1;
                               return (
                                 <div
                                   key={`${ccPair.connector.id}-${ccPair.credential.id}`}
@@ -291,7 +291,7 @@ export const DocumentSetCreationForm = ({
                           const ind = props.values.cc_pair_ids.indexOf(
                             ccPair.cc_pair_id
                           );
-                          let isSelected = ind !== -1;
+                          const isSelected = ind !== -1;
                           return (
                             <div
                               key={`${ccPair.connector.id}-${ccPair.credential.id}`}
diff --git a/web/src/app/admin/documents/sets/new/page.tsx b/web/src/app/admin/documents/sets/new/page.tsx
index 24b337460b8..c44f4b9502f 100644
--- a/web/src/app/admin/documents/sets/new/page.tsx
+++ b/web/src/app/admin/documents/sets/new/page.tsx
@@ -13,7 +13,6 @@ import { Card } from "@tremor/react";
 import { BackButton } from "@/components/BackButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { useRouter } from "next/navigation";
-import { UserGroup } from "@/lib/types";
 import { refreshDocumentSets } from "../hooks";
 
 function Main() {
diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx
index 41104f9c343..a4a3a227048 100644
--- a/web/src/app/admin/documents/sets/page.tsx
+++ b/web/src/app/admin/documents/sets/page.tsx
@@ -13,18 +13,17 @@ import {
   Title,
   Divider,
   Badge,
+  Button,
+  Text,
 } from "@tremor/react";
 import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
 import { ConnectorIndexingStatus, DocumentSet } from "@/lib/types";
-import { useState, useEffect } from "react";
-import { getCurrentUser } from "@/lib/user";
-import { User, UserRole } from "@/lib/types";
+import { useState } from "react";
 import { useDocumentSets } from "./hooks";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 import { deleteDocumentSet } from "./lib";
 import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { AdminPageTitle } from "@/components/admin/Title";
-import { Button, Text } from "@tremor/react";
 import {
   FiAlertTriangle,
   FiCheckCircle,
diff --git a/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx b/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
index 2b4394c56b5..54d180f6498 100644
--- a/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
+++ b/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
@@ -6,7 +6,6 @@ import { Dispatch, SetStateAction, useState } from "react";
 import {
   CloudEmbeddingProvider,
   CloudEmbeddingModel,
-  AVAILABLE_CLOUD_PROVIDERS,
   AVAILABLE_MODELS,
   INVALID_OLD_MODEL,
   HostedEmbeddingModel,
diff --git a/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx b/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
index 636aa562474..912bd569a98 100644
--- a/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
@@ -1,7 +1,7 @@
 import React, { useRef, useState } from "react";
 import { Modal } from "@/components/Modal";
 import { Button, Text, Callout, Subtitle, Divider } from "@tremor/react";
-import { Label, TextFormField } from "@/components/admin/connectors/Field";
+import { Label } from "@/components/admin/connectors/Field";
 import { CloudEmbeddingProvider } from "../../../../components/embedding/interfaces";
 import {
   EMBEDDING_PROVIDERS_ADMIN_URL,
diff --git a/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx b/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
index 3f59de11657..3e4c4936832 100644
--- a/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
@@ -1,9 +1,6 @@
 import { Modal } from "@/components/Modal";
 import { Button, Text, Callout } from "@tremor/react";
-import {
-  EmbeddingModelDescriptor,
-  HostedEmbeddingModel,
-} from "../../../../components/embedding/interfaces";
+import { HostedEmbeddingModel } from "../../../../components/embedding/interfaces";
 
 export function ModelSelectionConfirmationModal({
   selectedModel,
diff --git a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
index 2377d5d16d5..7f127d8ef47 100644
--- a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
@@ -167,6 +167,7 @@ export function ProviderCreationModal({
                   className="cursor-pointer underline"
                   target="_blank"
                   href={selectedProvider.docsLink}
+                  rel="noreferrer"
                 >
                   here
                 </a>{" "}
@@ -175,6 +176,7 @@ export function ProviderCreationModal({
                   className="cursor-pointer underline"
                   target="_blank"
                   href={selectedProvider.apiLink}
+                  rel="noreferrer"
                 >
                   {isProxy ? "API URL" : "API KEY"}
                 </a>
@@ -223,6 +225,7 @@ export function ProviderCreationModal({
                   href={selectedProvider.apiLink}
                   target="_blank"
                   className="underline cursor-pointer"
+                  rel="noreferrer"
                 >
                   Learn more here
                 </a>
diff --git a/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
index c965bdfabf8..b708675116b 100644
--- a/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
@@ -1,7 +1,6 @@
-import React, { Dispatch, forwardRef, SetStateAction } from "react";
+import React, { forwardRef } from "react";
 import { Formik, Form, FormikProps, FieldArray, Field } from "formik";
 import * as Yup from "yup";
-import CredentialSubText from "@/components/credentials/CredentialFields";
 import { TrashIcon } from "@/components/icons/icons";
 import { FaPlus } from "react-icons/fa";
 import { AdvancedSearchConfiguration } from "../interfaces";
diff --git a/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx b/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
index a6c71530f24..dd202dc4f4b 100644
--- a/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
+++ b/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { Button, Card, Text, Title } from "@tremor/react";
+import { Card, Text, Title } from "@tremor/react";
 
 import {
   CloudEmbeddingProvider,
@@ -56,7 +56,7 @@ export default function CloudEmbeddingPage({
     );
   }
 
-  let providers: CloudEmbeddingProviderFull[] = AVAILABLE_CLOUD_PROVIDERS.map(
+  const providers: CloudEmbeddingProviderFull[] = AVAILABLE_CLOUD_PROVIDERS.map(
     (model) => ({
       ...model,
       configured:
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index 196c99da0d2..6060868e85c 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -165,7 +165,7 @@ export default function EmbeddingForm() {
   }
 
   const updateSearch = async () => {
-    let values: SavedSearchSettings = {
+    const values: SavedSearchSettings = {
       ...rerankingDetails,
       ...advancedEmbeddingDetails,
       provider_type:
diff --git a/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx b/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
index 2e28ce8e4b8..1b573cdd3df 100644
--- a/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
+++ b/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
@@ -1,5 +1,5 @@
 "use client";
-import { Button, Card, Text } from "@tremor/react";
+import { Button, Card, Text, Title } from "@tremor/react";
 import { ModelSelector } from "../../../../components/embedding/ModelSelector";
 import {
   AVAILABLE_MODELS,
@@ -8,7 +8,6 @@ import {
 } from "../../../../components/embedding/interfaces";
 import { CustomModelForm } from "../../../../components/embedding/CustomModelForm";
 import { useState } from "react";
-import { Title } from "@tremor/react";
 export default function OpenEmbeddingPage({
   onSelectOpenSource,
   selectedProvider,
@@ -34,7 +33,12 @@ export default function OpenEmbeddingPage({
 
       <Text className="mt-6">
         Alternatively, (if you know what you&apos;re doing) you can specify a{" "}
-        <a target="_blank" href="https://www.sbert.net/" className="text-link">
+        <a
+          target="_blank"
+          href="https://www.sbert.net/"
+          className="text-link"
+          rel="noreferrer"
+        >
           SentenceTransformers
         </a>
         -compatible model of your choice below. The rough list of supported
@@ -43,6 +47,7 @@ export default function OpenEmbeddingPage({
           target="_blank"
           href="https://huggingface.co/models?library=sentence-transformers&sort=trending"
           className="text-link"
+          rel="noreferrer"
         >
           here
         </a>
diff --git a/web/src/app/admin/indexing/[id]/page.tsx b/web/src/app/admin/indexing/[id]/page.tsx
index 51fe694541c..041f7bda9ba 100644
--- a/web/src/app/admin/indexing/[id]/page.tsx
+++ b/web/src/app/admin/indexing/[id]/page.tsx
@@ -4,7 +4,6 @@ import { BackButton } from "@/components/BackButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { errorHandlingFetcher } from "@/lib/fetcher";
-import { ValidSources } from "@/lib/types";
 import { Title } from "@tremor/react";
 import useSWR from "swr";
 import { IndexAttemptErrorsTable } from "./IndexAttemptErrorsTable";
diff --git a/web/src/app/admin/indexing/status/page.tsx b/web/src/app/admin/indexing/status/page.tsx
index bb56ceaa74e..e5f1ea6ce0d 100644
--- a/web/src/app/admin/indexing/status/page.tsx
+++ b/web/src/app/admin/indexing/status/page.tsx
@@ -1,11 +1,7 @@
 "use client";
 
-import useSWR from "swr";
-
 import { LoadingAnimation } from "@/components/Loading";
 import { NotebookIcon } from "@/components/icons/icons";
-import { errorHandlingFetcher } from "@/lib/fetcher";
-import { ConnectorIndexingStatus } from "@/lib/types";
 import { CCPairIndexingStatusTable } from "./CCPairIndexingStatusTable";
 import { AdminPageTitle } from "@/components/admin/Title";
 import Link from "next/link";
diff --git a/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx b/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
index 0d385f48479..be83a19e926 100644
--- a/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
+++ b/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
@@ -1,8 +1,8 @@
 import React from "react";
-import { Formik, Form, Field, ErrorMessage } from "formik";
+import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { ModalWrapper } from "@/components/modals/ModalWrapper";
-import { Button, Textarea, TextInput } from "@tremor/react";
+import { Button } from "@tremor/react";
 
 import { BookstackIcon } from "@/components/icons/icons";
 import { AddPromptModalProps } from "../interfaces";
diff --git a/web/src/app/admin/prompt-library/promptLibrary.tsx b/web/src/app/admin/prompt-library/promptLibrary.tsx
index 18cb5927b50..a5427bdf04b 100644
--- a/web/src/app/admin/prompt-library/promptLibrary.tsx
+++ b/web/src/app/admin/prompt-library/promptLibrary.tsx
@@ -16,7 +16,6 @@ import { FilterDropdown } from "@/components/search/filtering/FilterDropdown";
 import { FiTag } from "react-icons/fi";
 import { PageSelector } from "@/components/PageSelector";
 import { InputPrompt } from "./interfaces";
-import { Modal } from "@/components/Modal";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
 
 const CategoryBubble = ({
diff --git a/web/src/app/admin/settings/SettingsForm.tsx b/web/src/app/admin/settings/SettingsForm.tsx
index 03a0171363e..9017c920af8 100644
--- a/web/src/app/admin/settings/SettingsForm.tsx
+++ b/web/src/app/admin/settings/SettingsForm.tsx
@@ -2,15 +2,13 @@
 
 import { Label, SubLabel } from "@/components/admin/connectors/Field";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { Title } from "@tremor/react";
+import { Title, Button } from "@tremor/react";
 import { Settings } from "./interfaces";
 import { useRouter } from "next/navigation";
 import { DefaultDropdown, Option } from "@/components/Dropdown";
-import { useContext } from "react";
+import React, { useContext, useState, useEffect } from "react";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
-import React, { useState, useEffect } from "react";
 import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
-import { Button } from "@tremor/react";
 
 function Checkbox({
   label,
diff --git a/web/src/app/admin/token-rate-limits/page.tsx b/web/src/app/admin/token-rate-limits/page.tsx
index fb4b711a2e0..c64a8cc5f1f 100644
--- a/web/src/app/admin/token-rate-limits/page.tsx
+++ b/web/src/app/admin/token-rate-limits/page.tsx
@@ -11,7 +11,7 @@ import {
   Text,
 } from "@tremor/react";
 import { useState } from "react";
-import { FiGlobe, FiShield, FiUser, FiUsers } from "react-icons/fi";
+import { FiGlobe, FiUser, FiUsers } from "react-icons/fi";
 import {
   insertGlobalTokenRateLimit,
   insertGroupTokenRateLimit,
diff --git a/web/src/app/admin/tools/ToolEditor.tsx b/web/src/app/admin/tools/ToolEditor.tsx
index 0ffb8750ef6..b6d080ebd15 100644
--- a/web/src/app/admin/tools/ToolEditor.tsx
+++ b/web/src/app/admin/tools/ToolEditor.tsx
@@ -13,7 +13,7 @@ import {
 import * as Yup from "yup";
 import { MethodSpec, ToolSnapshot } from "@/lib/tools/interfaces";
 import { TextFormField } from "@/components/admin/connectors/Field";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button, Divider } from "@tremor/react";
 import {
   createCustomTool,
   updateCustomTool,
@@ -64,28 +64,31 @@ function ToolForm({
   const [definitionError, setDefinitionError] = definitionErrorState;
   const [methodSpecs, setMethodSpecs] = methodSpecsState;
   const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
-
   const debouncedValidateDefinition = useCallback(
-    debounce(async (definition: string) => {
-      try {
-        const parsedDefinition = parseJsonWithTrailingCommas(definition);
-        const response = await validateToolDefinition({
-          definition: parsedDefinition,
-        });
-        if (response.error) {
+    (definition: string) => {
+      const validateDefinition = async () => {
+        try {
+          const parsedDefinition = parseJsonWithTrailingCommas(definition);
+          const response = await validateToolDefinition({
+            definition: parsedDefinition,
+          });
+          if (response.error) {
+            setMethodSpecs(null);
+            setDefinitionError(response.error);
+          } else {
+            setMethodSpecs(response.data);
+            setDefinitionError(null);
+          }
+        } catch (error) {
+          console.log(error);
           setMethodSpecs(null);
-          setDefinitionError(response.error);
-        } else {
-          setMethodSpecs(response.data);
-          setDefinitionError(null);
+          setDefinitionError("Invalid JSON format");
         }
-      } catch (error) {
-        console.log(error);
-        setMethodSpecs(null);
-        setDefinitionError("Invalid JSON format");
-      }
-    }, 300),
-    []
+      };
+
+      debounce(validateDefinition, 300)();
+    },
+    [setMethodSpecs, setDefinitionError]
   );
 
   useEffect(() => {
diff --git a/web/src/app/admin/tools/ToolsTable.tsx b/web/src/app/admin/tools/ToolsTable.tsx
index 88b91eddafa..a283718afc6 100644
--- a/web/src/app/admin/tools/ToolsTable.tsx
+++ b/web/src/app/admin/tools/ToolsTable.tsx
@@ -1,7 +1,6 @@
 "use client";
 
 import {
-  Text,
   Table,
   TableHead,
   TableRow,
diff --git a/web/src/app/admin/tools/edit/[toolId]/page.tsx b/web/src/app/admin/tools/edit/[toolId]/page.tsx
index 8ae1e908a2d..7a9e86b2fff 100644
--- a/web/src/app/admin/tools/edit/[toolId]/page.tsx
+++ b/web/src/app/admin/tools/edit/[toolId]/page.tsx
@@ -3,7 +3,6 @@ import { Card, Text, Title } from "@tremor/react";
 import { ToolEditor } from "@/app/admin/tools/ToolEditor";
 import { fetchToolByIdSS } from "@/lib/tools/fetchTools";
 import { DeleteToolButton } from "./DeleteToolButton";
-import { FiTool } from "react-icons/fi";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { BackButton } from "@/components/BackButton";
 import { ToolIcon } from "@/components/icons/icons";
diff --git a/web/src/app/admin/tools/new/page.tsx b/web/src/app/admin/tools/new/page.tsx
index efff155be58..79042eb670e 100644
--- a/web/src/app/admin/tools/new/page.tsx
+++ b/web/src/app/admin/tools/new/page.tsx
@@ -5,7 +5,6 @@ import { BackButton } from "@/components/BackButton";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { ToolIcon } from "@/components/icons/icons";
 import { Card } from "@tremor/react";
-import { FiTool } from "react-icons/fi";
 
 export default function NewToolPage() {
   return (
diff --git a/web/src/app/admin/tools/page.tsx b/web/src/app/admin/tools/page.tsx
index 543f89ac367..a1b20c79dfc 100644
--- a/web/src/app/admin/tools/page.tsx
+++ b/web/src/app/admin/tools/page.tsx
@@ -1,6 +1,6 @@
 import { ToolsTable } from "./ToolsTable";
 import { ToolSnapshot } from "@/lib/tools/interfaces";
-import { FiPlusSquare, FiTool } from "react-icons/fi";
+import { FiPlusSquare } from "react-icons/fi";
 import Link from "next/link";
 import { Divider, Text, Title } from "@tremor/react";
 import { fetchSS } from "@/lib/utilsSS";
diff --git a/web/src/app/admin/users/page.tsx b/web/src/app/admin/users/page.tsx
index 0c258cd857c..8d34594c881 100644
--- a/web/src/app/admin/users/page.tsx
+++ b/web/src/app/admin/users/page.tsx
@@ -12,7 +12,6 @@ import { AdminPageTitle } from "@/components/admin/Title";
 import { usePopup, PopupSpec } from "@/components/admin/connectors/Popup";
 import { UsersIcon } from "@/components/icons/icons";
 import { errorHandlingFetcher } from "@/lib/fetcher";
-import { type User, UserStatus } from "@/lib/types";
 import useSWR, { mutate } from "swr";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { HidableSection } from "@/app/admin/assistants/HidableSection";
diff --git a/web/src/app/assistants/SidebarWrapper.tsx b/web/src/app/assistants/SidebarWrapper.tsx
index 2feae589240..9a1d320d735 100644
--- a/web/src/app/assistants/SidebarWrapper.tsx
+++ b/web/src/app/assistants/SidebarWrapper.tsx
@@ -6,7 +6,14 @@ import { Folder } from "@/app/chat/folders/interfaces";
 import { User } from "@/lib/types";
 import Cookies from "js-cookie";
 import { SIDEBAR_TOGGLED_COOKIE_NAME } from "@/components/resizable/constants";
-import { ReactNode, useContext, useEffect, useRef, useState } from "react";
+import {
+  ReactNode,
+  useCallback,
+  useContext,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
 import { useSidebarVisibility } from "@/components/chat_search/hooks";
 import FunctionalHeader from "@/components/chat_search/Header";
 import { useRouter } from "next/navigation";
@@ -54,7 +61,7 @@ export default function SidebarWrapper<T extends object>({
     }, 200);
   };
 
-  const toggleSidebar = () => {
+  const toggleSidebar = useCallback(() => {
     Cookies.set(
       SIDEBAR_TOGGLED_COOKIE_NAME,
       String(!toggledSidebar).toLocaleLowerCase()
@@ -63,7 +70,7 @@ export default function SidebarWrapper<T extends object>({
         path: "/",
       };
     setToggledSidebar((toggledSidebar) => !toggledSidebar);
-  };
+  }, [toggledSidebar]);
 
   const sidebarElementRef = useRef<HTMLDivElement>(null);
 
diff --git a/web/src/app/assistants/gallery/AssistantsGallery.tsx b/web/src/app/assistants/gallery/AssistantsGallery.tsx
index 8926238b454..635bb80cf54 100644
--- a/web/src/app/assistants/gallery/AssistantsGallery.tsx
+++ b/web/src/app/assistants/gallery/AssistantsGallery.tsx
@@ -4,7 +4,6 @@ import { Persona } from "@/app/admin/assistants/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 import { User } from "@/lib/types";
 import { Button } from "@tremor/react";
-import Link from "next/link";
 import { useState } from "react";
 import { FiList, FiMinus, FiPlus } from "react-icons/fi";
 import { AssistantsPageTitle } from "../AssistantsPageTitle";
diff --git a/web/src/app/assistants/mine/AssistantSharingModal.tsx b/web/src/app/assistants/mine/AssistantSharingModal.tsx
index b0e96b00070..a30f3ce088d 100644
--- a/web/src/app/assistants/mine/AssistantSharingModal.tsx
+++ b/web/src/app/assistants/mine/AssistantSharingModal.tsx
@@ -1,7 +1,7 @@
 import { useState } from "react";
 import { Modal } from "@/components/Modal";
 import { MinimalUserSnapshot, User } from "@/lib/types";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button } from "@tremor/react";
 import { FiPlus, FiX } from "react-icons/fi";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { SearchMultiSelectDropdown } from "@/components/Dropdown";
diff --git a/web/src/app/assistants/mine/AssistantsList.tsx b/web/src/app/assistants/mine/AssistantsList.tsx
index a16c22d3ac8..84eca4d13cf 100644
--- a/web/src/app/assistants/mine/AssistantsList.tsx
+++ b/web/src/app/assistants/mine/AssistantsList.tsx
@@ -1,15 +1,9 @@
 "use client";
 
-import React, {
-  Dispatch,
-  ReactNode,
-  SetStateAction,
-  useEffect,
-  useState,
-} from "react";
+import React, { Dispatch, SetStateAction, useEffect, useState } from "react";
 import { MinimalUserSnapshot, User } from "@/lib/types";
 import { Persona } from "@/app/admin/assistants/interfaces";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button, Divider } from "@tremor/react";
 import {
   FiEdit2,
   FiList,
@@ -51,8 +45,8 @@ import {
   SortableContext,
   sortableKeyboardCoordinates,
   verticalListSortingStrategy,
+  useSortable,
 } from "@dnd-kit/sortable";
-import { useSortable } from "@dnd-kit/sortable";
 
 import { DragHandle } from "@/components/table/DragHandle";
 import {
diff --git a/web/src/app/assistants/mine/WrappedInputPrompts.tsx b/web/src/app/assistants/mine/WrappedInputPrompts.tsx
index 4428b5244c3..31bcd28d97e 100644
--- a/web/src/app/assistants/mine/WrappedInputPrompts.tsx
+++ b/web/src/app/assistants/mine/WrappedInputPrompts.tsx
@@ -5,7 +5,6 @@ import { Folder } from "@/app/chat/folders/interfaces";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { User } from "@/lib/types";
 
-import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import { AssistantsPageTitle } from "../AssistantsPageTitle";
 import { useInputPrompts } from "@/app/admin/prompt-library/hooks";
 import { PromptSection } from "@/app/admin/prompt-library/promptSection";
diff --git a/web/src/app/auth/login/LoginText.tsx b/web/src/app/auth/login/LoginText.tsx
index b465ad33530..a875b407a65 100644
--- a/web/src/app/auth/login/LoginText.tsx
+++ b/web/src/app/auth/login/LoginText.tsx
@@ -1,7 +1,6 @@
 "use client";
 
-import React from "react";
-import { useContext } from "react";
+import React, { useContext } from "react";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 export const LoginText = () => {
diff --git a/web/src/app/auth/verify-email/Verify.tsx b/web/src/app/auth/verify-email/Verify.tsx
index aea4d1bfefb..2b5fd1dcc6e 100644
--- a/web/src/app/auth/verify-email/Verify.tsx
+++ b/web/src/app/auth/verify-email/Verify.tsx
@@ -2,7 +2,7 @@
 
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { useRouter, useSearchParams } from "next/navigation";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
 import { Text } from "@tremor/react";
 import { RequestNewVerificationEmail } from "../waiting-on-verification/RequestNewVerificationEmail";
 import { User } from "@/lib/types";
@@ -14,7 +14,7 @@ export function Verify({ user }: { user: User | null }) {
 
   const [error, setError] = useState("");
 
-  async function verify() {
+  const verify = useCallback(async () => {
     const token = searchParams.get("token");
     if (!token) {
       setError(
@@ -39,11 +39,11 @@ export function Verify({ user }: { user: User | null }) {
         `Failed to verify your email - ${errorDetail}. Please try requesting a new verification email.`
       );
     }
-  }
+  }, [searchParams, router]);
 
   useEffect(() => {
     verify();
-  }, []);
+  }, [verify]);
 
   return (
     <main>
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 4f052e7388c..70749119349 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -123,7 +123,7 @@ export function ChatPage({
   const router = useRouter();
   const searchParams = useSearchParams();
 
-  let {
+  const {
     chatSessions,
     availableSources,
     availableDocumentSets,
@@ -243,7 +243,8 @@ export function ChatPage({
         destructureValue(user?.preferences.default_model)
       );
     }
-  }, [liveAssistant]);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [liveAssistant, llmProviders, user?.preferences.default_model]);
 
   const stopGenerating = () => {
     const currentSession = currentSessionId();
@@ -436,6 +437,7 @@ export function ChatPage({
     }
 
     initialSessionFetch();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [existingChatSessionId]);
 
   const [message, setMessage] = useState(
@@ -696,7 +698,7 @@ export function ChatPage({
         finalAssistants.find((persona) => persona.id === defaultAssistantId)
       );
     }
-  }, [defaultAssistantId]);
+  }, [defaultAssistantId, finalAssistants, messageHistory.length]);
 
   const [
     selectedDocuments,
@@ -772,7 +774,7 @@ export function ChatPage({
   const handleInputResize = () => {
     setTimeout(() => {
       if (inputRef.current && lastMessageRef.current) {
-        let newHeight: number =
+        const newHeight: number =
           inputRef.current?.getBoundingClientRect().height!;
         const heightDifference = newHeight - previousHeight.current;
         if (
@@ -984,7 +986,7 @@ export function ChatPage({
     setAlternativeGeneratingAssistant(alternativeAssistantOverride);
     clientScrollToBottom();
     let currChatSessionId: number;
-    let isNewSession = chatSessionIdRef.current === null;
+    const isNewSession = chatSessionIdRef.current === null;
     const searchParamBasedChatSessionName =
       searchParams.get(SEARCH_PARAM_NAMES.TITLE) || null;
 
@@ -1067,7 +1069,7 @@ export function ChatPage({
 
     let answer = "";
 
-    let stopReason: StreamStopReason | null = null;
+    const stopReason: StreamStopReason | null = null;
     let query: string | null = null;
     let retrievalType: RetrievalType =
       selectedDocuments.length > 0
@@ -1668,17 +1670,22 @@ export function ChatPage({
 
   useEffect(() => {
     initializeVisibleRange();
-  }, [router, messageHistory, chatSessionIdRef.current]);
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [router, messageHistory]);
 
   useLayoutEffect(() => {
+    const scrollableDiv = scrollableDivRef.current;
+
     const handleScroll = () => {
       updateVisibleRangeBasedOnScroll();
     };
-    scrollableDivRef.current?.addEventListener("scroll", handleScroll);
+
+    scrollableDiv?.addEventListener("scroll", handleScroll);
 
     return () => {
-      scrollableDivRef.current?.removeEventListener("scroll", handleScroll);
+      scrollableDiv?.removeEventListener("scroll", handleScroll);
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [messageHistory]);
 
   const currentVisibleRange = visibleRange.get(currentSessionId()) || {
@@ -1728,6 +1735,7 @@ export function ChatPage({
     return () => {
       window.removeEventListener("keydown", handleKeyDown);
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [router]);
   const [sharedChatSession, setSharedChatSession] =
     useState<ChatSession | null>();
diff --git a/web/src/app/chat/ChatPopup.tsx b/web/src/app/chat/ChatPopup.tsx
index b26bd19b9b3..664cfffacb0 100644
--- a/web/src/app/chat/ChatPopup.tsx
+++ b/web/src/app/chat/ChatPopup.tsx
@@ -17,7 +17,7 @@ export function ChatPopup() {
     setCompletedFlow(
       localStorage.getItem(ALL_USERS_INITIAL_POPUP_FLOW_COMPLETED) === "true"
     );
-  });
+  }, []);
 
   const settings = useContext(SettingsContext);
   const enterpriseSettings = settings?.enterpriseSettings;
diff --git a/web/src/app/chat/RegenerateOption.tsx b/web/src/app/chat/RegenerateOption.tsx
index 7c0f97676d2..f28a83b03fa 100644
--- a/web/src/app/chat/RegenerateOption.tsx
+++ b/web/src/app/chat/RegenerateOption.tsx
@@ -14,7 +14,6 @@ import { destructureValue, getFinalLLM, structureValue } from "@/lib/llm/utils";
 import { useState } from "react";
 import { Hoverable } from "@/components/Hoverable";
 import { Popover } from "@/components/popover/Popover";
-import { FiStar } from "react-icons/fi";
 import { StarFeedback } from "@/components/icons/icons";
 import { IconType } from "react-icons";
 
diff --git a/web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx b/web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx
index 2c6acf3710b..85ac429c497 100644
--- a/web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx
+++ b/web/src/app/chat/documentSidebar/ChatDocumentDisplay.tsx
@@ -51,6 +51,7 @@ export function ChatDocumentDisplay({
             "rounded-lg flex font-bold flex-shrink truncate" +
             (document.link ? "" : "pointer-events-none")
           }
+          rel="noreferrer"
         >
           {isInternet ? (
             <InternetSearchIcon url={document.link} />
diff --git a/web/src/app/chat/documentSidebar/DocumentSidebar.tsx b/web/src/app/chat/documentSidebar/DocumentSidebar.tsx
index 57bfee363a8..e20fd8e6735 100644
--- a/web/src/app/chat/documentSidebar/DocumentSidebar.tsx
+++ b/web/src/app/chat/documentSidebar/DocumentSidebar.tsx
@@ -3,7 +3,7 @@ import { Divider, Text } from "@tremor/react";
 import { ChatDocumentDisplay } from "./ChatDocumentDisplay";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { removeDuplicateDocs } from "@/lib/documentUtils";
-import { Message, RetrievalType } from "../interfaces";
+import { Message } from "../interfaces";
 import { ForwardedRef, forwardRef } from "react";
 
 interface DocumentSidebarProps {
diff --git a/web/src/app/chat/files/InputBarPreview.tsx b/web/src/app/chat/files/InputBarPreview.tsx
index 5d473b9f2dc..d218db04f51 100644
--- a/web/src/app/chat/files/InputBarPreview.tsx
+++ b/web/src/app/chat/files/InputBarPreview.tsx
@@ -1,5 +1,5 @@
 import { useEffect, useRef, useState } from "react";
-import { ChatFileType, FileDescriptor } from "../interfaces";
+import { FileDescriptor } from "../interfaces";
 
 import { FiX, FiLoader, FiFileText } from "react-icons/fi";
 import { InputBarPreviewImage } from "./images/InputBarPreviewImage";
diff --git a/web/src/app/chat/files/images/InputBarPreviewImage.tsx b/web/src/app/chat/files/images/InputBarPreviewImage.tsx
index 51260af1d2e..a46d3c09814 100644
--- a/web/src/app/chat/files/images/InputBarPreviewImage.tsx
+++ b/web/src/app/chat/files/images/InputBarPreviewImage.tsx
@@ -30,6 +30,7 @@ export function InputBarPreviewImage({ fileId }: { fileId: string }) {
       `}
       >
         <img
+          alt="preview"
           onClick={() => setFullImageShowing(true)}
           className="h-8 w-8 object-cover rounded-lg bg-background cursor-pointer"
           src={buildImgUrl(fileId)}
diff --git a/web/src/app/chat/folders/FolderManagement.tsx b/web/src/app/chat/folders/FolderManagement.tsx
index b1d245147ce..7e5abca0a42 100644
--- a/web/src/app/chat/folders/FolderManagement.tsx
+++ b/web/src/app/chat/folders/FolderManagement.tsx
@@ -1,5 +1,3 @@
-import { useState, useEffect, FC } from "react";
-
 // Function to create a new folder
 export async function createFolder(folderName: string): Promise<number> {
   const response = await fetch("/api/folder", {
diff --git a/web/src/app/chat/input/ChatInputBar.tsx b/web/src/app/chat/input/ChatInputBar.tsx
index cef0bc49f50..cce1bba53d9 100644
--- a/web/src/app/chat/input/ChatInputBar.tsx
+++ b/web/src/app/chat/input/ChatInputBar.tsx
@@ -98,7 +98,7 @@ export function ChatInputBar({
         MAX_INPUT_HEIGHT
       )}px`;
     }
-  }, [message]);
+  }, [message, textAreaRef]);
 
   const handlePaste = (event: React.ClipboardEvent) => {
     const items = event.clipboardData?.items;
diff --git a/web/src/app/chat/input/ChatInputOption.tsx b/web/src/app/chat/input/ChatInputOption.tsx
index d2d7bc5fde9..5d5e9d45f47 100644
--- a/web/src/app/chat/input/ChatInputOption.tsx
+++ b/web/src/app/chat/input/ChatInputOption.tsx
@@ -1,9 +1,5 @@
 import React, { useState, useRef, useEffect } from "react";
-import {
-  ChevronDownIcon,
-  ChevronRightIcon,
-  IconProps,
-} from "@/components/icons/icons";
+import { ChevronDownIcon, IconProps } from "@/components/icons/icons";
 
 interface ChatInputOptionProps {
   name?: string;
diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx
index 01090aa5637..73ddd63c6ee 100644
--- a/web/src/app/chat/lib.tsx
+++ b/web/src/app/chat/lib.tsx
@@ -4,16 +4,9 @@ import {
   Filters,
   StreamStopInfo,
 } from "@/lib/search/interfaces";
-import { handleSSEStream, handleStream } from "@/lib/search/streamingUtils";
+import { handleSSEStream } from "@/lib/search/streamingUtils";
 import { ChatState, FeedbackType } from "./types";
-import {
-  Dispatch,
-  MutableRefObject,
-  RefObject,
-  SetStateAction,
-  useEffect,
-  useRef,
-} from "react";
+import { MutableRefObject, RefObject, useEffect, useRef } from "react";
 import {
   BackendMessage,
   ChatSession,
@@ -664,7 +657,7 @@ export async function useScrollonStream({
 
   useEffect(() => {
     if (chatState != "input" && scrollableDivRef && scrollableDivRef.current) {
-      let newHeight: number = scrollableDivRef.current?.scrollTop!;
+      const newHeight: number = scrollableDivRef.current?.scrollTop!;
       const heightDifference = newHeight - previousScroll.current;
       previousScroll.current = newHeight;
 
@@ -729,5 +722,5 @@ export async function useScrollonStream({
         });
       }
     }
-  }, [chatState]);
+  }, [chatState, distance, scrollDist, scrollableDivRef]);
 }
diff --git a/web/src/app/chat/message/ContinueMessage.tsx b/web/src/app/chat/message/ContinueMessage.tsx
index 097b3e57e33..e60a13a5323 100644
--- a/web/src/app/chat/message/ContinueMessage.tsx
+++ b/web/src/app/chat/message/ContinueMessage.tsx
@@ -1,6 +1,6 @@
 import { EmphasizedClickable } from "@/components/BasicClickable";
 import { useEffect, useState } from "react";
-import { FiBook, FiPlayCircle } from "react-icons/fi";
+import { FiPlayCircle } from "react-icons/fi";
 
 export function ContinueGenerating({
   handleContinueGenerating,
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index e10f5cea03c..7a8a47ced54 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -202,7 +202,7 @@ export const AIMessage = ({
 
   const selectedDocumentIds =
     selectedDocuments?.map((document) => document.document_id) || [];
-  let citedDocumentIds: string[] = [];
+  const citedDocumentIds: string[] = [];
 
   citedDocuments?.forEach((doc) => {
     citedDocumentIds.push(doc[1].document_id);
@@ -272,7 +272,7 @@ export const AIMessage = ({
         );
       },
     }),
-    [messageId, content]
+    [finalContent]
   );
 
   const renderedMarkdown = useMemo(() => {
@@ -286,7 +286,7 @@ export const AIMessage = ({
         {finalContent as string}
       </ReactMarkdown>
     );
-  }, [finalContent]);
+  }, [finalContent, markdownComponents]);
 
   const includeMessageSwitcher =
     currentMessageInd !== undefined &&
@@ -412,6 +412,7 @@ export const AIMessage = ({
                                     href={doc.link || undefined}
                                     target="_blank"
                                     className="text-sm flex w-full pt-1 gap-x-1.5 overflow-hidden justify-between font-semibold text-text-700"
+                                    rel="noreferrer"
                                   >
                                     <Citation link={doc.link} index={ind + 1} />
                                     <p className="shrink truncate ellipsis break-all">
@@ -670,7 +671,7 @@ export const HumanMessage = ({
     if (!isEditing) {
       setEditedContent(content);
     }
-  }, [content]);
+  }, [content, isEditing]);
 
   useEffect(() => {
     if (textareaRef.current) {
diff --git a/web/src/app/chat/message/SearchSummary.tsx b/web/src/app/chat/message/SearchSummary.tsx
index 66f12aa2dcb..6b7ea248e82 100644
--- a/web/src/app/chat/message/SearchSummary.tsx
+++ b/web/src/app/chat/message/SearchSummary.tsx
@@ -83,7 +83,7 @@ export function SearchSummary({
     if (!isEditing) {
       setFinalQuery(query);
     }
-  }, [query]);
+  }, [query, isEditing]);
 
   const searchingForDisplay = (
     <div className={`flex p-1 rounded ${isOverflowed && "cursor-default"}`}>
diff --git a/web/src/app/chat/message/SkippedSearch.tsx b/web/src/app/chat/message/SkippedSearch.tsx
index b339ac784ab..05dc8f2d8e4 100644
--- a/web/src/app/chat/message/SkippedSearch.tsx
+++ b/web/src/app/chat/message/SkippedSearch.tsx
@@ -1,5 +1,5 @@
 import { EmphasizedClickable } from "@/components/BasicClickable";
-import { FiArchive, FiBook, FiSearch } from "react-icons/fi";
+import { FiBook } from "react-icons/fi";
 
 function ForceSearchButton({
   messageId,
diff --git a/web/src/app/chat/modal/FeedbackModal.tsx b/web/src/app/chat/modal/FeedbackModal.tsx
index 6b3df8793cf..64feffefc73 100644
--- a/web/src/app/chat/modal/FeedbackModal.tsx
+++ b/web/src/app/chat/modal/FeedbackModal.tsx
@@ -2,13 +2,8 @@
 
 import { useState } from "react";
 import { FeedbackType } from "../types";
-import { FiThumbsDown, FiThumbsUp } from "react-icons/fi";
 import { ModalWrapper } from "@/components/modals/ModalWrapper";
-import {
-  DislikeFeedbackIcon,
-  FilledLikeIcon,
-  LikeFeedbackIcon,
-} from "@/components/icons/icons";
+import { FilledLikeIcon } from "@/components/icons/icons";
 
 const predefinedPositiveFeedbackOptions =
   process.env.NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS?.split(",") ||
diff --git a/web/src/app/chat/modal/SetDefaultModelModal.tsx b/web/src/app/chat/modal/SetDefaultModelModal.tsx
index 97e05cf59b0..46482a9ee72 100644
--- a/web/src/app/chat/modal/SetDefaultModelModal.tsx
+++ b/web/src/app/chat/modal/SetDefaultModelModal.tsx
@@ -1,6 +1,6 @@
-import { Dispatch, SetStateAction, useState, useEffect, useRef } from "react";
+import { Dispatch, SetStateAction, useEffect, useRef } from "react";
 import { ModalWrapper } from "@/components/modals/ModalWrapper";
-import { Badge, Text } from "@tremor/react";
+import { Text } from "@tremor/react";
 import { getDisplayNameForModel, LlmOverride } from "@/lib/hooks";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 
diff --git a/web/src/app/chat/modal/ShareChatSessionModal.tsx b/web/src/app/chat/modal/ShareChatSessionModal.tsx
index 6c287a6ceb5..e220fdb6d22 100644
--- a/web/src/app/chat/modal/ShareChatSessionModal.tsx
+++ b/web/src/app/chat/modal/ShareChatSessionModal.tsx
@@ -3,7 +3,7 @@ import { ModalWrapper } from "@/components/modals/ModalWrapper";
 import { Button, Callout, Divider, Text } from "@tremor/react";
 import { Spinner } from "@/components/Spinner";
 import { ChatSessionSharedStatus } from "../interfaces";
-import { FiCopy, FiX } from "react-icons/fi";
+import { FiCopy } from "react-icons/fi";
 import { CopyButton } from "@/components/CopyButton";
 
 function buildShareLink(chatSessionId: number) {
@@ -82,6 +82,7 @@ export function ShareChatSessionModal({
                   href={shareLink}
                   target="_blank"
                   className="underline text-link mt-1 ml-1 text-sm my-auto"
+                  rel="noreferrer"
                 >
                   {shareLink}
                 </a>
diff --git a/web/src/app/chat/modal/configuration/AssistantsTab.tsx b/web/src/app/chat/modal/configuration/AssistantsTab.tsx
index 1db03cd605d..fca691d8fd2 100644
--- a/web/src/app/chat/modal/configuration/AssistantsTab.tsx
+++ b/web/src/app/chat/modal/configuration/AssistantsTab.tsx
@@ -19,7 +19,6 @@ import { getFinalLLM } from "@/lib/llm/utils";
 import React, { useState } from "react";
 import { updateUserAssistantList } from "@/lib/assistants/updateAssistantPreferences";
 import { DraggableAssistantCard } from "@/components/assistants/AssistantCards";
-import { useRouter } from "next/navigation";
 
 export function AssistantsTab({
   selectedAssistant,
diff --git a/web/src/app/chat/modal/configuration/LlmTab.tsx b/web/src/app/chat/modal/configuration/LlmTab.tsx
index 86ead4309f3..2cd40290db4 100644
--- a/web/src/app/chat/modal/configuration/LlmTab.tsx
+++ b/web/src/app/chat/modal/configuration/LlmTab.tsx
@@ -1,14 +1,10 @@
 import { useChatContext } from "@/components/context/ChatContext";
-import { getDisplayNameForModel, LlmOverrideManager } from "@/lib/hooks";
+import { LlmOverrideManager } from "@/lib/hooks";
 import React, { forwardRef, useCallback, useState } from "react";
 import { debounce } from "lodash";
 import { Text } from "@tremor/react";
 import { Persona } from "@/app/admin/assistants/interfaces";
-import {
-  checkLLMSupportsImageInput,
-  destructureValue,
-  structureValue,
-} from "@/lib/llm/utils";
+import { destructureValue } from "@/lib/llm/utils";
 import { updateModelOverrideForChatSession } from "../../lib";
 import { GearIcon } from "@/components/icons/icons";
 import { LlmList } from "@/components/llm/LLMList";
@@ -44,12 +40,14 @@ export const LlmTab = forwardRef<HTMLDivElement, LlmTabProps>(
     const [localTemperature, setLocalTemperature] = useState<number>(
       temperature || 0
     );
-
     const debouncedSetTemperature = useCallback(
-      debounce((value) => {
-        setTemperature(value);
-      }, 300),
-      []
+      (value: number) => {
+        const debouncedFunction = debounce((value: number) => {
+          setTemperature(value);
+        }, 300);
+        return debouncedFunction(value);
+      },
+      [setTemperature]
     );
 
     const handleTemperatureChange = (value: number) => {
diff --git a/web/src/app/chat/modifiers/SearchTypeSelector.tsx b/web/src/app/chat/modifiers/SearchTypeSelector.tsx
index c9f1f8a5ce3..94c1ec7047a 100644
--- a/web/src/app/chat/modifiers/SearchTypeSelector.tsx
+++ b/web/src/app/chat/modifiers/SearchTypeSelector.tsx
@@ -1,7 +1,7 @@
 import { BasicClickable } from "@/components/BasicClickable";
 import { ControlledPopup, DefaultDropdownElement } from "@/components/Dropdown";
 import { useState } from "react";
-import { FiCpu, FiFilter, FiSearch } from "react-icons/fi";
+import { FiCpu, FiSearch } from "react-icons/fi";
 
 export const QA = "Question Answering";
 export const SEARCH = "Search Only";
diff --git a/web/src/app/chat/modifiers/SelectedDocuments.tsx b/web/src/app/chat/modifiers/SelectedDocuments.tsx
index be3b81f37f1..fbae7029ce4 100644
--- a/web/src/app/chat/modifiers/SelectedDocuments.tsx
+++ b/web/src/app/chat/modifiers/SelectedDocuments.tsx
@@ -1,7 +1,6 @@
 import { BasicClickable } from "@/components/BasicClickable";
 import { DanswerDocument } from "@/lib/search/interfaces";
-import { useState } from "react";
-import { FiBook, FiFilter } from "react-icons/fi";
+import { FiBook } from "react-icons/fi";
 
 export function SelectedDocuments({
   selectedDocuments,
diff --git a/web/src/app/chat/page.tsx b/web/src/app/chat/page.tsx
index 72a1cf1aa57..c192c55aa38 100644
--- a/web/src/app/chat/page.tsx
+++ b/web/src/app/chat/page.tsx
@@ -5,8 +5,6 @@ import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrap
 import { ChatProvider } from "@/components/context/ChatContext";
 import { fetchChatData } from "@/lib/chat/fetchChatData";
 import WrappedChat from "./WrappedChat";
-import { ProviderContextProvider } from "@/components/chat_search/ProviderContext";
-import { orderAssistantsForUser } from "@/lib/assistants/utils";
 
 export default async function Page({
   searchParams,
diff --git a/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx b/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
index aeea9b97779..8b690968c74 100644
--- a/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
+++ b/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
@@ -3,11 +3,7 @@
 import { useRouter } from "next/navigation";
 import { ChatSession } from "../interfaces";
 import { useState, useEffect, useContext } from "react";
-import {
-  deleteChatSession,
-  getChatRetentionInfo,
-  renameChatSession,
-} from "../lib";
+import { getChatRetentionInfo, renameChatSession } from "../lib";
 import { BasicSelectable } from "@/components/BasicClickable";
 import Link from "next/link";
 import {
diff --git a/web/src/app/chat/sessionSidebar/HistorySidebar.tsx b/web/src/app/chat/sessionSidebar/HistorySidebar.tsx
index beb45e1775e..9e072050b5a 100644
--- a/web/src/app/chat/sessionSidebar/HistorySidebar.tsx
+++ b/web/src/app/chat/sessionSidebar/HistorySidebar.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import { FiEdit, FiFolderPlus } from "react-icons/fi";
-import { ForwardedRef, forwardRef, useContext, useState } from "react";
+import React, { ForwardedRef, forwardRef, useContext, useState } from "react";
 import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { ChatSession } from "../interfaces";
@@ -11,7 +11,6 @@ import { createFolder } from "../folders/FolderManagement";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 
-import React from "react";
 import {
   AssistantsIconSkeleton,
   ClosedBookIcon,
diff --git a/web/src/app/chat/sessionSidebar/PagesTab.tsx b/web/src/app/chat/sessionSidebar/PagesTab.tsx
index e6612a96c8d..8477e197d10 100644
--- a/web/src/app/chat/sessionSidebar/PagesTab.tsx
+++ b/web/src/app/chat/sessionSidebar/PagesTab.tsx
@@ -7,7 +7,7 @@ import { Folder } from "../folders/interfaces";
 import { CHAT_SESSION_ID_KEY, FOLDER_ID_KEY } from "@/lib/drag/constants";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { useRouter } from "next/navigation";
-import { useEffect, useState } from "react";
+import { useState } from "react";
 import { pageType } from "./types";
 
 export function PagesTab({
diff --git a/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx b/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
index 4232929c838..8703beb1d7b 100644
--- a/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
+++ b/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
@@ -6,7 +6,6 @@ import {
 } from "@/components/admin/connectors/Field";
 import { createApiKey, updateApiKey } from "./lib";
 import { Modal } from "@/components/Modal";
-import { XIcon } from "@/components/icons/icons";
 import { Button, Divider, Text } from "@tremor/react";
 import { UserRole } from "@/lib/types";
 import { APIKey } from "./types";
diff --git a/web/src/app/ee/admin/api-key/page.tsx b/web/src/app/ee/admin/api-key/page.tsx
index 4c5447932c5..2b7db0bce24 100644
--- a/web/src/app/ee/admin/api-key/page.tsx
+++ b/web/src/app/ee/admin/api-key/page.tsx
@@ -16,10 +16,10 @@ import {
   TableRow,
   Text,
   Title,
+  Table,
 } from "@tremor/react";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { useState } from "react";
-import { Table } from "@tremor/react";
 import { DeleteButton } from "@/components/DeleteButton";
 import { FiCopy, FiEdit2, FiRefreshCw, FiX } from "react-icons/fi";
 import { Modal } from "@/components/Modal";
diff --git a/web/src/app/ee/admin/groups/ConnectorEditor.tsx b/web/src/app/ee/admin/groups/ConnectorEditor.tsx
index ab6bbb0bec9..2283b71fb63 100644
--- a/web/src/app/ee/admin/groups/ConnectorEditor.tsx
+++ b/web/src/app/ee/admin/groups/ConnectorEditor.tsx
@@ -19,7 +19,7 @@ export const ConnectorEditor = ({
         .filter((ccPair) => !(ccPair.access_type === "public"))
         .map((ccPair) => {
           const ind = selectedCCPairIds.indexOf(ccPair.cc_pair_id);
-          let isSelected = ind !== -1;
+          const isSelected = ind !== -1;
           return (
             <div
               key={`${ccPair.connector.id}-${ccPair.credential.id}`}
diff --git a/web/src/app/ee/admin/groups/UserEditor.tsx b/web/src/app/ee/admin/groups/UserEditor.tsx
index b4e7354fd3b..799f35139c6 100644
--- a/web/src/app/ee/admin/groups/UserEditor.tsx
+++ b/web/src/app/ee/admin/groups/UserEditor.tsx
@@ -1,5 +1,4 @@
 import { User } from "@/lib/types";
-import { useState } from "react";
 import { FiPlus, FiX } from "react-icons/fi";
 import { SearchMultiSelectDropdown } from "@/components/Dropdown";
 import { UsersIcon } from "@/components/icons/icons";
diff --git a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
index 2963eb20b0c..1d761caebc4 100644
--- a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
+++ b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
@@ -3,12 +3,10 @@ import * as Yup from "yup";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
 import { ConnectorIndexingStatus, User, UserGroup } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
-import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
 import { createUserGroup } from "./lib";
 import { UserEditor } from "./UserEditor";
 import { ConnectorEditor } from "./ConnectorEditor";
 import { Modal } from "@/components/Modal";
-import { XIcon } from "@/components/icons/icons";
 import { Button, Divider } from "@tremor/react";
 
 interface UserGroupCreationFormProps {
diff --git a/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx b/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx
index 8b5166ed851..dee6d5ee17b 100644
--- a/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx
+++ b/web/src/app/ee/admin/groups/[groupId]/AddConnectorForm.tsx
@@ -1,7 +1,6 @@
 import { Button } from "@/components/Button";
 import { SearchMultiSelectDropdown } from "@/components/Dropdown";
 import { Modal } from "@/components/Modal";
-import { UsersIcon } from "@/components/icons/icons";
 import { useState } from "react";
 import { FiPlus, FiX } from "react-icons/fi";
 import { updateUserGroup } from "./lib";
diff --git a/web/src/app/ee/admin/groups/[groupId]/page.tsx b/web/src/app/ee/admin/groups/[groupId]/page.tsx
index 5c45bc5247f..82e9d747118 100644
--- a/web/src/app/ee/admin/groups/[groupId]/page.tsx
+++ b/web/src/app/ee/admin/groups/[groupId]/page.tsx
@@ -2,7 +2,6 @@
 
 import { GroupsIcon } from "@/components/icons/icons";
 import { GroupDisplay } from "./GroupDisplay";
-import { FiAlertCircle, FiChevronLeft } from "react-icons/fi";
 import { useSpecificUserGroup } from "./hook";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { useConnectorCredentialIndexingStatus, useUsers } from "@/lib/hooks";
diff --git a/web/src/app/ee/admin/groups/page.tsx b/web/src/app/ee/admin/groups/page.tsx
index 15c0950ba76..6ee00255fe7 100644
--- a/web/src/app/ee/admin/groups/page.tsx
+++ b/web/src/app/ee/admin/groups/page.tsx
@@ -4,9 +4,7 @@ import { GroupsIcon } from "@/components/icons/icons";
 import { UserGroupsTable } from "./UserGroupsTable";
 import { UserGroupCreationForm } from "./UserGroupCreationForm";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { useState, useEffect } from "react";
-import { getCurrentUser } from "@/lib/user";
-import { User, UserRole } from "@/lib/types";
+import { useState } from "react";
 import { ThreeDotsLoader } from "@/components/Loading";
 import {
   useConnectorCredentialIndexingStatus,
diff --git a/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx b/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx
index 0ee3522b4fa..43a206bc543 100644
--- a/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx
+++ b/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx
@@ -9,9 +9,10 @@ import {
   TableBody,
   TableCell,
   Text,
+  Divider,
+  Select,
+  SelectItem,
 } from "@tremor/react";
-import { Divider } from "@tremor/react";
-import { Select, SelectItem } from "@tremor/react";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { ChatSessionMinimal } from "../usage/types";
 import { timestampToReadableDate } from "@/lib/dateUtils";
diff --git a/web/src/app/ee/admin/performance/query-history/[id]/page.tsx b/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
index f96d070aefe..531387664d0 100644
--- a/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
+++ b/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
@@ -30,7 +30,12 @@ function MessageDisplay({ message }: { message: MessageSnapshot }) {
                   }
                 />
                 {document.link ? (
-                  <a href={document.link} target="_blank" className="text-link">
+                  <a
+                    href={document.link}
+                    target="_blank"
+                    className="text-link"
+                    rel="noreferrer"
+                  >
                     {document.semantic_identifier}
                   </a>
                 ) : (
diff --git a/web/src/app/ee/admin/performance/usage/UsageReports.tsx b/web/src/app/ee/admin/performance/usage/UsageReports.tsx
index 9dbeb6ca2f3..f0516254354 100644
--- a/web/src/app/ee/admin/performance/usage/UsageReports.tsx
+++ b/web/src/app/ee/admin/performance/usage/UsageReports.tsx
@@ -16,9 +16,9 @@ import {
   TableRow,
   Text,
   Title,
+  Button,
 } from "@tremor/react";
 import useSWR from "swr";
-import { Button } from "@tremor/react";
 import { useState } from "react";
 import { UsageReport } from "./types";
 import { ThreeDotsLoader } from "@/components/Loading";
@@ -36,7 +36,7 @@ function GenerateReportInput() {
   const [errorOccurred, setErrorOccurred] = useState<Error | null>(null);
 
   const download = (bytes: Blob) => {
-    let elm = document.createElement("a");
+    const elm = document.createElement("a");
     elm.href = URL.createObjectURL(bytes);
     elm.setAttribute("download", "usage_reports.zip");
     elm.click();
diff --git a/web/src/app/ee/admin/performance/usage/page.tsx b/web/src/app/ee/admin/performance/usage/page.tsx
index 4fd287eccc6..ccb2822e253 100644
--- a/web/src/app/ee/admin/performance/usage/page.tsx
+++ b/web/src/app/ee/admin/performance/usage/page.tsx
@@ -4,7 +4,6 @@ import { DateRangeSelector } from "../DateRangeSelector";
 import { DanswerBotChart } from "./DanswerBotChart";
 import { FeedbackChart } from "./FeedbackChart";
 import { QueryPerformanceChart } from "./QueryPerformanceChart";
-import { BarChartIcon } from "@/components/icons/icons";
 import { useTimeRange } from "../lib";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { FiActivity } from "react-icons/fi";
diff --git a/web/src/app/ee/admin/standard-answer/[id]/page.tsx b/web/src/app/ee/admin/standard-answer/[id]/page.tsx
index 6d949331b19..772b1507d33 100644
--- a/web/src/app/ee/admin/standard-answer/[id]/page.tsx
+++ b/web/src/app/ee/admin/standard-answer/[id]/page.tsx
@@ -3,7 +3,6 @@ import { StandardAnswerCreationForm } from "@/app/ee/admin/standard-answer/Stand
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { BackButton } from "@/components/BackButton";
-import { Text } from "@tremor/react";
 import { ClipboardIcon } from "@/components/icons/icons";
 import { StandardAnswer, StandardAnswerCategory } from "@/lib/types";
 
diff --git a/web/src/app/ee/admin/standard-answer/new/page.tsx b/web/src/app/ee/admin/standard-answer/new/page.tsx
index e671f5e1ae9..5a5cccb9c0a 100644
--- a/web/src/app/ee/admin/standard-answer/new/page.tsx
+++ b/web/src/app/ee/admin/standard-answer/new/page.tsx
@@ -3,7 +3,6 @@ import { StandardAnswerCreationForm } from "@/app/ee/admin/standard-answer/Stand
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { BackButton } from "@/components/BackButton";
-import { Text } from "@tremor/react";
 import { ClipboardIcon } from "@/components/icons/icons";
 import { StandardAnswerCategory } from "@/lib/types";
 
diff --git a/web/src/app/ee/admin/standard-answer/page.tsx b/web/src/app/ee/admin/standard-answer/page.tsx
index 867770ede48..0cff20af848 100644
--- a/web/src/app/ee/admin/standard-answer/page.tsx
+++ b/web/src/app/ee/admin/standard-answer/page.tsx
@@ -6,12 +6,10 @@ import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { useStandardAnswers, useStandardAnswerCategories } from "./hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { Button, Divider, Text } from "@tremor/react";
-import Link from "next/link";
-import { StandardAnswer, StandardAnswerCategory } from "@/lib/types";
-import { MagnifyingGlass } from "@phosphor-icons/react";
-import { useState } from "react";
 import {
+  Button,
+  Divider,
+  Text,
   Table,
   TableHead,
   TableRow,
@@ -19,12 +17,15 @@ import {
   TableBody,
   TableCell,
 } from "@tremor/react";
+import Link from "next/link";
+import { StandardAnswer, StandardAnswerCategory } from "@/lib/types";
+import { MagnifyingGlass } from "@phosphor-icons/react";
+import { useState } from "react";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
 import { deleteStandardAnswer } from "./lib";
 import { FilterDropdown } from "@/components/search/filtering/FilterDropdown";
 import { FiTag } from "react-icons/fi";
-import { SelectedBubble } from "@/components/search/filtering/Filters";
 import { PageSelector } from "@/components/PageSelector";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
 
diff --git a/web/src/app/ee/admin/whitelabeling/ImageUpload.tsx b/web/src/app/ee/admin/whitelabeling/ImageUpload.tsx
index 3158fca8c8d..f9be5343cfc 100644
--- a/web/src/app/ee/admin/whitelabeling/ImageUpload.tsx
+++ b/web/src/app/ee/admin/whitelabeling/ImageUpload.tsx
@@ -61,7 +61,11 @@ export function ImageUpload({
             {tmpImageUrl && (
               <div className="mt-4 mb-8">
                 <SubLabel>Uploaded Image:</SubLabel>
-                <img src={tmpImageUrl} className="mt-4 max-w-xs max-h-64" />
+                <img
+                  alt="Uploaded Image"
+                  src={tmpImageUrl}
+                  className="mt-4 max-w-xs max-h-64"
+                />
               </div>
             )}
           </section>
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index 415990373b0..f64edd17964 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -6,12 +6,11 @@ import {
 } from "@/components/settings/lib";
 import {
   CUSTOM_ANALYTICS_ENABLED,
-  EE_ENABLED,
   SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED,
 } from "@/lib/constants";
 import { SettingsProvider } from "@/components/settings/SettingsProvider";
 import { Metadata } from "next";
-import { buildClientUrl, fetchSS } from "@/lib/utilsSS";
+import { buildClientUrl } from "@/lib/utilsSS";
 import { Inter } from "next/font/google";
 import Head from "next/head";
 import { EnterpriseSettings } from "./admin/settings/interfaces";
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index c490e1971cf..818dfe6b965 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -136,7 +136,7 @@ export default async function Home() {
   const storedSearchType = cookies().get("searchType")?.value as
     | string
     | undefined;
-  let searchTypeDefault: SearchType =
+  const searchTypeDefault: SearchType =
     storedSearchType !== undefined &&
     SearchType.hasOwnProperty(storedSearchType)
       ? (storedSearchType as SearchType)
diff --git a/web/src/components/Dropdown.tsx b/web/src/components/Dropdown.tsx
index 5e36dca9843..78867b0dce9 100644
--- a/web/src/components/Dropdown.tsx
+++ b/web/src/components/Dropdown.tsx
@@ -2,6 +2,7 @@ import {
   ChangeEvent,
   FC,
   forwardRef,
+  useCallback,
   useEffect,
   useRef,
   useState,
@@ -411,14 +412,17 @@ export function ControlledPopup({
 }) {
   const filtersRef = useRef<HTMLDivElement>(null);
   // hides logout popup on any click outside
-  const handleClickOutside = (event: MouseEvent) => {
-    if (
-      filtersRef.current &&
-      !filtersRef.current.contains(event.target as Node)
-    ) {
-      setIsOpen(false);
-    }
-  };
+  const handleClickOutside = useCallback(
+    (event: MouseEvent) => {
+      if (
+        filtersRef.current &&
+        !filtersRef.current.contains(event.target as Node)
+      ) {
+        setIsOpen(false);
+      }
+    },
+    [filtersRef, setIsOpen]
+  );
 
   useEffect(() => {
     document.addEventListener("mousedown", handleClickOutside);
@@ -426,7 +430,7 @@ export function ControlledPopup({
     return () => {
       document.removeEventListener("mousedown", handleClickOutside);
     };
-  }, []);
+  }, [handleClickOutside]);
 
   return (
     <div ref={filtersRef} className="relative">
diff --git a/web/src/components/IsPublicGroupSelector.tsx b/web/src/components/IsPublicGroupSelector.tsx
index 37a9d47b3f6..7cb8ff8f830 100644
--- a/web/src/components/IsPublicGroupSelector.tsx
+++ b/web/src/components/IsPublicGroupSelector.tsx
@@ -49,12 +49,7 @@ export const IsPublicGroupSelector = <T extends IsPublicGroupSelectorFormType>({
         setShouldHideContent(false);
       }
     }
-  }, [
-    user,
-    userGroups,
-    formikProps.setFieldValue,
-    formikProps.values.is_public,
-  ]);
+  }, [user, userGroups, formikProps, isPaidEnterpriseFeaturesEnabled]);
 
   if (isLoadingUser || userGroupsIsLoading) {
     return <div>Loading...</div>;
diff --git a/web/src/components/SSRAutoRefresh.tsx b/web/src/components/SSRAutoRefresh.tsx
index 24f0edf5355..31e9fede46b 100644
--- a/web/src/components/SSRAutoRefresh.tsx
+++ b/web/src/components/SSRAutoRefresh.tsx
@@ -25,7 +25,7 @@ export function InstantSSRAutoRefresh() {
 
   useEffect(() => {
     router.refresh();
-  }, []);
+  }, [router]);
 
   return <></>;
 }
diff --git a/web/src/components/UserDropdown.tsx b/web/src/components/UserDropdown.tsx
index 8a8db182299..e59e7291c52 100644
--- a/web/src/components/UserDropdown.tsx
+++ b/web/src/components/UserDropdown.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { useState, useRef, useContext, useEffect } from "react";
+import { useState, useRef, useContext, useEffect, useMemo } from "react";
 import { FiLogOut } from "react-icons/fi";
 import Link from "next/link";
 import { useRouter } from "next/navigation";
@@ -67,8 +67,10 @@ export function UserDropdown({
   const router = useRouter();
 
   const combinedSettings = useContext(SettingsContext);
-  const customNavItems: NavigationItem[] =
-    combinedSettings?.enterpriseSettings?.custom_nav_items || [];
+  const customNavItems: NavigationItem[] = useMemo(
+    () => combinedSettings?.enterpriseSettings?.custom_nav_items || [],
+    [combinedSettings]
+  );
 
   useEffect(() => {
     const iconNames = customNavItems
diff --git a/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx b/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx
index 830ca5b5068..d90fbedbfac 100644
--- a/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx
+++ b/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx
@@ -46,7 +46,14 @@ export function AccessTypeGroupSelector({}: {}) {
         setShouldHideContent(false);
       }
     }
-  }, [user, userGroups, access_type.value]);
+  }, [
+    user,
+    userGroups,
+    access_type.value,
+    access_type_helpers,
+    groups_helpers,
+    isPaidEnterpriseFeaturesEnabled,
+  ]);
 
   if (isLoadingUser || userGroupsIsLoading) {
     return <div>Loading...</div>;
diff --git a/web/src/components/assistants/AssistantIcon.tsx b/web/src/components/assistants/AssistantIcon.tsx
index 07ab05aa145..58a5b11b237 100644
--- a/web/src/components/assistants/AssistantIcon.tsx
+++ b/web/src/components/assistants/AssistantIcon.tsx
@@ -36,6 +36,7 @@ export function AssistantIcon({
         // Prioritization order: image, graph, defaults
         assistant.uploaded_image_id ? (
           <img
+            alt={assistant.name}
             className={`object-cover object-center rounded-sm overflow-hidden transition-opacity duration-300 opacity-100
               ${size === "large" ? "w-8 h-8" : "w-6 h-6"}`}
             src={buildImgUrl(assistant.uploaded_image_id)}
diff --git a/web/src/components/chat_search/Header.tsx b/web/src/components/chat_search/Header.tsx
index 640393641d0..c19d46ce874 100644
--- a/web/src/components/chat_search/Header.tsx
+++ b/web/src/components/chat_search/Header.tsx
@@ -51,7 +51,7 @@ export default function FunctionalHeader({
     return () => {
       window.removeEventListener("keydown", handleKeyDown);
     };
-  }, []);
+  }, [page, currentChatSession]);
   const router = useRouter();
 
   const handleNewChat = () => {
diff --git a/web/src/components/chat_search/ProviderContext.tsx b/web/src/components/chat_search/ProviderContext.tsx
index 3907b98a68b..609713f874b 100644
--- a/web/src/components/chat_search/ProviderContext.tsx
+++ b/web/src/components/chat_search/ProviderContext.tsx
@@ -1,6 +1,12 @@
 "use client";
 import { WellKnownLLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
-import React, { createContext, useContext, useState, useEffect } from "react";
+import React, {
+  createContext,
+  useContext,
+  useState,
+  useEffect,
+  useCallback,
+} from "react";
 import { useUser } from "../user/UserProvider";
 import { useRouter } from "next/navigation";
 import { checkLlmProvider } from "../initialSetup/welcome/lib";
@@ -28,16 +34,16 @@ export function ProviderContextProvider({
     WellKnownLLMProviderDescriptor[]
   >([]);
 
-  const fetchProviderInfo = async () => {
+  const fetchProviderInfo = useCallback(async () => {
     const { providers, options, defaultCheckSuccessful } =
       await checkLlmProvider(user);
     setValidProviderExists(providers.length > 0 && defaultCheckSuccessful);
     setProviderOptions(options);
-  };
+  }, [user, setValidProviderExists, setProviderOptions]);
 
   useEffect(() => {
     fetchProviderInfo();
-  }, [router, user]);
+  }, [router, user, fetchProviderInfo]);
 
   const shouldShowConfigurationNeeded =
     !validProviderExists && providerOptions.length > 0;
diff --git a/web/src/components/chat_search/hooks.ts b/web/src/components/chat_search/hooks.ts
index 314810b55ee..d83059636bc 100644
--- a/web/src/components/chat_search/hooks.ts
+++ b/web/src/components/chat_search/hooks.ts
@@ -77,6 +77,7 @@ export const useSidebarVisibility = ({
       document.removeEventListener("mousemove", handleEvent);
       document.removeEventListener("mouseleave", handleMouseLeave);
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [showDocSidebar, toggledSidebar, sidebarElementRef, mobile]);
 
   return { showDocSidebar };
diff --git a/web/src/components/context/EmbeddingContext.tsx b/web/src/components/context/EmbeddingContext.tsx
index 3d03b99b1d5..33bb850a87c 100644
--- a/web/src/components/context/EmbeddingContext.tsx
+++ b/web/src/components/context/EmbeddingContext.tsx
@@ -74,7 +74,7 @@ export const EmbeddingFormProvider: React.FC<{
     if (stepFromUrl !== formStep) {
       setFormStep(stepFromUrl);
     }
-  }, [searchParams]);
+  }, [searchParams, formStep]);
 
   const contextValue: EmbeddingFormContextType = {
     formStep,
diff --git a/web/src/components/context/FormContext.tsx b/web/src/components/context/FormContext.tsx
index d445782f718..ee6e9dab786 100644
--- a/web/src/components/context/FormContext.tsx
+++ b/web/src/components/context/FormContext.tsx
@@ -73,7 +73,7 @@ export const FormProvider: React.FC<{
     if (stepFromUrl !== formStep) {
       setFormStep(stepFromUrl);
     }
-  }, [searchParams]);
+  }, [searchParams, formStep]);
 
   const contextValue: FormContextType = {
     formStep,
diff --git a/web/src/components/health/healthcheck.tsx b/web/src/components/health/healthcheck.tsx
index 037082ca0bd..d537f0577bc 100644
--- a/web/src/components/health/healthcheck.tsx
+++ b/web/src/components/health/healthcheck.tsx
@@ -3,7 +3,7 @@
 import { errorHandlingFetcher, RedirectError } from "@/lib/fetcher";
 import useSWR from "swr";
 import { Modal } from "../Modal";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useState } from "react";
 import { getSecondsUntilExpiration } from "@/lib/time";
 import { User } from "@/lib/types";
 import { mockedRefreshToken, refreshToken } from "./refreshUtils";
@@ -20,7 +20,7 @@ export const HealthCheckBanner = () => {
     errorHandlingFetcher
   );
 
-  const updateExpirationTime = async () => {
+  const updateExpirationTime = useCallback(async () => {
     const updatedUser = await mutateUser();
 
     if (updatedUser) {
@@ -28,11 +28,11 @@ export const HealthCheckBanner = () => {
       setSecondsUntilExpiration(seconds);
       console.debug(`Updated seconds until expiration:! ${seconds}`);
     }
-  };
+  }, [mutateUser]);
 
   useEffect(() => {
     updateExpirationTime();
-  }, [user]);
+  }, [user, updateExpirationTime]);
 
   useEffect(() => {
     if (CUSTOM_REFRESH_URL) {
@@ -89,7 +89,7 @@ export const HealthCheckBanner = () => {
         clearTimeout(expireTimeoutId);
       };
     }
-  }, [secondsUntilExpiration, user]);
+  }, [secondsUntilExpiration, user, mutateUser, updateExpirationTime]);
 
   if (!error && !expired) {
     return null;
diff --git a/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx b/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx
index 42aba7ab189..decf302e2f9 100644
--- a/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx
+++ b/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx
@@ -20,7 +20,7 @@ export function NoCompleteSourcesModal({
     }, 5000);
 
     return () => clearInterval(interval);
-  }, []);
+  }, [router]);
 
   if (isHidden) {
     return null;
diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index d2229ab9008..1fac3803025 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -48,7 +48,7 @@ export function _WelcomeModal({ user }: { user: User | null }) {
     }
 
     fetchProviderInfo();
-  }, []);
+  }, [user]);
 
   return (
     <>
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index a9641511af2..8ff4fdd3122 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { useContext, useEffect, useRef, useState } from "react";
+import { useCallback, useContext, useEffect, useRef, useState } from "react";
 import { FullSearchBar } from "./SearchBar";
 import { SearchResultsDisplay } from "./SearchResultsDisplay";
 import { SourceSelector } from "./filtering/Filters";
@@ -106,15 +106,15 @@ export const SearchSection = ({
 
   const [agentic, setAgentic] = useState(agenticSearchEnabled);
 
-  const toggleAgentic = () => {
+  const toggleAgentic = useCallback(() => {
     Cookies.set(
       AGENTIC_SEARCH_TYPE_COOKIE_NAME,
       String(!agentic).toLocaleLowerCase()
     );
     setAgentic((agentic) => !agentic);
-  };
+  }, [agentic]);
 
-  const toggleSidebar = () => {
+  const toggleSidebar = useCallback(() => {
     Cookies.set(
       SIDEBAR_TOGGLED_COOKIE_NAME,
       String(!toggledSidebar).toLocaleLowerCase()
@@ -123,7 +123,7 @@ export const SearchSection = ({
         path: "/",
       };
     toggle();
-  };
+  }, [toggledSidebar, toggle]);
 
   useEffect(() => {
     const handleKeyDown = (event: KeyboardEvent) => {
diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts
index 7347964a756..18bc3336ae7 100644
--- a/web/src/lib/sources.ts
+++ b/web/src/lib/sources.ts
@@ -27,7 +27,6 @@ import {
   SharepointIcon,
   TeamsIcon,
   SlabIcon,
-  SlackIcon,
   ZendeskIcon,
   ZulipIcon,
   MediaWikiIcon,
@@ -358,10 +357,3 @@ export function getSourcesForPersona(persona: Persona): ValidSources[] {
   });
   return personaSources;
 }
-
-function stripTrailingSlash(str: string) {
-  if (str.substr(-1) === "/") {
-    return str.substr(0, str.length - 1);
-  }
-  return str;
-}
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index f4751780880..c760074f403 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -3,7 +3,7 @@ import { Credential } from "./connectors/credentials";
 import { Connector } from "./connectors/connectors";
 import { ConnectorCredentialPairStatus } from "@/app/admin/connector/[ccPairId]/types";
 
-export interface UserPreferences {
+interface UserPreferences {
   chosen_assistants: number[] | null;
   visible_assistants: number[];
   hidden_assistants: number[];

From e4c7cfde42b846434f53c7d0c6b24b4522dd694d Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 7 Oct 2024 13:29:04 -0700
Subject: [PATCH 044/376] Minor update to initial modal (#2571)

* minor update

* nit: pretty
---
 .../initialSetup/welcome/WelcomeModal.tsx      | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index 1fac3803025..1c94ae22961 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -1,5 +1,6 @@
 "use client";
 
+import React from "react";
 import { Button, Divider, Text } from "@tremor/react";
 import { Modal } from "../../Modal";
 import Cookies from "js-cookie";
@@ -25,8 +26,8 @@ export function _CompletedWelcomeFlowDummyComponent() {
 
 export function _WelcomeModal({ user }: { user: User | null }) {
   const router = useRouter();
+
   const [canBegin, setCanBegin] = useState(false);
-  const [apiKeyVerified, setApiKeyVerified] = useState<boolean>(false);
   const [providerOptions, setProviderOptions] = useState<
     WellKnownLLMProviderDescriptor[]
   >([]);
@@ -41,19 +42,26 @@ export function _WelcomeModal({ user }: { user: User | null }) {
 
   useEffect(() => {
     async function fetchProviderInfo() {
-      const { providers, options, defaultCheckSuccessful } =
-        await checkLlmProvider(user);
-      setApiKeyVerified(providers.length > 0 && defaultCheckSuccessful);
+      const { options } = await checkLlmProvider(user);
       setProviderOptions(options);
     }
 
     fetchProviderInfo();
   }, [user]);
 
+  // We should always have options
+  if (providerOptions.length === 0) {
+    return null;
+  }
+
   return (
     <>
       {popup}
-      <Modal title={"Welcome to Danswer!"} width="w-full max-w-3xl">
+
+      <Modal
+        title={"Welcome to Danswer!"}
+        width="w-full max-h-[900px] overflow-y-scroll max-w-3xl"
+      >
         <div>
           <Text className="mb-4">
             Danswer brings all your company&apos;s knowledge to your fingertips,

From 5d356cc971cadf45c99718f6c7ca7788ff5390d4 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Mon, 7 Oct 2024 13:50:30 -0700
Subject: [PATCH 045/376] Remove Perm Sync Script Dev (#2712)

---
 backend/scripts/dev_run_background_jobs.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/backend/scripts/dev_run_background_jobs.py b/backend/scripts/dev_run_background_jobs.py
index a4a253a10df..84a92f4826b 100644
--- a/backend/scripts/dev_run_background_jobs.py
+++ b/backend/scripts/dev_run_background_jobs.py
@@ -122,26 +122,6 @@ def run_jobs(exclude_indexing: bool) -> None:
 
         indexing_thread.start()
         indexing_thread.join()
-    try:
-        update_env = os.environ.copy()
-        update_env["PYTHONPATH"] = "."
-        cmd_perm_sync = ["python", "ee/danswer/background/permission_sync.py"]
-
-        indexing_process = subprocess.Popen(
-            cmd_perm_sync,
-            env=update_env,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-        )
-
-        perm_sync_thread = threading.Thread(
-            target=monitor_process, args=("INDEXING", indexing_process)
-        )
-        perm_sync_thread.start()
-        perm_sync_thread.join()
-    except Exception:
-        pass
 
     worker_primary_thread.join()
     worker_light_thread.join()

From 30dc408028040859182f6160325ce679354b6efb Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 7 Oct 2024 14:30:03 -0700
Subject: [PATCH 046/376] rely on stdout redirection for supervisord logging
 (#2711)

---
 backend/supervisord.conf | 35 +++++++++++++++--------------------
 1 file changed, 15 insertions(+), 20 deletions(-)

diff --git a/backend/supervisord.conf b/backend/supervisord.conf
index 2c73545904f..32951067300 100644
--- a/backend/supervisord.conf
+++ b/backend/supervisord.conf
@@ -30,10 +30,10 @@ command=celery -A danswer.background.celery.celery_run:celery_app worker
     --concurrency=4
     --prefetch-multiplier=1 
     --loglevel=INFO 
-    --logfile=/var/log/celery_worker_primary_supervisor.log 
     --hostname=primary@%%n 
     -Q celery
-environment=LOG_FILE_NAME=celery_worker_primary
+stdout_logfile=/var/log/celery_worker_primary.log
+stdout_logfile_maxbytes=16MB
 redirect_stderr=true
 autorestart=true
 startsecs=10
@@ -48,10 +48,10 @@ command=bash -c "celery -A danswer.background.celery.celery_run:celery_app worke
     --concurrency=${CELERY_WORKER_LIGHT_CONCURRENCY:-24} \
     --prefetch-multiplier=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-8} \
     --loglevel=INFO \
-    --logfile=/var/log/celery_worker_light_supervisor.log \
     --hostname=light@%%n \
     -Q vespa_metadata_sync,connector_deletion"
-environment=LOG_FILE_NAME=celery_worker_light
+stdout_logfile=/var/log/celery_worker_light.log
+stdout_logfile_maxbytes=16MB
 redirect_stderr=true
 autorestart=true
 startsecs=10
@@ -63,10 +63,10 @@ command=celery -A danswer.background.celery.celery_run:celery_app worker
     --concurrency=4
     --prefetch-multiplier=1 
     --loglevel=INFO 
-    --logfile=/var/log/celery_worker_heavy_supervisor.log 
     --hostname=heavy@%%n
     -Q connector_pruning
-environment=LOG_FILE_NAME=celery_worker_heavy
+stdout_logfile=/var/log/celery_worker_heavy.log
+stdout_logfile_maxbytes=16MB
 redirect_stderr=true
 autorestart=true
 startsecs=10
@@ -74,9 +74,9 @@ stopasgroup=true
 
 # Job scheduler for periodic tasks
 [program:celery_beat]
-command=celery -A danswer.background.celery.celery_run:celery_app beat 
-    --logfile=/var/log/celery_beat_supervisor.log
-environment=LOG_FILE_NAME=celery_beat
+command=celery -A danswer.background.celery.celery_run:celery_app beat
+stdout_logfile=/var/log/celery_beat.log
+stdout_logfile_maxbytes=16MB
 redirect_stderr=true
 startsecs=10
 stopasgroup=true
@@ -97,17 +97,12 @@ startsecs=60
 # No log rotation here, since it's stdout it's handled by the Docker container logging
 [program:log-redirect-handler]
 command=tail -qF
+    /var/log/celery_beat.log
+    /var/log/celery_worker_primary.log
+    /var/log/celery_worker_light.log
+    /var/log/celery_worker_heavy.log
     /var/log/document_indexing_info.log
-    /var/log/celery_beat_supervisor.log
-    /var/log/celery_worker_primary_supervisor.log
-    /var/log/celery_worker_light_supervisor.log
-    /var/log/celery_worker_heavy_supervisor.log
-    /var/log/celery_beat_debug.log
-    /var/log/celery_worker_primary_debug.log
-    /var/log/celery_worker_light_debug.log
-    /var/log/celery_worker_heavy_debug.log
     /var/log/slack_bot_debug.log
 stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-redirect_stderr=true
-autorestart=true
\ No newline at end of file
+stdout_logfile_maxbytes = 0  # must be set to 0 when stdout_logfile=/dev/stdout
+autorestart=true

From 1a3469d2c5584f58581f9745de5baec4476f580d Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 7 Oct 2024 14:37:56 -0700
Subject: [PATCH 047/376] check before using fetch_versioned_implementation
 because it logs warnings that confuse users. (#2708)

Renamed get_is_ee_version to is_ee_version to be less redundant
---
 .../danswer/background/celery/celery_redis.py |  4 +++
 .../background/celery/tasks/vespa/tasks.py    | 30 +++++++++++--------
 backend/danswer/background/update.py          |  4 +--
 backend/danswer/main.py                       |  2 +-
 .../danswer/utils/variable_functionality.py   |  6 ++--
 5 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index d9d61b6c8a6..f08bfd17e2f 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -21,6 +21,7 @@
 )
 from danswer.db.document_set import construct_document_select_by_docset
 from danswer.utils.variable_functionality import fetch_versioned_implementation
+from danswer.utils.variable_functionality import global_version
 
 
 class RedisObjectHelper(ABC):
@@ -172,6 +173,9 @@ def generate_tasks(
 
         async_results = []
 
+        if not global_version.is_ee_version():
+            return 0
+
         try:
             construct_document_select_by_usergroup = fetch_versioned_implementation(
                 "danswer.db.user_group",
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 62806c7b81d..3f347cbab3d 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -48,6 +48,7 @@
 from danswer.utils.variable_functionality import (
     fetch_versioned_implementation_with_fallback,
 )
+from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import noop_fallback
 
 
@@ -87,21 +88,24 @@ def check_for_vespa_sync_task() -> None:
                 )
 
             # check if any user groups are not synced
-            try:
-                fetch_user_groups = fetch_versioned_implementation(
-                    "danswer.db.user_group", "fetch_user_groups"
-                )
+            if global_version.is_ee_version():
+                try:
+                    fetch_user_groups = fetch_versioned_implementation(
+                        "danswer.db.user_group", "fetch_user_groups"
+                    )
 
-                user_groups = fetch_user_groups(
-                    db_session=db_session, only_up_to_date=False
-                )
-                for usergroup in user_groups:
-                    try_generate_user_group_sync_tasks(
-                        usergroup, db_session, r, lock_beat
+                    user_groups = fetch_user_groups(
+                        db_session=db_session, only_up_to_date=False
                     )
-            except ModuleNotFoundError:
-                # Always exceptions on the MIT version, which is expected
-                pass
+                    for usergroup in user_groups:
+                        try_generate_user_group_sync_tasks(
+                            usergroup, db_session, r, lock_beat
+                        )
+                except ModuleNotFoundError:
+                    # Always exceptions on the MIT version, which is expected
+                    # We shouldn't actually get here if the ee version check works
+                    pass
+
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index 57d05513ac2..773165c5161 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -353,7 +353,7 @@ def kickoff_indexing_jobs(
                     run_indexing_entrypoint,
                     attempt.id,
                     attempt.connector_credential_pair_id,
-                    global_version.get_is_ee_version(),
+                    global_version.is_ee_version(),
                     pure=False,
                 )
                 if not run:
@@ -364,7 +364,7 @@ def kickoff_indexing_jobs(
                     run_indexing_entrypoint,
                     attempt.id,
                     attempt.connector_credential_pair_id,
-                    global_version.get_is_ee_version(),
+                    global_version.is_ee_version(),
                     pure=False,
                 )
                 if not run:
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index b9231a9c561..d3aa8b00efd 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -329,7 +329,7 @@ def get_application() -> FastAPI:
         f"Starting Danswer Backend version {__version__} on http://{APP_HOST}:{str(APP_PORT)}/"
     )
 
-    if global_version.get_is_ee_version():
+    if global_version.is_ee_version():
         logger.notice("Running Enterprise Edition")
 
     uvicorn.run(app, host=APP_HOST, port=APP_PORT)
diff --git a/backend/danswer/utils/variable_functionality.py b/backend/danswer/utils/variable_functionality.py
index 55f296aa8e7..dfe6def2a56 100644
--- a/backend/danswer/utils/variable_functionality.py
+++ b/backend/danswer/utils/variable_functionality.py
@@ -16,7 +16,7 @@ def __init__(self) -> None:
     def set_ee(self) -> None:
         self._is_ee = True
 
-    def get_is_ee_version(self) -> bool:
+    def is_ee_version(self) -> bool:
         return self._is_ee
 
 
@@ -24,7 +24,7 @@ def get_is_ee_version(self) -> bool:
 
 
 def set_is_ee_based_on_env_variable() -> None:
-    if ENTERPRISE_EDITION_ENABLED and not global_version.get_is_ee_version():
+    if ENTERPRISE_EDITION_ENABLED and not global_version.is_ee_version():
         logger.notice("Enterprise Edition enabled")
         global_version.set_ee()
 
@@ -54,7 +54,7 @@ def fetch_versioned_implementation(module: str, attribute: str) -> Any:
         implementation cannot be found or loaded.
     """
     logger.debug("Fetching versioned implementation for %s.%s", module, attribute)
-    is_ee = global_version.get_is_ee_version()
+    is_ee = global_version.is_ee_version()
 
     module_full = f"ee.{module}" if is_ee else module
     try:

From 4214a3a6e2c4127df4dd9ae839069ba292eb4816 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 7 Oct 2024 15:23:37 -0700
Subject: [PATCH 048/376] Inline code + effect clarity (#2715)

* cleaner code blocks + form context

* cleaner

* nit
---
 web/src/app/chat/message/CodeBlock.tsx        | 49 ++++++++++---------
 .../components/context/EmbeddingContext.tsx   |  2 +-
 web/src/components/context/FormContext.tsx    |  2 +-
 3 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/web/src/app/chat/message/CodeBlock.tsx b/web/src/app/chat/message/CodeBlock.tsx
index ee6b6e6eab6..c1f2f99397c 100644
--- a/web/src/app/chat/message/CodeBlock.tsx
+++ b/web/src/app/chat/message/CodeBlock.tsx
@@ -55,32 +55,33 @@ export const CodeBlock = memo(function CodeBlock({
     </div>
   );
 
+  if (typeof children === "string") {
+    return (
+      <span
+        className={`
+          font-mono 
+          text-gray-800 
+          bg-gray-50 
+          border 
+          border-gray-300 
+          rounded 
+          align-bottom
+          px-1
+          py-[3px]
+          text-xs 
+          inline-block
+          whitespace-pre-wrap 
+          break-words 
+          ${className}
+        `}
+      >
+        {children}
+      </span>
+    );
+  }
+
   const CodeContent = () => {
     if (!language) {
-      if (typeof children === "string") {
-        return (
-          <code
-            className={`
-            font-mono 
-            text-gray-800 
-            bg-gray-50 
-            border 
-            border-gray-300 
-            rounded 
-            px-1
-            py-[3px]
-            text-xs 
-            whitespace-pre-wrap 
-            break-words 
-            overflow-hidden
-            mb-1
-            ${className}
-          `}
-          >
-            {children}
-          </code>
-        );
-      }
       return (
         <pre style={CODE_BLOCK_PADDING}>
           <code className={`text-sm ${className}`}>
diff --git a/web/src/components/context/EmbeddingContext.tsx b/web/src/components/context/EmbeddingContext.tsx
index 33bb850a87c..3d03b99b1d5 100644
--- a/web/src/components/context/EmbeddingContext.tsx
+++ b/web/src/components/context/EmbeddingContext.tsx
@@ -74,7 +74,7 @@ export const EmbeddingFormProvider: React.FC<{
     if (stepFromUrl !== formStep) {
       setFormStep(stepFromUrl);
     }
-  }, [searchParams, formStep]);
+  }, [searchParams]);
 
   const contextValue: EmbeddingFormContextType = {
     formStep,
diff --git a/web/src/components/context/FormContext.tsx b/web/src/components/context/FormContext.tsx
index ee6e9dab786..d445782f718 100644
--- a/web/src/components/context/FormContext.tsx
+++ b/web/src/components/context/FormContext.tsx
@@ -73,7 +73,7 @@ export const FormProvider: React.FC<{
     if (stepFromUrl !== formStep) {
       setFormStep(stepFromUrl);
     }
-  }, [searchParams, formStep]);
+  }, [searchParams]);
 
   const contextValue: FormContextType = {
     formStep,

From 6fa8fabb473cd18a8ec8e6a41dd91857bffe4edb Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 7 Oct 2024 15:17:49 -0700
Subject: [PATCH 049/376] add one more retry and wait a little longer to allow
 ourselves to recover from infra issues (#2714)

---
 .../actions/custom-build-and-push/action.yml  | 47 ++++++++++++++++---
 1 file changed, 40 insertions(+), 7 deletions(-)

diff --git a/.github/actions/custom-build-and-push/action.yml b/.github/actions/custom-build-and-push/action.yml
index 48344237059..fbee0554995 100644
--- a/.github/actions/custom-build-and-push/action.yml
+++ b/.github/actions/custom-build-and-push/action.yml
@@ -32,16 +32,20 @@ inputs:
     description: 'Cache destinations'
     required: false
   retry-wait-time:
-    description: 'Time to wait before retry in seconds'
+    description: 'Time to wait before attempt 2 in seconds'
     required: false
-    default: '5'
+    default: '60'
+  retry-wait-time-2:
+    description: 'Time to wait before attempt 3 in seconds'
+    required: false
+    default: '120'
 
 runs:
   using: "composite"
   steps:
-    - name: Build and push Docker image (First Attempt)
+    - name: Build and push Docker image (Attempt 1 of 3)
       id: buildx1
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       continue-on-error: true
       with:
         context: ${{ inputs.context }}
@@ -54,16 +58,39 @@ runs:
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
 
-    - name: Wait to retry
+    - name: Wait before attempt 2
       if: steps.buildx1.outcome != 'success'
       run: |
         echo "First attempt failed. Waiting ${{ inputs.retry-wait-time }} seconds before retry..."
         sleep ${{ inputs.retry-wait-time }}
       shell: bash
 
-    - name: Build and push Docker image (Retry Attempt)
+    - name: Build and push Docker image (Attempt 2 of 3)
+      id: buildx2
       if: steps.buildx1.outcome != 'success'
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        pull: ${{ inputs.pull }}
+        push: ${{ inputs.push }}
+        load: ${{ inputs.load }}
+        tags: ${{ inputs.tags }}
+        cache-from: ${{ inputs.cache-from }}
+        cache-to: ${{ inputs.cache-to }}
+
+    - name: Wait before attempt 3
+      if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
+      run: |
+        echo "Second attempt failed. Waiting ${{ inputs.retry-wait-time-2 }} seconds before retry..."
+        sleep ${{ inputs.retry-wait-time-2 }}
+      shell: bash
+
+    - name: Build and push Docker image (Attempt 3 of 3)
+      id: buildx3
+      if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success'
+      uses: docker/build-push-action@v6
       with:
         context: ${{ inputs.context }}
         file: ${{ inputs.file }}
@@ -74,3 +101,9 @@ runs:
         tags: ${{ inputs.tags }}
         cache-from: ${{ inputs.cache-from }}
         cache-to: ${{ inputs.cache-to }}
+
+    - name: Report failure
+      if: steps.buildx1.outcome != 'success' && steps.buildx2.outcome != 'success' && steps.buildx3.outcome != 'success'
+      run: |
+        echo "All attempts failed. Possible transient infrastucture issues? Try again later or inspect logs for details."
+      shell: bash

From 79d37156c658e09f7a0597e848ddd19289b9a4c2 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 7 Oct 2024 15:21:16 -0700
Subject: [PATCH 050/376] better logging for actions being taken inside
 document_by_cc_pair_cleanup (#2713)

---
 .../background/celery/tasks/shared/tasks.py   | 18 +++++++++++++----
 backend/danswer/document_index/interfaces.py  |  4 ++--
 backend/danswer/document_index/vespa/index.py | 20 +++++++++++--------
 3 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 0ca4d5031b1..0977fb35d29 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -43,10 +43,11 @@ def document_by_cc_pair_cleanup_task(
     connector / credential pair from the access list
     (6) delete all relevant entries from postgres
     """
-    task_logger.info(f"document_id={document_id}")
-
     try:
         with Session(get_sqlalchemy_engine()) as db_session:
+            action = "skip"
+            chunks_affected = 0
+
             curr_ind_name, sec_ind_name = get_both_index_names(db_session)
             document_index = get_default_document_index(
                 primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
@@ -56,12 +57,16 @@ def document_by_cc_pair_cleanup_task(
             if count == 1:
                 # count == 1 means this is the only remaining cc_pair reference to the doc
                 # delete it from vespa and the db
-                document_index.delete(doc_ids=[document_id])
+                action = "delete"
+
+                chunks_affected = document_index.delete_single(document_id)
                 delete_documents_complete__no_commit(
                     db_session=db_session,
                     document_ids=[document_id],
                 )
             elif count > 1:
+                action = "update"
+
                 # count > 1 means the document still has cc_pair references
                 doc = get_document(document_id, db_session)
                 if not doc:
@@ -84,7 +89,9 @@ def document_by_cc_pair_cleanup_task(
                 )
 
                 # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-                document_index.update_single(document_id, fields=fields)
+                chunks_affected = document_index.update_single(
+                    document_id, fields=fields
+                )
 
                 # there are still other cc_pair references to the doc, so just resync to Vespa
                 delete_document_by_connector_credential_pair__no_commit(
@@ -100,6 +107,9 @@ def document_by_cc_pair_cleanup_task(
             else:
                 pass
 
+            task_logger.info(
+                f"document_id={document_id} refcount={count} action={action} chunks={chunks_affected}"
+            )
             db_session.commit()
     except SoftTimeLimitExceeded:
         task_logger.info(f"SoftTimeLimitExceeded exception. doc_id={document_id}")
diff --git a/backend/danswer/document_index/interfaces.py b/backend/danswer/document_index/interfaces.py
index e4ff90a40cf..42d763d0c98 100644
--- a/backend/danswer/document_index/interfaces.py
+++ b/backend/danswer/document_index/interfaces.py
@@ -172,7 +172,7 @@ class Deletable(abc.ABC):
     """
 
     @abc.abstractmethod
-    def delete_single(self, doc_id: str) -> None:
+    def delete_single(self, doc_id: str) -> int:
         """
         Given a single document id, hard delete it from the document index
 
@@ -203,7 +203,7 @@ class Updatable(abc.ABC):
     """
 
     @abc.abstractmethod
-    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
+    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> int:
         """
         Updates all chunks for a document with the specified fields.
         None values mean that the field does not need an update.
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 512eb932156..25663e966a3 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -384,12 +384,14 @@ def update(self, update_requests: list[UpdateRequest]) -> None:
             time.monotonic() - update_start,
         )
 
-    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
+    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> int:
         """Note: if the document id does not exist, the update will be a no-op and the
         function will complete with no errors or exceptions.
         Handle other exceptions if you wish to implement retry behavior
         """
 
+        total_chunks_updated = 0
+
         # Handle Vespa character limitations
         # Mutating update_request but it's not used later anyway
         normalized_doc_id = replace_invalid_doc_id_characters(doc_id)
@@ -411,7 +413,7 @@ def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
 
         if not update_dict["fields"]:
             logger.error("Update request received but nothing to update")
-            return
+            return 0
 
         index_names = [self.index_name]
         if self.secondary_index_name:
@@ -426,7 +428,6 @@ def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
                     }
                 )
 
-                total_chunks_updated = 0
                 while True:
                     try:
                         resp = http_client.put(
@@ -462,9 +463,10 @@ def update_single(self, doc_id: str, fields: VespaDocumentFields) -> None:
                     f"VespaIndex.update_single: "
                     f"index={index_name} "
                     f"doc={normalized_doc_id} "
-                    f"chunks_deleted={total_chunks_updated}"
+                    f"chunks_updated={total_chunks_updated}"
                 )
-        return
+
+        return total_chunks_updated
 
     def delete(self, doc_ids: list[str]) -> None:
         logger.info(f"Deleting {len(doc_ids)} documents from Vespa")
@@ -484,10 +486,12 @@ def delete(self, doc_ids: list[str]) -> None:
                 )
         return
 
-    def delete_single(self, doc_id: str) -> None:
+    def delete_single(self, doc_id: str) -> int:
         """Possibly faster overall than the delete method due to using a single
         delete call with a selection query."""
 
+        total_chunks_deleted = 0
+
         # Vespa deletion is poorly documented ... luckily we found this
         # https://docs.vespa.ai/en/operations/batch-delete.html#example
 
@@ -508,7 +512,6 @@ def delete_single(self, doc_id: str) -> None:
                     }
                 )
 
-                total_chunks_deleted = 0
                 while True:
                     try:
                         resp = http_client.delete(
@@ -543,7 +546,8 @@ def delete_single(self, doc_id: str) -> None:
                     f"doc={doc_id} "
                     f"chunks_deleted={total_chunks_deleted}"
                 )
-        return
+
+        return total_chunks_deleted
 
     def id_based_retrieval(
         self,

From a52485bda28abc855a67300da400b95da86204f8 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Mon, 7 Oct 2024 15:22:28 -0700
Subject: [PATCH 051/376] Fix all LegacyKeyValueFormat docker warnings

---
 backend/Dockerfile                   | 2 +-
 backend/Dockerfile.model_server      | 2 +-
 backend/tests/integration/Dockerfile | 2 +-
 web/Dockerfile                       | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/backend/Dockerfile b/backend/Dockerfile
index b8b71920f53..17f93979955 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -101,7 +101,7 @@ COPY ./scripts/force_delete_connector_by_id.py /app/scripts/force_delete_connect
 # Put logo in assets
 COPY ./assets /app/assets
 
-ENV PYTHONPATH /app
+ENV PYTHONPATH=/app
 
 # Default command which does nothing
 # This container is used by api server and background which specify their own CMD
diff --git a/backend/Dockerfile.model_server b/backend/Dockerfile.model_server
index 04dfd7d812b..05a284a2baa 100644
--- a/backend/Dockerfile.model_server
+++ b/backend/Dockerfile.model_server
@@ -55,6 +55,6 @@ COPY ./shared_configs /app/shared_configs
 # Model Server main code
 COPY ./model_server /app/model_server
 
-ENV PYTHONPATH /app
+ENV PYTHONPATH=/app
 
 CMD ["uvicorn", "model_server.main:app", "--host", "0.0.0.0", "--port", "9000"]
diff --git a/backend/tests/integration/Dockerfile b/backend/tests/integration/Dockerfile
index ac9a19da687..02cdcad0b44 100644
--- a/backend/tests/integration/Dockerfile
+++ b/backend/tests/integration/Dockerfile
@@ -81,6 +81,6 @@ RUN pip install --no-cache-dir --upgrade \
         -r /tmp/dev-requirements.txt
 COPY ./tests/integration /app/tests/integration
 
-ENV PYTHONPATH /app
+ENV PYTHONPATH=/app
 
 CMD ["pytest", "-s", "/app/tests/integration"]
diff --git a/web/Dockerfile b/web/Dockerfile
index 6ea85752b82..48c13f57be1 100644
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -25,10 +25,10 @@ COPY . .
 RUN npm ci
 
 # needed to get the `standalone` dir we expect later
-ENV NEXT_PRIVATE_STANDALONE true
+ENV NEXT_PRIVATE_STANDALONE=true
 
 # Disable automatic telemetry collection
-ENV NEXT_TELEMETRY_DISABLED 1
+ENV NEXT_TELEMETRY_DISABLED=1
 
 # Environment variables must be present at build time
 # https://github.com/vercel/next.js/discussions/14030
@@ -77,7 +77,7 @@ RUN rm -rf /usr/local/lib/node_modules
 # ENV NODE_ENV production  
 
 # Disable automatic telemetry collection
-ENV NEXT_TELEMETRY_DISABLED 1
+ENV NEXT_TELEMETRY_DISABLED=1
 
 # Don't run production as root
 RUN addgroup --system --gid 1001 nodejs

From a0124e4e5015eaa3f206c818d0c183ee0177b0e9 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 8 Oct 2024 08:48:38 -0700
Subject: [PATCH 052/376] ensure all timeout -> hook (#2718)

---
 .../app/admin/connector/[ccPairId]/page.tsx   | 19 ++----------
 web/src/app/admin/indexing/status/page.tsx    | 29 ++++++++++---------
 2 files changed, 19 insertions(+), 29 deletions(-)

diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index 91c2e197b4e..530b11332b2 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -50,14 +50,8 @@ function Main({ ccPairId }: { ccPairId: number }) {
   const { popup, setPopup } = usePopup();
 
   const finishConnectorDeletion = useCallback(() => {
-    setPopup({
-      message: "Connector deleted successfully",
-      type: "success",
-    });
-    setTimeout(() => {
-      router.push("/admin/indexing/status");
-    }, 2000);
-  }, [router, setPopup]);
+    router.push("/admin/indexing/status?message=connector-deleted");
+  }, [router]);
 
   useEffect(() => {
     if (isEditing && inputRef.current) {
@@ -80,14 +74,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
     ) {
       finishConnectorDeletion();
     }
-  }, [
-    isLoading,
-    ccPair,
-    error,
-    hasLoadedOnce,
-    router,
-    finishConnectorDeletion,
-  ]);
+  }, [isLoading, ccPair, error, hasLoadedOnce]);
 
   const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
     setEditableName(e.target.value);
diff --git a/web/src/app/admin/indexing/status/page.tsx b/web/src/app/admin/indexing/status/page.tsx
index e5f1ea6ce0d..5b77d5151a0 100644
--- a/web/src/app/admin/indexing/status/page.tsx
+++ b/web/src/app/admin/indexing/status/page.tsx
@@ -10,12 +10,6 @@ import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
 import { usePopupFromQuery } from "@/components/popup/PopupFromQuery";
 
 function Main() {
-  const { popup } = usePopupFromQuery({
-    "connector-created": {
-      message: "Connector created successfully",
-      type: "success",
-    },
-  });
   const {
     data: indexAttemptData,
     isLoading: indexAttemptIsLoading,
@@ -70,19 +64,28 @@ function Main() {
   });
 
   return (
-    <>
-      {popup}
-      <CCPairIndexingStatusTable
-        ccPairsIndexingStatuses={indexAttemptData}
-        editableCcPairsIndexingStatuses={editableIndexAttemptData}
-      />
-    </>
+    <CCPairIndexingStatusTable
+      ccPairsIndexingStatuses={indexAttemptData}
+      editableCcPairsIndexingStatuses={editableIndexAttemptData}
+    />
   );
 }
 
 export default function Status() {
+  const { popup } = usePopupFromQuery({
+    "connector-created": {
+      message: "Connector created successfully",
+      type: "success",
+    },
+    "connector-deleted": {
+      message: "Connector deleted successfully",
+      type: "success",
+    },
+  });
+
   return (
     <div className="mx-auto container">
+      {popup}
       <AdminPageTitle
         icon={<NotebookIcon size={32} />}
         title="Existing Connectors"

From 3ef72b8d1a2eb19a0fc7e1a4ad2db31cd9f95185 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 8 Oct 2024 09:33:29 -0700
Subject: [PATCH 053/376] k (#2721)

---
 web/src/app/admin/assistants/PersonaTable.tsx | 2 +-
 web/src/components/IsPublicGroupSelector.tsx  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index fad5853d88e..15775c7cdfb 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -175,7 +175,7 @@ export function PersonasTable({
               </div>,
               <div key="edit" className="flex">
                 <div className="mr-auto my-auto">
-                  {!persona.is_default_persona && isEditable ? (
+                  {!persona.builtin_persona && isEditable ? (
                     <div
                       className="hover:bg-hover rounded p-1 cursor-pointer"
                       onClick={async () => {
diff --git a/web/src/components/IsPublicGroupSelector.tsx b/web/src/components/IsPublicGroupSelector.tsx
index 7cb8ff8f830..adb2496ed8a 100644
--- a/web/src/components/IsPublicGroupSelector.tsx
+++ b/web/src/components/IsPublicGroupSelector.tsx
@@ -49,7 +49,7 @@ export const IsPublicGroupSelector = <T extends IsPublicGroupSelectorFormType>({
         setShouldHideContent(false);
       }
     }
-  }, [user, userGroups, formikProps, isPaidEnterpriseFeaturesEnabled]);
+  }, [user, userGroups, isPaidEnterpriseFeaturesEnabled]);
 
   if (isLoadingUser || userGroupsIsLoading) {
     return <div>Loading...</div>;

From aa69fe762be67082980dd35887dae4dca0231fa3 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Tue, 8 Oct 2024 11:08:45 -0700
Subject: [PATCH 054/376] Temp patch to remove multiple tool calls (#2720)

---
 backend/danswer/llm/answering/answer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/llm/answering/answer.py b/backend/danswer/llm/answering/answer.py
index 12c1bc25f4f..4648e0fe821 100644
--- a/backend/danswer/llm/answering/answer.py
+++ b/backend/danswer/llm/answering/answer.py
@@ -316,7 +316,9 @@ def _raw_output_for_explicit_tool_calling_llms(
 
                 yield from self._process_llm_stream(
                     prompt=prompt,
-                    tools=[tool.tool_definition() for tool in self.tools],
+                    # as of now, we don't support multiple tool calls in sequence, which is why
+                    # we don't need to pass this in here
+                    # tools=[tool.tool_definition() for tool in self.tools],
                 )
 
             return

From 3586f9b5655e0c73e62c203c15b79b25842cfd60 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 11:23:10 -0700
Subject: [PATCH 055/376] experimental workflow to auto merge hotfixes to
 release branches.

---
 .../workflows/hotfix-to-release-branches.yml  | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 .github/workflows/hotfix-to-release-branches.yml

diff --git a/.github/workflows/hotfix-to-release-branches.yml b/.github/workflows/hotfix-to-release-branches.yml
new file mode 100644
index 00000000000..dc8b50794d6
--- /dev/null
+++ b/.github/workflows/hotfix-to-release-branches.yml
@@ -0,0 +1,100 @@
+# This workflow is intended to be manually triggered via the GitHub Action tab.
+# Given a hotfix branch, it will attempt to open a PR to all release branches and
+# by default auto merge them
+
+name: Open PRs from hotfix to release branches
+
+on:
+  workflow_dispatch:
+    inputs:
+      hotfix_branch:
+        description: 'Hotfix branch name'
+        required: true
+      release_branch_pattern:
+        description: 'Release branch pattern (regex)'
+        required: true
+        default: 'release/.*'
+      auto_merge:
+        description: 'Automatically merge the PRs if set to true'
+        required: false
+        default: 'true'
+        
+jobs:
+  hotfix_to_release:
+    # See https://runs-on.com/runners/linux/
+    # use a lower powered instance since this just does i/o to docker hub
+    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Fetch All Branches
+        run: |
+          git fetch --all --prune
+
+      - name: Get Release Branches
+        id: get_release_branches
+        run: |
+          BRANCHES=$(git branch -r | grep -E "${{ github.event.inputs.release_branch_pattern }}" | sed 's|origin/||' | tr -d ' ')
+          if [ -z "$BRANCHES" ]; then
+            echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'."
+            exit 1
+          fi
+          echo "Found release branches:"
+          echo "$BRANCHES"
+          # Set the branches as an output
+          echo "branches=$BRANCHES" >> $GITHUB_OUTPUT
+
+      - name: Ensure Hotfix Branch Exists Locally
+        run: |
+          git fetch origin "${{ github.event.inputs.hotfix_branch }}":"${{ github.event.inputs.hotfix_branch }}" || true
+
+      - name: Create and Merge Pull Requests to Matching Release Branches
+        env:
+          HOTFIX_BRANCH: ${{ github.event.inputs.hotfix_branch }}
+          AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
+        run: |
+          # Get the branches from the previous step
+          BRANCHES="${{ steps.get_release_branches.outputs.branches }}"
+
+          # Convert BRANCHES to an array
+          IFS=$'\n' read -rd '' -a BRANCH_ARRAY <<<"$BRANCHES"
+
+          # Loop through each release branch and create and merge a PR
+          for RELEASE_BRANCH in $BRANCHES; do
+            echo "Creating PR from $HOTFIX_BRANCH to $RELEASE_BRANCH"
+
+            # Check if PR already exists
+            EXISTING_PR=$(gh pr list --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --state open --json number --jq '.[0].number')
+
+            if [ -n "$EXISTING_PR" ]; then
+              echo "An open PR already exists: #$EXISTING_PR. Skipping..."
+              continue
+            fi
+            
+            # Create a new PR
+            PR_URL=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \
+              --body "Automated PR to merge \`$HOTFIX_BRANCH\` into \`$RELEASE_BRANCH\`." \
+              --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --json url --jq '.url')
+
+            echo "Pull request created: $PR_URL"
+
+            # Extract PR number from URL
+            PR_NUMBER=$(basename "$PR_URL")
+
+            if [ "$AUTO_MERGE" == "true" ]; then
+              echo "Attempting to merge pull request #$PR_NUMBER"
+
+              # Attempt to merge the PR
+              gh pr merge "$PR_NUMBER" --merge --admin --yes
+
+              if [ $? -eq 0 ]; then
+                echo "Pull request #$PR_NUMBER merged successfully."
+              else
+                echo "Failed to merge pull request #$PR_NUMBER."
+                # Optionally, handle the error or continue
+              fi
+            fi
+          done
\ No newline at end of file

From 21a3921790c29da9cf37d04070e06e4ac9a81b5e Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Tue, 8 Oct 2024 12:41:14 -0700
Subject: [PATCH 056/376] Better support for image generation capable models
 (#2725)

---
 .../app/admin/assistants/AssistantEditor.tsx  | 20 +++------
 web/src/lib/llm/utils.ts                      | 45 +++++++++----------
 2 files changed, 28 insertions(+), 37 deletions(-)

diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index b9bd5152149..8c295b31a48 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -26,7 +26,7 @@ import { getDisplayNameForModel } from "@/lib/hooks";
 import { DocumentSetSelectable } from "@/components/documentSet/DocumentSetSelectable";
 import { Option } from "@/components/Dropdown";
 import { addAssistantToList } from "@/lib/assistants/updateAssistantPreferences";
-import { checkLLMSupportsImageOutput, destructureValue } from "@/lib/llm/utils";
+import { checkLLMSupportsImageInput, destructureValue } from "@/lib/llm/utils";
 import { ToolSnapshot } from "@/lib/tools/interfaces";
 import { checkUserIsNoAuthUser } from "@/lib/user";
 
@@ -349,12 +349,9 @@ export function AssistantEditor({
 
           if (imageGenerationToolEnabled) {
             if (
-              !checkLLMSupportsImageOutput(
-                providerDisplayNameToProviderName.get(
-                  values.llm_model_provider_override || ""
-                ) ||
-                  defaultProviderName ||
-                  "",
+              // model must support image input for image generation
+              // to work
+              !checkLLMSupportsImageInput(
                 values.llm_model_version_override || defaultModelName || ""
               )
             ) {
@@ -469,12 +466,9 @@ export function AssistantEditor({
               : false;
           }
 
-          const currentLLMSupportsImageOutput = checkLLMSupportsImageOutput(
-            providerDisplayNameToProviderName.get(
-              values.llm_model_provider_override || ""
-            ) ||
-              defaultProviderName ||
-              "",
+          // model must support image input for image generation
+          // to work
+          const currentLLMSupportsImageOutput = checkLLMSupportsImageInput(
             values.llm_model_version_override || defaultModelName || ""
           );
 
diff --git a/web/src/lib/llm/utils.ts b/web/src/lib/llm/utils.ts
index b020854d4fe..2adfbfc0543 100644
--- a/web/src/lib/llm/utils.ts
+++ b/web/src/lib/llm/utils.ts
@@ -68,15 +68,17 @@ const MODEL_NAMES_SUPPORTING_IMAGE_INPUT = [
   "gpt-4-vision-preview",
   "gpt-4-turbo",
   "gpt-4-1106-vision-preview",
-  "gpt-4o",
-  "gpt-4o-mini",
-  "gpt-4-vision-preview",
-  "gpt-4-turbo",
-  "gpt-4-1106-vision-preview",
+  // standard claude names
   "claude-3-5-sonnet-20240620",
   "claude-3-opus-20240229",
   "claude-3-sonnet-20240229",
   "claude-3-haiku-20240307",
+  // claude names with AWS Bedrock Suffix
+  "claude-3-opus-20240229-v1:0",
+  "claude-3-sonnet-20240229-v1:0",
+  "claude-3-haiku-20240307-v1:0",
+  "claude-3-5-sonnet-20240620-v1:0",
+  // claude names with full AWS Bedrock names
   "anthropic.claude-3-opus-20240229-v1:0",
   "anthropic.claude-3-sonnet-20240229-v1:0",
   "anthropic.claude-3-haiku-20240307-v1:0",
@@ -84,29 +86,24 @@ const MODEL_NAMES_SUPPORTING_IMAGE_INPUT = [
 ];
 
 export function checkLLMSupportsImageInput(model: string) {
-  return MODEL_NAMES_SUPPORTING_IMAGE_INPUT.some(
+  // Original exact match check
+  const exactMatch = MODEL_NAMES_SUPPORTING_IMAGE_INPUT.some(
     (modelName) => modelName === model
   );
-}
 
-const MODEL_PROVIDER_PAIRS_SUPPORTING_IMAGE_OUTPUT = [
-  ["openai", "gpt-4o"],
-  ["openai", "gpt-4o-mini"],
-  ["openai", "gpt-4-vision-preview"],
-  ["openai", "gpt-4-turbo"],
-  ["openai", "gpt-4-1106-vision-preview"],
-  ["azure", "gpt-4o"],
-  ["azure", "gpt-4o-mini"],
-  ["azure", "gpt-4-vision-preview"],
-  ["azure", "gpt-4-turbo"],
-  ["azure", "gpt-4-1106-vision-preview"],
-];
+  if (exactMatch) {
+    return true;
+  }
 
-export function checkLLMSupportsImageOutput(provider: string, model: string) {
-  return MODEL_PROVIDER_PAIRS_SUPPORTING_IMAGE_OUTPUT.some(
-    (modelProvider) =>
-      modelProvider[0] === provider && modelProvider[1] === model
-  );
+  // Additional check for the last part of the model name
+  const modelParts = model.split(/[/.]/);
+  const lastPart = modelParts[modelParts.length - 1];
+
+  return MODEL_NAMES_SUPPORTING_IMAGE_INPUT.some((modelName) => {
+    const modelNameParts = modelName.split(/[/.]/);
+    const modelNameLastPart = modelNameParts[modelNameParts.length - 1];
+    return modelNameLastPart === lastPart;
+  });
 }
 
 export const structureValue = (

From 5cc46341f7d1b388f778e9b60ec50332fb89f0aa Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 13:11:59 -0700
Subject: [PATCH 057/376] try porting docker web build to runs-on

---
 .../workflows/docker-build-push-web-container-on-tag.yml   | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/docker-build-push-web-container-on-tag.yml b/.github/workflows/docker-build-push-web-container-on-tag.yml
index 1c901613563..4f1fd804969 100644
--- a/.github/workflows/docker-build-push-web-container-on-tag.yml
+++ b/.github/workflows/docker-build-push-web-container-on-tag.yml
@@ -11,8 +11,11 @@ env:
   
 jobs:
   build:
-    runs-on: 
-      group: ${{ matrix.platform == 'linux/amd64' && 'amd64-image-builders' || 'arm64-image-builders' }}
+    runs-on:
+      - runs-on
+      - runner=${{ matrix.platform == 'linux/amd64' && '8cpu-linux-x64' || '8cpu-linux-arm64' }}
+      - run-id=${{ github.run_id }}
+      - tag=platform-${{ matrix.platform }}
     strategy:
       fail-fast: false
       matrix:

From 057321a59fe5f96e7816b103698aecfccd825f4a Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 13:40:35 -0700
Subject: [PATCH 058/376] disable flaky test

---
 .../connector_job_tests/slack/test_permission_sync.py           | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
index 20f2c1913d6..9114c2ddecf 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
@@ -3,6 +3,7 @@
 from datetime import timezone
 from typing import Any
 
+import pytest
 import requests
 
 from danswer.connectors.models import InputType
@@ -26,6 +27,7 @@
 from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
 
 
+@pytest.mark.skip(reason="flaky - see DAN-789 for example")
 def test_slack_permission_sync(
     reset: None,
     vespa_client: vespa_fixture,

From 672f5cc5ce8853334e8baf755dbac386e8284c71 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 13:46:11 -0700
Subject: [PATCH 059/376] urlencode the password part properly before putting
 it in the broker url (#2719)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 backend/danswer/background/celery/celeryconfig.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/background/celery/celeryconfig.py b/backend/danswer/background/celery/celeryconfig.py
index 1b1aa092d17..31d36d99533 100644
--- a/backend/danswer/background/celery/celeryconfig.py
+++ b/backend/danswer/background/celery/celeryconfig.py
@@ -1,4 +1,6 @@
 # docs: https://docs.celeryq.dev/en/stable/userguide/configuration.html
+import urllib.parse
+
 from danswer.configs.app_configs import CELERY_BROKER_POOL_LIMIT
 from danswer.configs.app_configs import CELERY_RESULT_EXPIRES
 from danswer.configs.app_configs import REDIS_DB_NUMBER_CELERY
@@ -17,7 +19,7 @@
 
 CELERY_PASSWORD_PART = ""
 if REDIS_PASSWORD:
-    CELERY_PASSWORD_PART = f":{REDIS_PASSWORD}@"
+    CELERY_PASSWORD_PART = ":" + urllib.parse.quote(REDIS_PASSWORD, safe="") + "@"
 
 REDIS_SCHEME = "redis"
 

From 78e7710f1777caced16e18602b479bdabfbb4061 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Tue, 8 Oct 2024 14:01:37 -0700
Subject: [PATCH 060/376] Handle bug with initial connector page display
 (#2727)

* Handle bug with initial connector page display

* Casing consistency
---
 .../[ccPairId]/IndexingAttemptsTable.tsx      | 32 ++++++++++++++++---
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index 517fb1585c4..4bb532981b4 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -9,6 +9,7 @@ import {
   TableBody,
   TableCell,
   Text,
+  Callout,
 } from "@tremor/react";
 import { CCPairFullInfo, PaginatedIndexAttempts } from "./types";
 import { IndexAttemptStatus } from "@/components/Status";
@@ -23,6 +24,7 @@ import Link from "next/link";
 import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
 import { useRouter } from "next/navigation";
 import { Tooltip } from "@/components/tooltip/Tooltip";
+import { FiInfo } from "react-icons/fi";
 
 // This is the number of index attempts to display per page
 const NUM_IN_PAGE = 8;
@@ -61,7 +63,9 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
 
   const batchRetrievalUrlBuilder = useCallback(
     (batchNum: number) => {
-      return `${buildCCPairInfoUrl(ccPair.id)}/index-attempts?page=${batchNum}&page_size=${BATCH_SIZE * NUM_IN_PAGE}`;
+      return `${buildCCPairInfoUrl(
+        ccPair.id
+      )}/index-attempts?page=${batchNum}&page_size=${BATCH_SIZE * NUM_IN_PAGE}`;
     },
     [ccPair.id]
   );
@@ -124,9 +128,9 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
       setIsCurrentPageLoading(false);
     }
 
-    const nextBatchNum = Math.min(
-      batchNum + 1,
-      Math.ceil(totalPages / BATCH_SIZE) - 1
+    const nextBatchNum = Math.max(
+      Math.min(batchNum + 1, Math.ceil(totalPages / BATCH_SIZE) - 1),
+      0
     );
     if (!cachedBatches[nextBatchNum]) {
       fetchBatchData(nextBatchNum);
@@ -182,6 +186,26 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
     );
   }
 
+  // if no indexing attempts have been scheduled yet, let the user know why
+  if (
+    Object.keys(cachedBatches).length === 0 ||
+    Object.values(cachedBatches).every((batch) =>
+      batch.every((page) => page.index_attempts.length === 0)
+    )
+  ) {
+    return (
+      <Callout
+        className="mt-4"
+        title="No indexing attempts scheduled yet"
+        icon={FiInfo}
+        color="blue"
+      >
+        Index attempts are scheduled in the background, and may take some time
+        to appear. Try refreshing the page in ~30 seconds!
+      </Callout>
+    );
+  }
+
   // This is the index attempt that the user wants to view the trace for
   const indexAttemptToDisplayTraceFor = currentPageData?.index_attempts?.find(
     (indexAttempt) => indexAttempt.id === indexAttemptTracePopupId

From c72c5619f04beddaf68a8e90b7314f5cbb894ca6 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 14:42:04 -0700
Subject: [PATCH 061/376] remove more flaky tests

---
 .../integration/tests/dev_apis/test_knowledge_chat.py  | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py b/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
index 2911c0f11f1..475085c6777 100644
--- a/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
+++ b/backend/tests/integration/tests/dev_apis/test_knowledge_chat.py
@@ -83,8 +83,9 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     # FLAKY - check that the cited documents are correct
     # assert cc_pair_1.documents[0].id in response_json["cited_documents"].values()
 
-    # check that the top documents are correct
-    assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[0].id
+    # flakiness likely due to non-deterministic rephrasing
+    # FLAKY - check that the top documents are correct
+    # assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[0].id
     print("response 1/3 passed")
 
     # TESTING RESPONSE FOR QUESTION 2
@@ -129,8 +130,9 @@ def test_all_stream_chat_message_objects_outputs(reset: None) -> None:
     # FLAKY - check that the cited documents are correct
     # assert cc_pair_1.documents[1].id in response_json["cited_documents"].values()
 
-    # check that the top documents are correct
-    assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[1].id
+    # flakiness likely due to non-deterministic rephrasing
+    # FLAKY - check that the top documents are correct
+    # assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[1].id
     print("response 2/3 passed")
 
     # TESTING RESPONSE FOR QUESTION 3

From a47d27de6c7efc3b26870b641934494f47a08a17 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 14:42:59 -0700
Subject: [PATCH 062/376] experimental workflow to auto merge hotfixes to
 release branches. (#2723)

---
 .../workflows/hotfix-to-release-branches.yml  | 100 ++++++++++++++++++
 1 file changed, 100 insertions(+)
 create mode 100644 .github/workflows/hotfix-to-release-branches.yml

diff --git a/.github/workflows/hotfix-to-release-branches.yml b/.github/workflows/hotfix-to-release-branches.yml
new file mode 100644
index 00000000000..dc8b50794d6
--- /dev/null
+++ b/.github/workflows/hotfix-to-release-branches.yml
@@ -0,0 +1,100 @@
+# This workflow is intended to be manually triggered via the GitHub Action tab.
+# Given a hotfix branch, it will attempt to open a PR to all release branches and
+# by default auto merge them
+
+name: Open PRs from hotfix to release branches
+
+on:
+  workflow_dispatch:
+    inputs:
+      hotfix_branch:
+        description: 'Hotfix branch name'
+        required: true
+      release_branch_pattern:
+        description: 'Release branch pattern (regex)'
+        required: true
+        default: 'release/.*'
+      auto_merge:
+        description: 'Automatically merge the PRs if set to true'
+        required: false
+        default: 'true'
+        
+jobs:
+  hotfix_to_release:
+    # See https://runs-on.com/runners/linux/
+    # use a lower powered instance since this just does i/o to docker hub
+    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Fetch All Branches
+        run: |
+          git fetch --all --prune
+
+      - name: Get Release Branches
+        id: get_release_branches
+        run: |
+          BRANCHES=$(git branch -r | grep -E "${{ github.event.inputs.release_branch_pattern }}" | sed 's|origin/||' | tr -d ' ')
+          if [ -z "$BRANCHES" ]; then
+            echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'."
+            exit 1
+          fi
+          echo "Found release branches:"
+          echo "$BRANCHES"
+          # Set the branches as an output
+          echo "branches=$BRANCHES" >> $GITHUB_OUTPUT
+
+      - name: Ensure Hotfix Branch Exists Locally
+        run: |
+          git fetch origin "${{ github.event.inputs.hotfix_branch }}":"${{ github.event.inputs.hotfix_branch }}" || true
+
+      - name: Create and Merge Pull Requests to Matching Release Branches
+        env:
+          HOTFIX_BRANCH: ${{ github.event.inputs.hotfix_branch }}
+          AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
+        run: |
+          # Get the branches from the previous step
+          BRANCHES="${{ steps.get_release_branches.outputs.branches }}"
+
+          # Convert BRANCHES to an array
+          IFS=$'\n' read -rd '' -a BRANCH_ARRAY <<<"$BRANCHES"
+
+          # Loop through each release branch and create and merge a PR
+          for RELEASE_BRANCH in $BRANCHES; do
+            echo "Creating PR from $HOTFIX_BRANCH to $RELEASE_BRANCH"
+
+            # Check if PR already exists
+            EXISTING_PR=$(gh pr list --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --state open --json number --jq '.[0].number')
+
+            if [ -n "$EXISTING_PR" ]; then
+              echo "An open PR already exists: #$EXISTING_PR. Skipping..."
+              continue
+            fi
+            
+            # Create a new PR
+            PR_URL=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \
+              --body "Automated PR to merge \`$HOTFIX_BRANCH\` into \`$RELEASE_BRANCH\`." \
+              --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --json url --jq '.url')
+
+            echo "Pull request created: $PR_URL"
+
+            # Extract PR number from URL
+            PR_NUMBER=$(basename "$PR_URL")
+
+            if [ "$AUTO_MERGE" == "true" ]; then
+              echo "Attempting to merge pull request #$PR_NUMBER"
+
+              # Attempt to merge the PR
+              gh pr merge "$PR_NUMBER" --merge --admin --yes
+
+              if [ $? -eq 0 ]; then
+                echo "Pull request #$PR_NUMBER merged successfully."
+              else
+                echo "Failed to merge pull request #$PR_NUMBER."
+                # Optionally, handle the error or continue
+              fi
+            fi
+          done
\ No newline at end of file

From 8f61505437ba7b8d6afa5f23e315a48203e85edc Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 8 Oct 2024 16:13:45 -0700
Subject: [PATCH 063/376] Fix azure (#2665)

* fix azure

* nit

* nit

* nit

* nit pretty
---
 ...33ba_add_deployment_name_to_llmprovider.py | 26 +++++++
 backend/danswer/chat/process_message.py       | 28 +++++++-
 backend/danswer/configs/app_configs.py        |  6 ++
 backend/danswer/db/llm.py                     |  1 +
 backend/danswer/db/models.py                  |  2 +
 backend/danswer/llm/chat_llm.py               |  5 +-
 backend/danswer/llm/factory.py                |  2 +
 backend/danswer/llm/interfaces.py             |  2 +-
 backend/danswer/llm/llm_provider_options.py   |  7 +-
 .../danswer/server/features/persona/api.py    |  7 +-
 backend/danswer/server/features/tool/api.py   |  9 ++-
 backend/danswer/server/manage/llm/api.py      | 15 ++++
 backend/danswer/server/manage/llm/models.py   |  2 +
 backend/danswer/tools/utils.py                | 21 ++++++
 .../app/admin/assistants/AssistantEditor.tsx  | 10 +--
 .../llm/ConfiguredLLMProviderDisplay.tsx      |  1 +
 .../llm/CustomLLMProviderUpdateForm.tsx       | 71 +++++++++++--------
 .../llm/LLMProviderUpdateForm.tsx             | 68 +++++++++++-------
 .../app/admin/configuration/llm/interfaces.ts |  5 +-
 web/src/lib/chat/fetchChatData.ts             | 10 ++-
 web/src/lib/chat/fetchSomeChatData.ts         | 12 ++--
 21 files changed, 234 insertions(+), 76 deletions(-)
 create mode 100644 backend/alembic/versions/e4334d5b33ba_add_deployment_name_to_llmprovider.py

diff --git a/backend/alembic/versions/e4334d5b33ba_add_deployment_name_to_llmprovider.py b/backend/alembic/versions/e4334d5b33ba_add_deployment_name_to_llmprovider.py
new file mode 100644
index 00000000000..e837b87e3e0
--- /dev/null
+++ b/backend/alembic/versions/e4334d5b33ba_add_deployment_name_to_llmprovider.py
@@ -0,0 +1,26 @@
+"""add_deployment_name_to_llmprovider
+
+Revision ID: e4334d5b33ba
+Revises: ac5eaac849f9
+Create Date: 2024-10-04 09:52:34.896867
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "e4334d5b33ba"
+down_revision = "ac5eaac849f9"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "llm_provider", sa.Column("deployment_name", sa.String(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("llm_provider", "deployment_name")
diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index f09ac18f32a..19787545e4a 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -18,6 +18,10 @@
 from danswer.chat.models import MessageSpecificCitations
 from danswer.chat.models import QADocsResponse
 from danswer.chat.models import StreamingError
+from danswer.configs.app_configs import AZURE_DALLE_API_BASE
+from danswer.configs.app_configs import AZURE_DALLE_API_KEY
+from danswer.configs.app_configs import AZURE_DALLE_API_VERSION
+from danswer.configs.app_configs import AZURE_DALLE_DEPLOYMENT_NAME
 from danswer.configs.chat_configs import BING_API_KEY
 from danswer.configs.chat_configs import CHAT_TARGET_CHUNK_PERCENTAGE
 from danswer.configs.chat_configs import DISABLE_LLM_CHOOSE_SEARCH
@@ -560,7 +564,26 @@ def stream_chat_message_objects(
                         and llm.config.api_key
                         and llm.config.model_provider == "openai"
                     ):
-                        img_generation_llm_config = llm.config
+                        img_generation_llm_config = LLMConfig(
+                            model_provider=llm.config.model_provider,
+                            model_name="dall-e-3",
+                            temperature=GEN_AI_TEMPERATURE,
+                            api_key=llm.config.api_key,
+                            api_base=llm.config.api_base,
+                            api_version=llm.config.api_version,
+                        )
+                    elif (
+                        llm.config.model_provider == "azure"
+                        and AZURE_DALLE_API_KEY is not None
+                    ):
+                        img_generation_llm_config = LLMConfig(
+                            model_provider="azure",
+                            model_name=f"azure/{AZURE_DALLE_DEPLOYMENT_NAME}",
+                            temperature=GEN_AI_TEMPERATURE,
+                            api_key=AZURE_DALLE_API_KEY,
+                            api_base=AZURE_DALLE_API_BASE,
+                            api_version=AZURE_DALLE_API_VERSION,
+                        )
                     else:
                         llm_providers = fetch_existing_llm_providers(db_session)
                         openai_provider = next(
@@ -579,7 +602,7 @@ def stream_chat_message_objects(
                             )
                         img_generation_llm_config = LLMConfig(
                             model_provider=openai_provider.provider,
-                            model_name=openai_provider.default_model_name,
+                            model_name="dall-e-3",
                             temperature=GEN_AI_TEMPERATURE,
                             api_key=openai_provider.api_key,
                             api_base=openai_provider.api_base,
@@ -591,6 +614,7 @@ def stream_chat_message_objects(
                             api_base=img_generation_llm_config.api_base,
                             api_version=img_generation_llm_config.api_version,
                             additional_headers=litellm_additional_headers,
+                            model=img_generation_llm_config.model_name,
                         )
                     ]
                 elif tool_cls.__name__ == InternetSearchTool.__name__:
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 4559fed6b87..1174c8d060f 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -413,6 +413,12 @@
     os.environ.get("ENABLE_PAID_ENTERPRISE_EDITION_FEATURES", "").lower() == "true"
 )
 
+# Azure DALL-E Configurations
+AZURE_DALLE_API_VERSION = os.environ.get("AZURE_DALLE_API_VERSION")
+AZURE_DALLE_API_KEY = os.environ.get("AZURE_DALLE_API_KEY")
+AZURE_DALLE_API_BASE = os.environ.get("AZURE_DALLE_API_BASE")
+AZURE_DALLE_DEPLOYMENT_NAME = os.environ.get("AZURE_DALLE_DEPLOYMENT_NAME")
+
 
 MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
 SECRET_JWT_KEY = os.environ.get("SECRET_JWT_KEY", "")
diff --git a/backend/danswer/db/llm.py b/backend/danswer/db/llm.py
index af2ded9562a..c03ed99e412 100644
--- a/backend/danswer/db/llm.py
+++ b/backend/danswer/db/llm.py
@@ -83,6 +83,7 @@ def upsert_llm_provider(
     existing_llm_provider.model_names = llm_provider.model_names
     existing_llm_provider.is_public = llm_provider.is_public
     existing_llm_provider.display_model_names = llm_provider.display_model_names
+    existing_llm_provider.deployment_name = llm_provider.deployment_name
 
     if not existing_llm_provider.id:
         # If its not already in the db, we need to generate an ID by flushing
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index 4777577d0fd..392c7a28b2e 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -1143,6 +1143,8 @@ class LLMProvider(Base):
         postgresql.ARRAY(String), nullable=True
     )
 
+    deployment_name: Mapped[str | None] = mapped_column(String, nullable=True)
+
     # should only be set for a single provider
     is_default_provider: Mapped[bool | None] = mapped_column(Boolean, unique=True)
     # EE only
diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
index 1021f82abc6..90136a76bd8 100644
--- a/backend/danswer/llm/chat_llm.py
+++ b/backend/danswer/llm/chat_llm.py
@@ -204,6 +204,7 @@ def __init__(
         model_name: str,
         api_base: str | None = None,
         api_version: str | None = None,
+        deployment_name: str | None = None,
         max_output_tokens: int | None = None,
         custom_llm_provider: str | None = None,
         temperature: float = GEN_AI_TEMPERATURE,
@@ -215,6 +216,7 @@ def __init__(
         self._model_version = model_name
         self._temperature = temperature
         self._api_key = api_key
+        self._deployment_name = deployment_name
         self._api_base = api_base
         self._api_version = api_version
         self._custom_llm_provider = custom_llm_provider
@@ -290,7 +292,7 @@ def _completion(
         try:
             return litellm.completion(
                 # model choice
-                model=f"{self.config.model_provider}/{self.config.model_name}",
+                model=f"{self.config.model_provider}/{self.config.deployment_name or self.config.model_name}",
                 # NOTE: have to pass in None instead of empty string for these
                 # otherwise litellm can have some issues with bedrock
                 api_key=self._api_key or None,
@@ -325,6 +327,7 @@ def config(self) -> LLMConfig:
             api_key=self._api_key,
             api_base=self._api_base,
             api_version=self._api_version,
+            deployment_name=self._deployment_name,
         )
 
     def _invoke_implementation(
diff --git a/backend/danswer/llm/factory.py b/backend/danswer/llm/factory.py
index f57bfb524b9..904735d5ffe 100644
--- a/backend/danswer/llm/factory.py
+++ b/backend/danswer/llm/factory.py
@@ -88,6 +88,7 @@ def _create_llm(model: str) -> LLM:
         return get_llm(
             provider=llm_provider.provider,
             model=model,
+            deployment_name=llm_provider.deployment_name,
             api_key=llm_provider.api_key,
             api_base=llm_provider.api_base,
             api_version=llm_provider.api_version,
@@ -103,6 +104,7 @@ def _create_llm(model: str) -> LLM:
 def get_llm(
     provider: str,
     model: str,
+    deployment_name: str | None = None,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
diff --git a/backend/danswer/llm/interfaces.py b/backend/danswer/llm/interfaces.py
index 5e39792c393..6cb58e46c6b 100644
--- a/backend/danswer/llm/interfaces.py
+++ b/backend/danswer/llm/interfaces.py
@@ -24,7 +24,7 @@ class LLMConfig(BaseModel):
     api_key: str | None = None
     api_base: str | None = None
     api_version: str | None = None
-
+    deployment_name: str | None = None
     # This disables the "model_" protected namespace for pydantic
     model_config = {"protected_namespaces": ()}
 
diff --git a/backend/danswer/llm/llm_provider_options.py b/backend/danswer/llm/llm_provider_options.py
index 8fc1de73955..3cb6157d6da 100644
--- a/backend/danswer/llm/llm_provider_options.py
+++ b/backend/danswer/llm/llm_provider_options.py
@@ -16,10 +16,13 @@ class WellKnownLLMProviderDescriptor(BaseModel):
     api_base_required: bool
     api_version_required: bool
     custom_config_keys: list[CustomConfigKey] | None = None
-
     llm_names: list[str]
     default_model: str | None = None
     default_fast_model: str | None = None
+    # set for providers like Azure, which require a deployment name.
+    deployment_name_required: bool = False
+    # set for providers like Azure, which support a single model per deployment.
+    single_model_supported: bool = False
 
 
 OPENAI_PROVIDER_NAME = "openai"
@@ -108,6 +111,8 @@ def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
             api_version_required=True,
             custom_config_keys=[],
             llm_names=fetch_models_for_provider(AZURE_PROVIDER_NAME),
+            deployment_name_required=True,
+            single_model_supported=True,
         ),
         WellKnownLLMProviderDescriptor(
             name=BEDROCK_PROVIDER_NAME,
diff --git a/backend/danswer/server/features/persona/api.py b/backend/danswer/server/features/persona/api.py
index bcc4800b860..8b4305755dc 100644
--- a/backend/danswer/server/features/persona/api.py
+++ b/backend/danswer/server/features/persona/api.py
@@ -31,9 +31,9 @@
 from danswer.server.features.persona.models import PersonaSnapshot
 from danswer.server.features.persona.models import PromptTemplateResponse
 from danswer.server.models import DisplayPriorityRequest
+from danswer.tools.utils import is_image_generation_available
 from danswer.utils.logger import setup_logger
 
-
 logger = setup_logger()
 
 
@@ -226,6 +226,11 @@ def list_personas(
             get_editable=False,
             joinedload_all=True,
         )
+        # If the persona has an image generation tool and it's not available, don't include it
+        if not (
+            any(tool.in_code_tool_id == "ImageGenerationTool" for tool in persona.tools)
+            and not is_image_generation_available(db_session=db_session)
+        )
     ]
 
 
diff --git a/backend/danswer/server/features/tool/api.py b/backend/danswer/server/features/tool/api.py
index 1d441593784..7e15c048826 100644
--- a/backend/danswer/server/features/tool/api.py
+++ b/backend/danswer/server/features/tool/api.py
@@ -21,6 +21,8 @@
 from danswer.tools.custom.openapi_parsing import MethodSpec
 from danswer.tools.custom.openapi_parsing import openapi_to_method_specs
 from danswer.tools.custom.openapi_parsing import validate_openapi_schema
+from danswer.tools.images.image_generation_tool import ImageGenerationTool
+from danswer.tools.utils import is_image_generation_available
 
 router = APIRouter(prefix="/tool")
 admin_router = APIRouter(prefix="/admin/tool")
@@ -127,4 +129,9 @@ def list_tools(
     _: User | None = Depends(current_user),
 ) -> list[ToolSnapshot]:
     tools = get_tools(db_session)
-    return [ToolSnapshot.from_model(tool) for tool in tools]
+    return [
+        ToolSnapshot.from_model(tool)
+        for tool in tools
+        if tool.in_code_tool_id != ImageGenerationTool.name
+        or is_image_generation_available(db_session=db_session)
+    ]
diff --git a/backend/danswer/server/manage/llm/api.py b/backend/danswer/server/manage/llm/api.py
index 23f16047e91..06501d6834c 100644
--- a/backend/danswer/server/manage/llm/api.py
+++ b/backend/danswer/server/manage/llm/api.py
@@ -55,6 +55,7 @@ def test_llm_configuration(
         api_version=test_llm_request.api_version,
         custom_config=test_llm_request.custom_config,
     )
+
     functions_with_args: list[tuple[Callable, tuple]] = [(test_llm, (llm,))]
 
     if (
@@ -141,6 +142,20 @@ def put_llm_provider(
             detail=f"LLM Provider with name {llm_provider.name} already exists",
         )
 
+    # Ensure default_model_name and fast_default_model_name are in display_model_names
+    # This is necessary for custom models and Bedrock/Azure models
+    if llm_provider.display_model_names is None:
+        llm_provider.display_model_names = []
+
+    if llm_provider.default_model_name not in llm_provider.display_model_names:
+        llm_provider.display_model_names.append(llm_provider.default_model_name)
+
+    if (
+        llm_provider.fast_default_model_name
+        and llm_provider.fast_default_model_name not in llm_provider.display_model_names
+    ):
+        llm_provider.display_model_names.append(llm_provider.fast_default_model_name)
+
     try:
         return upsert_llm_provider(
             llm_provider=llm_provider,
diff --git a/backend/danswer/server/manage/llm/models.py b/backend/danswer/server/manage/llm/models.py
index 3ef66971003..2e3b3844807 100644
--- a/backend/danswer/server/manage/llm/models.py
+++ b/backend/danswer/server/manage/llm/models.py
@@ -66,6 +66,7 @@ class LLMProvider(BaseModel):
     is_public: bool = True
     groups: list[int] = Field(default_factory=list)
     display_model_names: list[str] | None = None
+    deployment_name: str | None = None
 
 
 class LLMProviderUpsertRequest(LLMProvider):
@@ -100,4 +101,5 @@ def from_model(cls, llm_provider_model: "LLMProviderModel") -> "FullLLMProvider"
             ),
             is_public=llm_provider_model.is_public,
             groups=[group.id for group in llm_provider_model.groups],
+            deployment_name=llm_provider_model.deployment_name,
         )
diff --git a/backend/danswer/tools/utils.py b/backend/danswer/tools/utils.py
index 9e20105edef..157d4bb6ec9 100644
--- a/backend/danswer/tools/utils.py
+++ b/backend/danswer/tools/utils.py
@@ -1,5 +1,11 @@
 import json
 
+from sqlalchemy.orm import Session
+
+from danswer.configs.app_configs import AZURE_DALLE_API_KEY
+from danswer.db.connector import check_connectors_exist
+from danswer.db.document import check_docs_exist
+from danswer.db.models import LLMProvider
 from danswer.natural_language_processing.utils import BaseTokenizer
 from danswer.tools.tool import Tool
 
@@ -26,3 +32,18 @@ def compute_tool_tokens(tool: Tool, llm_tokenizer: BaseTokenizer) -> int:
 
 def compute_all_tool_tokens(tools: list[Tool], llm_tokenizer: BaseTokenizer) -> int:
     return sum(compute_tool_tokens(tool, llm_tokenizer) for tool in tools)
+
+
+def is_image_generation_available(db_session: Session) -> bool:
+    providers = db_session.query(LLMProvider).all()
+    for provider in providers:
+        if provider.name == "OpenAI":
+            return True
+
+    return bool(AZURE_DALLE_API_KEY)
+
+
+def is_document_search_available(db_session: Session) -> bool:
+    docs_exist = check_docs_exist(db_session)
+    connectors_exist = check_connectors_exist(db_session)
+    return docs_exist or connectors_exist
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 8c295b31a48..7ca1d087641 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -192,15 +192,11 @@ export function AssistantEditor({
     modelOptionsByProvider.set(llmProvider.name, providerOptions);
   });
 
-  const providerSupportingImageGenerationExists =
-    providersContainImageGeneratingSupport(llmProviders);
-
   const personaCurrentToolIds =
     existingPersona?.tools.map((tool) => tool.id) || [];
+
   const searchTool = findSearchTool(tools);
-  const imageGenerationTool = providerSupportingImageGenerationExists
-    ? findImageGenerationTool(tools)
-    : undefined;
+  const imageGenerationTool = findImageGenerationTool(tools);
   const internetSearchTool = findInternetSearchTool(tools);
 
   const customTools = tools.filter(
@@ -997,7 +993,7 @@ export function AssistantEditor({
                           alignTop={tool.description != null}
                           key={tool.id}
                           name={`enabled_tools_map.${tool.id}`}
-                          label={tool.name}
+                          label={tool.display_name}
                           subtext={tool.description}
                           onChange={() => {
                             toggleToolInValues(tool.id);
diff --git a/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx b/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
index aa8c0f9725d..850eb9690b7 100644
--- a/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
+++ b/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
@@ -133,6 +133,7 @@ function LLMProviderDisplay({
           </Button>
         </div>
       </div>
+
       {formIsVisible && (
         <LLMProviderUpdateModal
           llmProviderDescriptor={llmProviderDescriptor}
diff --git a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
index 18ef2202c47..1b86f9ef8fa 100644
--- a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
@@ -67,6 +67,7 @@ export function CustomLLMProviderUpdateForm({
       : [],
     is_public: existingLlmProvider?.is_public ?? true,
     groups: existingLlmProvider?.groups ?? [],
+    deployment_name: existingLlmProvider?.deployment_name ?? null,
   };
 
   // Setup validation schema if required
@@ -83,6 +84,7 @@ export function CustomLLMProviderUpdateForm({
     // EE Only
     is_public: Yup.boolean().required(),
     groups: Yup.array().of(Yup.number()),
+    deployment_name: Yup.string().nullable(),
   });
 
   return (
@@ -208,6 +210,7 @@ export function CustomLLMProviderUpdateForm({
               label="Display Name"
               subtext="A name which you can use to identify this provider when selecting it in the UI."
               placeholder="Display Name"
+              disabled={existingLlmProvider ? true : false}
             />
 
             <TextFormField
@@ -245,6 +248,14 @@ export function CustomLLMProviderUpdateForm({
               type="password"
             />
 
+            {existingLlmProvider?.deployment_name && (
+              <TextFormField
+                name="deployment_name"
+                label="[Optional] Deployment Name"
+                placeholder="Deployment Name"
+              />
+            )}
+
             <TextFormField
               name="api_base"
               label="[Optional] API Base"
@@ -360,28 +371,30 @@ export function CustomLLMProviderUpdateForm({
 
             <Divider />
 
-            <TextArrayField
-              name="model_names"
-              label="Model Names"
-              values={formikProps.values}
-              subtext={
-                <>
-                  List the individual models that you want to make available as
-                  a part of this provider. At least one must be specified. For
-                  the best experience your [Provider Name]/[Model Name] should
-                  match one of the pairs listed{" "}
-                  <a
-                    target="_blank"
-                    href="https://models.litellm.ai/"
-                    className="text-link"
-                    rel="noreferrer"
-                  >
-                    here
-                  </a>
-                  .
-                </>
-              }
-            />
+            {!existingLlmProvider?.deployment_name && (
+              <TextArrayField
+                name="model_names"
+                label="Model Names"
+                values={formikProps.values}
+                subtext={
+                  <>
+                    List the individual models that you want to make available
+                    as a part of this provider. At least one must be specified.
+                    For the best experience your [Provider Name]/[Model Name]
+                    should match one of the pairs listed{" "}
+                    <a
+                      target="_blank"
+                      href="https://models.litellm.ai/"
+                      className="text-link"
+                      rel="noreferrer"
+                    >
+                      here
+                    </a>
+                    .
+                  </>
+                }
+              />
+            )}
 
             <Divider />
 
@@ -395,14 +408,16 @@ export function CustomLLMProviderUpdateForm({
               placeholder="E.g. gpt-4"
             />
 
-            <TextFormField
-              name="fast_default_model_name"
-              subtext={`The model to use for lighter flows like \`LLM Chunk Filter\` 
+            {!existingLlmProvider?.deployment_name && (
+              <TextFormField
+                name="fast_default_model_name"
+                subtext={`The model to use for lighter flows like \`LLM Chunk Filter\` 
                 for this provider. If not set, will use 
                 the Default Model configured above.`}
-              label="[Optional] Fast Model"
-              placeholder="E.g. gpt-4"
-            />
+                label="[Optional] Fast Model"
+                placeholder="E.g. gpt-4"
+              />
+            )}
 
             <Divider />
 
diff --git a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
index 70a3ce7ff99..b072083662f 100644
--- a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
@@ -68,6 +68,7 @@ export function LLMProviderUpdateForm({
       existingLlmProvider?.display_model_names ||
       defaultModelsByProvider[llmProviderDescriptor.name] ||
       [],
+    deployment_name: existingLlmProvider?.deployment_name,
   };
 
   // Setup validation schema if required
@@ -99,6 +100,9 @@ export function LLMProviderUpdateForm({
           ),
         }
       : {}),
+    deployment_name: llmProviderDescriptor.deployment_name_required
+      ? Yup.string().required("Deployment Name is required")
+      : Yup.string(),
     default_model_name: Yup.string().required("Model name is required"),
     fast_default_model_name: Yup.string().nullable(),
     // EE Only
@@ -289,38 +293,50 @@ export function LLMProviderUpdateForm({
                 />
               )}
 
-              {llmProviderDescriptor.llm_names.length > 0 ? (
-                <SelectorFormField
-                  name="fast_default_model_name"
-                  subtext={`The model to use for lighter flows like \`LLM Chunk Filter\` 
-                for this provider. If \`Default\` is specified, will use 
-                the Default Model configured above.`}
-                  label="[Optional] Fast Model"
-                  options={llmProviderDescriptor.llm_names.map((name) => ({
-                    name: getDisplayNameForModel(name),
-                    value: name,
-                  }))}
-                  includeDefault
-                  maxHeight="max-h-56"
-                />
-              ) : (
+              {llmProviderDescriptor.deployment_name_required && (
                 <TextFormField
-                  name="fast_default_model_name"
-                  subtext={`The model to use for lighter flows like \`LLM Chunk Filter\` 
-                for this provider. If \`Default\` is specified, will use 
-                the Default Model configured above.`}
-                  label="[Optional] Fast Model"
-                  placeholder="E.g. gpt-4"
+                  small={hideAdvanced}
+                  name="deployment_name"
+                  label="Deployment Name"
+                  placeholder="Deployment Name"
                 />
               )}
 
-              <Divider />
+              {!llmProviderDescriptor.single_model_supported &&
+                (llmProviderDescriptor.llm_names.length > 0 ? (
+                  <SelectorFormField
+                    name="fast_default_model_name"
+                    subtext={`The model to use for lighter flows like \`LLM Chunk Filter\` 
+                for this provider. If \`Default\` is specified, will use 
+                the Default Model configured above.`}
+                    label="[Optional] Fast Model"
+                    options={llmProviderDescriptor.llm_names.map((name) => ({
+                      name: getDisplayNameForModel(name),
+                      value: name,
+                    }))}
+                    includeDefault
+                    maxHeight="max-h-56"
+                  />
+                ) : (
+                  <TextFormField
+                    name="fast_default_model_name"
+                    subtext={`The model to use for lighter flows like \`LLM Chunk Filter\` 
+                for this provider. If \`Default\` is specified, will use 
+                the Default Model configured above.`}
+                    label="[Optional] Fast Model"
+                    placeholder="E.g. gpt-4"
+                  />
+                ))}
 
               {llmProviderDescriptor.name != "azure" && (
-                <AdvancedOptionsToggle
-                  showAdvancedOptions={showAdvancedOptions}
-                  setShowAdvancedOptions={setShowAdvancedOptions}
-                />
+                <>
+                  <Divider />
+
+                  <AdvancedOptionsToggle
+                    showAdvancedOptions={showAdvancedOptions}
+                    setShowAdvancedOptions={setShowAdvancedOptions}
+                  />
+                </>
               )}
 
               {showAdvancedOptions && (
diff --git a/web/src/app/admin/configuration/llm/interfaces.ts b/web/src/app/admin/configuration/llm/interfaces.ts
index 33fa94d7f15..61f81311ecf 100644
--- a/web/src/app/admin/configuration/llm/interfaces.ts
+++ b/web/src/app/admin/configuration/llm/interfaces.ts
@@ -19,11 +19,13 @@ export interface WellKnownLLMProviderDescriptor {
   name: string;
   display_name: string;
 
+  deployment_name_required: boolean;
   api_key_required: boolean;
   api_base_required: boolean;
   api_version_required: boolean;
-  custom_config_keys: CustomConfigKey[] | null;
 
+  single_model_supported: boolean;
+  custom_config_keys: CustomConfigKey[] | null;
   llm_names: string[];
   default_model: string | null;
   default_fast_model: string | null;
@@ -43,6 +45,7 @@ export interface LLMProvider {
   is_public: boolean;
   groups: number[];
   display_model_names: string[] | null;
+  deployment_name: string | null;
 }
 
 export interface FullLLMProvider extends LLMProvider {
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index d17c3da01b8..1416f787cc7 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -28,6 +28,7 @@ import {
 import { hasCompletedWelcomeFlowSS } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 import { fetchAssistantsSS } from "../assistants/fetchAssistantsSS";
 import { NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN } from "../constants";
+import { checkLLMSupportsImageInput } from "../llm/utils";
 
 interface FetchChatDataResult {
   user: User | null;
@@ -195,10 +196,13 @@ export async function fetchChatData(searchParams: {
     assistants = assistants.filter((assistant) => assistant.num_chunks === 0);
   }
 
-  const hasOpenAIProvider = llmProviders.some(
-    (provider) => provider.provider === "openai"
+  const hasImageCompatibleModel = llmProviders.some(
+    (provider) =>
+      provider.provider === "openai" ||
+      provider.model_names.some((model) => checkLLMSupportsImageInput(model))
   );
-  if (!hasOpenAIProvider) {
+
+  if (!hasImageCompatibleModel) {
     assistants = assistants.filter(
       (assistant) =>
         !assistant.tools.some(
diff --git a/web/src/lib/chat/fetchSomeChatData.ts b/web/src/lib/chat/fetchSomeChatData.ts
index 827cf0c21fc..fdfa55b5ea1 100644
--- a/web/src/lib/chat/fetchSomeChatData.ts
+++ b/web/src/lib/chat/fetchSomeChatData.ts
@@ -28,6 +28,7 @@ import {
 import { hasCompletedWelcomeFlowSS } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 import { fetchAssistantsSS } from "../assistants/fetchAssistantsSS";
 import { NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN } from "../constants";
+import { checkLLMSupportsImageInput } from "../llm/utils";
 
 interface FetchChatDataResult {
   user?: User | null;
@@ -178,10 +179,13 @@ export async function fetchSomeChatData(
       );
     }
 
-    const hasOpenAIProvider =
-      result.llmProviders &&
-      result.llmProviders.some((provider) => provider.provider === "openai");
-    if (!hasOpenAIProvider) {
+    const hasImageCompatibleModel = result.llmProviders?.some(
+      (provider) =>
+        provider.provider === "openai" ||
+        provider.model_names.some((model) => checkLLMSupportsImageInput(model))
+    );
+
+    if (!hasImageCompatibleModel) {
       result.assistants = result.assistants.filter(
         (assistant) =>
           !assistant.tools.some(

From f83e6806b66599fb112fd524ffa26c7331937d05 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 8 Oct 2024 18:07:51 -0700
Subject: [PATCH 064/376] More robust edge detection (#2710)

* more robust edge detection

* nit

* k
---
 web/src/app/chat/ChatBanner.tsx | 46 +++++++++++++++++++--------------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/web/src/app/chat/ChatBanner.tsx b/web/src/app/chat/ChatBanner.tsx
index 72a2cac5652..59fc8bd32d5 100644
--- a/web/src/app/chat/ChatBanner.tsx
+++ b/web/src/app/chat/ChatBanner.tsx
@@ -16,9 +16,13 @@ export function ChatBanner() {
   useLayoutEffect(() => {
     const checkOverflow = () => {
       if (contentRef.current && fullContentRef.current) {
-        setIsOverflowing(
-          fullContentRef.current.scrollHeight > contentRef.current.clientHeight
-        );
+        const contentRect = contentRef.current.getBoundingClientRect();
+        const fullContentRect = fullContentRef.current.getBoundingClientRect();
+
+        const isWidthOverflowing = fullContentRect.width > contentRect.width;
+        const isHeightOverflowing = fullContentRect.height > contentRect.height;
+
+        setIsOverflowing(isWidthOverflowing || isHeightOverflowing);
       }
     };
 
@@ -53,23 +57,27 @@ export function ChatBanner() {
     >
       <div className="text-emphasis text-sm w-full">
         <div className="relative">
-          <div
-            ref={contentRef}
-            className={`${settings.enterpriseSettings.two_lines_for_chat_header ? "line-clamp-2" : "line-clamp-1"} text-center w-full overflow-hidden pr-8`}
-          >
-            <MinimalMarkdown
-              className="prose text-sm max-w-full"
-              content={settings.enterpriseSettings.custom_header_content}
-            />
+          <div className={`flex justify-center w-full overflow-hidden pr-8`}>
+            <div
+              ref={contentRef}
+              className={`overflow-hidden ${settings.enterpriseSettings.two_lines_for_chat_header ? "line-clamp-2" : "line-clamp-1"} text-center max-w-full`}
+            >
+              <MinimalMarkdown
+                className="prose text-sm max-w-full"
+                content={settings.enterpriseSettings.custom_header_content}
+              />
+            </div>
           </div>
-          <div
-            ref={fullContentRef}
-            className="absolute top-0 left-0 invisible w-full"
-          >
-            <MinimalMarkdown
-              className="prose text-sm max-w-full"
-              content={settings.enterpriseSettings.custom_header_content}
-            />
+          <div className="absolute top-0 left-0 invisible flex justify-center max-w-full">
+            <div
+              ref={fullContentRef}
+              className={`overflow-hidden invisible ${settings.enterpriseSettings.two_lines_for_chat_header ? "line-clamp-2" : "line-clamp-1"} text-center max-w-full`}
+            >
+              <MinimalMarkdown
+                className="prose text-sm max-w-full"
+                content={settings.enterpriseSettings.custom_header_content}
+              />
+            </div>
           </div>
           <div className="absolute bottom-0 right-0">
             {isOverflowing && (

From 10f221cd37adb39e4a3f3bebb92dc3427900e65f Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 8 Oct 2024 20:13:19 -0700
Subject: [PATCH 065/376] Remove mildly annoying groups fetch (#2733)

* remove mildly annoying groups fetch

* ensure in client component
---
 web/src/lib/hooks.ts | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/web/src/lib/hooks.ts b/web/src/lib/hooks.ts
index a760c082471..eb9741b7516 100644
--- a/web/src/lib/hooks.ts
+++ b/web/src/lib/hooks.ts
@@ -1,3 +1,4 @@
+"use client";
 import {
   ConnectorIndexingStatus,
   DocumentBoostStatus,
@@ -6,14 +7,14 @@ import {
 } from "@/lib/types";
 import useSWR, { mutate, useSWRConfig } from "swr";
 import { errorHandlingFetcher } from "./fetcher";
-import { useEffect, useState } from "react";
+import { useContext, useEffect, useState } from "react";
 import { DateRangePickerValue } from "@tremor/react";
 import { SourceMetadata } from "./search/interfaces";
 import { destructureValue } from "./llm/utils";
 import { ChatSession } from "@/app/chat/interfaces";
 import { UsersResponse } from "./users/interfaces";
-import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import { Credential } from "./connectors/credentials";
+import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 const CREDENTIAL_URL = "/api/manage/admin/credential";
 
@@ -220,8 +221,14 @@ export const useUserGroups = (): {
   error: string;
   refreshUserGroups: () => void;
 } => {
-  const swrResponse = useSWR<UserGroup[]>(USER_GROUP_URL, errorHandlingFetcher);
-  const isPaidEnterpriseFeaturesEnabled = usePaidEnterpriseFeaturesEnabled();
+  const combinedSettings = useContext(SettingsContext);
+  const isPaidEnterpriseFeaturesEnabled =
+    combinedSettings && combinedSettings.enterpriseSettings !== null;
+
+  const swrResponse = useSWR<UserGroup[]>(
+    isPaidEnterpriseFeaturesEnabled ? USER_GROUP_URL : null,
+    errorHandlingFetcher
+  );
 
   if (!isPaidEnterpriseFeaturesEnabled) {
     return {

From d5b9a6e552c0e77e975b17402ac1885a3f6698a4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 8 Oct 2024 20:20:28 -0700
Subject: [PATCH 066/376] add vespa + embedding timeout env variables (#2689)

* add vespa + embedding timeout env variables

* nit: integration test

* add dangerous override

* k

* add additional clarity

* nit

* nit
---
 backend/danswer/configs/app_configs.py               |  3 +++
 backend/danswer/document_index/vespa/index.py        | 11 ++++++-----
 backend/danswer/indexing/chunker.py                  |  1 +
 backend/danswer/main.py                              |  7 +++++++
 backend/model_server/encoders.py                     |  3 ++-
 backend/shared_configs/configs.py                    |  3 +++
 deployment/docker_compose/docker-compose.dev.yml     |  1 +
 deployment/docker_compose/docker-compose.gpu-dev.yml |  4 ++++
 8 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 1174c8d060f..eaa231e88b7 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -401,6 +401,9 @@
     os.environ.get("CUSTOM_ANSWER_VALIDITY_CONDITIONS", "[]")
 )
 
+VESPA_REQUEST_TIMEOUT = int(os.environ.get("VESPA_REQUEST_TIMEOUT") or "5")
+
+SYSTEM_RECURSION_LIMIT = int(os.environ.get("SYSTEM_RECURSION_LIMIT") or "1000")
 
 #####
 # Enterprise Edition Configs
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 25663e966a3..44a5918d756 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -15,6 +15,7 @@
 import requests
 
 from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
+from danswer.configs.app_configs import VESPA_REQUEST_TIMEOUT
 from danswer.configs.chat_configs import DOC_TIME_DECAY
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
 from danswer.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -211,7 +212,7 @@ def index(
         # indexing / updates / deletes since we have to make a large volume of requests.
         with (
             concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor,
-            httpx.Client(http2=True) as http_client,
+            httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client,
         ):
             # Check for existing documents, existing documents need to have all of their chunks deleted
             # prior to indexing as the document size (num chunks) may have shrunk
@@ -275,7 +276,7 @@ def _update_chunk(
         # indexing / updates / deletes since we have to make a large volume of requests.
         with (
             concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor,
-            httpx.Client(http2=True) as http_client,
+            httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client,
         ):
             for update_batch in batch_generator(updates, batch_size):
                 future_to_document_id = {
@@ -419,7 +420,7 @@ def update_single(self, doc_id: str, fields: VespaDocumentFields) -> int:
         if self.secondary_index_name:
             index_names.append(self.secondary_index_name)
 
-        with httpx.Client(http2=True) as http_client:
+        with httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client:
             for index_name in index_names:
                 params = httpx.QueryParams(
                     {
@@ -475,7 +476,7 @@ def delete(self, doc_ids: list[str]) -> None:
 
         # NOTE: using `httpx` here since `requests` doesn't support HTTP2. This is beneficial for
         # indexing / updates / deletes since we have to make a large volume of requests.
-        with httpx.Client(http2=True) as http_client:
+        with httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client:
             index_names = [self.index_name]
             if self.secondary_index_name:
                 index_names.append(self.secondary_index_name)
@@ -503,7 +504,7 @@ def delete_single(self, doc_id: str) -> int:
         if self.secondary_index_name:
             index_names.append(self.secondary_index_name)
 
-        with httpx.Client(http2=True) as http_client:
+        with httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client:
             for index_name in index_names:
                 params = httpx.QueryParams(
                     {
diff --git a/backend/danswer/indexing/chunker.py b/backend/danswer/indexing/chunker.py
index a25cfc3d32b..9cb4b3e1954 100644
--- a/backend/danswer/indexing/chunker.py
+++ b/backend/danswer/indexing/chunker.py
@@ -27,6 +27,7 @@
 MAX_METADATA_PERCENTAGE = 0.25
 CHUNK_MIN_CONTENT = 256
 
+
 logger = setup_logger()
 
 
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index d3aa8b00efd..d7ac6b3c3ed 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -1,3 +1,4 @@
+import sys
 import traceback
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
@@ -32,6 +33,7 @@
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_SIZE
+from danswer.configs.app_configs import SYSTEM_RECURSION_LIMIT
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import AuthType
@@ -140,6 +142,11 @@ def include_router_with_global_prefix_prepended(
 
 @asynccontextmanager
 async def lifespan(app: FastAPI) -> AsyncGenerator:
+    # Set recursion limit
+    if SYSTEM_RECURSION_LIMIT is not None:
+        sys.setrecursionlimit(SYSTEM_RECURSION_LIMIT)
+        logger.notice(f"System recursion limit set to {SYSTEM_RECURSION_LIMIT}")
+
     SqlEngine.set_app_name(POSTGRES_WEB_APP_NAME)
     SqlEngine.init_engine(
         pool_size=POSTGRES_API_SERVER_POOL_SIZE,
diff --git a/backend/model_server/encoders.py b/backend/model_server/encoders.py
index 860151b3dc4..e2e167520ba 100644
--- a/backend/model_server/encoders.py
+++ b/backend/model_server/encoders.py
@@ -25,6 +25,7 @@
 from model_server.constants import EmbeddingProvider
 from model_server.utils import simple_log_function_time
 from shared_configs.configs import INDEXING_ONLY
+from shared_configs.configs import OPENAI_EMBEDDING_TIMEOUT
 from shared_configs.enums import EmbedTextType
 from shared_configs.enums import RerankerProvider
 from shared_configs.model_server_models import Embedding
@@ -56,7 +57,7 @@ def _initialize_client(
     api_key: str, provider: EmbeddingProvider, model: str | None = None
 ) -> Any:
     if provider == EmbeddingProvider.OPENAI:
-        return openai.OpenAI(api_key=api_key)
+        return openai.OpenAI(api_key=api_key, timeout=OPENAI_EMBEDDING_TIMEOUT)
     elif provider == EmbeddingProvider.COHERE:
         return CohereClient(api_key=api_key)
     elif provider == EmbeddingProvider.VOYAGE:
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index e8b599b7795..50233ab6878 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -60,6 +60,9 @@
 # notset, debug, info, notice, warning, error, or critical
 LOG_LEVEL = os.environ.get("LOG_LEVEL", "notice")
 
+# Only used for OpenAI
+OPENAI_EMBEDDING_TIMEOUT = int(os.environ.get("OPENAI_EMBEDDING_TIMEOUT", "600"))
+
 
 # Fields which should only be set on new search setting
 PRESERVED_SEARCH_FIELDS = [
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 86d988e7d90..4d0eff8612d 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -281,6 +281,7 @@ services:
       - INDEXING_ONLY=True
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
+      - CLIENT_EMBEDDING_TIMEOUT=${CLIENT_EMBEDDING_TIMEOUT:-}
     volumes:
       # Not necessary, this is just to reduce download time during startup
       - indexing_huggingface_model_cache:/root/.cache/huggingface/
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index ebce01eadb2..6397f657c19 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -70,6 +70,9 @@ services:
       - DISABLE_RERANK_FOR_STREAMING=${DISABLE_RERANK_FOR_STREAMING:-}
       - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
       - MODEL_SERVER_PORT=${MODEL_SERVER_PORT:-}
+      - VESPA_REQUEST_TIMEOUT=${VESPA_REQUEST_TIMEOUT:-}
+      # We do not recommend changing this value
+      - SYSTEM_RECURSION_LIMIT=${SYSTEM_RECURSION_LIMIT:-}
       # Leave this on pretty please? Nothing sensitive is collected!
       # https://docs.danswer.dev/more/telemetry
       - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-}
@@ -252,6 +255,7 @@ services:
       - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-}
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
+      - CLIENT_EMBEDDING_TIMEOUT=${CLIENT_EMBEDDING_TIMEOUT:-}
     volumes:
       # Not necessary, this is just to reduce download time during startup
       - model_cache_huggingface:/root/.cache/huggingface/

From 80343d6d75b1e69d21681d6c925ad9dcfdbe9fd0 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 20:31:17 -0700
Subject: [PATCH 067/376] update hotfix to use commas

---
 .github/workflows/hotfix-to-release-branches.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/hotfix-to-release-branches.yml b/.github/workflows/hotfix-to-release-branches.yml
index dc8b50794d6..22260fe01a7 100644
--- a/.github/workflows/hotfix-to-release-branches.yml
+++ b/.github/workflows/hotfix-to-release-branches.yml
@@ -2,7 +2,7 @@
 # Given a hotfix branch, it will attempt to open a PR to all release branches and
 # by default auto merge them
 
-name: Open PRs from hotfix to release branches
+name: Hotfix release branches
 
 on:
   workflow_dispatch:
@@ -42,10 +42,15 @@ jobs:
             echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'."
             exit 1
           fi
+          
           echo "Found release branches:"
           echo "$BRANCHES"
+          
+          # Join the branches into a single line separated by commas
+            BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//')
+
           # Set the branches as an output
-          echo "branches=$BRANCHES" >> $GITHUB_OUTPUT
+          echo "branches=$BRANCHES_JOINED" >> $GITHUB_OUTPUT
 
       - name: Ensure Hotfix Branch Exists Locally
         run: |
@@ -60,7 +65,7 @@ jobs:
           BRANCHES="${{ steps.get_release_branches.outputs.branches }}"
 
           # Convert BRANCHES to an array
-          IFS=$'\n' read -rd '' -a BRANCH_ARRAY <<<"$BRANCHES"
+          IFS=$',' read -ra BRANCH_ARRAY <<< "$BRANCHES"
 
           # Loop through each release branch and create and merge a PR
           for RELEASE_BRANCH in $BRANCHES; do
@@ -88,7 +93,7 @@ jobs:
               echo "Attempting to merge pull request #$PR_NUMBER"
 
               # Attempt to merge the PR
-              gh pr merge "$PR_NUMBER" --merge --admin --yes
+              gh pr merge "$PR_NUMBER" --merge --yes
 
               if [ $? -eq 0 ]; then
                 echo "Pull request #$PR_NUMBER merged successfully."

From cbd4481838b79d586ca8c8239df4e9416627ddcd Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 8 Oct 2024 20:32:39 -0700
Subject: [PATCH 068/376] rename

---
 ...hotfix-to-release-branches.yml => hotfix-release-branches.yml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/workflows/{hotfix-to-release-branches.yml => hotfix-release-branches.yml} (100%)

diff --git a/.github/workflows/hotfix-to-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
similarity index 100%
rename from .github/workflows/hotfix-to-release-branches.yml
rename to .github/workflows/hotfix-release-branches.yml

From 95df136104ae3ee2dc8725a44606331d8bed405d Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 09:40:27 -0700
Subject: [PATCH 069/376] another cut

---
 .github/workflows/hotfix-release-branches.yml | 56 +++++++++++++++----
 1 file changed, 45 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 22260fe01a7..5b6dfbd36a0 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -7,8 +7,11 @@ name: Hotfix release branches
 on:
   workflow_dispatch:
     inputs:
-      hotfix_branch:
-        description: 'Hotfix branch name'
+      hotfix_commit:
+        description: 'Hotfix commit hash'
+        required: true
+      hotfix_suffix:
+        description: 'Hotfix branch suffix (e.g. hotfix/v0.8-{suffix})'
         required: true
       release_branch_pattern:
         description: 'Release branch pattern (regex)'
@@ -20,7 +23,7 @@ on:
         default: 'true'
         
 jobs:
-  hotfix_to_release:
+  hotfix_release_branches:
     # See https://runs-on.com/runners/linux/
     # use a lower powered instance since this just does i/o to docker hub
     runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
@@ -34,6 +37,10 @@ jobs:
         run: |
           git fetch --all --prune
 
+      - name: Verify Hotfix Commit Exists
+        run: |
+          git rev-parse --verify "${{ github.event.inputs.hotfix_commit }}" || { echo "Commit not found: ${{ github.event.inputs.hotfix_commit }}"; exit 1; }
+
       - name: Get Release Branches
         id: get_release_branches
         run: |
@@ -47,18 +54,15 @@ jobs:
           echo "$BRANCHES"
           
           # Join the branches into a single line separated by commas
-            BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//')
+          BRANCHES_JOINED=$(echo "$BRANCHES" | tr '\n' ',' | sed 's/,$//')
 
           # Set the branches as an output
           echo "branches=$BRANCHES_JOINED" >> $GITHUB_OUTPUT
 
-      - name: Ensure Hotfix Branch Exists Locally
-        run: |
-          git fetch origin "${{ github.event.inputs.hotfix_branch }}":"${{ github.event.inputs.hotfix_branch }}" || true
-
       - name: Create and Merge Pull Requests to Matching Release Branches
         env:
-          HOTFIX_BRANCH: ${{ github.event.inputs.hotfix_branch }}
+          HOTFIX_COMMIT: ${{ github.event.inputs.hotfix_commit }}
+          HOTFIX_SUFFIX: ${{ github.event.inputs.hotfix_suffix }}
           AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
         run: |
           # Get the branches from the previous step
@@ -68,9 +72,39 @@ jobs:
           IFS=$',' read -ra BRANCH_ARRAY <<< "$BRANCHES"
 
           # Loop through each release branch and create and merge a PR
-          for RELEASE_BRANCH in $BRANCHES; do
+          for RELEASE_BRANCH in "${BRANCH_ARRAY[@]}"; do
+            echo "Processing $RELEASE_BRANCH..."
+            
+            # Parse out the release version by removing "release/" from the branch name
+            RELEASE_VERSION=${RELEASE_BRANCH#release/}
+            echo "Release version parsed: $RELEASE_VERSION"
+            
+            HOTFIX_BRANCH="hotfix/${RELEASE_VERSION}-${HOTFIX_SUFFIX}"
             echo "Creating PR from $HOTFIX_BRANCH to $RELEASE_BRANCH"
 
+            # Checkout the release branch
+            echo "Checking out $RELEASE_BRANCH"
+            git checkout "$RELEASE_BRANCH"
+
+            # Create the new hotfix branch
+            echo "Branching $RELEASE_BRANCH to $HOTFIX_BRANCH"
+            git checkout -b "$HOTFIX_BRANCH"
+
+            # Cherry-pick the hotfix commit
+            echo "Cherry picking commit hash $HOTFIX_COMMIT onto $HOTFIX_BRANCH"
+            git cherry-pick "$HOTFIX_COMMIT"
+
+            if [ $? -ne 0 ]; then
+              echo "Cherry-pick failed for $HOTFIX_COMMIT on $HOTFIX_BRANCH. Aborting..."
+              git cherry-pick --abort
+              continue
+            fi
+
+            # Push the hotfix branch to the remote
+            echo "Pushing $HOTFIX_BRANCH..."
+            git push origin "$HOTFIX_BRANCH"
+            echo "Hotfix branch $HOTFIX_BRANCH created and pushed."
+            
             # Check if PR already exists
             EXISTING_PR=$(gh pr list --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --state open --json number --jq '.[0].number')
 
@@ -98,8 +132,8 @@ jobs:
               if [ $? -eq 0 ]; then
                 echo "Pull request #$PR_NUMBER merged successfully."
               else
-                echo "Failed to merge pull request #$PR_NUMBER."
                 # Optionally, handle the error or continue
+                echo "Failed to merge pull request #$PR_NUMBER."
               fi
             fi
           done
\ No newline at end of file

From af8e361fc2e7af0dcb8361d67720dfdb515923ae Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:16:36 -0700
Subject: [PATCH 070/376] handle merge commits during cherry picking

---
 .github/workflows/hotfix-release-branches.yml | 33 ++++++++++++++-----
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 5b6dfbd36a0..ee54127f071 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -18,10 +18,14 @@ on:
         required: true
         default: 'release/.*'
       auto_merge:
-        description: 'Automatically merge the PRs if set to true'
-        required: false
+        description: 'Automatically merge the hotfix PRs'
+        required: true
+        type: choice
         default: 'true'
-        
+        options:
+          - true
+          - false
+          
 jobs:
   hotfix_release_branches:
     # See https://runs-on.com/runners/linux/
@@ -87,12 +91,25 @@ jobs:
             git checkout "$RELEASE_BRANCH"
 
             # Create the new hotfix branch
-            echo "Branching $RELEASE_BRANCH to $HOTFIX_BRANCH"
-            git checkout -b "$HOTFIX_BRANCH"
+            if git rev-parse --verify "$HOTFIX_BRANCH" >/dev/null 2>&1; then
+              echo "Hotfix branch $HOTFIX_BRANCH already exists. Skipping branch creation."
+            else
+              echo "Branching $RELEASE_BRANCH to $HOTFIX_BRANCH"
+              git checkout -b "$HOTFIX_BRANCH"
+            fi
+            
+            # Check if the hotfix commit is a merge commit
+            if git rev-list --merges -n 1 "$HOTFIX_COMMIT" >/dev/null 2>&1; then
+              # -m 1 uses the target branch as the base (which is what we want)
+              echo "Hotfix commit $HOTFIX_COMMIT is a merge commit, using -m 1 for cherry-pick"
+              CHERRY_PICK_CMD="git cherry-pick -m 1 $HOTFIX_COMMIT"
+            else
+              CHERRY_PICK_CMD="git cherry-pick $HOTFIX_COMMIT"
+            fi
 
-            # Cherry-pick the hotfix commit
-            echo "Cherry picking commit hash $HOTFIX_COMMIT onto $HOTFIX_BRANCH"
-            git cherry-pick "$HOTFIX_COMMIT"
+            # Perform the cherry-pick
+            echo "Executing: $CHERRY_PICK_CMD"
+            eval "$CHERRY_PICK_CMD"
 
             if [ $? -ne 0 ]; then
               echo "Cherry-pick failed for $HOTFIX_COMMIT on $HOTFIX_BRANCH. Aborting..."

From d6b4c08d242025a01fd9255147dc82d9734ca61d Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:21:31 -0700
Subject: [PATCH 071/376] need git user

---
 .github/workflows/hotfix-release-branches.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index ee54127f071..7932aa3ca27 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -36,6 +36,11 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          
+      - name: Set up Git user
+        run: |
+          git config user.name "Richard Kuo [bot]"
+          git config user.email "rkuo[bot]@danswer.ai"
 
       - name: Fetch All Branches
         run: |

From 8bbf5053deb1bd1867f1d01fba760ea1d95c54e0 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:29:41 -0700
Subject: [PATCH 072/376] add deploy key

---
 .github/workflows/hotfix-release-branches.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 7932aa3ca27..0a33710445d 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -32,9 +32,12 @@ jobs:
     # use a lower powered instance since this just does i/o to docker hub
     runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
     steps:
+    
+      # needs RKUO_DEPLOY_KEY for write access to merge PR's
       - name: Checkout Repository
         uses: actions/checkout@v4
         with:
+          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
           fetch-depth: 0
           
       - name: Set up Git user

From 03807688e662f3a4532465b77151afde408dc7ba Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:35:07 -0700
Subject: [PATCH 073/376] gh cli needs its token

---
 .github/workflows/hotfix-release-branches.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 0a33710445d..6b8d2340c72 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -45,6 +45,10 @@ jobs:
           git config user.name "Richard Kuo [bot]"
           git config user.email "rkuo[bot]@danswer.ai"
 
+      - name: Set up GH_TOKEN
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          
       - name: Fetch All Branches
         run: |
           git fetch --all --prune

From 174dabf52ffe505afa3ea70fd4e30566623fb789 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:38:59 -0700
Subject: [PATCH 074/376] edit step name

---
 .github/workflows/hotfix-release-branches.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 6b8d2340c72..54621ab528b 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -45,7 +45,7 @@ jobs:
           git config user.name "Richard Kuo [bot]"
           git config user.email "rkuo[bot]@danswer.ai"
 
-      - name: Set up GH_TOKEN
+      - name: Set up GitHub CLI Token
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           

From d40384050744da9bbfd8357c637946cae49cb511 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:40:33 -0700
Subject: [PATCH 075/376] fix where GH_TOKEN is set

---
 .github/workflows/hotfix-release-branches.yml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 54621ab528b..2fd3c543979 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -45,10 +45,6 @@ jobs:
           git config user.name "Richard Kuo [bot]"
           git config user.email "rkuo[bot]@danswer.ai"
 
-      - name: Set up GitHub CLI Token
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          
       - name: Fetch All Branches
         run: |
           git fetch --all --prune
@@ -80,6 +76,7 @@ jobs:
           HOTFIX_COMMIT: ${{ github.event.inputs.hotfix_commit }}
           HOTFIX_SUFFIX: ${{ github.event.inputs.hotfix_suffix }}
           AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Get the branches from the previous step
           BRANCHES="${{ steps.get_release_branches.outputs.branches }}"

From 28fe0d12ca9fb2dc83d873bbabfd78429e6e361c Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 10:55:47 -0700
Subject: [PATCH 076/376] try capturing gh output and parsing

---
 .github/workflows/hotfix-release-branches.yml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index 2fd3c543979..bdadc9f7413 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -139,10 +139,13 @@ jobs:
               continue
             fi
             
-            # Create a new PR
-            PR_URL=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \
+            # Create a new PR and capture the output
+            PR_OUTPUT=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \
               --body "Automated PR to merge \`$HOTFIX_BRANCH\` into \`$RELEASE_BRANCH\`." \
-              --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --json url --jq '.url')
+              --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH")
+
+            # Extract the URL from the output
+            PR_URL=$(echo "$PR_OUTPUT" | grep -Eo 'https://github.com/[^ ]+')
 
             echo "Pull request created: $PR_URL"
 

From fbf09c78599a5cba372bde8d34074f28dfd0d7a2 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 11:07:20 -0700
Subject: [PATCH 077/376] try to update token permissions

---
 .github/workflows/hotfix-release-branches.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index bdadc9f7413..d660acc0378 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -28,6 +28,7 @@ on:
           
 jobs:
   hotfix_release_branches:
+    permissions: write-all
     # See https://runs-on.com/runners/linux/
     # use a lower powered instance since this just does i/o to docker hub
     runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]

From 7c9f605a99823f0a2bffe3d8a6bdf7956e4e4fb3 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 11:44:47 -0700
Subject: [PATCH 078/376] fix pr merge command

---
 .github/workflows/hotfix-release-branches.yml |   4 +-
 .../workflows/hotfix-to-release-branches.yml  | 100 ------------------
 2 files changed, 2 insertions(+), 102 deletions(-)
 delete mode 100644 .github/workflows/hotfix-to-release-branches.yml

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index d660acc0378..b94c4490ab2 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -147,17 +147,17 @@ jobs:
 
             # Extract the URL from the output
             PR_URL=$(echo "$PR_OUTPUT" | grep -Eo 'https://github.com/[^ ]+')
-
             echo "Pull request created: $PR_URL"
 
             # Extract PR number from URL
             PR_NUMBER=$(basename "$PR_URL")
+            echo "Pull request created: $PR_NUMBER"
 
             if [ "$AUTO_MERGE" == "true" ]; then
               echo "Attempting to merge pull request #$PR_NUMBER"
 
               # Attempt to merge the PR
-              gh pr merge "$PR_NUMBER" --merge --yes
+              gh pr merge "$PR_NUMBER" --merge --delete-branch
 
               if [ $? -eq 0 ]; then
                 echo "Pull request #$PR_NUMBER merged successfully."
diff --git a/.github/workflows/hotfix-to-release-branches.yml b/.github/workflows/hotfix-to-release-branches.yml
deleted file mode 100644
index dc8b50794d6..00000000000
--- a/.github/workflows/hotfix-to-release-branches.yml
+++ /dev/null
@@ -1,100 +0,0 @@
-# This workflow is intended to be manually triggered via the GitHub Action tab.
-# Given a hotfix branch, it will attempt to open a PR to all release branches and
-# by default auto merge them
-
-name: Open PRs from hotfix to release branches
-
-on:
-  workflow_dispatch:
-    inputs:
-      hotfix_branch:
-        description: 'Hotfix branch name'
-        required: true
-      release_branch_pattern:
-        description: 'Release branch pattern (regex)'
-        required: true
-        default: 'release/.*'
-      auto_merge:
-        description: 'Automatically merge the PRs if set to true'
-        required: false
-        default: 'true'
-        
-jobs:
-  hotfix_to_release:
-    # See https://runs-on.com/runners/linux/
-    # use a lower powered instance since this just does i/o to docker hub
-    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Fetch All Branches
-        run: |
-          git fetch --all --prune
-
-      - name: Get Release Branches
-        id: get_release_branches
-        run: |
-          BRANCHES=$(git branch -r | grep -E "${{ github.event.inputs.release_branch_pattern }}" | sed 's|origin/||' | tr -d ' ')
-          if [ -z "$BRANCHES" ]; then
-            echo "No release branches found matching pattern '${{ github.event.inputs.release_branch_pattern }}'."
-            exit 1
-          fi
-          echo "Found release branches:"
-          echo "$BRANCHES"
-          # Set the branches as an output
-          echo "branches=$BRANCHES" >> $GITHUB_OUTPUT
-
-      - name: Ensure Hotfix Branch Exists Locally
-        run: |
-          git fetch origin "${{ github.event.inputs.hotfix_branch }}":"${{ github.event.inputs.hotfix_branch }}" || true
-
-      - name: Create and Merge Pull Requests to Matching Release Branches
-        env:
-          HOTFIX_BRANCH: ${{ github.event.inputs.hotfix_branch }}
-          AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
-        run: |
-          # Get the branches from the previous step
-          BRANCHES="${{ steps.get_release_branches.outputs.branches }}"
-
-          # Convert BRANCHES to an array
-          IFS=$'\n' read -rd '' -a BRANCH_ARRAY <<<"$BRANCHES"
-
-          # Loop through each release branch and create and merge a PR
-          for RELEASE_BRANCH in $BRANCHES; do
-            echo "Creating PR from $HOTFIX_BRANCH to $RELEASE_BRANCH"
-
-            # Check if PR already exists
-            EXISTING_PR=$(gh pr list --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --state open --json number --jq '.[0].number')
-
-            if [ -n "$EXISTING_PR" ]; then
-              echo "An open PR already exists: #$EXISTING_PR. Skipping..."
-              continue
-            fi
-            
-            # Create a new PR
-            PR_URL=$(gh pr create --title "Merge $HOTFIX_BRANCH into $RELEASE_BRANCH" \
-              --body "Automated PR to merge \`$HOTFIX_BRANCH\` into \`$RELEASE_BRANCH\`." \
-              --head "$HOTFIX_BRANCH" --base "$RELEASE_BRANCH" --json url --jq '.url')
-
-            echo "Pull request created: $PR_URL"
-
-            # Extract PR number from URL
-            PR_NUMBER=$(basename "$PR_URL")
-
-            if [ "$AUTO_MERGE" == "true" ]; then
-              echo "Attempting to merge pull request #$PR_NUMBER"
-
-              # Attempt to merge the PR
-              gh pr merge "$PR_NUMBER" --merge --admin --yes
-
-              if [ $? -eq 0 ]; then
-                echo "Pull request #$PR_NUMBER merged successfully."
-              else
-                echo "Failed to merge pull request #$PR_NUMBER."
-                # Optionally, handle the error or continue
-              fi
-            fi
-          done
\ No newline at end of file

From ca88100f38c4fd4dcc48393cdc6dd0a30e027b15 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 12:27:28 -0700
Subject: [PATCH 079/376] add branching

---
 .github/workflows/hotfix-release-branches.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/hotfix-release-branches.yml b/.github/workflows/hotfix-release-branches.yml
index b94c4490ab2..0e921f8d694 100644
--- a/.github/workflows/hotfix-release-branches.yml
+++ b/.github/workflows/hotfix-release-branches.yml
@@ -72,12 +72,15 @@ jobs:
           # Set the branches as an output
           echo "branches=$BRANCHES_JOINED" >> $GITHUB_OUTPUT
 
+      # notes on all the vagaries of wiring up automated PR's
+      # https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs
+      # we must use a custom token for GH_TOKEN to trigger the subsequent PR checks
       - name: Create and Merge Pull Requests to Matching Release Branches
         env:
           HOTFIX_COMMIT: ${{ github.event.inputs.hotfix_commit }}
           HOTFIX_SUFFIX: ${{ github.event.inputs.hotfix_suffix }}
           AUTO_MERGE: ${{ github.event.inputs.auto_merge }}
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.RKUO_PERSONAL_ACCESS_TOKEN }}
         run: |
           # Get the branches from the previous step
           BRANCHES="${{ steps.get_release_branches.outputs.branches }}"
@@ -157,7 +160,7 @@ jobs:
               echo "Attempting to merge pull request #$PR_NUMBER"
 
               # Attempt to merge the PR
-              gh pr merge "$PR_NUMBER" --merge --delete-branch
+              gh pr merge "$PR_NUMBER" --merge --auto --delete-branch
 
               if [ $? -eq 0 ]; then
                 echo "Pull request #$PR_NUMBER merged successfully."

From 6c0a0b6454cd3f3de22d71d67802ea3c36cae294 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 9 Oct 2024 12:52:34 -0700
Subject: [PATCH 080/376] Add sync status (#2743)

* add sync status

* nit
---
 .../admin/indexing/status/CCPairIndexingStatusTable.tsx  | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
index 2b78e11de2d..39c238870e5 100644
--- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
+++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
@@ -23,6 +23,7 @@ import {
   FiSettings,
   FiLock,
   FiUnlock,
+  FiRefreshCw,
 } from "react-icons/fi";
 import { Tooltip } from "@/components/tooltip/Tooltip";
 import { SourceIcon } from "@/components/SourceIcon";
@@ -240,6 +241,14 @@ function ConnectorRow({
             >
               Public
             </Badge>
+          ) : ccPairsIndexingStatus.access_type === "sync" ? (
+            <Badge
+              size="md"
+              color={isEditable ? "orange" : "gray"}
+              icon={FiRefreshCw}
+            >
+              Sync
+            </Badge>
           ) : (
             <Badge size="md" color={isEditable ? "blue" : "gray"} icon={FiLock}>
               Private

From 1cbc067483ffc9935b761b22c1d2977f01cabf5a Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 13:37:34 -0700
Subject: [PATCH 081/376] print various celery queue lengths (#2729)

* print various celery queue lengths

* use the correct redis client

* mypy ignore
---
 .../background/celery/tasks/vespa/tasks.py    | 30 +++++++++++++++++--
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 3f347cbab3d..39b6f8a91e0 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -11,6 +11,7 @@
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.celery_app import celery_app
 from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.celery_redis import celery_get_queue_length
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.background.celery.celery_redis import RedisConnectorPruning
@@ -18,6 +19,7 @@
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryQueues
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.db.connector import fetch_connector_by_id
 from danswer.db.connector import mark_ccpair_as_pruned
@@ -468,8 +470,8 @@ def monitor_ccpair_pruning_taskset(
     r.delete(rcp.fence_key)
 
 
-@shared_task(name="monitor_vespa_sync", soft_time_limit=300)
-def monitor_vespa_sync() -> None:
+@shared_task(name="monitor_vespa_sync", soft_time_limit=300, bind=True)
+def monitor_vespa_sync(self: Task) -> None:
     """This is a celery beat task that monitors and finalizes metadata sync tasksets.
     It scans for fence values and then gets the counts of any associated tasksets.
     If the count is 0, that means all tasks finished and we should clean up.
@@ -479,7 +481,7 @@ def monitor_vespa_sync() -> None:
     """
     r = get_redis_client()
 
-    lock_beat = r.lock(
+    lock_beat: redis.lock.Lock = r.lock(
         DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK,
         timeout=CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT,
     )
@@ -489,16 +491,37 @@ def monitor_vespa_sync() -> None:
         if not lock_beat.acquire(blocking=False):
             return
 
+        # print current queue lengths
+        r_celery = self.app.broker_connection().channel().client  # type: ignore
+        n_celery = celery_get_queue_length("celery", r)
+        n_sync = celery_get_queue_length(
+            DanswerCeleryQueues.VESPA_METADATA_SYNC, r_celery
+        )
+        n_deletion = celery_get_queue_length(
+            DanswerCeleryQueues.CONNECTOR_DELETION, r_celery
+        )
+        n_pruning = celery_get_queue_length(
+            DanswerCeleryQueues.CONNECTOR_PRUNING, r_celery
+        )
+
+        task_logger.info(
+            f"Queue lengths: celery={n_celery} sync={n_sync} deletion={n_deletion} pruning={n_pruning}"
+        )
+
+        lock_beat.reacquire()
         if r.exists(RedisConnectorCredentialPair.get_fence_key()):
             monitor_connector_taskset(r)
 
+        lock_beat.reacquire()
         for key_bytes in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
             monitor_connector_deletion_taskset(key_bytes, r)
 
         with Session(get_sqlalchemy_engine()) as db_session:
+            lock_beat.reacquire()
             for key_bytes in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
                 monitor_document_set_taskset(key_bytes, r, db_session)
 
+            lock_beat.reacquire()
             for key_bytes in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
                 monitor_usergroup_taskset = (
                     fetch_versioned_implementation_with_fallback(
@@ -509,6 +532,7 @@ def monitor_vespa_sync() -> None:
                 )
                 monitor_usergroup_taskset(key_bytes, r, db_session)
 
+            lock_beat.reacquire()
             for key_bytes in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
                 monitor_ccpair_pruning_taskset(key_bytes, r, db_session)
 

From 804de3248e9da6f803fb137be342ffc559874f1e Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Wed, 9 Oct 2024 14:17:22 -0700
Subject: [PATCH 082/376] google drive permission sync cleanup (#2749)

---
 .../google_drive/doc_sync.py                  | 41 +++++++------------
 1 file changed, 14 insertions(+), 27 deletions(-)

diff --git a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
index a957558a99e..19dbb845323 100644
--- a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
@@ -14,7 +14,6 @@
 from danswer.connectors.google_drive.connector_auth import (
     get_google_drive_creds,
 )
-from danswer.connectors.google_drive.constants import FETCH_PERMISSIONS_SCOPES
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.models import InputType
 from danswer.db.models import ConnectorCredentialPair
@@ -72,25 +71,6 @@ def _fetch_permissions_paginated(
 ) -> Iterator[dict[str, Any]]:
     next_token = None
 
-    # Check if the file is trashed
-    # Returning nothing here will cause the external permissions to
-    # be empty which will get written to vespa (failing shut)
-    try:
-        file_metadata = add_retries(
-            lambda: drive_service.files()
-            .get(fileId=drive_file_id, fields="id, trashed")
-            .execute()
-        )()
-    except HttpError as e:
-        if e.resp.status == 404 or e.resp.status == 403:
-            return
-        logger.error(f"Failed to fetch permissions: {e}")
-        raise
-
-    if file_metadata.get("trashed", False):
-        logger.debug(f"File with ID {drive_file_id} is trashed")
-        return
-
     # Get paginated permissions for the file id
     while True:
         try:
@@ -99,7 +79,7 @@ def _fetch_permissions_paginated(
                     drive_service.permissions()
                     .list(
                         fileId=drive_file_id,
-                        fields="permissions(id, emailAddress, role, type, domain)",
+                        fields="permissions(emailAddress, type, domain)",
                         supportsAllDrives=True,
                         pageToken=next_token,
                     )
@@ -107,10 +87,17 @@ def _fetch_permissions_paginated(
                 )
             )()
         except HttpError as e:
-            if e.resp.status == 404 or e.resp.status == 403:
+            if e.resp.status == 404:
+                logger.warning(f"Document with id {drive_file_id} not found: {e}")
+                break
+            elif e.resp.status == 403:
+                logger.warning(
+                    f"Access denied for retrieving document permissions: {e}"
+                )
                 break
-            logger.error(f"Failed to fetch permissions: {e}")
-            raise
+            else:
+                logger.error(f"Failed to fetch permissions: {e}")
+                raise
 
         for permission in permissions_resp.get("permissions", []):
             yield permission
@@ -123,12 +110,12 @@ def _fetch_permissions_paginated(
 def _fetch_google_permissions_for_document_id(
     db_session: Session,
     drive_file_id: str,
-    raw_credentials_json: dict[str, str],
+    credentials_json: dict[str, str],
     company_google_domains: list[str],
 ) -> ExternalAccess:
     # Authenticate and construct service
     google_drive_creds, _ = get_google_drive_creds(
-        raw_credentials_json, scopes=FETCH_PERMISSIONS_SCOPES
+        credentials_json,
     )
     if not google_drive_creds.valid:
         raise ValueError("Invalid Google Drive credentials")
@@ -187,7 +174,7 @@ def gdrive_doc_sync(
         ext_access = _fetch_google_permissions_for_document_id(
             db_session=db_session,
             drive_file_id=doc_additional_info,
-            raw_credentials_json=cc_pair.credential.credential_json,
+            credentials_json=cc_pair.credential.credential_json,
             company_google_domains=[
                 cast(dict[str, str], sync_details)["company_domain"]
             ],

From 30d17ef9ee22ddd7d73fb40819109dc61587c9c4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 9 Oct 2024 14:56:45 -0700
Subject: [PATCH 083/376] Convert images to jpeg (#2737)

* convert to jpeg

* k

* typing
---
 .../server/query_and_chat/chat_backend.py     | 29 +++++++++++++++++--
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index 36a09afde19..91efe6cb874 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -3,6 +3,7 @@
 import uuid
 from collections.abc import Callable
 from collections.abc import Generator
+from typing import Tuple
 
 from fastapi import APIRouter
 from fastapi import Depends
@@ -11,6 +12,7 @@
 from fastapi import Response
 from fastapi import UploadFile
 from fastapi.responses import StreamingResponse
+from PIL import Image
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
@@ -508,6 +510,21 @@ def seed_chat(
 """File upload"""
 
 
+def convert_to_jpeg(file: UploadFile) -> Tuple[io.BytesIO, str]:
+    try:
+        with Image.open(file.file) as img:
+            if img.mode != "RGB":
+                img = img.convert("RGB")
+            jpeg_io = io.BytesIO()
+            img.save(jpeg_io, format="JPEG", quality=85)
+            jpeg_io.seek(0)
+        return jpeg_io, "image/jpeg"
+    except Exception as e:
+        raise HTTPException(
+            status_code=400, detail=f"Failed to convert image: {str(e)}"
+        )
+
+
 @router.post("/file")
 def upload_files_for_chat(
     files: list[UploadFile],
@@ -570,19 +587,25 @@ def upload_files_for_chat(
     for file in files:
         if file.content_type in image_content_types:
             file_type = ChatFileType.IMAGE
+            # Convert image to JPEG
+            file_content, new_content_type = convert_to_jpeg(file)
         elif file.content_type in document_content_types:
             file_type = ChatFileType.DOC
+            file_content = io.BytesIO(file.file.read())
+            new_content_type = file.content_type or ""
         else:
             file_type = ChatFileType.PLAIN_TEXT
+            file_content = io.BytesIO(file.file.read())
+            new_content_type = file.content_type or ""
 
-        # store the raw file
+        # store the file (now JPEG for images)
         file_id = str(uuid.uuid4())
         file_store.save_file(
             file_name=file_id,
-            content=file.file,
+            content=file_content,
             display_name=file.filename,
             file_origin=FileOrigin.CHAT_UPLOAD,
-            file_type=file.content_type or file_type.value,
+            file_type=new_content_type or file_type.value,
         )
 
         # if the file is a doc, extract text and store that so we don't need

From 2d74d44538fd6adc22b537c92e44bf8ef69dea88 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 9 Oct 2024 17:31:54 -0700
Subject: [PATCH 084/376] update indexing and slack bot to use stdout options
 (#2752)

---
 backend/supervisord.conf | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/backend/supervisord.conf b/backend/supervisord.conf
index 32951067300..ebe56761381 100644
--- a/backend/supervisord.conf
+++ b/backend/supervisord.conf
@@ -7,12 +7,13 @@ logfile=/var/log/supervisord.log
 # Cannot place this in Celery for now because Celery must run as a single process (see note below)
 # Indexing uses multi-processing to speed things up
 [program:document_indexing]
-environment=CURRENT_PROCESS_IS_AN_INDEXING_JOB=true,LOG_FILE_NAME=document_indexing
+environment=CURRENT_PROCESS_IS_AN_INDEXING_JOB=true
 command=python danswer/background/update.py
+stdout_logfile=/var/log/document_indexing.log
+stdout_logfile_maxbytes=16MB
 redirect_stderr=true
 autorestart=true
 
-
 # Background jobs that must be run async due to long time to completion
 # NOTE: due to an issue with Celery + SQLAlchemy 
 # (https://github.com/celery/celery/issues/7007#issuecomment-1740139367)
@@ -87,7 +88,8 @@ stopasgroup=true
 # More details on setup here: https://docs.danswer.dev/slack_bot_setup
 [program:slack_bot]
 command=python danswer/danswerbot/slack/listener.py
-environment=LOG_FILE_NAME=slack_bot
+stdout_logfile=/var/log/slack_bot.log
+stdout_logfile_maxbytes=16MB
 redirect_stderr=true
 autorestart=true
 startretries=5
@@ -101,8 +103,8 @@ command=tail -qF
     /var/log/celery_worker_primary.log
     /var/log/celery_worker_light.log
     /var/log/celery_worker_heavy.log
-    /var/log/document_indexing_info.log
-    /var/log/slack_bot_debug.log
+    /var/log/document_indexing.log
+    /var/log/slack_bot.log
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes = 0  # must be set to 0 when stdout_logfile=/dev/stdout
 autorestart=true

From b4417fabd7b0d83eda19d7dee16057c04ab52087 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 9 Oct 2024 18:47:38 -0700
Subject: [PATCH 085/376] ensure shared assistants accessible via query params
 (#2740)

---
 web/src/app/chat/ChatPage.tsx     | 6 +++---
 web/src/lib/chat/fetchChatData.ts | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 70749119349..4f7363a3b66 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -187,7 +187,6 @@ export function ChatPage({
           )
         : undefined
   );
-
   // Gather default temperature settings
   const search_param_temperature = searchParams.get(
     SEARCH_PARAM_NAMES.TEMPERATURE
@@ -694,11 +693,12 @@ export function ChatPage({
 
   useEffect(() => {
     if (messageHistory.length === 0 && chatSessionIdRef.current === null) {
+      // Select from available assistants so shared assistants appear.
       setSelectedAssistant(
-        finalAssistants.find((persona) => persona.id === defaultAssistantId)
+        availableAssistants.find((persona) => persona.id === defaultAssistantId)
       );
     }
-  }, [defaultAssistantId, finalAssistants, messageHistory.length]);
+  }, [defaultAssistantId, availableAssistants, messageHistory.length]);
 
   const [
     selectedDocuments,
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index 1416f787cc7..144a839cd73 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -153,6 +153,7 @@ export async function fetchChatData(searchParams: {
     console.log(`Failed to fetch assistants - ${assistantsFetchError}`);
   }
   // remove those marked as hidden by an admin
+
   assistants = assistants.filter((assistant) => assistant.is_visible);
 
   // sort them in priority order

From 9be54a2b4ce880b8387006b1aca97015f32596dc Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Wed, 9 Oct 2024 19:08:09 -0700
Subject: [PATCH 086/376] Fix slack bot follow up questions (#2756)

---
 .../slack/handlers/handle_regular_answer.py           | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
index 5bd920c4b6c..e864d92c702 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
@@ -5,7 +5,6 @@
 from typing import Optional
 from typing import TypeVar
 
-from fastapi import HTTPException
 from retry import retry
 from slack_sdk import WebClient
 from slack_sdk.models.blocks import DividerBlock
@@ -155,11 +154,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
         with Session(get_sqlalchemy_engine()) as db_session:
             if len(new_message_request.messages) > 1:
                 if new_message_request.persona_config:
-                    raise HTTPException(
-                        status_code=403,
-                        detail="Slack bot does not support persona config",
-                    )
-
+                    raise RuntimeError("Slack bot does not support persona config")
                 elif new_message_request.persona_id is not None:
                     persona = cast(
                         Persona,
@@ -170,6 +165,10 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
                             get_editable=False,
                         ),
                     )
+                else:
+                    raise RuntimeError(
+                        "No persona id provided, this should never happen."
+                    )
 
                 llm, _ = get_llms_for_persona(persona)
 

From f40c5ca9bdc6fc9a9adbb6599d5155c9eade4810 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 10 Oct 2024 09:34:32 -0700
Subject: [PATCH 087/376] Add tenant context (#2596)

* add proper tenant context to background tasks

* update for new session logic

* remove unnecessary functions

* add additional tenant context

* update ports

* proper format / directory structure

* update ports

* ensure tenant context properly passed to ee bg tasks

* add user provisioning

* nit

* validated for multi tenant

* auth

* nit

* nit

* nit

* nit

* validate pruning

* evaluate integration tests

* at long last, validated celery beat

* nit: minor edge case patched

* minor

* validate update

* nit
---
 backend/alembic.ini                           |  14 +-
 backend/alembic/env.py                        |  48 ++--
 ...9164_chosen_assistants_changed_to_jsonb.py |   8 +-
 backend/alembic_tenants/README.md             |   3 +
 backend/alembic_tenants/env.py                | 111 ++++++++
 backend/alembic_tenants/script.py.mako        |  24 ++
 ...3a331951_create_usertenantmapping_table.py |  24 ++
 backend/danswer/auth/schemas.py               |   1 +
 backend/danswer/auth/users.py                 | 265 ++++++++++++++----
 .../danswer/background/celery/celery_app.py   |  86 +++---
 .../danswer/background/celery/celery_redis.py |  14 +-
 .../celery/tasks/connector_deletion/tasks.py  |   9 +-
 .../background/celery/tasks/pruning/tasks.py  |  30 +-
 .../background/celery/tasks/shared/tasks.py   |  11 +-
 .../background/celery/tasks/vespa/tasks.py    |  47 +++-
 .../background/indexing/run_indexing.py       |  58 ++--
 backend/danswer/background/update.py          | 192 ++++++++-----
 backend/danswer/configs/app_configs.py        |   2 +
 backend/danswer/configs/constants.py          |   3 +
 backend/danswer/db/auth.py                    |   4 +-
 .../danswer/db/connector_credential_pair.py   |   1 +
 backend/danswer/db/engine.py                  | 128 ++++++---
 backend/danswer/db/models.py                  |  20 ++
 backend/danswer/indexing/indexing_pipeline.py |   6 +
 backend/danswer/indexing/models.py            |   3 +
 backend/danswer/key_value_store/store.py      |  16 ++
 backend/danswer/main.py                       |  12 +-
 backend/danswer/server/documents/cc_pair.py   |   7 +-
 .../danswer/server/manage/administrative.py   |   3 +
 backend/danswer/server/manage/users.py        |  69 ++++-
 .../server/query_and_chat/chat_backend.py     |   1 +
 backend/danswer/server/utils.py               |  35 +++
 backend/danswer/setup.py                      |   4 +-
 .../danswer/background/celery/celery_app.py   |  93 ++++--
 .../danswer/background/task_name_builders.py  |   8 +-
 backend/ee/danswer/main.py                    |   5 +
 .../server/middleware/tenant_tracking.py      |  60 ++++
 backend/ee/danswer/server/tenants/api.py      |  20 +-
 .../ee/danswer/server/tenants/provisioning.py |  47 ++++
 .../query_time_check/seed_dummy_docs.py       |   1 +
 backend/shared_configs/configs.py             |   3 +
 .../docker_compose/docker-compose.dev.yml     |   1 +
 web/src/app/auth/create-account/page.tsx      |  45 +++
 web/src/app/auth/error/page.tsx               |  46 ++-
 web/src/app/auth/login/LoginText.tsx          |  12 +-
 web/src/app/auth/login/page.tsx               |  70 +++--
 web/src/app/auth/oauth/callback/route.ts      |   6 +
 web/src/app/auth/signup/page.tsx              |   4 +
 web/src/app/layout.tsx                        |   4 +-
 web/src/components/auth/AuthFlowContainer.tsx |  16 ++
 web/src/components/settings/lib.ts            |   4 +-
 web/src/lib/constants.ts                      |   4 +
 52 files changed, 1319 insertions(+), 389 deletions(-)
 create mode 100644 backend/alembic_tenants/README.md
 create mode 100644 backend/alembic_tenants/env.py
 create mode 100644 backend/alembic_tenants/script.py.mako
 create mode 100644 backend/alembic_tenants/versions/14a83a331951_create_usertenantmapping_table.py
 create mode 100644 backend/ee/danswer/server/middleware/tenant_tracking.py
 create mode 100644 web/src/app/auth/create-account/page.tsx
 create mode 100644 web/src/components/auth/AuthFlowContainer.tsx

diff --git a/backend/alembic.ini b/backend/alembic.ini
index 10ae5cfdd27..599c46fadd7 100644
--- a/backend/alembic.ini
+++ b/backend/alembic.ini
@@ -1,6 +1,6 @@
 # A generic, single database configuration.
 
-[alembic]
+[DEFAULT]
 # path to migration scripts
 script_location = alembic
 
@@ -47,7 +47,8 @@ prepend_sys_path = .
 # version_path_separator = :
 # version_path_separator = ;
 # version_path_separator = space
-version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+version_path_separator = os  
+# Use os.pathsep. Default configuration used for new projects.
 
 # set to 'true' to search source files recursively
 # in each "version_locations" directory
@@ -106,3 +107,12 @@ formatter = generic
 [formatter_generic]
 format = %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %H:%M:%S
+
+
+[alembic]
+script_location = alembic
+version_locations = %(script_location)s/versions
+
+[schema_private]
+script_location = alembic_tenants
+version_locations = %(script_location)s/versions
diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index d7ac37af562..afa5a9669c1 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -1,21 +1,22 @@
+from typing import Any
 import asyncio
 from logging.config import fileConfig
 
 from alembic import context
-from danswer.db.engine import build_connection_string
-from danswer.db.models import Base
 from sqlalchemy import pool
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import create_async_engine
-from celery.backends.database.session import ResultModelBase  # type: ignore
-from sqlalchemy.schema import SchemaItem
 from sqlalchemy.sql import text
 
+from danswer.configs.app_configs import MULTI_TENANT
+from danswer.db.engine import build_connection_string
+from danswer.db.models import Base
+from celery.backends.database.session import ResultModelBase  # type: ignore
+
 # Alembic Config object
 config = context.config
 
 # Interpret the config file for Python logging.
-# This line sets up loggers basically.
 if config.config_file_name is not None and config.attributes.get(
     "configure_logger", True
 ):
@@ -35,8 +36,7 @@ def get_schema_options() -> tuple[str, bool]:
         for pair in arg.split(","):
             if "=" in pair:
                 key, value = pair.split("=", 1)
-                x_args[key] = value
-
+                x_args[key.strip()] = value.strip()
     schema_name = x_args.get("schema", "public")
     create_schema = x_args.get("create_schema", "true").lower() == "true"
     return schema_name, create_schema
@@ -46,11 +46,7 @@ def get_schema_options() -> tuple[str, bool]:
 
 
 def include_object(
-    object: SchemaItem,
-    name: str,
-    type_: str,
-    reflected: bool,
-    compare_to: SchemaItem | None,
+    object: Any, name: str, type_: str, reflected: bool, compare_to: Any
 ) -> bool:
     if type_ == "table" and name in EXCLUDE_TABLES:
         return False
@@ -59,7 +55,6 @@ def include_object(
 
 def run_migrations_offline() -> None:
     """Run migrations in 'offline' mode.
-
     This configures the context with just a URL
     and not an Engine, though an Engine is acceptable
     here as well.  By skipping the Engine creation
@@ -67,17 +62,18 @@ def run_migrations_offline() -> None:
     Calls to context.execute() here emit the given string to the
     script output.
     """
+    schema_name, _ = get_schema_options()
     url = build_connection_string()
-    schema, _ = get_schema_options()
 
     context.configure(
         url=url,
         target_metadata=target_metadata,  # type: ignore
         literal_binds=True,
         include_object=include_object,
-        dialect_opts={"paramstyle": "named"},
-        version_table_schema=schema,
+        version_table_schema=schema_name,
         include_schemas=True,
+        script_location=config.get_main_option("script_location"),
+        dialect_opts={"paramstyle": "named"},
     )
 
     with context.begin_transaction():
@@ -85,20 +81,30 @@ def run_migrations_offline() -> None:
 
 
 def do_run_migrations(connection: Connection) -> None:
-    schema, create_schema = get_schema_options()
+    schema_name, create_schema = get_schema_options()
+
+    if MULTI_TENANT and schema_name == "public":
+        raise ValueError(
+            "Cannot run default migrations in public schema when multi-tenancy is enabled. "
+            "Please specify a tenant-specific schema."
+        )
+
     if create_schema:
-        connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema}"'))
+        connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema_name}"'))
         connection.execute(text("COMMIT"))
 
-    connection.execute(text(f'SET search_path TO "{schema}"'))
+    # Set search_path to the target schema
+    connection.execute(text(f'SET search_path TO "{schema_name}"'))
 
     context.configure(
         connection=connection,
         target_metadata=target_metadata,  # type: ignore
-        version_table_schema=schema,
+        include_object=include_object,
+        version_table_schema=schema_name,
         include_schemas=True,
         compare_type=True,
         compare_server_default=True,
+        script_location=config.get_main_option("script_location"),
     )
 
     with context.begin_transaction():
@@ -106,7 +112,6 @@ def do_run_migrations(connection: Connection) -> None:
 
 
 async def run_async_migrations() -> None:
-    """Run migrations in 'online' mode."""
     connectable = create_async_engine(
         build_connection_string(),
         poolclass=pool.NullPool,
@@ -119,7 +124,6 @@ async def run_async_migrations() -> None:
 
 
 def run_migrations_online() -> None:
-    """Run migrations in 'online' mode."""
     asyncio.run(run_async_migrations())
 
 
diff --git a/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py b/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py
index 95b53cbeb41..8e0a8e6072d 100644
--- a/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py
+++ b/backend/alembic/versions/da4c21c69164_chosen_assistants_changed_to_jsonb.py
@@ -20,7 +20,7 @@
 def upgrade() -> None:
     conn = op.get_bind()
     existing_ids_and_chosen_assistants = conn.execute(
-        sa.text("select id, chosen_assistants from public.user")
+        sa.text('select id, chosen_assistants from "user"')
     )
     op.drop_column(
         "user",
@@ -37,7 +37,7 @@ def upgrade() -> None:
     for id, chosen_assistants in existing_ids_and_chosen_assistants:
         conn.execute(
             sa.text(
-                "update public.user set chosen_assistants = :chosen_assistants where id = :id"
+                'update "user" set chosen_assistants = :chosen_assistants where id = :id'
             ),
             {"chosen_assistants": json.dumps(chosen_assistants), "id": id},
         )
@@ -46,7 +46,7 @@ def upgrade() -> None:
 def downgrade() -> None:
     conn = op.get_bind()
     existing_ids_and_chosen_assistants = conn.execute(
-        sa.text("select id, chosen_assistants from public.user")
+        sa.text('select id, chosen_assistants from "user"')
     )
     op.drop_column(
         "user",
@@ -59,7 +59,7 @@ def downgrade() -> None:
     for id, chosen_assistants in existing_ids_and_chosen_assistants:
         conn.execute(
             sa.text(
-                "update public.user set chosen_assistants = :chosen_assistants where id = :id"
+                'update "user" set chosen_assistants = :chosen_assistants where id = :id'
             ),
             {"chosen_assistants": chosen_assistants, "id": id},
         )
diff --git a/backend/alembic_tenants/README.md b/backend/alembic_tenants/README.md
new file mode 100644
index 00000000000..f075b958305
--- /dev/null
+++ b/backend/alembic_tenants/README.md
@@ -0,0 +1,3 @@
+These files are for public table migrations when operating with multi tenancy.
+
+If you are not a Danswer developer, you can ignore this directory entirely.
\ No newline at end of file
diff --git a/backend/alembic_tenants/env.py b/backend/alembic_tenants/env.py
new file mode 100644
index 00000000000..f0f1178ce09
--- /dev/null
+++ b/backend/alembic_tenants/env.py
@@ -0,0 +1,111 @@
+import asyncio
+from logging.config import fileConfig
+
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.schema import SchemaItem
+
+from alembic import context
+from danswer.db.engine import build_connection_string
+from danswer.db.models import PublicBase
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None and config.attributes.get(
+    "configure_logger", True
+):
+    fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = [PublicBase.metadata]
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+EXCLUDE_TABLES = {"kombu_queue", "kombu_message"}
+
+
+def include_object(
+    object: SchemaItem,
+    name: str,
+    type_: str,
+    reflected: bool,
+    compare_to: SchemaItem | None,
+) -> bool:
+    if type_ == "table" and name in EXCLUDE_TABLES:
+        return False
+    return True
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = build_connection_string()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,  # type: ignore
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+    context.configure(
+        connection=connection,
+        target_metadata=target_metadata,  # type: ignore
+        include_object=include_object,
+    )  # type: ignore
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+    """In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+
+    connectable = create_async_engine(
+        build_connection_string(),
+        poolclass=pool.NullPool,
+    )
+
+    async with connectable.connect() as connection:
+        await connection.run_sync(do_run_migrations)
+
+    await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode."""
+
+    asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()
diff --git a/backend/alembic_tenants/script.py.mako b/backend/alembic_tenants/script.py.mako
new file mode 100644
index 00000000000..55df2863d20
--- /dev/null
+++ b/backend/alembic_tenants/script.py.mako
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}
diff --git a/backend/alembic_tenants/versions/14a83a331951_create_usertenantmapping_table.py b/backend/alembic_tenants/versions/14a83a331951_create_usertenantmapping_table.py
new file mode 100644
index 00000000000..f8f3016bab1
--- /dev/null
+++ b/backend/alembic_tenants/versions/14a83a331951_create_usertenantmapping_table.py
@@ -0,0 +1,24 @@
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "14a83a331951"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "user_tenant_mapping",
+        sa.Column("email", sa.String(), nullable=False),
+        sa.Column("tenant_id", sa.String(), nullable=False),
+        sa.UniqueConstraint("email", "tenant_id", name="uq_user_tenant"),
+        sa.UniqueConstraint("email", name="uq_email"),
+        schema="public",
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("user_tenant_mapping", schema="public")
diff --git a/backend/danswer/auth/schemas.py b/backend/danswer/auth/schemas.py
index db8a97ceb04..9c81899a421 100644
--- a/backend/danswer/auth/schemas.py
+++ b/backend/danswer/auth/schemas.py
@@ -34,6 +34,7 @@ class UserRead(schemas.BaseUser[uuid.UUID]):
 class UserCreate(schemas.BaseUserCreate):
     role: UserRole = UserRole.BASIC
     has_web_login: bool | None = True
+    tenant_id: str | None = None
 
 
 class UserUpdate(schemas.BaseUserUpdate):
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 81607aab884..3fc117b31a0 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -26,11 +26,14 @@
 from fastapi_users import UUIDIDMixin
 from fastapi_users.authentication import AuthenticationBackend
 from fastapi_users.authentication import CookieTransport
+from fastapi_users.authentication import JWTStrategy
 from fastapi_users.authentication import Strategy
 from fastapi_users.authentication.strategy.db import AccessTokenDatabase
 from fastapi_users.authentication.strategy.db import DatabaseStrategy
 from fastapi_users.openapi import OpenAPIResponseType
 from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase
+from sqlalchemy import select
+from sqlalchemy.orm import attributes
 from sqlalchemy.orm import Session
 
 from danswer.auth.invited_users import get_invited_users
@@ -42,7 +45,9 @@
 from danswer.configs.app_configs import DISABLE_AUTH
 from danswer.configs.app_configs import EMAIL_FROM
 from danswer.configs.app_configs import EXPECTED_API_KEY
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
+from danswer.configs.app_configs import SECRET_JWT_KEY
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import SMTP_PASS
 from danswer.configs.app_configs import SMTP_PORT
@@ -60,15 +65,21 @@
 from danswer.db.auth import get_default_admin_user_emails
 from danswer.db.auth import get_user_count
 from danswer.db.auth import get_user_db
+from danswer.db.auth import SQLAlchemyUserAdminDB
+from danswer.db.engine import get_async_session_with_tenant
 from danswer.db.engine import get_session
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.models import AccessToken
+from danswer.db.models import OAuthAccount
 from danswer.db.models import User
+from danswer.db.models import UserTenantMapping
 from danswer.db.users import get_user_by_email
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import optional_telemetry
 from danswer.utils.telemetry import RecordType
 from danswer.utils.variable_functionality import fetch_versioned_implementation
+from shared_configs.configs import current_tenant_id
 
 logger = setup_logger()
 
@@ -136,8 +147,8 @@ def verify_email_is_invited(email: str) -> None:
     raise PermissionError("User not on allowed user whitelist")
 
 
-def verify_email_in_whitelist(email: str) -> None:
-    with Session(get_sqlalchemy_engine()) as db_session:
+def verify_email_in_whitelist(email: str, tenant_id: str | None = None) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         if not get_user_by_email(email, db_session):
             verify_email_is_invited(email)
 
@@ -157,6 +168,20 @@ def verify_email_domain(email: str) -> None:
             )
 
 
+def get_tenant_id_for_email(email: str) -> str:
+    if not MULTI_TENANT:
+        return "public"
+    # Implement logic to get tenant_id from the mapping table
+    with Session(get_sqlalchemy_engine()) as db_session:
+        result = db_session.execute(
+            select(UserTenantMapping.tenant_id).where(UserTenantMapping.email == email)
+        )
+        tenant_id = result.scalar_one_or_none()
+    if tenant_id is None:
+        raise exceptions.UserNotExists()
+    return tenant_id
+
+
 def send_user_verification_email(
     user_email: str,
     token: str,
@@ -221,6 +246,29 @@ async def create(
                 raise exceptions.UserAlreadyExists()
         return user
 
+    async def on_after_login(
+        self,
+        user: User,
+        request: Request | None = None,
+        response: Response | None = None,
+    ) -> None:
+        if response is None or not MULTI_TENANT:
+            return
+
+        tenant_id = get_tenant_id_for_email(user.email)
+
+        tenant_token = jwt.encode(
+            {"tenant_id": tenant_id}, SECRET_JWT_KEY, algorithm="HS256"
+        )
+
+        response.set_cookie(
+            key="tenant_details",
+            value=tenant_token,
+            httponly=True,
+            secure=WEB_DOMAIN.startswith("https"),
+            samesite="lax",
+        )
+
     async def oauth_callback(
         self: "BaseUserManager[models.UOAP, models.ID]",
         oauth_name: str,
@@ -234,45 +282,111 @@ async def oauth_callback(
         associate_by_email: bool = False,
         is_verified_by_default: bool = False,
     ) -> models.UOAP:
-        verify_email_in_whitelist(account_email)
-        verify_email_domain(account_email)
-
-        user = await super().oauth_callback(  # type: ignore
-            oauth_name=oauth_name,
-            access_token=access_token,
-            account_id=account_id,
-            account_email=account_email,
-            expires_at=expires_at,
-            refresh_token=refresh_token,
-            request=request,
-            associate_by_email=associate_by_email,
-            is_verified_by_default=is_verified_by_default,
-        )
-
-        # NOTE: Most IdPs have very short expiry times, and we don't want to force the user to
-        # re-authenticate that frequently, so by default this is disabled
-        if expires_at and TRACK_EXTERNAL_IDP_EXPIRY:
-            oidc_expiry = datetime.fromtimestamp(expires_at, tz=timezone.utc)
-            await self.user_db.update(user, update_dict={"oidc_expiry": oidc_expiry})
-
-        # this is needed if an organization goes from `TRACK_EXTERNAL_IDP_EXPIRY=true` to `false`
-        # otherwise, the oidc expiry will always be old, and the user will never be able to login
-        if user.oidc_expiry and not TRACK_EXTERNAL_IDP_EXPIRY:
-            await self.user_db.update(user, update_dict={"oidc_expiry": None})
-
-        # Handle case where user has used product outside of web and is now creating an account through web
-        if not user.has_web_login:
-            await self.user_db.update(
-                user,
-                update_dict={
-                    "is_verified": is_verified_by_default,
-                    "has_web_login": True,
-                },
+        # Get tenant_id from mapping table
+        try:
+            tenant_id = (
+                get_tenant_id_for_email(account_email) if MULTI_TENANT else "public"
             )
-            user.is_verified = is_verified_by_default
-            user.has_web_login = True
+        except exceptions.UserNotExists:
+            raise HTTPException(status_code=401, detail="User not found")
+
+        if not tenant_id:
+            raise HTTPException(status_code=401, detail="User not found")
+
+        token = None
+        async with get_async_session_with_tenant(tenant_id) as db_session:
+            token = current_tenant_id.set(tenant_id)
+            # Print a list of tables in the current database session
+            verify_email_in_whitelist(account_email, tenant_id)
+            verify_email_domain(account_email)
+            if MULTI_TENANT:
+                tenant_user_db = SQLAlchemyUserAdminDB(db_session, User, OAuthAccount)
+                self.user_db = tenant_user_db
+                self.database = tenant_user_db
+
+            oauth_account_dict = {
+                "oauth_name": oauth_name,
+                "access_token": access_token,
+                "account_id": account_id,
+                "account_email": account_email,
+                "expires_at": expires_at,
+                "refresh_token": refresh_token,
+            }
+
+            try:
+                # Attempt to get user by OAuth account
+                user = await self.get_by_oauth_account(oauth_name, account_id)
+
+            except exceptions.UserNotExists:
+                try:
+                    # Attempt to get user by email
+                    user = await self.get_by_email(account_email)
+                    if not associate_by_email:
+                        raise exceptions.UserAlreadyExists()
+
+                    user = await self.user_db.add_oauth_account(
+                        user, oauth_account_dict
+                    )
+
+                    # If user not found by OAuth account or email, create a new user
+                except exceptions.UserNotExists:
+                    password = self.password_helper.generate()
+                    user_dict = {
+                        "email": account_email,
+                        "hashed_password": self.password_helper.hash(password),
+                        "is_verified": is_verified_by_default,
+                    }
+
+                    user = await self.user_db.create(user_dict)
+                    user = await self.user_db.add_oauth_account(
+                        user, oauth_account_dict
+                    )
+                    await self.on_after_register(user, request)
 
-        return user
+            else:
+                for existing_oauth_account in user.oauth_accounts:
+                    if (
+                        existing_oauth_account.account_id == account_id
+                        and existing_oauth_account.oauth_name == oauth_name
+                    ):
+                        user = await self.user_db.update_oauth_account(
+                            user, existing_oauth_account, oauth_account_dict
+                        )
+
+            # NOTE: Most IdPs have very short expiry times, and we don't want to force the user to
+            # re-authenticate that frequently, so by default this is disabled
+
+            if expires_at and TRACK_EXTERNAL_IDP_EXPIRY:
+                oidc_expiry = datetime.fromtimestamp(expires_at, tz=timezone.utc)
+                await self.user_db.update(
+                    user, update_dict={"oidc_expiry": oidc_expiry}
+                )
+
+            # Handle case where user has used product outside of web and is now creating an account through web
+            if not user.has_web_login:  # type: ignore
+                await self.user_db.update(
+                    user,
+                    {
+                        "is_verified": is_verified_by_default,
+                        "has_web_login": True,
+                    },
+                )
+                user.is_verified = is_verified_by_default
+                user.has_web_login = True  # type: ignore
+
+            # this is needed if an organization goes from `TRACK_EXTERNAL_IDP_EXPIRY=true` to `false`
+            # otherwise, the oidc expiry will always be old, and the user will never be able to login
+            if (
+                user.oidc_expiry is not None  # type: ignore
+                and not TRACK_EXTERNAL_IDP_EXPIRY
+            ):
+                await self.user_db.update(user, {"oidc_expiry": None})
+                user.oidc_expiry = None  # type: ignore
+
+            if token:
+                current_tenant_id.reset(token)
+
+            return user
 
     async def on_after_register(
         self, user: User, request: Optional[Request] = None
@@ -303,28 +417,51 @@ async def on_after_request_verify(
     async def authenticate(
         self, credentials: OAuth2PasswordRequestForm
     ) -> Optional[User]:
-        try:
-            user = await self.get_by_email(credentials.username)
-        except exceptions.UserNotExists:
+        email = credentials.username
+
+        # Get tenant_id from mapping table
+
+        tenant_id = get_tenant_id_for_email(email)
+        if not tenant_id:
+            # User not found in mapping
             self.password_helper.hash(credentials.password)
             return None
 
-        if not user.has_web_login:
-            raise HTTPException(
-                status_code=status.HTTP_403_FORBIDDEN,
-                detail="NO_WEB_LOGIN_AND_HAS_NO_PASSWORD",
+        # Create a tenant-specific session
+        async with get_async_session_with_tenant(tenant_id) as tenant_session:
+            tenant_user_db: SQLAlchemyUserDatabase = SQLAlchemyUserDatabase(
+                tenant_session, User
             )
+            self.user_db = tenant_user_db
 
-        verified, updated_password_hash = self.password_helper.verify_and_update(
-            credentials.password, user.hashed_password
-        )
-        if not verified:
-            return None
+            # Proceed with authentication
+            try:
+                user = await self.get_by_email(email)
 
-        if updated_password_hash is not None:
-            await self.user_db.update(user, {"hashed_password": updated_password_hash})
+            except exceptions.UserNotExists:
+                self.password_helper.hash(credentials.password)
+                return None
 
-        return user
+            has_web_login = attributes.get_attribute(user, "has_web_login")
+
+            if not has_web_login:
+                raise HTTPException(
+                    status_code=status.HTTP_403_FORBIDDEN,
+                    detail="NO_WEB_LOGIN_AND_HAS_NO_PASSWORD",
+                )
+
+            verified, updated_password_hash = self.password_helper.verify_and_update(
+                credentials.password, user.hashed_password
+            )
+            if not verified:
+                return None
+
+            if updated_password_hash is not None:
+                await self.user_db.update(
+                    user, {"hashed_password": updated_password_hash}
+                )
+
+            return user
 
 
 async def get_user_manager(
@@ -339,20 +476,26 @@ async def get_user_manager(
 )
 
 
+def get_jwt_strategy() -> JWTStrategy:
+    return JWTStrategy(
+        secret=USER_AUTH_SECRET,
+        lifetime_seconds=SESSION_EXPIRE_TIME_SECONDS,
+    )
+
+
 def get_database_strategy(
     access_token_db: AccessTokenDatabase[AccessToken] = Depends(get_access_token_db),
 ) -> DatabaseStrategy:
-    strategy = DatabaseStrategy(
+    return DatabaseStrategy(
         access_token_db, lifetime_seconds=SESSION_EXPIRE_TIME_SECONDS  # type: ignore
     )
-    return strategy
 
 
 auth_backend = AuthenticationBackend(
-    name="database",
+    name="jwt" if MULTI_TENANT else "database",
     transport=cookie_transport,
-    get_strategy=get_database_strategy,
-)
+    get_strategy=get_jwt_strategy if MULTI_TENANT else get_database_strategy,  # type: ignore
+)  # type: ignore
 
 
 class FastAPIUserWithLogoutRouter(FastAPIUsers[models.UP, models.ID]):
@@ -366,9 +509,11 @@ def get_logout_router(
         This way the login router does not need to be included
         """
         router = APIRouter()
+
         get_current_user_token = self.authenticator.current_user_token(
             active=True, verified=requires_verification
         )
+
         logout_responses: OpenAPIResponseType = {
             **{
                 status.HTTP_401_UNAUTHORIZED: {
@@ -415,8 +560,8 @@ async def optional_user_(
 
 async def optional_user(
     request: Request,
-    user: User | None = Depends(optional_fastapi_current_user),
     db_session: Session = Depends(get_session),
+    user: User | None = Depends(optional_fastapi_current_user),
 ) -> User | None:
     versioned_fetch_user = fetch_versioned_implementation(
         "danswer.auth.users", "optional_user_"
diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 5d5450315b5..0e9fb00b1fd 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -23,6 +23,7 @@
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
+from danswer.background.update import get_all_tenant_ids
 from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerRedisLocks
@@ -70,7 +71,6 @@ def celery_task_postrun(
         return
 
     task_logger.debug(f"Task {task.name} (ID: {task_id}) completed with state: {state}")
-    # logger.debug(f"Result: {retval}")
 
     if state not in READY_STATES:
         return
@@ -437,48 +437,58 @@ def stop(self, worker: Any) -> None:
 #####
 # Celery Beat (Periodic Tasks) Settings
 #####
-celery_app.conf.beat_schedule = {
-    "check-for-vespa-sync": {
+
+tenant_ids = get_all_tenant_ids()
+
+tasks_to_schedule = [
+    {
+        "name": "check-for-vespa-sync",
         "task": "check_for_vespa_sync_task",
         "schedule": timedelta(seconds=5),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
-}
-celery_app.conf.beat_schedule.update(
     {
-        "check-for-connector-deletion-task": {
-            "task": "check_for_connector_deletion_task",
-            # don't need to check too often, since we kick off a deletion initially
-            # during the API call that actually marks the CC pair for deletion
-            "schedule": timedelta(seconds=60),
-            "options": {"priority": DanswerCeleryPriority.HIGH},
-        },
-    }
-)
-celery_app.conf.beat_schedule.update(
+        "name": "check-for-connector-deletion",
+        "task": "check_for_connector_deletion_task",
+        "schedule": timedelta(seconds=60),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
     {
-        "check-for-prune": {
-            "task": "check_for_prune_task_2",
-            "schedule": timedelta(seconds=60),
-            "options": {"priority": DanswerCeleryPriority.HIGH},
-        },
-    }
-)
-celery_app.conf.beat_schedule.update(
+        "name": "check-for-prune",
+        "task": "check_for_prune_task_2",
+        "schedule": timedelta(seconds=10),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
     {
-        "kombu-message-cleanup": {
-            "task": "kombu_message_cleanup_task",
-            "schedule": timedelta(seconds=3600),
-            "options": {"priority": DanswerCeleryPriority.LOWEST},
-        },
-    }
-)
-celery_app.conf.beat_schedule.update(
+        "name": "kombu-message-cleanup",
+        "task": "kombu_message_cleanup_task",
+        "schedule": timedelta(seconds=3600),
+        "options": {"priority": DanswerCeleryPriority.LOWEST},
+    },
     {
-        "monitor-vespa-sync": {
-            "task": "monitor_vespa_sync",
-            "schedule": timedelta(seconds=5),
-            "options": {"priority": DanswerCeleryPriority.HIGH},
-        },
-    }
-)
+        "name": "monitor-vespa-sync",
+        "task": "monitor_vespa_sync",
+        "schedule": timedelta(seconds=5),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+]
+
+# Build the celery beat schedule dynamically
+beat_schedule = {}
+
+for tenant_id in tenant_ids:
+    for task in tasks_to_schedule:
+        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
+        beat_schedule[task_name] = {
+            "task": task["task"],
+            "schedule": task["schedule"],
+            "options": task["options"],
+            "args": (tenant_id,),  # Must pass tenant_id as an argument
+        }
+
+# Include any existing beat schedules
+existing_beat_schedule = celery_app.conf.beat_schedule or {}
+beat_schedule.update(existing_beat_schedule)
+
+# Update the Celery app configuration once
+celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index f08bfd17e2f..1506a4b9be1 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -107,6 +107,7 @@ def generate_tasks(
         db_session: Session,
         redis_client: Redis,
         lock: redis.lock.Lock,
+        tenant_id: str | None,
     ) -> int | None:
         pass
 
@@ -122,6 +123,7 @@ def generate_tasks(
         db_session: Session,
         redis_client: Redis,
         lock: redis.lock.Lock,
+        tenant_id: str | None,
     ) -> int | None:
         last_lock_time = time.monotonic()
 
@@ -146,7 +148,7 @@ def generate_tasks(
 
             result = celery_app.send_task(
                 "vespa_metadata_sync_task",
-                kwargs=dict(document_id=doc.id),
+                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
                 queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
                 task_id=custom_task_id,
                 priority=DanswerCeleryPriority.LOW,
@@ -168,6 +170,7 @@ def generate_tasks(
         db_session: Session,
         redis_client: Redis,
         lock: redis.lock.Lock,
+        tenant_id: str | None,
     ) -> int | None:
         last_lock_time = time.monotonic()
 
@@ -204,7 +207,7 @@ def generate_tasks(
 
             result = celery_app.send_task(
                 "vespa_metadata_sync_task",
-                kwargs=dict(document_id=doc.id),
+                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
                 queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
                 task_id=custom_task_id,
                 priority=DanswerCeleryPriority.LOW,
@@ -244,6 +247,7 @@ def generate_tasks(
         db_session: Session,
         redis_client: Redis,
         lock: redis.lock.Lock,
+        tenant_id: str | None,
     ) -> int | None:
         last_lock_time = time.monotonic()
 
@@ -278,7 +282,7 @@ def generate_tasks(
             # Priority on sync's triggered by new indexing should be medium
             result = celery_app.send_task(
                 "vespa_metadata_sync_task",
-                kwargs=dict(document_id=doc.id),
+                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
                 queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
                 task_id=custom_task_id,
                 priority=DanswerCeleryPriority.MEDIUM,
@@ -300,6 +304,7 @@ def generate_tasks(
         db_session: Session,
         redis_client: Redis,
         lock: redis.lock.Lock,
+        tenant_id: str | None,
     ) -> int | None:
         last_lock_time = time.monotonic()
 
@@ -336,6 +341,7 @@ def generate_tasks(
                     document_id=doc.id,
                     connector_id=cc_pair.connector_id,
                     credential_id=cc_pair.credential_id,
+                    tenant_id=tenant_id,
                 ),
                 queue=DanswerCeleryQueues.CONNECTOR_DELETION,
                 task_id=custom_task_id,
@@ -409,6 +415,7 @@ def generate_tasks(
         db_session: Session,
         redis_client: Redis,
         lock: redis.lock.Lock | None,
+        tenant_id: str | None,
     ) -> int | None:
         last_lock_time = time.monotonic()
 
@@ -442,6 +449,7 @@ def generate_tasks(
                     document_id=doc_id,
                     connector_id=cc_pair.connector_id,
                     credential_id=cc_pair.credential_id,
+                    tenant_id=tenant_id,
                 ),
                 queue=DanswerCeleryQueues.CONNECTOR_DELETION,
                 task_id=custom_task_id,
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index a3223aacc9f..6a4c4da8243 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -23,7 +23,7 @@
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
 )
-def check_for_connector_deletion_task() -> None:
+def check_for_connector_deletion_task(tenant_id: str | None) -> None:
     r = get_redis_client()
 
     lock_beat = r.lock(
@@ -40,7 +40,7 @@ def check_for_connector_deletion_task() -> None:
             cc_pairs = get_connector_credential_pairs(db_session)
             for cc_pair in cc_pairs:
                 try_generate_document_cc_pair_cleanup_tasks(
-                    cc_pair, db_session, r, lock_beat
+                    cc_pair, db_session, r, lock_beat, tenant_id
                 )
     except SoftTimeLimitExceeded:
         task_logger.info(
@@ -58,6 +58,7 @@ def try_generate_document_cc_pair_cleanup_tasks(
     db_session: Session,
     r: Redis,
     lock_beat: redis.lock.Lock,
+    tenant_id: str | None,
 ) -> int | None:
     """Returns an int if syncing is needed. The int represents the number of sync tasks generated.
     Note that syncing can still be required even if the number of sync tasks generated is zero.
@@ -90,7 +91,9 @@ def try_generate_document_cc_pair_cleanup_tasks(
     task_logger.info(
         f"RedisConnectorDeletion.generate_tasks starting. cc_pair_id={cc_pair.id}"
     )
-    tasks_generated = rcd.generate_tasks(celery_app, db_session, r, lock_beat)
+    tasks_generated = rcd.generate_tasks(
+        celery_app, db_session, r, lock_beat, tenant_id
+    )
     if tasks_generated is None:
         return None
 
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index f72229b7d8c..28149bb82a3 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -24,17 +24,21 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pair
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.document import get_documents_for_connector_credential_pair
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.models import ConnectorCredentialPair
 from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
 
 
 @shared_task(
     name="check_for_prune_task_2",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_for_prune_task_2() -> None:
+def check_for_prune_task_2(tenant_id: str | None) -> None:
     r = get_redis_client()
 
     lock_beat = r.lock(
@@ -47,11 +51,11 @@ def check_for_prune_task_2() -> None:
         if not lock_beat.acquire(blocking=False):
             return
 
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = get_connector_credential_pairs(db_session)
             for cc_pair in cc_pairs:
                 tasks_created = ccpair_pruning_generator_task_creation_helper(
-                    cc_pair, db_session, r, lock_beat
+                    cc_pair, db_session, tenant_id, r, lock_beat
                 )
                 if not tasks_created:
                     continue
@@ -71,6 +75,7 @@ def check_for_prune_task_2() -> None:
 def ccpair_pruning_generator_task_creation_helper(
     cc_pair: ConnectorCredentialPair,
     db_session: Session,
+    tenant_id: str | None,
     r: Redis,
     lock_beat: redis.lock.Lock,
 ) -> int | None:
@@ -101,13 +106,14 @@ def ccpair_pruning_generator_task_creation_helper(
     if datetime.now(timezone.utc) < next_prune:
         return None
 
-    return try_creating_prune_generator_task(cc_pair, db_session, r)
+    return try_creating_prune_generator_task(cc_pair, db_session, r, tenant_id)
 
 
 def try_creating_prune_generator_task(
     cc_pair: ConnectorCredentialPair,
     db_session: Session,
     r: Redis,
+    tenant_id: str | None,
 ) -> int | None:
     """Checks for any conditions that should block the pruning generator task from being
     created, then creates the task.
@@ -140,7 +146,9 @@ def try_creating_prune_generator_task(
     celery_app.send_task(
         "connector_pruning_generator_task",
         kwargs=dict(
-            connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id
+            connector_id=cc_pair.connector_id,
+            credential_id=cc_pair.credential_id,
+            tenant_id=tenant_id,
         ),
         queue=DanswerCeleryQueues.CONNECTOR_PRUNING,
         task_id=custom_task_id,
@@ -153,14 +161,16 @@ def try_creating_prune_generator_task(
 
 
 @shared_task(name="connector_pruning_generator_task", soft_time_limit=JOB_TIMEOUT)
-def connector_pruning_generator_task(connector_id: int, credential_id: int) -> None:
+def connector_pruning_generator_task(
+    connector_id: int, credential_id: int, tenant_id: str | None
+) -> None:
     """connector pruning task. For a cc pair, this task pulls all document IDs from the source
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
     from the most recently pulled document ID list"""
 
     r = get_redis_client()
 
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         try:
             cc_pair = get_connector_credential_pair(
                 db_session=db_session,
@@ -218,7 +228,9 @@ def redis_increment_callback(amount: int) -> None:
             task_logger.info(
                 f"RedisConnectorPruning.generate_tasks starting. cc_pair_id={cc_pair.id}"
             )
-            tasks_generated = rcp.generate_tasks(celery_app, db_session, r, None)
+            tasks_generated = rcp.generate_tasks(
+                celery_app, db_session, r, None, tenant_id
+            )
             if tasks_generated is None:
                 return None
 
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 0977fb35d29..b065122be84 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -1,7 +1,6 @@
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from sqlalchemy.orm import Session
 
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.celery_app import task_logger
@@ -11,7 +10,7 @@
 from danswer.db.document import get_document_connector_count
 from danswer.db.document import mark_document_as_synced
 from danswer.db.document_set import fetch_document_sets_for_document
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import VespaDocumentFields
@@ -26,7 +25,11 @@
     max_retries=3,
 )
 def document_by_cc_pair_cleanup_task(
-    self: Task, document_id: str, connector_id: int, credential_id: int
+    self: Task,
+    document_id: str,
+    connector_id: int,
+    credential_id: int,
+    tenant_id: str | None,
 ) -> bool:
     """A lightweight subtask used to clean up document to cc pair relationships.
     Created by connection deletion and connector pruning parent tasks."""
@@ -44,7 +47,7 @@ def document_by_cc_pair_cleanup_task(
     (6) delete all relevant entries from postgres
     """
     try:
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             action = "skip"
             chunks_affected = 0
 
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 39b6f8a91e0..e6a017b7ac7 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -38,6 +38,7 @@
 from danswer.db.document_set import fetch_document_sets_for_document
 from danswer.db.document_set import get_document_set_by_id
 from danswer.db.document_set import mark_document_set_as_synced
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.index_attempt import delete_index_attempts
 from danswer.db.models import DocumentSet
@@ -61,7 +62,7 @@
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
 )
-def check_for_vespa_sync_task() -> None:
+def check_for_vespa_sync_task(tenant_id: str | None) -> None:
     """Runs periodically to check if any document needs syncing.
     Generates sets of tasks for Celery if syncing is needed."""
 
@@ -77,8 +78,8 @@ def check_for_vespa_sync_task() -> None:
         if not lock_beat.acquire(blocking=False):
             return
 
-        with Session(get_sqlalchemy_engine()) as db_session:
-            try_generate_stale_document_sync_tasks(db_session, r, lock_beat)
+        with get_session_with_tenant(tenant_id) as db_session:
+            try_generate_stale_document_sync_tasks(db_session, r, lock_beat, tenant_id)
 
             # check if any document sets are not synced
             document_set_info = fetch_document_sets(
@@ -86,7 +87,7 @@ def check_for_vespa_sync_task() -> None:
             )
             for document_set, _ in document_set_info:
                 try_generate_document_set_sync_tasks(
-                    document_set, db_session, r, lock_beat
+                    document_set, db_session, r, lock_beat, tenant_id
                 )
 
             # check if any user groups are not synced
@@ -101,7 +102,7 @@ def check_for_vespa_sync_task() -> None:
                     )
                     for usergroup in user_groups:
                         try_generate_user_group_sync_tasks(
-                            usergroup, db_session, r, lock_beat
+                            usergroup, db_session, r, lock_beat, tenant_id
                         )
                 except ModuleNotFoundError:
                     # Always exceptions on the MIT version, which is expected
@@ -120,7 +121,7 @@ def check_for_vespa_sync_task() -> None:
 
 
 def try_generate_stale_document_sync_tasks(
-    db_session: Session, r: Redis, lock_beat: redis.lock.Lock
+    db_session: Session, r: Redis, lock_beat: redis.lock.Lock, tenant_id: str | None
 ) -> int | None:
     # the fence is up, do nothing
     if r.exists(RedisConnectorCredentialPair.get_fence_key()):
@@ -145,7 +146,9 @@ def try_generate_stale_document_sync_tasks(
     cc_pairs = get_connector_credential_pairs(db_session)
     for cc_pair in cc_pairs:
         rc = RedisConnectorCredentialPair(cc_pair.id)
-        tasks_generated = rc.generate_tasks(celery_app, db_session, r, lock_beat)
+        tasks_generated = rc.generate_tasks(
+            celery_app, db_session, r, lock_beat, tenant_id
+        )
 
         if tasks_generated is None:
             continue
@@ -169,7 +172,11 @@ def try_generate_stale_document_sync_tasks(
 
 
 def try_generate_document_set_sync_tasks(
-    document_set: DocumentSet, db_session: Session, r: Redis, lock_beat: redis.lock.Lock
+    document_set: DocumentSet,
+    db_session: Session,
+    r: Redis,
+    lock_beat: redis.lock.Lock,
+    tenant_id: str | None,
 ) -> int | None:
     lock_beat.reacquire()
 
@@ -193,7 +200,9 @@ def try_generate_document_set_sync_tasks(
     )
 
     # Add all documents that need to be updated into the queue
-    tasks_generated = rds.generate_tasks(celery_app, db_session, r, lock_beat)
+    tasks_generated = rds.generate_tasks(
+        celery_app, db_session, r, lock_beat, tenant_id
+    )
     if tasks_generated is None:
         return None
 
@@ -214,7 +223,11 @@ def try_generate_document_set_sync_tasks(
 
 
 def try_generate_user_group_sync_tasks(
-    usergroup: UserGroup, db_session: Session, r: Redis, lock_beat: redis.lock.Lock
+    usergroup: UserGroup,
+    db_session: Session,
+    r: Redis,
+    lock_beat: redis.lock.Lock,
+    tenant_id: str | None,
 ) -> int | None:
     lock_beat.reacquire()
 
@@ -236,7 +249,9 @@ def try_generate_user_group_sync_tasks(
     task_logger.info(
         f"RedisUserGroup.generate_tasks starting. usergroup_id={usergroup.id}"
     )
-    tasks_generated = rug.generate_tasks(celery_app, db_session, r, lock_beat)
+    tasks_generated = rug.generate_tasks(
+        celery_app, db_session, r, lock_beat, tenant_id
+    )
     if tasks_generated is None:
         return None
 
@@ -471,7 +486,7 @@ def monitor_ccpair_pruning_taskset(
 
 
 @shared_task(name="monitor_vespa_sync", soft_time_limit=300, bind=True)
-def monitor_vespa_sync(self: Task) -> None:
+def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
     """This is a celery beat task that monitors and finalizes metadata sync tasksets.
     It scans for fence values and then gets the counts of any associated tasksets.
     If the count is 0, that means all tasks finished and we should clean up.
@@ -516,7 +531,7 @@ def monitor_vespa_sync(self: Task) -> None:
         for key_bytes in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
             monitor_connector_deletion_taskset(key_bytes, r)
 
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             lock_beat.reacquire()
             for key_bytes in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
                 monitor_document_set_taskset(key_bytes, r, db_session)
@@ -556,11 +571,13 @@ def monitor_vespa_sync(self: Task) -> None:
     time_limit=60,
     max_retries=3,
 )
-def vespa_metadata_sync_task(self: Task, document_id: str) -> bool:
+def vespa_metadata_sync_task(
+    self: Task, document_id: str, tenant_id: str | None
+) -> bool:
     task_logger.info(f"document_id={document_id}")
 
     try:
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             curr_ind_name, sec_ind_name = get_both_index_names(db_session)
             document_index = get_default_document_index(
                 primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index b3d011a422b..d5e14675c65 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -4,6 +4,7 @@
 from datetime import timedelta
 from datetime import timezone
 
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from danswer.background.indexing.checkpointing import get_time_windows_for_index_attempt
@@ -17,7 +18,7 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.connector_credential_pair import get_last_successful_attempt_time
 from danswer.db.connector_credential_pair import update_connector_credential_pair
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import mark_attempt_failed
@@ -46,6 +47,7 @@ def _get_connector_runner(
     attempt: IndexAttempt,
     start_time: datetime,
     end_time: datetime,
+    tenant_id: str | None,
 ) -> ConnectorRunner:
     """
     NOTE: `start_time` and `end_time` are only used for poll connectors
@@ -87,8 +89,7 @@ def _get_connector_runner(
 
 
 def _run_indexing(
-    db_session: Session,
-    index_attempt: IndexAttempt,
+    db_session: Session, index_attempt: IndexAttempt, tenant_id: str | None
 ) -> None:
     """
     1. Get documents which are either new or updated from specified application
@@ -129,6 +130,7 @@ def _run_indexing(
             or (search_settings.status == IndexModelStatus.FUTURE)
         ),
         db_session=db_session,
+        tenant_id=tenant_id,
     )
 
     db_cc_pair = index_attempt.connector_credential_pair
@@ -185,6 +187,7 @@ def _run_indexing(
                 attempt=index_attempt,
                 start_time=window_start,
                 end_time=window_end,
+                tenant_id=tenant_id,
             )
 
             all_connector_doc_ids: set[str] = set()
@@ -212,7 +215,9 @@ def _run_indexing(
                 db_session.refresh(index_attempt)
                 if index_attempt.status != IndexingStatus.IN_PROGRESS:
                     # Likely due to user manually disabling it or model swap
-                    raise RuntimeError("Index Attempt was canceled")
+                    raise RuntimeError(
+                        f"Index Attempt was canceled, status is {index_attempt.status}"
+                    )
 
                 batch_description = []
                 for doc in doc_batch:
@@ -373,12 +378,21 @@ def _run_indexing(
         )
 
 
-def _prepare_index_attempt(db_session: Session, index_attempt_id: int) -> IndexAttempt:
+def _prepare_index_attempt(
+    db_session: Session, index_attempt_id: int, tenant_id: str | None
+) -> IndexAttempt:
     # make sure that the index attempt can't change in between checking the
     # status and marking it as in_progress. This setting will be discarded
     # after the next commit:
     # https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#setting-isolation-for-individual-transactions
     db_session.connection(execution_options={"isolation_level": "SERIALIZABLE"})  # type: ignore
+    if tenant_id is not None:
+        # Explicitly set the search path for the given tenant
+        db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
+        # Verify the search path was set correctly
+        result = db_session.execute(text("SHOW search_path"))
+        current_search_path = result.scalar()
+        logger.info(f"Current search path set to: {current_search_path}")
 
     attempt = get_index_attempt(
         db_session=db_session,
@@ -401,12 +415,11 @@ def _prepare_index_attempt(db_session: Session, index_attempt_id: int) -> IndexA
 
 
 def run_indexing_entrypoint(
-    index_attempt_id: int, connector_credential_pair_id: int, is_ee: bool = False
+    index_attempt_id: int,
+    tenant_id: str | None,
+    connector_credential_pair_id: int,
+    is_ee: bool = False,
 ) -> None:
-    """Entrypoint for indexing run when using dask distributed.
-    Wraps the actual logic in a `try` block so that we can catch any exceptions
-    and mark the attempt as failed."""
-
     try:
         if is_ee:
             global_version.set_ee()
@@ -416,26 +429,29 @@ def run_indexing_entrypoint(
         IndexAttemptSingleton.set_cc_and_index_id(
             index_attempt_id, connector_credential_pair_id
         )
-
-        with Session(get_sqlalchemy_engine()) as db_session:
-            # make sure that it is valid to run this indexing attempt + mark it
-            # as in progress
-            attempt = _prepare_index_attempt(db_session, index_attempt_id)
+        with get_session_with_tenant(tenant_id) as db_session:
+            attempt = _prepare_index_attempt(db_session, index_attempt_id, tenant_id)
 
             logger.info(
-                f"Indexing starting: "
-                f"connector='{attempt.connector_credential_pair.connector.name}' "
+                f"Indexing starting for tenant {tenant_id}: "
+                if tenant_id is not None
+                else ""
+                + f"connector='{attempt.connector_credential_pair.connector.name}' "
                 f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
                 f"credentials='{attempt.connector_credential_pair.connector_id}'"
             )
 
-            _run_indexing(db_session, attempt)
+            _run_indexing(db_session, attempt, tenant_id)
 
             logger.info(
-                f"Indexing finished: "
-                f"connector='{attempt.connector_credential_pair.connector.name}' "
+                f"Indexing finished for tenant {tenant_id}: "
+                if tenant_id is not None
+                else ""
+                + f"connector='{attempt.connector_credential_pair.connector.name}' "
                 f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
                 f"credentials='{attempt.connector_credential_pair.connector_id}'"
             )
     except Exception as e:
-        logger.exception(f"Indexing job with ID '{index_attempt_id}' failed due to {e}")
+        logger.exception(
+            f"Indexing job with ID '{index_attempt_id}' for tenant {tenant_id} failed due to {e}"
+        )
diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index 773165c5161..f7a00687c43 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -6,6 +6,8 @@
 from dask.distributed import Client
 from dask.distributed import Future
 from distributed import LocalCluster
+from sqlalchemy import text
+from sqlalchemy.exc import ProgrammingError
 from sqlalchemy.orm import Session
 
 from danswer.background.indexing.dask_utils import ResourceLogger
@@ -15,14 +17,16 @@
 from danswer.configs.app_configs import CLEANUP_INDEXING_JOBS_TIMEOUT
 from danswer.configs.app_configs import DASK_JOB_CLIENT_ENABLED
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import NUM_INDEXING_WORKERS
 from danswer.configs.app_configs import NUM_SECONDARY_INDEXING_WORKERS
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import POSTGRES_INDEXER_APP_NAME
+from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.db.connector import fetch_connectors
 from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
 from danswer.db.engine import get_db_current_time
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import SqlEngine
 from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import get_index_attempt
@@ -153,13 +157,15 @@ def _mark_run_failed(
 """Main funcs"""
 
 
-def create_indexing_jobs(existing_jobs: dict[int, Future | SimpleJob]) -> None:
+def create_indexing_jobs(
+    existing_jobs: dict[int, Future | SimpleJob], tenant_id: str | None
+) -> None:
     """Creates new indexing jobs for each connector / credential pair which is:
     1. Enabled
     2. `refresh_frequency` time has passed since the last indexing run for this pair
     3. There is not already an ongoing indexing attempt for this pair
     """
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         ongoing: set[tuple[int | None, int]] = set()
         for attempt_id in existing_jobs:
             attempt = get_index_attempt(
@@ -214,11 +220,12 @@ def create_indexing_jobs(existing_jobs: dict[int, Future | SimpleJob]) -> None:
 
 def cleanup_indexing_jobs(
     existing_jobs: dict[int, Future | SimpleJob],
+    tenant_id: str | None,
     timeout_hours: int = CLEANUP_INDEXING_JOBS_TIMEOUT,
 ) -> dict[int, Future | SimpleJob]:
     existing_jobs_copy = existing_jobs.copy()
     # clean up completed jobs
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         for attempt_id, job in existing_jobs.items():
             index_attempt = get_index_attempt(
                 db_session=db_session, index_attempt_id=attempt_id
@@ -256,38 +263,41 @@ def cleanup_indexing_jobs(
                 )
 
         # clean up in-progress jobs that were never completed
-        connectors = fetch_connectors(db_session)
-        for connector in connectors:
-            in_progress_indexing_attempts = get_inprogress_index_attempts(
-                connector.id, db_session
-            )
-            for index_attempt in in_progress_indexing_attempts:
-                if index_attempt.id in existing_jobs:
-                    # If index attempt is canceled, stop the run
-                    if index_attempt.status == IndexingStatus.FAILED:
-                        existing_jobs[index_attempt.id].cancel()
-                    # check to see if the job has been updated in last `timeout_hours` hours, if not
-                    # assume it to frozen in some bad state and just mark it as failed. Note: this relies
-                    # on the fact that the `time_updated` field is constantly updated every
-                    # batch of documents indexed
-                    current_db_time = get_db_current_time(db_session=db_session)
-                    time_since_update = current_db_time - index_attempt.time_updated
-                    if time_since_update.total_seconds() > 60 * 60 * timeout_hours:
-                        existing_jobs[index_attempt.id].cancel()
+        try:
+            connectors = fetch_connectors(db_session)
+            for connector in connectors:
+                in_progress_indexing_attempts = get_inprogress_index_attempts(
+                    connector.id, db_session
+                )
+
+                for index_attempt in in_progress_indexing_attempts:
+                    if index_attempt.id in existing_jobs:
+                        # If index attempt is canceled, stop the run
+                        if index_attempt.status == IndexingStatus.FAILED:
+                            existing_jobs[index_attempt.id].cancel()
+                        # check to see if the job has been updated in last `timeout_hours` hours, if not
+                        # assume it to frozen in some bad state and just mark it as failed. Note: this relies
+                        # on the fact that the `time_updated` field is constantly updated every
+                        # batch of documents indexed
+                        current_db_time = get_db_current_time(db_session=db_session)
+                        time_since_update = current_db_time - index_attempt.time_updated
+                        if time_since_update.total_seconds() > 60 * 60 * timeout_hours:
+                            existing_jobs[index_attempt.id].cancel()
+                            _mark_run_failed(
+                                db_session=db_session,
+                                index_attempt=index_attempt,
+                                failure_reason="Indexing run frozen - no updates in the last three hours. "
+                                "The run will be re-attempted at next scheduled indexing time.",
+                            )
+                    else:
+                        # If job isn't known, simply mark it as failed
                         _mark_run_failed(
                             db_session=db_session,
                             index_attempt=index_attempt,
-                            failure_reason="Indexing run frozen - no updates in the last three hours. "
-                            "The run will be re-attempted at next scheduled indexing time.",
+                            failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
                         )
-                else:
-                    # If job isn't known, simply mark it as failed
-                    _mark_run_failed(
-                        db_session=db_session,
-                        index_attempt=index_attempt,
-                        failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
-                    )
-
+        except ProgrammingError:
+            logger.debug(f"No Connector Table exists for: {tenant_id}")
     return existing_jobs_copy
 
 
@@ -295,13 +305,15 @@ def kickoff_indexing_jobs(
     existing_jobs: dict[int, Future | SimpleJob],
     client: Client | SimpleJobClient,
     secondary_client: Client | SimpleJobClient,
+    tenant_id: str | None,
 ) -> dict[int, Future | SimpleJob]:
     existing_jobs_copy = existing_jobs.copy()
-    engine = get_sqlalchemy_engine()
+
+    current_session = get_session_with_tenant(tenant_id)
 
     # Don't include jobs waiting in the Dask queue that just haven't started running
     # Also (rarely) don't include for jobs that started but haven't updated the indexing tables yet
-    with Session(engine) as db_session:
+    with current_session as db_session:
         # get_not_started_index_attempts orders its returned results from oldest to newest
         # we must process attempts in a FIFO manner to prevent connector starvation
         new_indexing_attempts = [
@@ -332,7 +344,7 @@ def kickoff_indexing_jobs(
             logger.warning(
                 f"Skipping index attempt as Connector has been deleted: {attempt}"
             )
-            with Session(engine) as db_session:
+            with current_session as db_session:
                 mark_attempt_failed(
                     attempt, db_session, failure_reason="Connector is null"
                 )
@@ -341,7 +353,7 @@ def kickoff_indexing_jobs(
             logger.warning(
                 f"Skipping index attempt as Credential has been deleted: {attempt}"
             )
-            with Session(engine) as db_session:
+            with current_session as db_session:
                 mark_attempt_failed(
                     attempt, db_session, failure_reason="Credential is null"
                 )
@@ -352,6 +364,7 @@ def kickoff_indexing_jobs(
                 run = client.submit(
                     run_indexing_entrypoint,
                     attempt.id,
+                    tenant_id,
                     attempt.connector_credential_pair_id,
                     global_version.is_ee_version(),
                     pure=False,
@@ -363,6 +376,7 @@ def kickoff_indexing_jobs(
                 run = secondary_client.submit(
                     run_indexing_entrypoint,
                     attempt.id,
+                    tenant_id,
                     attempt.connector_credential_pair_id,
                     global_version.is_ee_version(),
                     pure=False,
@@ -398,42 +412,40 @@ def kickoff_indexing_jobs(
     return existing_jobs_copy
 
 
+def get_all_tenant_ids() -> list[str] | list[None]:
+    if not MULTI_TENANT:
+        return [None]
+    with get_session_with_tenant(tenant_id="public") as session:
+        result = session.execute(
+            text(
+                """
+            SELECT schema_name
+            FROM information_schema.schemata
+            WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
+            )
+        )
+        tenant_ids = [row[0] for row in result]
+
+    valid_tenants = [
+        tenant
+        for tenant in tenant_ids
+        if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
+    ]
+
+    return valid_tenants
+
+
 def update_loop(
     delay: int = 10,
     num_workers: int = NUM_INDEXING_WORKERS,
     num_secondary_workers: int = NUM_SECONDARY_INDEXING_WORKERS,
 ) -> None:
-    engine = get_sqlalchemy_engine()
-    with Session(engine) as db_session:
-        check_index_swap(db_session=db_session)
-        search_settings = get_current_search_settings(db_session)
-
-        # So that the first time users aren't surprised by really slow speed of first
-        # batch of documents indexed
-
-        if search_settings.provider_type is None:
-            logger.notice("Running a first inference to warm up embedding model")
-            embedding_model = EmbeddingModel.from_db_model(
-                search_settings=search_settings,
-                server_host=INDEXING_MODEL_SERVER_HOST,
-                server_port=MODEL_SERVER_PORT,
-            )
-
-            warm_up_bi_encoder(
-                embedding_model=embedding_model,
-            )
-            logger.notice("First inference complete.")
-
     client_primary: Client | SimpleJobClient
     client_secondary: Client | SimpleJobClient
     if DASK_JOB_CLIENT_ENABLED:
         cluster_primary = LocalCluster(
             n_workers=num_workers,
             threads_per_worker=1,
-            # there are warning about high memory usage + "Event loop unresponsive"
-            # which are not relevant to us since our workers are expected to use a
-            # lot of memory + involve CPU intensive tasks that will not relinquish
-            # the event loop
             silence_logs=logging.ERROR,
         )
         cluster_secondary = LocalCluster(
@@ -449,7 +461,7 @@ def update_loop(
         client_primary = SimpleJobClient(n_workers=num_workers)
         client_secondary = SimpleJobClient(n_workers=num_secondary_workers)
 
-    existing_jobs: dict[int, Future | SimpleJob] = {}
+    existing_jobs: dict[str | None, dict[int, Future | SimpleJob]] = {}
 
     logger.notice("Startup complete. Waiting for indexing jobs...")
     while True:
@@ -458,24 +470,58 @@ def update_loop(
         logger.debug(f"Running update, current UTC time: {start_time_utc}")
 
         if existing_jobs:
-            # TODO: make this debug level once the "no jobs are being scheduled" issue is resolved
             logger.debug(
                 "Found existing indexing jobs: "
-                f"{[(attempt_id, job.status) for attempt_id, job in existing_jobs.items()]}"
+                f"{[(tenant_id, list(jobs.keys())) for tenant_id, jobs in existing_jobs.items()]}"
             )
 
         try:
-            with Session(get_sqlalchemy_engine()) as db_session:
-                check_index_swap(db_session)
-            existing_jobs = cleanup_indexing_jobs(existing_jobs=existing_jobs)
-            create_indexing_jobs(existing_jobs=existing_jobs)
-            existing_jobs = kickoff_indexing_jobs(
-                existing_jobs=existing_jobs,
-                client=client_primary,
-                secondary_client=client_secondary,
-            )
+            tenants = get_all_tenant_ids()
+
+            for tenant_id in tenants:
+                try:
+                    logger.debug(
+                        f"Processing {'index attempts' if tenant_id is None else f'tenant {tenant_id}'}"
+                    )
+                    with get_session_with_tenant(tenant_id) as db_session:
+                        check_index_swap(db_session=db_session)
+                        if not MULTI_TENANT:
+                            search_settings = get_current_search_settings(db_session)
+                            if search_settings.provider_type is None:
+                                logger.notice(
+                                    "Running a first inference to warm up embedding model"
+                                )
+                                embedding_model = EmbeddingModel.from_db_model(
+                                    search_settings=search_settings,
+                                    server_host=INDEXING_MODEL_SERVER_HOST,
+                                    server_port=MODEL_SERVER_PORT,
+                                )
+                                warm_up_bi_encoder(embedding_model=embedding_model)
+                                logger.notice("First inference complete.")
+
+                    tenant_jobs = existing_jobs.get(tenant_id, {})
+
+                    tenant_jobs = cleanup_indexing_jobs(
+                        existing_jobs=tenant_jobs, tenant_id=tenant_id
+                    )
+                    create_indexing_jobs(existing_jobs=tenant_jobs, tenant_id=tenant_id)
+                    tenant_jobs = kickoff_indexing_jobs(
+                        existing_jobs=tenant_jobs,
+                        client=client_primary,
+                        secondary_client=client_secondary,
+                        tenant_id=tenant_id,
+                    )
+
+                    existing_jobs[tenant_id] = tenant_jobs
+
+                except Exception as e:
+                    logger.exception(
+                        f"Failed to process tenant {tenant_id or 'default'}: {e}"
+                    )
+
         except Exception as e:
             logger.exception(f"Failed to run update due to {e}")
+
         sleep_time = delay - (time.time() - start)
         if sleep_time > 0:
             time.sleep(sleep_time)
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index eaa231e88b7..04925262196 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -429,3 +429,5 @@
 
 DATA_PLANE_SECRET = os.environ.get("DATA_PLANE_SECRET", "")
 EXPECTED_API_KEY = os.environ.get("EXPECTED_API_KEY", "")
+
+ENABLE_EMAIL_INVITES = os.environ.get("ENABLE_EMAIL_INVITES", "").lower() == "true"
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 4c43dfcf634..e4aeb88c279 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -31,6 +31,9 @@
     "You can still use Danswer as a search engine."
 )
 
+# Prefix used for all tenant ids
+TENANT_ID_PREFIX = "tenant_"
+
 # Postgres connection constants for application_name
 POSTGRES_WEB_APP_NAME = "web"
 POSTGRES_INDEXER_APP_NAME = "indexer"
diff --git a/backend/danswer/db/auth.py b/backend/danswer/db/auth.py
index 6d150b106cb..dc3f5a837bd 100644
--- a/backend/danswer/db/auth.py
+++ b/backend/danswer/db/auth.py
@@ -13,7 +13,7 @@
 
 from danswer.auth.schemas import UserRole
 from danswer.db.engine import get_async_session
-from danswer.db.engine import get_sqlalchemy_async_engine
+from danswer.db.engine import get_async_session_with_tenant
 from danswer.db.models import AccessToken
 from danswer.db.models import OAuthAccount
 from danswer.db.models import User
@@ -34,7 +34,7 @@ def get_default_admin_user_emails() -> list[str]:
 
 
 async def get_user_count() -> int:
-    async with AsyncSession(get_sqlalchemy_async_engine()) as asession:
+    async with get_async_session_with_tenant() as asession:
         stmt = select(func.count(User.id))
         result = await asession.execute(stmt)
         user_count = result.scalar()
diff --git a/backend/danswer/db/connector_credential_pair.py b/backend/danswer/db/connector_credential_pair.py
index f9d79df96ae..b3e1de7647a 100644
--- a/backend/danswer/db/connector_credential_pair.py
+++ b/backend/danswer/db/connector_credential_pair.py
@@ -390,6 +390,7 @@ def add_credential_to_connector(
     )
     db_session.add(association)
     db_session.flush()  # make sure the association has an id
+    db_session.refresh(association)
 
     if groups and access_type != AccessType.SYNC:
         _relate_groups_to_cc_pair__no_commit(
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index af7aad23669..a1f2335d348 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -1,16 +1,16 @@
 import contextlib
-import contextvars
 import re
 import threading
 import time
 from collections.abc import AsyncGenerator
 from collections.abc import Generator
+from contextlib import asynccontextmanager
+from contextlib import contextmanager
 from datetime import datetime
 from typing import Any
 from typing import ContextManager
 
 import jwt
-from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Request
 from sqlalchemy import event
@@ -39,7 +39,7 @@
 from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
 from danswer.utils.logger import setup_logger
-
+from shared_configs.configs import current_tenant_id
 
 logger = setup_logger()
 
@@ -230,18 +230,8 @@ def get_sqlalchemy_async_engine() -> AsyncEngine:
     return _ASYNC_ENGINE
 
 
-# Context variable to store the current tenant ID
-# This allows us to maintain tenant-specific context throughout the request lifecycle
-# The default value is set to POSTGRES_DEFAULT_SCHEMA for non-multi-tenant setups
-# This context variable works in both synchronous and asynchronous contexts
-# In async code, it's automatically carried across coroutines
-# In sync code, it's managed per thread
-current_tenant_id = contextvars.ContextVar(
-    "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
-)
-
-
-# Dependency to get the current tenant ID and set the context variable
+# Dependency to get the current tenant ID
+# If no token is present, uses the default schema for this use case
 def get_current_tenant_id(request: Request) -> str:
     """Dependency that extracts the tenant ID from the JWT token in the request and sets the context variable."""
     if not MULTI_TENANT:
@@ -251,32 +241,31 @@ def get_current_tenant_id(request: Request) -> str:
 
     token = request.cookies.get("tenant_details")
     if not token:
+        current_value = current_tenant_id.get()
         # If no token is present, use the default schema or handle accordingly
-        tenant_id = POSTGRES_DEFAULT_SCHEMA
-        current_tenant_id.set(tenant_id)
-        return tenant_id
+        return current_value
 
     try:
         payload = jwt.decode(token, SECRET_JWT_KEY, algorithms=["HS256"])
         tenant_id = payload.get("tenant_id")
         if not tenant_id:
-            raise HTTPException(
-                status_code=400, detail="Invalid token: tenant_id missing"
-            )
+            return current_tenant_id.get()
         if not is_valid_schema_name(tenant_id):
-            raise ValueError("Invalid tenant ID format")
+            raise HTTPException(status_code=400, detail="Invalid tenant ID format")
         current_tenant_id.set(tenant_id)
+
         return tenant_id
     except jwt.InvalidTokenError:
-        raise HTTPException(status_code=401, detail="Invalid token format")
-    except ValueError as e:
-        # Let the 400 error bubble up
-        raise HTTPException(status_code=400, detail=str(e))
-    except Exception:
+        return current_tenant_id.get()
+    except Exception as e:
+        logger.error(f"Unexpected error in get_current_tenant_id: {str(e)}")
         raise HTTPException(status_code=500, detail="Internal server error")
 
 
-def get_session_with_tenant(tenant_id: str | None = None) -> Session:
+@asynccontextmanager
+async def get_async_session_with_tenant(
+    tenant_id: str | None = None,
+) -> AsyncGenerator[AsyncSession, None]:
     if tenant_id is None:
         tenant_id = current_tenant_id.get()
 
@@ -284,20 +273,78 @@ def get_session_with_tenant(tenant_id: str | None = None) -> Session:
         logger.error(f"Invalid tenant ID: {tenant_id}")
         raise Exception("Invalid tenant ID")
 
-    engine = SqlEngine.get_engine()
-    session = Session(engine, expire_on_commit=False)
+    engine = get_sqlalchemy_async_engine()
+    async_session_factory = sessionmaker(
+        bind=engine, expire_on_commit=False, class_=AsyncSession
+    )  # type: ignore
 
-    @event.listens_for(session, "after_begin")
-    def set_search_path(session: Session, transaction: Any, connection: Any) -> None:
-        connection.execute(text("SET search_path TO :schema"), {"schema": tenant_id})
+    async with async_session_factory() as session:
+        try:
+            # Set the search_path to the tenant's schema
+            await session.execute(text(f'SET search_path = "{tenant_id}"'))
+        except Exception as e:
+            logger.error(f"Error setting search_path: {str(e)}")
+            # You can choose to re-raise the exception or handle it
+            # Here, we'll re-raise to prevent proceeding with an incorrect session
+            raise
+        else:
+            yield session
+
+
+@contextmanager
+def get_session_with_tenant(
+    tenant_id: str | None = None,
+) -> Generator[Session, None, None]:
+    """Generate a database session with the appropriate tenant schema set."""
+    engine = get_sqlalchemy_engine()
+    if tenant_id is None:
+        tenant_id = current_tenant_id.get()
 
-    return session
+    if not is_valid_schema_name(tenant_id):
+        raise HTTPException(status_code=400, detail="Invalid tenant ID")
+
+    # Establish a raw connection without starting a transaction
+    with engine.connect() as connection:
+        # Access the raw DBAPI connection
+        dbapi_connection = connection.connection
+
+        # Execute SET search_path outside of any transaction
+        cursor = dbapi_connection.cursor()
+        try:
+            cursor.execute(f'SET search_path TO "{tenant_id}"')
+            # Optionally verify the search_path was set correctly
+            cursor.execute("SHOW search_path")
+            cursor.fetchone()
+        finally:
+            cursor.close()
+
+        # Proceed to create a session using the connection
+        with Session(bind=connection, expire_on_commit=False) as session:
+            try:
+                yield session
+            finally:
+                # Reset search_path to default after the session is used
+                if MULTI_TENANT:
+                    cursor = dbapi_connection.cursor()
+                    try:
+                        cursor.execute('SET search_path TO "$user", public')
+                    finally:
+                        cursor.close()
+
+
+def get_session_generator_with_tenant(
+    tenant_id: str | None = None,
+) -> Generator[Session, None, None]:
+    with get_session_with_tenant(tenant_id) as session:
+        yield session
 
 
-def get_session(
-    tenant_id: str = Depends(get_current_tenant_id),
-) -> Generator[Session, None, None]:
+def get_session() -> Generator[Session, None, None]:
     """Generate a database session with the appropriate tenant schema set."""
+    tenant_id = current_tenant_id.get()
+    if tenant_id == "public" and MULTI_TENANT:
+        raise HTTPException(status_code=401, detail="User must authenticate")
+
     engine = get_sqlalchemy_engine()
     with Session(engine, expire_on_commit=False) as session:
         if MULTI_TENANT:
@@ -308,10 +355,9 @@ def get_session(
         yield session
 
 
-async def get_async_session(
-    tenant_id: str = Depends(get_current_tenant_id),
-) -> AsyncGenerator[AsyncSession, None]:
+async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
     """Generate an async database session with the appropriate tenant schema set."""
+    tenant_id = current_tenant_id.get()
     engine = get_sqlalchemy_async_engine()
     async with AsyncSession(engine, expire_on_commit=False) as async_session:
         if MULTI_TENANT:
@@ -324,7 +370,7 @@ async def get_async_session(
 
 def get_session_context_manager() -> ContextManager[Session]:
     """Context manager for database sessions."""
-    return contextlib.contextmanager(get_session)()
+    return contextlib.contextmanager(get_session_generator_with_tenant)()
 
 
 def get_session_factory() -> sessionmaker[Session]:
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index 392c7a28b2e..fc7dad7793f 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -1763,3 +1763,23 @@ class UsageReport(Base):
 
     requestor = relationship("User")
     file = relationship("PGFileStore")
+
+
+"""
+Multi-tenancy related tables
+"""
+
+
+class PublicBase(DeclarativeBase):
+    __abstract__ = True
+
+
+class UserTenantMapping(Base):
+    __tablename__ = "user_tenant_mapping"
+    __table_args__ = (
+        UniqueConstraint("email", "tenant_id", name="uq_user_tenant"),
+        {"schema": "public"},
+    )
+
+    email: Mapped[str] = mapped_column(String, nullable=False, primary_key=True)
+    tenant_id: Mapped[str] = mapped_column(String, nullable=False)
diff --git a/backend/danswer/indexing/indexing_pipeline.py b/backend/danswer/indexing/indexing_pipeline.py
index 992bce2dccf..d40bd341fdf 100644
--- a/backend/danswer/indexing/indexing_pipeline.py
+++ b/backend/danswer/indexing/indexing_pipeline.py
@@ -137,6 +137,7 @@ def index_doc_batch_with_handler(
     attempt_id: int | None,
     db_session: Session,
     ignore_time_skip: bool = False,
+    tenant_id: str | None = None,
 ) -> tuple[int, int]:
     r = (0, 0)
     try:
@@ -148,6 +149,7 @@ def index_doc_batch_with_handler(
             index_attempt_metadata=index_attempt_metadata,
             db_session=db_session,
             ignore_time_skip=ignore_time_skip,
+            tenant_id=tenant_id,
         )
     except Exception as e:
         if INDEXING_EXCEPTION_LIMIT == 0:
@@ -261,6 +263,7 @@ def index_doc_batch(
     index_attempt_metadata: IndexAttemptMetadata,
     db_session: Session,
     ignore_time_skip: bool = False,
+    tenant_id: str | None = None,
 ) -> tuple[int, int]:
     """Takes different pieces of the indexing pipeline and applies it to a batch of documents
     Note that the documents should already be batched at this point so that it does not inflate the
@@ -324,6 +327,7 @@ def index_doc_batch(
                     if chunk.source_document.id in ctx.id_to_db_doc_map
                     else DEFAULT_BOOST
                 ),
+                tenant_id=tenant_id,
             )
             for chunk in chunks_with_embeddings
         ]
@@ -373,6 +377,7 @@ def build_indexing_pipeline(
     chunker: Chunker | None = None,
     ignore_time_skip: bool = False,
     attempt_id: int | None = None,
+    tenant_id: str | None = None,
 ) -> IndexingPipelineProtocol:
     """Builds a pipeline which takes in a list (batch) of docs and indexes them."""
     search_settings = get_current_search_settings(db_session)
@@ -416,4 +421,5 @@ def build_indexing_pipeline(
         ignore_time_skip=ignore_time_skip,
         attempt_id=attempt_id,
         db_session=db_session,
+        tenant_id=tenant_id,
     )
diff --git a/backend/danswer/indexing/models.py b/backend/danswer/indexing/models.py
index c789a2b351b..39cfa2cca0c 100644
--- a/backend/danswer/indexing/models.py
+++ b/backend/danswer/indexing/models.py
@@ -75,6 +75,7 @@ class DocMetadataAwareIndexChunk(IndexChunk):
            negative -> ranked lower.
     """
 
+    tenant_id: str | None = None
     access: "DocumentAccess"
     document_sets: set[str]
     boost: int
@@ -86,6 +87,7 @@ def from_index_chunk(
         access: "DocumentAccess",
         document_sets: set[str],
         boost: int,
+        tenant_id: str | None,
     ) -> "DocMetadataAwareIndexChunk":
         index_chunk_data = index_chunk.model_dump()
         return cls(
@@ -93,6 +95,7 @@ def from_index_chunk(
             access=access,
             document_sets=document_sets,
             boost=boost,
+            tenant_id=tenant_id,
         )
 
 
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index 4306743f875..240ff355b5b 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -3,15 +3,21 @@
 from contextlib import contextmanager
 from typing import cast
 
+from fastapi import HTTPException
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import is_valid_schema_name
 from danswer.db.models import KVStore
 from danswer.key_value_store.interface import JSON_ro
 from danswer.key_value_store.interface import KeyValueStore
 from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import current_tenant_id
+
 
 logger = setup_logger()
 
@@ -28,6 +34,16 @@ def __init__(self) -> None:
     def get_session(self) -> Iterator[Session]:
         engine = get_sqlalchemy_engine()
         with Session(engine, expire_on_commit=False) as session:
+            if MULTI_TENANT:
+                tenant_id = current_tenant_id.get()
+                if tenant_id == "public":
+                    raise HTTPException(
+                        status_code=401, detail="User must authenticate"
+                    )
+                if not is_valid_schema_name(tenant_id):
+                    raise HTTPException(status_code=400, detail="Invalid tenant ID")
+                # Set the search_path to the tenant's schema
+                session.execute(text(f'SET search_path = "{tenant_id}"'))
             yield session
 
     def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index d7ac6b3c3ed..cd0c5c195a6 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -29,6 +29,7 @@
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.app_configs import LOG_ENDPOINT_LATENCY
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
@@ -157,6 +158,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
     verify_auth = fetch_versioned_implementation(
         "danswer.auth.users", "verify_auth_setting"
     )
+
     # Will throw exception if an issue is found
     verify_auth()
 
@@ -169,11 +171,13 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
     # fill up Postgres connection pools
     await warm_up_connections()
 
-    # We cache this at the beginning so there is no delay in the first telemetry
-    get_or_generate_uuid()
+    if not MULTI_TENANT:
+        # We cache this at the beginning so there is no delay in the first telemetry
+        get_or_generate_uuid()
 
-    with Session(engine) as db_session:
-        setup_danswer(db_session)
+        # If we are multi-tenant, we need to only set up initial public tables
+        with Session(engine) as db_session:
+            setup_danswer(db_session)
 
     optional_telemetry(record_type=RecordType.VERSION, data={"version": __version__})
     yield
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index ce3131d050f..ea513b5c21d 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -22,6 +22,7 @@
     update_connector_credential_pair_from_id,
 )
 from danswer.db.document import get_document_counts_for_cc_pairs
+from danswer.db.engine import current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import AccessType
 from danswer.db.enums import ConnectorCredentialPairStatus
@@ -257,7 +258,9 @@ def prune_cc_pair(
         f"credential_id={cc_pair.credential_id} "
         f"{cc_pair.connector.name} connector."
     )
-    tasks_created = try_creating_prune_generator_task(cc_pair, db_session, r)
+    tasks_created = try_creating_prune_generator_task(
+        cc_pair, db_session, r, current_tenant_id.get()
+    )
     if not tasks_created:
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
@@ -342,7 +345,7 @@ def sync_cc_pair(
 
     logger.info(f"Syncing the {cc_pair.connector.name} connector.")
     sync_external_doc_permissions_task.apply_async(
-        kwargs=dict(cc_pair_id=cc_pair_id),
+        kwargs=dict(cc_pair_id=cc_pair_id, tenant_id=current_tenant_id.get()),
     )
 
     return StatusResponse(
diff --git a/backend/danswer/server/manage/administrative.py b/backend/danswer/server/manage/administrative.py
index 9c87e60b18a..7771c1ed824 100644
--- a/backend/danswer/server/manage/administrative.py
+++ b/backend/danswer/server/manage/administrative.py
@@ -20,6 +20,7 @@
     update_connector_credential_pair_from_id,
 )
 from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed
+from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.feedback import fetch_docs_ranked_by_boost
@@ -146,6 +147,7 @@ def create_deletion_attempt_for_connector_id(
     connector_credential_pair_identifier: ConnectorCredentialPairIdentifier,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
+    tenant_id: str = Depends(get_current_tenant_id),
 ) -> None:
     connector_id = connector_credential_pair_identifier.connector_id
     credential_id = connector_credential_pair_identifier.credential_id
@@ -196,6 +198,7 @@ def create_deletion_attempt_for_connector_id(
     celery_app.send_task(
         "check_for_connector_deletion_task",
         priority=DanswerCeleryPriority.HIGH,
+        kwargs={"tenant_id": tenant_id},
     )
 
     if cc_pair.connector.source == DocumentSource.FILE:
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index 2a43542460a..0614a4beb85 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -2,17 +2,21 @@
 from datetime import datetime
 from datetime import timezone
 
+import jwt
 from email_validator import validate_email
 from fastapi import APIRouter
 from fastapi import Body
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Request
 from fastapi import status
+from psycopg2.errors import UniqueViolation
 from pydantic import BaseModel
 from sqlalchemy import Column
 from sqlalchemy import desc
 from sqlalchemy import select
 from sqlalchemy import update
+from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
 
 from danswer.auth.invited_users import get_invited_users
@@ -26,9 +30,12 @@
 from danswer.auth.users import current_user
 from danswer.auth.users import optional_user
 from danswer.configs.app_configs import AUTH_TYPE
+from danswer.configs.app_configs import ENABLE_EMAIL_INVITES
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
+from danswer.db.engine import current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.models import AccessToken
 from danswer.db.models import DocumentSet__User
@@ -48,10 +55,13 @@
 from danswer.server.models import FullUserSnapshot
 from danswer.server.models import InvitedUserSnapshot
 from danswer.server.models import MinimalUserSnapshot
+from danswer.server.utils import send_user_email_invite
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.api_key import is_api_key_email_address
 from ee.danswer.db.external_perm import delete_user__ext_group_for_user__no_commit
 from ee.danswer.db.user_group import remove_curator_status__no_commit
+from ee.danswer.server.tenants.provisioning import add_users_to_tenant
+from ee.danswer.server.tenants.provisioning import remove_users_from_tenant
 
 logger = setup_logger()
 
@@ -171,12 +181,33 @@ def bulk_invite_users(
         raise HTTPException(
             status_code=400, detail="Auth is disabled, cannot invite users"
         )
+    tenant_id = current_tenant_id.get()
 
     normalized_emails = []
     for email in emails:
         email_info = validate_email(email)  # can raise EmailNotValidError
         normalized_emails.append(email_info.normalized)  # type: ignore
+
+    if MULTI_TENANT:
+        try:
+            add_users_to_tenant(normalized_emails, tenant_id)
+        except IntegrityError as e:
+            if isinstance(e.orig, UniqueViolation):
+                raise HTTPException(
+                    status_code=400,
+                    detail="User has already been invited to a Danswer organization",
+                )
+            raise
+
     all_emails = list(set(normalized_emails) | set(get_invited_users()))
+
+    if MULTI_TENANT and ENABLE_EMAIL_INVITES:
+        try:
+            for email in all_emails:
+                send_user_email_invite(email, current_user)
+        except Exception as e:
+            logger.error(f"Error sending email invite to invited users: {e}")
+
     return write_invited_users(all_emails)
 
 
@@ -187,6 +218,10 @@ def remove_invited_user(
 ) -> int:
     user_emails = get_invited_users()
     remaining_users = [user for user in user_emails if user != user_email.user_email]
+
+    tenant_id = current_tenant_id.get()
+    remove_users_from_tenant([user_email.user_email], tenant_id)
+
     return write_invited_users(remaining_users)
 
 
@@ -330,6 +365,35 @@ async def get_user_role(user: User = Depends(current_user)) -> UserRoleResponse:
     return UserRoleResponse(role=user.role)
 
 
+def get_current_token_expiration_jwt(
+    user: User | None, request: Request
+) -> datetime | None:
+    if user is None:
+        return None
+
+    try:
+        # Get the JWT from the cookie
+        jwt_token = request.cookies.get("fastapiusersauth")
+        if not jwt_token:
+            logger.error("No JWT token found in cookies")
+            return None
+
+        # Decode the JWT
+        decoded_token = jwt.decode(jwt_token, options={"verify_signature": False})
+
+        # Get the 'exp' (expiration) claim from the token
+        exp = decoded_token.get("exp")
+        if exp:
+            return datetime.fromtimestamp(exp)
+        else:
+            logger.error("No 'exp' claim found in JWT")
+            return None
+
+    except Exception as e:
+        logger.error(f"Error decoding JWT: {e}")
+        return None
+
+
 def get_current_token_creation(
     user: User | None, db_session: Session
 ) -> datetime | None:
@@ -357,6 +421,7 @@ def get_current_token_creation(
 
 @router.get("/me")
 def verify_user_logged_in(
+    request: Request,
     user: User | None = Depends(optional_user),
     db_session: Session = Depends(get_session),
 ) -> UserInfo:
@@ -380,7 +445,9 @@ def verify_user_logged_in(
             detail="Access denied. User's OIDC token has expired.",
         )
 
-    token_created_at = get_current_token_creation(user, db_session)
+    token_created_at = (
+        None if MULTI_TENANT else get_current_token_creation(user, db_session)
+    )
     user_info = UserInfo.from_model(
         user,
         current_token_created_at=token_created_at,
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index 91efe6cb874..49603fa3971 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -73,6 +73,7 @@
 from danswer.server.query_and_chat.token_limit import check_token_rate_limits
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 router = APIRouter(prefix="/chat")
diff --git a/backend/danswer/server/utils.py b/backend/danswer/server/utils.py
index 53ed5b426ba..70404537f70 100644
--- a/backend/danswer/server/utils.py
+++ b/backend/danswer/server/utils.py
@@ -1,7 +1,17 @@
 import json
+import smtplib
 from datetime import datetime
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
 from typing import Any
 
+from danswer.configs.app_configs import SMTP_PASS
+from danswer.configs.app_configs import SMTP_PORT
+from danswer.configs.app_configs import SMTP_SERVER
+from danswer.configs.app_configs import SMTP_USER
+from danswer.configs.app_configs import WEB_DOMAIN
+from danswer.db.models import User
+
 
 class DateTimeEncoder(json.JSONEncoder):
     """Custom JSON encoder that converts datetime objects to ISO format strings."""
@@ -43,3 +53,28 @@ def mask_credential_dict(credential_dict: dict[str, Any]) -> dict[str, str]:
 
         masked_creds[key] = mask_string(val)
     return masked_creds
+
+
+def send_user_email_invite(user_email: str, current_user: User) -> None:
+    msg = MIMEMultipart()
+    msg["Subject"] = "Invitation to Join Danswer Workspace"
+    msg["To"] = user_email
+    msg["From"] = current_user.email
+
+    email_body = f"""
+Hello,
+
+You have been invited to join a workspace on Danswer.
+
+To join the workspace, please do so at the following link:
+{WEB_DOMAIN}/auth/login
+
+Best regards,
+The Danswer Team"""
+
+    msg.attach(MIMEText(email_body, "plain"))
+
+    with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as smtp_server:
+        smtp_server.starttls()
+        smtp_server.login(SMTP_USER, SMTP_PASS)
+        smtp_server.send_message(msg)
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index 2baeda4a811..443ab501d6b 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -4,6 +4,7 @@
 
 from danswer.chat.load_yamls import load_chat_yamls
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.configs.constants import KV_SEARCH_SETTINGS
 from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
@@ -98,7 +99,8 @@ def setup_danswer(db_session: Session) -> None:
 
     # Does the user need to trigger a reindexing to bring the document index
     # into a good state, marked in the kv store
-    mark_reindex_flag(db_session)
+    if not MULTI_TENANT:
+        mark_reindex_flag(db_session)
 
     # ensure Vespa is setup correctly
     logger.notice("Verifying Document Index(s) is/are available.")
diff --git a/backend/ee/danswer/background/celery/celery_app.py b/backend/ee/danswer/background/celery/celery_app.py
index 5dd0f72009f..de57794ee5a 100644
--- a/backend/ee/danswer/background/celery/celery_app.py
+++ b/backend/ee/danswer/background/celery/celery_app.py
@@ -1,12 +1,12 @@
 from datetime import timedelta
 
-from sqlalchemy.orm import Session
-
 from danswer.background.celery.celery_app import celery_app
 from danswer.background.task_utils import build_celery_task_wrapper
+from danswer.background.update import get_all_tenant_ids
 from danswer.configs.app_configs import JOB_TIMEOUT
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.chat import delete_chat_sessions_older_than
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.server.settings.store import load_settings
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import global_version
@@ -32,6 +32,7 @@
     run_external_group_permission_sync,
 )
 from ee.danswer.server.reporting.usage_export_generation import create_new_usage_report
+from shared_configs.configs import current_tenant_id
 
 logger = setup_logger()
 
@@ -41,22 +42,26 @@
 
 @build_celery_task_wrapper(name_sync_external_doc_permissions_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
-def sync_external_doc_permissions_task(cc_pair_id: int) -> None:
-    with Session(get_sqlalchemy_engine()) as db_session:
+def sync_external_doc_permissions_task(cc_pair_id: int, tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         run_external_doc_permission_sync(db_session=db_session, cc_pair_id=cc_pair_id)
 
 
 @build_celery_task_wrapper(name_sync_external_group_permissions_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
-def sync_external_group_permissions_task(cc_pair_id: int) -> None:
-    with Session(get_sqlalchemy_engine()) as db_session:
+def sync_external_group_permissions_task(
+    cc_pair_id: int, tenant_id: str | None
+) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         run_external_group_permission_sync(db_session=db_session, cc_pair_id=cc_pair_id)
 
 
 @build_celery_task_wrapper(name_chat_ttl_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
-def perform_ttl_management_task(retention_limit_days: int) -> None:
-    with Session(get_sqlalchemy_engine()) as db_session:
+def perform_ttl_management_task(
+    retention_limit_days: int, tenant_id: str | None
+) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         delete_chat_sessions_older_than(retention_limit_days, db_session)
 
 
@@ -67,16 +72,16 @@ def perform_ttl_management_task(retention_limit_days: int) -> None:
     name="check_sync_external_doc_permissions_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_sync_external_doc_permissions_task() -> None:
+def check_sync_external_doc_permissions_task(tenant_id: str | None) -> None:
     """Runs periodically to sync external permissions"""
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         cc_pairs = get_all_auto_sync_cc_pairs(db_session)
         for cc_pair in cc_pairs:
             if should_perform_external_doc_permissions_check(
                 cc_pair=cc_pair, db_session=db_session
             ):
                 sync_external_doc_permissions_task.apply_async(
-                    kwargs=dict(cc_pair_id=cc_pair.id),
+                    kwargs=dict(cc_pair_id=cc_pair.id, tenant_id=tenant_id),
                 )
 
 
@@ -84,16 +89,16 @@ def check_sync_external_doc_permissions_task() -> None:
     name="check_sync_external_group_permissions_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_sync_external_group_permissions_task() -> None:
+def check_sync_external_group_permissions_task(tenant_id: str | None) -> None:
     """Runs periodically to sync external group permissions"""
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         cc_pairs = get_all_auto_sync_cc_pairs(db_session)
         for cc_pair in cc_pairs:
             if should_perform_external_group_permissions_check(
                 cc_pair=cc_pair, db_session=db_session
             ):
                 sync_external_group_permissions_task.apply_async(
-                    kwargs=dict(cc_pair_id=cc_pair.id),
+                    kwargs=dict(cc_pair_id=cc_pair.id, tenant_id=tenant_id),
                 )
 
 
@@ -101,25 +106,33 @@ def check_sync_external_group_permissions_task() -> None:
     name="check_ttl_management_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_ttl_management_task() -> None:
+def check_ttl_management_task(tenant_id: str | None) -> None:
     """Runs periodically to check if any ttl tasks should be run and adds them
     to the queue"""
+    token = None
+    if MULTI_TENANT and tenant_id is not None:
+        token = current_tenant_id.set(tenant_id)
+
     settings = load_settings()
     retention_limit_days = settings.maximum_chat_retention_days
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         if should_perform_chat_ttl_check(retention_limit_days, db_session):
             perform_ttl_management_task.apply_async(
-                kwargs=dict(retention_limit_days=retention_limit_days),
+                kwargs=dict(
+                    retention_limit_days=retention_limit_days, tenant_id=tenant_id
+                ),
             )
+    if token is not None:
+        current_tenant_id.reset(token)
 
 
 @celery_app.task(
     name="autogenerate_usage_report_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def autogenerate_usage_report_task() -> None:
+def autogenerate_usage_report_task(tenant_id: str | None) -> None:
     """This generates usage report under the /admin/generate-usage/report endpoint"""
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         create_new_usage_report(
             db_session=db_session,
             user_id=None,
@@ -130,22 +143,48 @@ def autogenerate_usage_report_task() -> None:
 #####
 # Celery Beat (Periodic Tasks) Settings
 #####
-celery_app.conf.beat_schedule = {
-    "sync-external-doc-permissions": {
+
+
+tenant_ids = get_all_tenant_ids()
+
+tasks_to_schedule = [
+    {
+        "name": "sync-external-doc-permissions",
         "task": "check_sync_external_doc_permissions_task",
         "schedule": timedelta(seconds=5),  # TODO: optimize this
     },
-    "sync-external-group-permissions": {
+    {
+        "name": "sync-external-group-permissions",
         "task": "check_sync_external_group_permissions_task",
         "schedule": timedelta(seconds=5),  # TODO: optimize this
     },
-    "autogenerate_usage_report": {
+    {
+        "name": "autogenerate_usage_report",
         "task": "autogenerate_usage_report_task",
         "schedule": timedelta(days=30),  # TODO: change this to config flag
     },
-    "check-ttl-management": {
+    {
+        "name": "check-ttl-management",
         "task": "check_ttl_management_task",
         "schedule": timedelta(hours=1),
     },
-    **(celery_app.conf.beat_schedule or {}),
-}
+]
+
+# Build the celery beat schedule dynamically
+beat_schedule = {}
+
+for tenant_id in tenant_ids:
+    for task in tasks_to_schedule:
+        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
+        beat_schedule[task_name] = {
+            "task": task["task"],
+            "schedule": task["schedule"],
+            "args": (tenant_id,),  # Must pass tenant_id as an argument
+        }
+
+# Include any existing beat schedules
+existing_beat_schedule = celery_app.conf.beat_schedule or {}
+beat_schedule.update(existing_beat_schedule)
+
+# Update the Celery app configuration
+celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/ee/danswer/background/task_name_builders.py b/backend/ee/danswer/background/task_name_builders.py
index c494329d366..7a8eee0cd70 100644
--- a/backend/ee/danswer/background/task_name_builders.py
+++ b/backend/ee/danswer/background/task_name_builders.py
@@ -2,9 +2,13 @@ def name_chat_ttl_task(retention_limit_days: int) -> str:
     return f"chat_ttl_{retention_limit_days}_days"
 
 
-def name_sync_external_doc_permissions_task(cc_pair_id: int) -> str:
+def name_sync_external_doc_permissions_task(
+    cc_pair_id: int, tenant_id: str | None = None
+) -> str:
     return f"sync_external_doc_permissions_task__{cc_pair_id}"
 
 
-def name_sync_external_group_permissions_task(cc_pair_id: int) -> str:
+def name_sync_external_group_permissions_task(
+    cc_pair_id: int, tenant_id: str | None = None
+) -> str:
     return f"sync_external_group_permissions_task__{cc_pair_id}"
diff --git a/backend/ee/danswer/main.py b/backend/ee/danswer/main.py
index 8422d5494ae..e6483f75ae1 100644
--- a/backend/ee/danswer/main.py
+++ b/backend/ee/danswer/main.py
@@ -4,6 +4,7 @@
 from danswer.auth.users import auth_backend
 from danswer.auth.users import fastapi_users
 from danswer.configs.app_configs import AUTH_TYPE
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import USER_AUTH_SECRET
@@ -24,6 +25,7 @@
     basic_router as enterprise_settings_router,
 )
 from ee.danswer.server.manage.standard_answer import router as standard_answer_router
+from ee.danswer.server.middleware.tenant_tracking import add_tenant_id_middleware
 from ee.danswer.server.query_and_chat.chat_backend import (
     router as chat_router,
 )
@@ -53,6 +55,9 @@ def get_application() -> FastAPI:
 
     application = get_application_base()
 
+    if MULTI_TENANT:
+        add_tenant_id_middleware(application, logger)
+
     if AUTH_TYPE == AuthType.OIDC:
         include_router_with_global_prefix_prepended(
             application,
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
new file mode 100644
index 00000000000..f564a4fc683
--- /dev/null
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -0,0 +1,60 @@
+import logging
+from collections.abc import Awaitable
+from collections.abc import Callable
+
+import jwt
+from fastapi import FastAPI
+from fastapi import HTTPException
+from fastapi import Request
+from fastapi import Response
+
+from danswer.configs.app_configs import MULTI_TENANT
+from danswer.configs.app_configs import SECRET_JWT_KEY
+from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
+from danswer.db.engine import is_valid_schema_name
+from shared_configs.configs import current_tenant_id
+
+
+def add_tenant_id_middleware(app: FastAPI, logger: logging.LoggerAdapter) -> None:
+    @app.middleware("http")
+    async def set_tenant_id(
+        request: Request, call_next: Callable[[Request], Awaitable[Response]]
+    ) -> Response:
+        try:
+            logger.info(f"Request route: {request.url.path}")
+
+            if not MULTI_TENANT:
+                tenant_id = POSTGRES_DEFAULT_SCHEMA
+            else:
+                token = request.cookies.get("tenant_details")
+                if token:
+                    try:
+                        payload = jwt.decode(
+                            token, SECRET_JWT_KEY, algorithms=["HS256"]
+                        )
+                        tenant_id = payload.get("tenant_id", POSTGRES_DEFAULT_SCHEMA)
+                        if not is_valid_schema_name(tenant_id):
+                            raise HTTPException(
+                                status_code=400, detail="Invalid tenant ID format"
+                            )
+                    except jwt.InvalidTokenError:
+                        tenant_id = POSTGRES_DEFAULT_SCHEMA
+                    except Exception as e:
+                        logger.error(
+                            f"Unexpected error in set_tenant_id_middleware: {str(e)}"
+                        )
+                        raise HTTPException(
+                            status_code=500, detail="Internal server error"
+                        )
+                else:
+                    tenant_id = POSTGRES_DEFAULT_SCHEMA
+
+            current_tenant_id.set(tenant_id)
+            logger.info(f"Middleware set current_tenant_id to: {tenant_id}")
+
+            response = await call_next(request)
+            return response
+
+        except Exception as e:
+            logger.error(f"Error in tenant ID middleware: {str(e)}")
+            raise
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index ec96351856b..b522112ae06 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -8,8 +8,11 @@
 from danswer.setup import setup_danswer
 from danswer.utils.logger import setup_logger
 from ee.danswer.server.tenants.models import CreateTenantRequest
+from ee.danswer.server.tenants.provisioning import add_users_to_tenant
 from ee.danswer.server.tenants.provisioning import ensure_schema_exists
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
+from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
+from shared_configs.configs import current_tenant_id
 
 logger = setup_logger()
 router = APIRouter(prefix="/tenants")
@@ -19,9 +22,15 @@
 def create_tenant(
     create_tenant_request: CreateTenantRequest, _: None = Depends(control_plane_dep)
 ) -> dict[str, str]:
-    try:
-        tenant_id = create_tenant_request.tenant_id
+    tenant_id = create_tenant_request.tenant_id
+    email = create_tenant_request.initial_admin_email
+    token = None
+    if user_owns_a_tenant(email):
+        raise HTTPException(
+            status_code=409, detail="User already belongs to an organization"
+        )
 
+    try:
         if not MULTI_TENANT:
             raise HTTPException(status_code=403, detail="Multi-tenancy is not enabled")
 
@@ -31,10 +40,14 @@ def create_tenant(
             logger.info(f"Schema already exists for tenant {tenant_id}")
 
         run_alembic_migrations(tenant_id)
+        token = current_tenant_id.set(tenant_id)
+        print("getting session", tenant_id)
         with get_session_with_tenant(tenant_id) as db_session:
             setup_danswer(db_session)
 
         logger.info(f"Tenant {tenant_id} created successfully")
+        add_users_to_tenant([email], tenant_id)
+
         return {
             "status": "success",
             "message": f"Tenant {tenant_id} created successfully",
@@ -44,3 +57,6 @@ def create_tenant(
         raise HTTPException(
             status_code=500, detail=f"Failed to create tenant: {str(e)}"
         )
+    finally:
+        if token is not None:
+            current_tenant_id.reset(token)
diff --git a/backend/ee/danswer/server/tenants/provisioning.py b/backend/ee/danswer/server/tenants/provisioning.py
index 62436c92e17..77d27e7a551 100644
--- a/backend/ee/danswer/server/tenants/provisioning.py
+++ b/backend/ee/danswer/server/tenants/provisioning.py
@@ -8,7 +8,9 @@
 from alembic import command
 from alembic.config import Config
 from danswer.db.engine import build_connection_string
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.models import UserTenantMapping
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -61,3 +63,48 @@ def ensure_schema_exists(tenant_id: str) -> bool:
                 db_session.execute(stmt)
                 return True
             return False
+
+
+# For now, we're implementing a primitive mapping between users and tenants.
+# This function is only used to determine a user's relationship to a tenant upon creation (implying ownership).
+def user_owns_a_tenant(email: str) -> bool:
+    with get_session_with_tenant("public") as db_session:
+        result = (
+            db_session.query(UserTenantMapping)
+            .filter(UserTenantMapping.email == email)
+            .first()
+        )
+        return result is not None
+
+
+def add_users_to_tenant(emails: list[str], tenant_id: str) -> None:
+    with get_session_with_tenant("public") as db_session:
+        try:
+            for email in emails:
+                db_session.add(UserTenantMapping(email=email, tenant_id=tenant_id))
+        except Exception as e:
+            logger.exception(f"Failed to add users to tenant {tenant_id}: {str(e)}")
+        db_session.commit()
+
+
+def remove_users_from_tenant(emails: list[str], tenant_id: str) -> None:
+    with get_session_with_tenant("public") as db_session:
+        try:
+            mappings_to_delete = (
+                db_session.query(UserTenantMapping)
+                .filter(
+                    UserTenantMapping.email.in_(emails),
+                    UserTenantMapping.tenant_id == tenant_id,
+                )
+                .all()
+            )
+
+            for mapping in mappings_to_delete:
+                db_session.delete(mapping)
+
+            db_session.commit()
+        except Exception as e:
+            logger.exception(
+                f"Failed to remove users from tenant {tenant_id}: {str(e)}"
+            )
+            db_session.rollback()
diff --git a/backend/scripts/query_time_check/seed_dummy_docs.py b/backend/scripts/query_time_check/seed_dummy_docs.py
index 96b6b4a0133..70cb2a4a6a8 100644
--- a/backend/scripts/query_time_check/seed_dummy_docs.py
+++ b/backend/scripts/query_time_check/seed_dummy_docs.py
@@ -94,6 +94,7 @@ def generate_dummy_chunk(
         ),
         document_sets={document_set for document_set in document_set_names},
         boost=random.randint(-1, 1),
+        tenant_id="public",
     )
 
 
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index 50233ab6878..898c01509aa 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -1,3 +1,4 @@
+import contextvars
 import os
 from typing import List
 from urllib.parse import urlparse
@@ -109,3 +110,5 @@ def validate_cors_origin(origin: str) -> None:
 else:
     # If the environment variable is empty, allow all origins
     CORS_ALLOWED_ORIGIN = ["*"]
+
+current_tenant_id = contextvars.ContextVar("current_tenant_id", default="public")
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 4d0eff8612d..5298859d13d 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -29,6 +29,7 @@ services:
       - SMTP_PORT=${SMTP_PORT:-587} # For sending verification emails, if unspecified then defaults to '587'
       - SMTP_USER=${SMTP_USER:-}
       - SMTP_PASS=${SMTP_PASS:-}
+      - ENABLE_EMAIL_INVITES=${ENABLE_EMAIL_INVITES:-} # If enabled, will send users (using SMTP settings) an email to join the workspace
       - EMAIL_FROM=${EMAIL_FROM:-}
       - OAUTH_CLIENT_ID=${OAUTH_CLIENT_ID:-}
       - OAUTH_CLIENT_SECRET=${OAUTH_CLIENT_SECRET:-}
diff --git a/web/src/app/auth/create-account/page.tsx b/web/src/app/auth/create-account/page.tsx
new file mode 100644
index 00000000000..b5c340afb6a
--- /dev/null
+++ b/web/src/app/auth/create-account/page.tsx
@@ -0,0 +1,45 @@
+"use client";
+
+import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
+import { REGISTRATION_URL } from "@/lib/constants";
+import { Button } from "@tremor/react";
+import Link from "next/link";
+import { FiLogIn } from "react-icons/fi";
+
+const Page = () => {
+  return (
+    <AuthFlowContainer>
+      <div className="flex flex-col space-y-6">
+        <h2 className="text-2xl font-bold text-text-900 text-center">
+          Account Not Found
+        </h2>
+        <p className="text-text-700 max-w-md text-center">
+          We couldn&apos;t find your account in our records. To access Danswer,
+          you need to either:
+        </p>
+        <ul className="list-disc text-left text-text-600 w-full pl-6 mx-auto">
+          <li>Be invited to an existing Danswer organization</li>
+          <li>Create a new Danswer organization</li>
+        </ul>
+        <div className="flex justify-center">
+          <Link
+            href={`${REGISTRATION_URL}/register`}
+            className="w-full max-w-xs"
+          >
+            <Button size="lg" icon={FiLogIn} color="indigo" className="w-full">
+              Create New Organization
+            </Button>
+          </Link>
+        </div>
+        <p className="text-sm text-text-500 text-center">
+          Have an account with a different email?{" "}
+          <Link href="/auth/login" className="text-indigo-600 hover:underline">
+            Sign in
+          </Link>
+        </p>
+      </div>
+    </AuthFlowContainer>
+  );
+};
+
+export default Page;
diff --git a/web/src/app/auth/error/page.tsx b/web/src/app/auth/error/page.tsx
index 4f288cd205f..c75e620068e 100644
--- a/web/src/app/auth/error/page.tsx
+++ b/web/src/app/auth/error/page.tsx
@@ -1,21 +1,49 @@
 "use client";
 
+import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 import { Button } from "@tremor/react";
 import Link from "next/link";
 import { FiLogIn } from "react-icons/fi";
 
 const Page = () => {
   return (
-    <div className="flex flex-col items-center justify-center h-screen">
-      <div className="font-bold">
-        Unable to login, please try again and/or contact an administrator.
+    <AuthFlowContainer>
+      <div className="flex flex-col space-y-6 max-w-md mx-auto">
+        <h2 className="text-2xl font-bold text-text-900 text-center">
+          Authentication Error
+        </h2>
+        <p className="text-text-700 text-center">
+          We encountered an issue while attempting to log you in.
+        </p>
+        <div className="bg-red-50 border border-red-200 rounded-lg p-4 shadow-sm">
+          <h3 className="text-red-800 font-semibold mb-2">Possible Issues:</h3>
+          <ul className="space-y-2">
+            <li className="flex items-center text-red-700">
+              <div className="w-2 h-2 bg-red-500 rounded-full mr-2"></div>
+              Incorrect or expired login credentials
+            </li>
+            <li className="flex items-center text-red-700">
+              <div className="w-2 h-2 bg-red-500 rounded-full mr-2"></div>
+              Temporary authentication system disruption
+            </li>
+            <li className="flex items-center text-red-700">
+              <div className="w-2 h-2 bg-red-500 rounded-full mr-2"></div>
+              Account access restrictions or permissions
+            </li>
+          </ul>
+        </div>
+
+        <Link href="/auth/login" className="w-full">
+          <Button size="lg" icon={FiLogIn} color="indigo" className="w-full">
+            Return to Login Page
+          </Button>
+        </Link>
+        <p className="text-sm text-text-500 text-center">
+          We recommend trying again. If you continue to experience problems,
+          please reach out to your system administrator for assistance.
+        </p>
       </div>
-      <Link href="/auth/login" className="w-fit">
-        <Button className="mt-4" size="xs" icon={FiLogIn}>
-          Back to login
-        </Button>
-      </Link>
-    </div>
+    </AuthFlowContainer>
   );
 };
 
diff --git a/web/src/app/auth/login/LoginText.tsx b/web/src/app/auth/login/LoginText.tsx
index a875b407a65..e31aeb81321 100644
--- a/web/src/app/auth/login/LoginText.tsx
+++ b/web/src/app/auth/login/LoginText.tsx
@@ -6,11 +6,15 @@ import { SettingsContext } from "@/components/settings/SettingsProvider";
 export const LoginText = () => {
   const settings = useContext(SettingsContext);
 
-  if (!settings) {
-    throw new Error("SettingsContext is not available");
-  }
+  // if (!settings) {
+  //   throw new Error("SettingsContext is not available");
+  // }
 
   return (
-    <>Log In to {settings?.enterpriseSettings?.application_name || "Danswer"}</>
+    <>
+      Log In to{" "}
+      {(settings && settings?.enterpriseSettings?.application_name) ||
+        "Danswer"}
+    </>
   );
 };
diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index 50f1d42d9b4..9ec047d61e2 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -14,6 +14,7 @@ import Link from "next/link";
 import { Logo } from "@/components/Logo";
 import { LoginText } from "./LoginText";
 import { getSecondsUntilExpiration } from "@/lib/time";
+import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 
 const Page = async ({
   searchParams,
@@ -51,7 +52,6 @@ const Page = async ({
     if (authTypeMetadata?.requiresVerification && !currentUser.is_verified) {
       return redirect("/auth/waiting-on-verification");
     }
-
     return redirect("/");
   }
 
@@ -70,46 +70,44 @@ const Page = async ({
   }
 
   return (
-    <main>
+    <AuthFlowContainer>
       <div className="absolute top-10x w-full">
         <HealthCheckBanner />
       </div>
-      <div className="min-h-screen flex items-center justify-center py-12 px-4 sm:px-6 lg:px-8">
-        <div>
-          <Logo height={64} width={64} className="mx-auto w-fit" />
-          {authUrl && authTypeMetadata && (
-            <>
-              <h2 className="text-center text-xl text-strong font-bold mt-6">
-                <LoginText />
-              </h2>
 
-              <SignInButton
-                authorizeUrl={authUrl}
-                authType={authTypeMetadata?.authType}
-              />
-            </>
-          )}
-          {authTypeMetadata?.authType === "basic" && (
-            <Card className="mt-4 w-96">
-              <div className="flex">
-                <Title className="mb-2 mx-auto font-bold">
-                  <LoginText />
-                </Title>
-              </div>
-              <EmailPasswordForm />
-              <div className="flex">
-                <Text className="mt-4 mx-auto">
-                  Don&apos;t have an account?{" "}
-                  <Link href="/auth/signup" className="text-link font-medium">
-                    Create an account
-                  </Link>
-                </Text>
-              </div>
-            </Card>
-          )}
-        </div>
+      <div>
+        {authUrl && authTypeMetadata && (
+          <>
+            <h2 className="text-center text-xl text-strong font-bold">
+              <LoginText />
+            </h2>
+
+            <SignInButton
+              authorizeUrl={authUrl}
+              authType={authTypeMetadata?.authType}
+            />
+          </>
+        )}
+        {authTypeMetadata?.authType === "basic" && (
+          <Card className="mt-4 w-96">
+            <div className="flex">
+              <Title className="mb-2 mx-auto font-bold">
+                <LoginText />
+              </Title>
+            </div>
+            <EmailPasswordForm />
+            <div className="flex">
+              <Text className="mt-4 mx-auto">
+                Don&apos;t have an account?{" "}
+                <Link href="/auth/signup" className="text-link font-medium">
+                  Create an account
+                </Link>
+              </Text>
+            </div>
+          </Card>
+        )}
       </div>
-    </main>
+    </AuthFlowContainer>
   );
 };
 
diff --git a/web/src/app/auth/oauth/callback/route.ts b/web/src/app/auth/oauth/callback/route.ts
index 0b4157731a1..6e8f290a65f 100644
--- a/web/src/app/auth/oauth/callback/route.ts
+++ b/web/src/app/auth/oauth/callback/route.ts
@@ -11,6 +11,12 @@ export const GET = async (request: NextRequest) => {
   const response = await fetch(url.toString());
   const setCookieHeader = response.headers.get("set-cookie");
 
+  if (response.status === 401) {
+    return NextResponse.redirect(
+      new URL("/auth/create-account", getDomain(request))
+    );
+  }
+
   if (!setCookieHeader) {
     return NextResponse.redirect(new URL("/auth/error", getDomain(request)));
   }
diff --git a/web/src/app/auth/signup/page.tsx b/web/src/app/auth/signup/page.tsx
index 9a2631c4350..ec276a09672 100644
--- a/web/src/app/auth/signup/page.tsx
+++ b/web/src/app/auth/signup/page.tsx
@@ -10,6 +10,7 @@ import { EmailPasswordForm } from "../login/EmailPasswordForm";
 import { Card, Title, Text } from "@tremor/react";
 import Link from "next/link";
 import { Logo } from "@/components/Logo";
+import { CLOUD_ENABLED } from "@/lib/constants";
 
 const Page = async () => {
   // catch cases where the backend is completely unreachable here
@@ -25,6 +26,9 @@ const Page = async () => {
   } catch (e) {
     console.log(`Some fetch failed for the login page - ${e}`);
   }
+  if (CLOUD_ENABLED) {
+    return redirect("/auth/login");
+  }
 
   // simply take the user to the home page if Auth is disabled
   if (authTypeMetadata?.authType === "disabled") {
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index f64edd17964..f49864aac75 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -19,6 +19,8 @@ import { HeaderTitle } from "@/components/header/HeaderTitle";
 import { Logo } from "@/components/Logo";
 import { UserProvider } from "@/components/user/UserProvider";
 import { ProviderContextProvider } from "@/components/chat_search/ProviderContext";
+import { redirect } from "next/navigation";
+import { headers } from "next/headers";
 
 const inter = Inter({
   subsets: ["latin"],
@@ -56,8 +58,6 @@ export default async function RootLayout({
   const combinedSettings = await fetchSettingsSS();
 
   if (!combinedSettings) {
-    // Just display a simple full page error if fetching fails.
-
     return (
       <html lang="en" className={`${inter.variable} font-sans`}>
         <Head>
diff --git a/web/src/components/auth/AuthFlowContainer.tsx b/web/src/components/auth/AuthFlowContainer.tsx
new file mode 100644
index 00000000000..35fd3d6f3c3
--- /dev/null
+++ b/web/src/components/auth/AuthFlowContainer.tsx
@@ -0,0 +1,16 @@
+import { Logo } from "../Logo";
+
+export default function AuthFlowContainer({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <div className="flex flex-col items-center justify-center min-h-screen bg-background">
+      <div className="w-full max-w-md p-8 gap-y-4 bg-white flex items-center flex-col rounded-xl shadow-lg border border-bacgkround-100">
+        <Logo width={70} height={70} />
+        {children}
+      </div>
+    </div>
+  );
+}
diff --git a/web/src/components/settings/lib.ts b/web/src/components/settings/lib.ts
index f4e14699f1a..1c1ec9249f1 100644
--- a/web/src/components/settings/lib.ts
+++ b/web/src/components/settings/lib.ts
@@ -40,7 +40,7 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
 
     let settings: Settings;
     if (!results[0].ok) {
-      if (results[0].status === 403) {
+      if (results[0].status === 403 || results[0].status === 401) {
         settings = {
           gpu_enabled: false,
           chat_page_enabled: true,
@@ -62,7 +62,7 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
     let enterpriseSettings: EnterpriseSettings | null = null;
     if (tasks.length > 1) {
       if (!results[1].ok) {
-        if (results[1].status !== 403) {
+        if (results[1].status !== 403 && results[1].status !== 401) {
           throw new Error(
             `fetchEnterpriseSettingsSS failed: status=${results[1].status} body=${await results[1].text()}`
           );
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index 974695a8350..4fe1d616dcd 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -55,3 +55,7 @@ export const CUSTOM_ANALYTICS_ENABLED = process.env.CUSTOM_ANALYTICS_SECRET_KEY
 
 export const DISABLE_LLM_DOC_RELEVANCE =
   process.env.DISABLE_LLM_DOC_RELEVANCE?.toLowerCase() === "true";
+
+export const CLOUD_ENABLED = process.env.NEXT_PUBLIC_CLOUD_ENABLED;
+export const REGISTRATION_URL =
+  process.env.INTERNAL_URL || "http://127.0.0.1:3001";

From 85d5e6c02fe479ccd1ce97470530d9f843544a9b Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 10 Oct 2024 10:17:18 -0700
Subject: [PATCH 088/376] PDF Encrypted Case (#2764)

---
 backend/danswer/file_processing/extract_file_text.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/file_processing/extract_file_text.py b/backend/danswer/file_processing/extract_file_text.py
index 0f8c4e782c6..d8ac04547f4 100644
--- a/backend/danswer/file_processing/extract_file_text.py
+++ b/backend/danswer/file_processing/extract_file_text.py
@@ -208,8 +208,9 @@ def read_pdf_file(
                 # By user request, keep files that are unreadable just so they
                 # can be discoverable by title.
                 return "", metadata
-        else:
-            logger.warning("No Password available to to decrypt pdf")
+        elif pdf_reader.is_encrypted:
+            logger.warning("No Password available to to decrypt pdf, returning empty")
+            return "", metadata
 
         # Extract metadata from the PDF, removing leading '/' from keys if present
         # This standardizes the metadata keys for consistency

From b212b228fb483425b70194192817931149b193cb Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 10 Oct 2024 10:21:30 -0700
Subject: [PATCH 089/376] Typo Fix (#2766)

---
 backend/danswer/file_processing/extract_file_text.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/file_processing/extract_file_text.py b/backend/danswer/file_processing/extract_file_text.py
index d8ac04547f4..9effad5b4e0 100644
--- a/backend/danswer/file_processing/extract_file_text.py
+++ b/backend/danswer/file_processing/extract_file_text.py
@@ -209,7 +209,7 @@ def read_pdf_file(
                 # can be discoverable by title.
                 return "", metadata
         elif pdf_reader.is_encrypted:
-            logger.warning("No Password available to to decrypt pdf, returning empty")
+            logger.warning("No Password available to decrypt pdf, returning empty")
             return "", metadata
 
         # Extract metadata from the PDF, removing leading '/' from keys if present

From 101b010c5cf2287966e81e1821eb1787e51ebe90 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 10 Oct 2024 10:37:27 -0700
Subject: [PATCH 090/376] Improved logging and added comments (#2763)

* Improved logging and added comments

* fix exception logging

* cleanup
---
 .../confluence/doc_sync.py                    | 156 ++++++++++++------
 .../external_permissions/permission_sync.py   |   8 +-
 2 files changed, 107 insertions(+), 57 deletions(-)

diff --git a/backend/ee/danswer/external_permissions/confluence/doc_sync.py b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
index b6812adb9e7..e87dcf1d79f 100644
--- a/backend/ee/danswer/external_permissions/confluence/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
@@ -24,6 +24,57 @@
 _REQUEST_PAGINATION_LIMIT = 100
 
 
+def _extract_user_email(subjects: dict[str, Any]) -> str | None:
+    # If the subject is a user, then return the user's email
+    user = subjects.get("user", {})
+    result = user.get("results", [{}])[0]
+    return result.get("email")
+
+
+def _extract_group_name(subjects: dict[str, Any]) -> str | None:
+    # If the subject is a group, then return the group's name
+    group = subjects.get("group", {})
+    result = group.get("results", [{}])[0]
+    return result.get("name")
+
+
+def _is_public_read_permission(permission: dict[str, Any]) -> bool:
+    # If the permission is a public read permission, then return True
+    operation = permission.get("operation", {})
+    operation_value = operation.get("operation")
+    anonymous_access = permission.get("anonymousAccess", False)
+    return operation_value == "read" and anonymous_access
+
+
+def _extract_read_access_restrictions(
+    restrictions: dict[str, Any]
+) -> tuple[list[str], list[str]]:
+    """
+    WARNING: This function includes no paginated retrieval. So if a page is private
+    within the space and has over 200 users or over 200 groups with explicitly read
+    access, this function will leave out some users or groups.
+    200 is a large amount so it is unlikely, but just be aware.
+    """
+    read_access = restrictions.get("read", {})
+    read_access_restrictions = read_access.get("restrictions", {})
+
+    # Extract the users with read access
+    read_access_user = read_access_restrictions.get("user", {})
+    read_access_user_jsons = read_access_user.get("results", [])
+    read_access_user_emails = [
+        user["email"] for user in read_access_user_jsons if user.get("email")
+    ]
+
+    # Extract the groups with read access
+    read_access_group = read_access_restrictions.get("group", {})
+    read_access_group_jsons = read_access_group.get("results", [])
+    read_access_group_names = [
+        group["name"] for group in read_access_group_jsons if group.get("name")
+    ]
+
+    return read_access_user_emails, read_access_group_names
+
+
 def _get_space_permissions(
     db_session: Session,
     confluence_client: Confluence,
@@ -33,27 +84,29 @@ def _get_space_permissions(
         confluence_client.get_space_permissions
     )
 
-    space_permissions = get_space_permissions(space_id).get("permissions", [])
+    space_permissions_result = get_space_permissions(space_id)
+    logger.debug(f"space_permissions_result: {space_permissions_result}")
+
+    space_permissions = space_permissions_result.get("permissions", [])
     user_emails = set()
     # Confluence enforces that group names are unique
     group_names = set()
     is_externally_public = False
     for permission in space_permissions:
-        subs = permission.get("subjects")
-        if subs:
+        subjects = permission.get("subjects")
+        if subjects:
             # If there are subjects, then there are explicit users or groups with access
-            if email := subs.get("user", {}).get("results", [{}])[0].get("email"):
+            if email := _extract_user_email(subjects):
                 user_emails.add(email)
-            if group_name := subs.get("group", {}).get("results", [{}])[0].get("name"):
+            if group_name := _extract_group_name(subjects):
                 group_names.add(group_name)
         else:
             # If there are no subjects, then the permission is for everyone
-            if permission.get("operation", {}).get(
-                "operation"
-            ) == "read" and permission.get("anonymousAccess", False):
+            if _is_public_read_permission(permission):
                 # If the permission specifies read access for anonymous users, then
                 # the space is publicly accessible
                 is_externally_public = True
+
     batch_add_non_web_user_if_not_exists__no_commit(
         db_session=db_session, emails=list(user_emails)
     )
@@ -64,42 +117,29 @@ def _get_space_permissions(
     )
 
 
-def _get_restrictions_for_page(
+def _get_page_specific_restrictions(
     db_session: Session,
     page: dict[str, Any],
-    space_permissions: ExternalAccess,
-) -> ExternalAccess:
-    """
-    WARNING: This function includes no pagination. So if a page is private within
-    the space and has over 200 users or over 200 groups with explicitly read access,
-    this function will leave out some users or groups.
-    200 is a large amount so it is unlikely, but just be aware.
-    """
-    restrictions_json = page.get("restrictions", {})
-    read_access_dict = restrictions_json.get("read", {}).get("restrictions", {})
-
-    read_access_user_jsons = read_access_dict.get("user", {}).get("results", [])
-    read_access_group_jsons = read_access_dict.get("group", {}).get("results", [])
+) -> ExternalAccess | None:
+    user_emails, group_names = _extract_read_access_restrictions(
+        restrictions=page.get("restrictions", {})
+    )
 
-    is_space_public = read_access_user_jsons == [] and read_access_group_jsons == []
+    # If there are no restrictions found, then the page
+    # inherits the space's restrictions so return None
+    is_space_public = user_emails == [] and group_names == []
+    if is_space_public:
+        return None
 
-    if not is_space_public:
-        read_access_user_emails = [
-            user["email"] for user in read_access_user_jsons if user.get("email")
-        ]
-        read_access_groups = [group["name"] for group in read_access_group_jsons]
-        batch_add_non_web_user_if_not_exists__no_commit(
-            db_session=db_session, emails=list(read_access_user_emails)
-        )
-        external_access = ExternalAccess(
-            external_user_emails=set(read_access_user_emails),
-            external_user_group_ids=set(read_access_groups),
-            is_public=False,
-        )
-    else:
-        external_access = space_permissions
-
-    return external_access
+    batch_add_non_web_user_if_not_exists__no_commit(
+        db_session=db_session, emails=list(user_emails)
+    )
+    return ExternalAccess(
+        external_user_emails=set(user_emails),
+        external_user_group_ids=set(group_names),
+        # there is no way for a page to be individually public if the space isn't public
+        is_public=False,
+    )
 
 
 def _fetch_attachment_document_ids_for_page_paginated(
@@ -182,14 +222,13 @@ def _fetch_all_page_restrictions_for_space(
     db_session: Session,
     confluence_client: Confluence,
     space_id: str,
-    space_permissions: ExternalAccess,
-) -> dict[str, ExternalAccess]:
+) -> dict[str, ExternalAccess | None]:
     all_pages = _fetch_all_pages_paginated(
         confluence_client=confluence_client,
         space_id=space_id,
     )
 
-    document_restrictions: dict[str, ExternalAccess] = {}
+    document_restrictions: dict[str, ExternalAccess | None] = {}
     for page in all_pages:
         """
         This assigns the same permissions to all attachments of a page and
@@ -199,24 +238,32 @@ def _fetch_all_page_restrictions_for_space(
         may not be their own standalone documents. This is likely fine as we just upsert a
         document with just permissions.
         """
-        attachment_document_ids = [
+        document_ids = []
+
+        # Add the page's document id
+        document_ids.append(
             build_confluence_document_id(
                 base_url=confluence_client.url,
                 content_url=page["_links"]["webui"],
             )
-        ]
-        attachment_document_ids.extend(
+        )
+
+        # Add the page's attachments document ids
+        document_ids.extend(
             _fetch_attachment_document_ids_for_page_paginated(
                 confluence_client=confluence_client, page=page
             )
         )
-        page_permissions = _get_restrictions_for_page(
+
+        # Get the page's specific restrictions
+        page_permissions = _get_page_specific_restrictions(
             db_session=db_session,
             page=page,
-            space_permissions=space_permissions,
         )
-        for attachment_document_id in attachment_document_ids:
-            document_restrictions[attachment_document_id] = page_permissions
+
+        # Apply the page's specific restrictions to the page and its attachments
+        for document_id in document_ids:
+            document_restrictions[document_id] = page_permissions
 
     return document_restrictions
 
@@ -243,12 +290,15 @@ def confluence_doc_sync(
         db_session=db_session,
         confluence_client=confluence_client,
         space_id=cc_pair.connector.connector_specific_config["space"],
-        space_permissions=space_permissions,
     )
-    for doc_id, ext_access in fresh_doc_permissions.items():
+    for doc_id, page_specific_access in fresh_doc_permissions.items():
+        # If there are no page specific restrictions, then
+        # the page inherits the space's restrictions
+        page_access = page_specific_access or space_permissions
+
         upsert_document_external_perms__no_commit(
             db_session=db_session,
             doc_id=doc_id,
-            external_access=ext_access,
+            external_access=page_access,
             source_type=cc_pair.connector.source,
         )
diff --git a/backend/ee/danswer/external_permissions/permission_sync.py b/backend/ee/danswer/external_permissions/permission_sync.py
index 0f07411da52..3f5e66f875c 100644
--- a/backend/ee/danswer/external_permissions/permission_sync.py
+++ b/backend/ee/danswer/external_permissions/permission_sync.py
@@ -43,8 +43,8 @@ def run_external_group_permission_sync(
 
         # update postgres
         db_session.commit()
-    except Exception as e:
-        logger.error(f"Error updating document index: {e}")
+    except Exception:
+        logger.exception("Error Syncing Group Permissions")
         db_session.rollback()
 
 
@@ -107,6 +107,6 @@ def run_external_doc_permission_sync(
 
         # update postgres
         db_session.commit()
-    except Exception as e:
-        logger.error(f"Error Syncing Permissions: {e}")
+    except Exception:
+        logger.exception("Error Syncing Document Permissions")
         db_session.rollback()

From 1f4fe42f4ba9de5fae50dd1f27862c4dc7b05060 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 10 Oct 2024 12:16:56 -0700
Subject: [PATCH 091/376] Add cql support for confluence connector (#2679)

* Added CQL support for Confluence

* changed string substitutions for CQL

* final cleanup

* updated string fixes

* remove print statements

* Update description
---
 .../connectors/confluence/connector.py        | 216 +++++++++++++-----
 .../[connector]/AddConnectorPage.tsx          |  12 +-
 .../pages/ConnectorInput/ListInput.tsx        |  86 ++-----
 .../pages/DynamicConnectorCreationForm.tsx    | 125 +++++-----
 .../{connectors.ts => connectors.tsx}         |  96 ++++++--
 5 files changed, 338 insertions(+), 197 deletions(-)
 rename web/src/lib/connectors/{connectors.ts => connectors.tsx} (92%)

diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index f800aa49520..623e742ef0a 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -1,5 +1,6 @@
 import io
 import os
+import re
 from collections.abc import Callable
 from collections.abc import Collection
 from datetime import datetime
@@ -56,8 +57,101 @@
 )
 
 
+class DanswerConfluence(Confluence):
+    """
+    This is a custom Confluence class that overrides the default Confluence class to add a custom CQL method.
+    This is necessary because the default Confluence class does not properly support cql expansions.
+    """
+
+    def __init__(self, url: str, *args: Any, **kwargs: Any) -> None:
+        super(DanswerConfluence, self).__init__(url, *args, **kwargs)
+
+    def danswer_cql(
+        self,
+        cql: str,
+        expand: str | None = None,
+        start: int = 0,
+        limit: int = 500,
+        include_archived_spaces: bool = False,
+    ) -> list[dict[str, Any]]:
+        # Performs the query expansion and start/limit url additions
+        url_suffix = f"rest/api/content/search?cql={cql}"
+        if expand:
+            url_suffix += f"&expand={expand}"
+        url_suffix += f"&start={start}&limit={limit}"
+        if include_archived_spaces:
+            url_suffix += "&includeArchivedSpaces=true"
+        try:
+            response = self.get(url_suffix)
+            return response.get("results", [])
+        except Exception as e:
+            raise e
+
+
+def _replace_cql_time_filter(
+    cql_query: str, start_time: datetime, end_time: datetime
+) -> str:
+    """
+    This function replaces the lastmodified filter in the CQL query with the start and end times.
+    This selects the more restrictive time range.
+    """
+    # Extract existing lastmodified >= and <= filters
+    existing_start_match = re.search(
+        r'lastmodified\s*>=\s*["\']?(\d{4}-\d{2}-\d{2}(?:\s+\d{2}:\d{2})?)["\']?',
+        cql_query,
+        flags=re.IGNORECASE,
+    )
+    existing_end_match = re.search(
+        r'lastmodified\s*<=\s*["\']?(\d{4}-\d{2}-\d{2}(?:\s+\d{2}:\d{2})?)["\']?',
+        cql_query,
+        flags=re.IGNORECASE,
+    )
+
+    # Remove all existing lastmodified and updated filters
+    cql_query = re.sub(
+        r'\s*AND\s+(lastmodified|updated)\s*[<>=]+\s*["\']?[\d-]+(?:\s+[\d:]+)?["\']?',
+        "",
+        cql_query,
+        flags=re.IGNORECASE,
+    )
+
+    # Determine the start time to use
+    if existing_start_match:
+        existing_start_str = existing_start_match.group(1)
+        existing_start = datetime.strptime(
+            existing_start_str,
+            "%Y-%m-%d %H:%M" if " " in existing_start_str else "%Y-%m-%d",
+        )
+        existing_start = existing_start.replace(
+            tzinfo=timezone.utc
+        )  # Make offset-aware
+        start_time_to_use = max(start_time.astimezone(timezone.utc), existing_start)
+    else:
+        start_time_to_use = start_time.astimezone(timezone.utc)
+
+    # Determine the end time to use
+    if existing_end_match:
+        existing_end_str = existing_end_match.group(1)
+        existing_end = datetime.strptime(
+            existing_end_str,
+            "%Y-%m-%d %H:%M" if " " in existing_end_str else "%Y-%m-%d",
+        )
+        existing_end = existing_end.replace(tzinfo=timezone.utc)  # Make offset-aware
+        end_time_to_use = min(end_time.astimezone(timezone.utc), existing_end)
+    else:
+        end_time_to_use = end_time.astimezone(timezone.utc)
+
+    # Add new time filters
+    cql_query += (
+        f" and lastmodified >= '{start_time_to_use.strftime('%Y-%m-%d %H:%M')}'"
+    )
+    cql_query += f" and lastmodified <= '{end_time_to_use.strftime('%Y-%m-%d %H:%M')}'"
+
+    return cql_query.strip()
+
+
 @lru_cache()
-def _get_user(user_id: str, confluence_client: Confluence) -> str:
+def _get_user(user_id: str, confluence_client: DanswerConfluence) -> str:
     """Get Confluence Display Name based on the account-id or userkey value
 
     Args:
@@ -81,7 +175,7 @@ def _get_user(user_id: str, confluence_client: Confluence) -> str:
     return user_not_found
 
 
-def parse_html_page(text: str, confluence_client: Confluence) -> str:
+def parse_html_page(text: str, confluence_client: DanswerConfluence) -> str:
     """Parse a Confluence html page and replace the 'user Id' by the real
         User Display Name
 
@@ -112,7 +206,7 @@ def parse_html_page(text: str, confluence_client: Confluence) -> str:
 def _comment_dfs(
     comments_str: str,
     comment_pages: Collection[dict[str, Any]],
-    confluence_client: Confluence,
+    confluence_client: DanswerConfluence,
 ) -> str:
     get_page_child_by_type = make_confluence_call_handle_rate_limit(
         confluence_client.get_page_child_by_type
@@ -159,7 +253,7 @@ class RecursiveIndexer:
     def __init__(
         self,
         batch_size: int,
-        confluence_client: Confluence,
+        confluence_client: DanswerConfluence,
         index_recursively: bool,
         origin_page_id: str,
     ) -> None:
@@ -285,8 +379,8 @@ class ConfluenceConnector(LoadConnector, PollConnector):
     def __init__(
         self,
         wiki_base: str,
-        space: str,
         is_cloud: bool,
+        space: str = "",
         page_id: str = "",
         index_recursively: bool = True,
         batch_size: int = INDEX_BATCH_SIZE,
@@ -295,35 +389,44 @@ def __init__(
         # skip it. This is generally used to avoid indexing extra sensitive
         # pages.
         labels_to_skip: list[str] = CONFLUENCE_CONNECTOR_LABELS_TO_SKIP,
+        cql_query: str | None = None,
     ) -> None:
         self.batch_size = batch_size
         self.continue_on_failure = continue_on_failure
         self.labels_to_skip = set(labels_to_skip)
         self.recursive_indexer: RecursiveIndexer | None = None
-        self.index_recursively = index_recursively
+        self.index_recursively = False if cql_query else index_recursively
 
         # Remove trailing slash from wiki_base if present
         self.wiki_base = wiki_base.rstrip("/")
         self.space = space
-        self.page_id = page_id
+        self.page_id = "" if cql_query else page_id
+        self.space_level_scan = bool(not self.page_id)
 
         self.is_cloud = is_cloud
 
-        self.space_level_scan = False
-        self.confluence_client: Confluence | None = None
+        self.confluence_client: DanswerConfluence | None = None
 
-        if self.page_id is None or self.page_id == "":
-            self.space_level_scan = True
+        # if a cql_query is provided, we will use it to fetch the pages
+        # if no cql_query is provided, we will use the space to fetch the pages
+        # if no space is provided, we will default to fetching all pages, regardless of space
+        if cql_query:
+            self.cql_query = cql_query
+        elif self.space:
+            self.cql_query = f"type=page and space={self.space}"
+        else:
+            self.cql_query = "type=page"
 
         logger.info(
             f"wiki_base: {self.wiki_base}, space: {self.space}, page_id: {self.page_id},"
-            + f" space_level_scan: {self.space_level_scan}, index_recursively: {self.index_recursively}"
+            + f" space_level_scan: {self.space_level_scan}, index_recursively: {self.index_recursively},"
+            + f" cql_query: {self.cql_query}"
         )
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         username = credentials["confluence_username"]
         access_token = credentials["confluence_access_token"]
-        self.confluence_client = Confluence(
+        self.confluence_client = DanswerConfluence(
             url=self.wiki_base,
             # passing in username causes issues for Confluence data center
             username=username if self.is_cloud else None,
@@ -334,26 +437,33 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
 
     def _fetch_pages(
         self,
-        confluence_client: Confluence,
         start_ind: int,
     ) -> list[dict[str, Any]]:
         def _fetch_space(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
-            get_all_pages_from_space = make_confluence_call_handle_rate_limit(
-                confluence_client.get_all_pages_from_space
+            if self.confluence_client is None:
+                raise ConnectorMissingCredentialError("Confluence")
+
+            get_all_pages = make_confluence_call_handle_rate_limit(
+                self.confluence_client.danswer_cql
+            )
+
+            include_archived_spaces = (
+                CONFLUENCE_CONNECTOR_INDEX_ARCHIVED_PAGES
+                if not self.is_cloud
+                else False
             )
+
             try:
-                return get_all_pages_from_space(
-                    self.space,
+                return get_all_pages(
+                    cql=self.cql_query,
                     start=start_ind,
                     limit=batch_size,
-                    status=(
-                        None if CONFLUENCE_CONNECTOR_INDEX_ARCHIVED_PAGES else "current"
-                    ),
                     expand="body.storage.value,version",
+                    include_archived_spaces=include_archived_spaces,
                 )
             except Exception:
                 logger.warning(
-                    f"Batch failed with space {self.space} at offset {start_ind} "
+                    f"Batch failed with cql {self.cql_query} at offset {start_ind} "
                     f"with size {batch_size}, processing pages individually..."
                 )
 
@@ -363,27 +473,23 @@ def _fetch_space(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
                         # Could be that one of the pages here failed due to this bug:
                         # https://jira.atlassian.com/browse/CONFCLOUD-76433
                         view_pages.extend(
-                            get_all_pages_from_space(
-                                self.space,
+                            get_all_pages(
+                                cql=self.cql_query,
                                 start=start_ind + i,
                                 limit=1,
-                                status=(
-                                    None
-                                    if CONFLUENCE_CONNECTOR_INDEX_ARCHIVED_PAGES
-                                    else "current"
-                                ),
                                 expand="body.storage.value,version",
+                                include_archived_spaces=include_archived_spaces,
                             )
                         )
                     except HTTPError as e:
                         logger.warning(
-                            f"Page failed with space {self.space} at offset {start_ind + i}, "
+                            f"Page failed with cql {self.cql_query} at offset {start_ind + i}, "
                             f"trying alternative expand option: {e}"
                         )
                         # Use view instead, which captures most info but is less complete
                         view_pages.extend(
-                            get_all_pages_from_space(
-                                self.space,
+                            get_all_pages(
+                                cql=self.cql_query,
                                 start=start_ind + i,
                                 limit=1,
                                 expand="body.view.value,version",
@@ -393,6 +499,9 @@ def _fetch_space(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
                 return view_pages
 
         def _fetch_page(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
+            if self.confluence_client is None:
+                raise ConnectorMissingCredentialError("Confluence")
+
             if self.recursive_indexer is None:
                 self.recursive_indexer = RecursiveIndexer(
                     origin_page_id=self.page_id,
@@ -421,7 +530,7 @@ def _fetch_page(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
                 raise e
 
         # error checking phase, only reachable if `self.continue_on_failure=True`
-        for i in range(self.batch_size):
+        for _ in range(self.batch_size):
             try:
                 pages = (
                     _fetch_space(start_ind, self.batch_size)
@@ -437,7 +546,9 @@ def _fetch_page(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
 
         return pages
 
-    def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
+    def _fetch_comments(
+        self, confluence_client: DanswerConfluence, page_id: str
+    ) -> str:
         get_page_child_by_type = make_confluence_call_handle_rate_limit(
             confluence_client.get_page_child_by_type
         )
@@ -463,7 +574,9 @@ def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
             )
             return ""
 
-    def _fetch_labels(self, confluence_client: Confluence, page_id: str) -> list[str]:
+    def _fetch_labels(
+        self, confluence_client: DanswerConfluence, page_id: str
+    ) -> list[str]:
         get_page_labels = make_confluence_call_handle_rate_limit(
             confluence_client.get_page_labels
         )
@@ -577,22 +690,20 @@ def _fetch_attachments(
         return "\n".join(files_attachment_content), unused_attachments
 
     def _get_doc_batch(
-        self, start_ind: int, time_filter: Callable[[datetime], bool] | None = None
+        self, start_ind: int
     ) -> tuple[list[Document], list[dict[str, Any]], int]:
+        if self.confluence_client is None:
+            raise ConnectorMissingCredentialError("Confluence")
+
         doc_batch: list[Document] = []
         unused_attachments: list[dict[str, Any]] = []
 
-        if self.confluence_client is None:
-            raise ConnectorMissingCredentialError("Confluence")
-        batch = self._fetch_pages(self.confluence_client, start_ind)
+        batch = self._fetch_pages(start_ind)
 
         for page in batch:
             last_modified = _datetime_from_string(page["version"]["when"])
             author = cast(str | None, page["version"].get("by", {}).get("email"))
 
-            if time_filter and not time_filter(last_modified):
-                continue
-
             page_id = page["id"]
 
             if self.labels_to_skip or not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING:
@@ -715,17 +826,12 @@ def _get_attachment_batch(
         return doc_batch, end_ind - start_ind
 
     def load_from_state(self) -> GenerateDocumentsOutput:
-        unused_attachments = []
-
-        if self.confluence_client is None:
-            raise ConnectorMissingCredentialError("Confluence")
+        unused_attachments: list[dict[str, Any]] = []
 
         start_ind = 0
         while True:
-            doc_batch, unused_attachments_batch, num_pages = self._get_doc_batch(
-                start_ind
-            )
-            unused_attachments.extend(unused_attachments_batch)
+            doc_batch, unused_attachments, num_pages = self._get_doc_batch(start_ind)
+            unused_attachments.extend(unused_attachments)
             start_ind += num_pages
             if doc_batch:
                 yield doc_batch
@@ -748,7 +854,7 @@ def load_from_state(self) -> GenerateDocumentsOutput:
     def poll_source(
         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
     ) -> GenerateDocumentsOutput:
-        unused_attachments = []
+        unused_attachments: list[dict[str, Any]] = []
 
         if self.confluence_client is None:
             raise ConnectorMissingCredentialError("Confluence")
@@ -756,12 +862,12 @@ def poll_source(
         start_time = datetime.fromtimestamp(start, tz=timezone.utc)
         end_time = datetime.fromtimestamp(end, tz=timezone.utc)
 
+        self.cql_query = _replace_cql_time_filter(self.cql_query, start_time, end_time)
+
         start_ind = 0
         while True:
-            doc_batch, unused_attachments_batch, num_pages = self._get_doc_batch(
-                start_ind, time_filter=lambda t: start_time <= t <= end_time
-            )
-            unused_attachments.extend(unused_attachments_batch)
+            doc_batch, unused_attachments, num_pages = self._get_doc_batch(start_ind)
+            unused_attachments.extend(unused_attachments)
 
             start_ind += num_pages
             if doc_batch:
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index b01347edab6..fee2042b3b4 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { errorHandlingFetcher } from "@/lib/fetcher";
+import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR, { mutate } from "swr";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 
@@ -209,7 +209,15 @@ export default function AddConnector({
 
   return (
     <Formik
-      initialValues={createConnectorInitialValues(connector)}
+      initialValues={{
+        ...createConnectorInitialValues(connector),
+        ...Object.fromEntries(
+          connectorConfigs[connector].advanced_values.map((field) => [
+            field.name,
+            field.default || "",
+          ])
+        ),
+      }}
       validationSchema={createConnectorValidationSchema(connector)}
       onSubmit={async (values) => {
         const {
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
index 059d08d539d..edb057a3e3e 100644
--- a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
@@ -1,74 +1,24 @@
-import CredentialSubText from "@/components/credentials/CredentialFields";
-import { TrashIcon } from "@/components/icons/icons";
+import React from "react";
 import { ListOption } from "@/lib/connectors/connectors";
-import { Field, FieldArray, useField } from "formik";
-import { FaPlus } from "react-icons/fa";
+import { TextArrayField } from "@/components/admin/connectors/Field";
+import { useFormikContext } from "formik";
 
-export default function ListInput({
-  field,
-  onUpdate,
-}: {
+interface ListInputProps {
   field: ListOption;
-  onUpdate?: (values: string[]) => void;
-}) {
-  const [fieldProps, , helpers] = useField(field.name);
-
-  return (
-    <FieldArray name={field.name}>
-      {({ push, remove }) => (
-        <div>
-          <label
-            htmlFor={field.name}
-            className="block text-sm font-medium text-text-700 mb-1"
-          >
-            {field.label}
-            {field.optional && (
-              <span className="text-text-500 ml-1">(optional)</span>
-            )}
-          </label>
-          {field.description && (
-            <CredentialSubText>{field.description}</CredentialSubText>
-          )}
+}
 
-          {fieldProps.value.map((value: string, index: number) => (
-            <div key={index} className="w-full flex mb-4">
-              <Field
-                name={`${field.name}.${index}`}
-                className="w-full bg-input text-sm p-2 border border-border-medium rounded-md focus:ring-2 focus:ring-indigo-500 focus:border-indigo-500 mr-2"
-              />
-              <button
-                className="p-2 my-auto bg-input flex-none rounded-md bg-red-500 text-white hover:bg-red-600 focus:outline-none focus:ring-2 focus:ring-red-500 focus:ring-opacity-50"
-                type="button"
-                onClick={() => {
-                  remove(index);
-                  if (onUpdate) {
-                    const newValue = fieldProps.value.filter(
-                      (_: any, i: number) => i !== index
-                    );
-                    onUpdate(newValue);
-                  }
-                }}
-              >
-                <TrashIcon className="text-white my-auto" />
-              </button>
-            </div>
-          ))}
+const ListInput: React.FC<ListInputProps> = ({ field }) => {
+  const { values } = useFormikContext();
 
-          <button
-            type="button"
-            onClick={() => {
-              push("");
-              if (onUpdate) {
-                onUpdate([...fieldProps.value, ""]);
-              }
-            }}
-            className="mt-2 p-2 bg-rose-500 text-xs text-white rounded-md hover:bg-rose-600 focus:outline-none focus:ring-2 focus:ring-rose-500 focus:ring-opacity-50 flex items-center"
-          >
-            <FaPlus className="mr-2" />
-            Add {field.label}
-          </button>
-        </div>
-      )}
-    </FieldArray>
+  return (
+    <TextArrayField
+      name={field.name}
+      label={field.label}
+      values={Array.isArray(values) ? values : []}
+      subtext={field.description}
+      placeholder={`Enter ${field.label.toLowerCase()}`}
+    />
   );
-}
+};
+
+export default ListInput;
diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
index cd7e90f6167..bc064b727ec 100644
--- a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
@@ -1,4 +1,4 @@
-import React, { Dispatch, FC, SetStateAction } from "react";
+import React, { Dispatch, FC, SetStateAction, useState } from "react";
 import CredentialSubText, {
   AdminBooleanFormField,
 } from "@/components/credentials/CredentialFields";
@@ -9,6 +9,7 @@ import NumberInput from "./ConnectorInput/NumberInput";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import ListInput from "./ConnectorInput/ListInput";
 import FileInput from "./ConnectorInput/FileInput";
+import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 
 export interface DynamicConnectionFormProps {
   config: ConnectionConfiguration;
@@ -23,6 +24,61 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
   setSelectedFiles,
   values,
 }) => {
+  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+
+  const renderField = (field: any) => (
+    <div key={field.name}>
+      {field.type === "file" ? (
+        <FileUpload
+          name={field.name}
+          selectedFiles={selectedFiles}
+          setSelectedFiles={setSelectedFiles}
+        />
+      ) : field.type === "zip" ? (
+        <FileInput
+          name={field.name}
+          label={field.label}
+          optional={field.optional}
+          description={field.description}
+          selectedFiles={selectedFiles}
+          setSelectedFiles={setSelectedFiles}
+        />
+      ) : field.type === "list" ? (
+        <ListInput field={field} />
+      ) : field.type === "select" ? (
+        <SelectInput
+          name={field.name}
+          optional={field.optional}
+          description={field.description}
+          options={field.options || []}
+          label={field.label}
+        />
+      ) : field.type === "number" ? (
+        <NumberInput
+          label={field.label}
+          optional={field.optional}
+          description={field.description}
+          name={field.name}
+        />
+      ) : field.type === "checkbox" ? (
+        <AdminBooleanFormField
+          checked={values[field.name]}
+          subtext={field.description}
+          name={field.name}
+          label={field.label}
+        />
+      ) : (
+        <TextFormField
+          subtext={field.description}
+          optional={field.optional}
+          type={field.type}
+          label={field.label}
+          name={field.name}
+        />
+      )}
+    </div>
+  );
+
   return (
     <>
       <h2 className="text-2xl font-bold text-text-800">{config.description}</h2>
@@ -38,62 +94,17 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
         name={"name"}
       />
 
-      {config.values.map((field) => {
-        if (!field.hidden) {
-          return (
-            <div key={field.name}>
-              {field.type == "file" ? (
-                <FileUpload
-                  name={field.name}
-                  selectedFiles={selectedFiles}
-                  setSelectedFiles={setSelectedFiles}
-                />
-              ) : field.type == "zip" ? (
-                <FileInput
-                  name={field.name}
-                  label={field.label}
-                  optional={field.optional}
-                  description={field.description}
-                  selectedFiles={selectedFiles}
-                  setSelectedFiles={setSelectedFiles}
-                />
-              ) : field.type === "list" ? (
-                <ListInput field={field} />
-              ) : field.type === "select" ? (
-                <SelectInput
-                  name={field.name}
-                  optional={field.optional}
-                  description={field.description}
-                  options={field.options || []}
-                  label={field.label}
-                />
-              ) : field.type === "number" ? (
-                <NumberInput
-                  label={field.label}
-                  optional={field.optional}
-                  description={field.description}
-                  name={field.name}
-                />
-              ) : field.type === "checkbox" ? (
-                <AdminBooleanFormField
-                  checked={values[field.name]}
-                  subtext={field.description}
-                  name={field.name}
-                  label={field.label}
-                />
-              ) : (
-                <TextFormField
-                  subtext={field.description}
-                  optional={field.optional}
-                  type={field.type}
-                  label={field.label}
-                  name={field.name}
-                />
-              )}
-            </div>
-          );
-        }
-      })}
+      {config.values.map((field) => !field.hidden && renderField(field))}
+
+      {config.advanced_values.length > 0 && (
+        <>
+          <AdvancedOptionsToggle
+            showAdvancedOptions={showAdvancedOptions}
+            setShowAdvancedOptions={setShowAdvancedOptions}
+          />
+          {showAdvancedOptions && config.advanced_values.map(renderField)}
+        </>
+      )}
     </>
   );
 };
diff --git a/web/src/lib/connectors/connectors.ts b/web/src/lib/connectors/connectors.tsx
similarity index 92%
rename from web/src/lib/connectors/connectors.ts
rename to web/src/lib/connectors/connectors.tsx
index 61ae2b076cf..959d981dd96 100644
--- a/web/src/lib/connectors/connectors.ts
+++ b/web/src/lib/connectors/connectors.tsx
@@ -86,6 +86,15 @@ export interface ConnectionConfiguration {
     | FileOption
     | ZipOption
   )[];
+  advanced_values: (
+    | BooleanOption
+    | ListOption
+    | TextOption
+    | NumberOption
+    | SelectOption
+    | FileOption
+    | ZipOption
+  )[];
   overrideDefaultFreq?: number;
 }
 
@@ -116,6 +125,17 @@ export const connectorConfigs: Record<
         ],
       },
     ],
+    advanced_values: [
+      {
+        type: "number",
+        query: "Enter the maximum depth to crawl:",
+        label: "Max Depth",
+        name: "max_depth",
+        optional: true,
+        description:
+          "The maximum depth to crawl from the base URL. Default is 2.",
+      },
+    ],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   github: {
@@ -152,6 +172,7 @@ export const connectorConfigs: Record<
         optional: true,
       },
     ],
+    advanced_values: [],
   },
   gitlab: {
     description: "Configure GitLab connector",
@@ -187,6 +208,7 @@ export const connectorConfigs: Record<
         hidden: true,
       },
     ],
+    advanced_values: [],
   },
   google_drive: {
     description: "Configure Google Drive connector",
@@ -223,22 +245,21 @@ export const connectorConfigs: Record<
         default: false,
       },
     ],
+    advanced_values: [],
   },
   gmail: {
     description: "Configure Gmail connector",
     values: [],
+    advanced_values: [],
   },
   bookstack: {
     description: "Configure Bookstack connector",
     values: [],
+    advanced_values: [],
   },
   confluence: {
     description: "Configure Confluence connector",
-    subtext: `Specify the base URL of your Confluence instance, the space name, and optionally a specific page ID to index. If no page ID is provided, the entire space will be indexed.
-
-For example, entering "https://your-company.atlassian.net/wiki" as the Wiki Base URL, "KB" as the Space, and "164331" as the Page ID will index the specific page at https:///your-company.atlassian.net/wiki/spaces/KB/pages/164331/Page. If you leave the Page ID empty, it will index the entire KB space.
-
-Selecting the "Index Recursively" checkbox will index the specified page and all of its children.`,
+    subtext: `Specify the base URL of your Confluence instance, the space name, and optionally a specific page ID to index. If no page ID is provided, the entire space will be indexed. If no space is specified, all available Confluence spaces will be indexed.`,
     values: [
       {
         type: "text",
@@ -254,9 +275,22 @@ Selecting the "Index Recursively" checkbox will index the specified page and all
         query: "Enter the space:",
         label: "Space",
         name: "space",
+        optional: true,
+        description:
+          "The Confluence space name to index (e.g. `KB`). If no space is specified, all available Confluence spaces will be indexed.",
+      },
+      {
+        type: "checkbox",
+        query: "Is this a Confluence Cloud instance?",
+        label: "Is Cloud",
+        name: "is_cloud",
         optional: false,
-        description: "The Confluence space name to index (e.g. `KB`)",
+        default: true,
+        description:
+          "Check if this is a Confluence Cloud instance, uncheck for Confluence Server/Data Center",
       },
+    ],
+    advanced_values: [
       {
         type: "text",
         query: "Enter the page ID (optional):",
@@ -276,14 +310,13 @@ Selecting the "Index Recursively" checkbox will index the specified page and all
         optional: false,
       },
       {
-        type: "checkbox",
-        query: "Is this a Confluence Cloud instance?",
-        label: "Is Cloud",
-        name: "is_cloud",
-        optional: false,
-        default: true,
+        type: "text",
+        query: "Enter the CQL query (optional):",
+        label: "CQL Query",
+        name: "cql_query",
+        optional: true,
         description:
-          "Check if this is a Confluence Cloud instance, uncheck for Confluence Server/Data Center",
+          "IMPORTANT: This will overwrite all other selected connector settings (besides Wiki Base URL). We currently only support CQL queries that return objects of type 'page'. This means all CQL queries must contain 'type=page' as the only type filter. We will still get all attachments and comments for the pages returned by the CQL query. Any 'lastmodified' filters will be overwritten. See https://developer.atlassian.com/server/confluence/advanced-searching-using-cql/ for more details.",
       },
     ],
   },
@@ -308,6 +341,7 @@ Selecting the "Index Recursively" checkbox will index the specified page and all
         optional: true,
       },
     ],
+    advanced_values: [],
   },
   salesforce: {
     description: "Configure Salesforce connector",
@@ -323,6 +357,7 @@ Selecting the "Index Recursively" checkbox will index the specified page and all
 Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of 'Opportunities').`,
       },
     ],
+    advanced_values: [],
   },
   sharepoint: {
     description: "Configure SharePoint connector",
@@ -339,6 +374,7 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
 `,
       },
     ],
+    advanced_values: [],
   },
   teams: {
     description: "Configure Teams connector",
@@ -352,6 +388,7 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
         description: `Specify 0 or more Teams to index. For example, specifying the Team 'Support' for the 'danswerai' Org will cause us to only index messages sent in channels belonging to the 'Support' Team. If no Teams are specified, all Teams in your organization will be indexed.`,
       },
     ],
+    advanced_values: [],
   },
   discourse: {
     description: "Configure Discourse connector",
@@ -371,6 +408,7 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
         optional: true,
       },
     ],
+    advanced_values: [],
   },
   axero: {
     description: "Configure Axero connector",
@@ -385,11 +423,13 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
           "Specify zero or more Spaces to index (by the Space IDs). If no Space IDs are specified, all Spaces will be indexed.",
       },
     ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   productboard: {
     description: "Configure Productboard connector",
     values: [],
+    advanced_values: [],
   },
   slack: {
     description: "Configure Slack connector",
@@ -401,6 +441,8 @@ Hint: Use the singular form of the object name (e.g., 'Opportunity' instead of '
         name: "workspace",
         optional: false,
       },
+    ],
+    advanced_values: [
       {
         type: "list",
         query: "Enter channels to include:",
@@ -434,10 +476,12 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         description: `Specify the base URL for your Slab team. This will look something like: https://danswer.slab.com/`,
       },
     ],
+    advanced_values: [],
   },
   guru: {
     description: "Configure Guru connector",
     values: [],
+    advanced_values: [],
   },
   gong: {
     description: "Configure Gong connector",
@@ -452,6 +496,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
           "Specify 0 or more workspaces to index. Provide the workspace ID or the EXACT workspace name from Gong. If no workspaces are specified, transcripts from all workspaces will be indexed.",
       },
     ],
+    advanced_values: [],
   },
   loopio: {
     description: "Configure Loopio connector",
@@ -466,6 +511,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: true,
       },
     ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   file: {
@@ -479,6 +525,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: false,
       },
     ],
+    advanced_values: [],
   },
   zulip: {
     description: "Configure Zulip connector",
@@ -498,6 +545,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: false,
       },
     ],
+    advanced_values: [],
   },
   notion: {
     description: "Configure Notion connector",
@@ -512,14 +560,17 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
           "If specified, will only index the specified page + all of its child pages. If left blank, will index all pages the integration has been given access to.",
       },
     ],
+    advanced_values: [],
   },
   requesttracker: {
     description: "Configure HubSpot connector",
     values: [],
+    advanced_values: [],
   },
   hubspot: {
     description: "Configure HubSpot connector",
     values: [],
+    advanced_values: [],
   },
   document360: {
     description: "Configure Document360 connector",
@@ -541,6 +592,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
           "Specify 0 or more categories to index. For instance, specifying the category 'Help' will cause us to only index all content within the 'Help' category. If no categories are specified, all categories in your workspace will be indexed.",
       },
     ],
+    advanced_values: [],
   },
   clickup: {
     description: "Configure ClickUp connector",
@@ -576,6 +628,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: false,
       },
     ],
+    advanced_values: [],
   },
   google_sites: {
     description: "Configure Google Sites connector",
@@ -597,6 +650,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: false,
       },
     ],
+    advanced_values: [],
   },
   zendesk: {
     description: "Configure Zendesk connector",
@@ -614,14 +668,17 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         default: "articles",
       },
     ],
+    advanced_values: [],
   },
   linear: {
-    description: "Configure Linear connector",
+    description: "Configure Dropbox connector",
     values: [],
+    advanced_values: [],
   },
   dropbox: {
     description: "Configure Dropbox connector",
     values: [],
+    advanced_values: [],
   },
   s3: {
     description: "Configure S3 connector",
@@ -649,6 +706,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         hidden: true,
       },
     ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   r2: {
@@ -677,6 +735,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         hidden: true,
       },
     ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   google_cloud_storage: {
@@ -706,6 +765,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         hidden: true,
       },
     ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   oci_storage: {
@@ -734,6 +794,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         hidden: true,
       },
     ],
+    advanced_values: [],
   },
   wikipedia: {
     description: "Configure Wikipedia connector",
@@ -773,6 +834,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: false,
       },
     ],
+    advanced_values: [],
   },
   xenforo: {
     description: "Configure Xenforo connector",
@@ -787,6 +849,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
           "The XenForo v2.2 forum URL to index. Can be board or thread.",
       },
     ],
+    advanced_values: [],
   },
   asana: {
     description: "Configure Asana connector",
@@ -819,6 +882,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
           "ID of a team to use for accessing team-visible tasks. This allows indexing of team-visible tasks in addition to public tasks. Leave empty if you don't want to use this feature.",
       },
     ],
+    advanced_values: [],
   },
   mediawiki: {
     description: "Configure MediaWiki connector",
@@ -866,6 +930,7 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
         optional: true,
       },
     ],
+    advanced_values: [],
   },
 };
 export function createConnectorInitialValues(
@@ -987,10 +1052,11 @@ export interface BookstackConfig {}
 
 export interface ConfluenceConfig {
   wiki_base: string;
-  space: string;
+  space?: string;
   page_id?: string;
   is_cloud?: boolean;
   index_recursively?: boolean;
+  cql_query?: string;
 }
 
 export interface JiraConfig {

From 1581d354760723b97cb1786f91c4be68949709e6 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 10 Oct 2024 12:34:30 -0700
Subject: [PATCH 092/376] account for no visible assistants (#2765)

---
 web/src/app/chat/ChatPage.tsx                 | 16 +++++++-
 web/src/components/BasicClickable.tsx         |  6 ++-
 .../components/modals/NoAssistantModal.tsx    | 37 +++++++++++++++++++
 3 files changed, 56 insertions(+), 3 deletions(-)
 create mode 100644 web/src/components/modals/NoAssistantModal.tsx

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 4f7363a3b66..8c7eccc0677 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -106,6 +106,7 @@ import {
   orderAssistantsForUser,
 } from "@/lib/assistants/utils";
 import BlurBackground from "./shared_chat_search/BlurBackground";
+import { NoAssistantModal } from "@/components/modals/NoAssistantModal";
 
 const TEMP_USER_MESSAGE_ID = -1;
 const TEMP_ASSISTANT_MESSAGE_ID = -2;
@@ -139,7 +140,7 @@ export function ChatPage({
 
   const [showApiKeyModal, setShowApiKeyModal] = useState(true);
 
-  const { user, refreshUser, isLoadingUser } = useUser();
+  const { user, refreshUser, isAdmin, isLoadingUser } = useUser();
 
   const existingChatIdRaw = searchParams.get("chatId");
   const currentPersonaId = searchParams.get(SEARCH_PARAM_NAMES.PERSONA_ID);
@@ -191,6 +192,7 @@ export function ChatPage({
   const search_param_temperature = searchParams.get(
     SEARCH_PARAM_NAMES.TEMPERATURE
   );
+
   const defaultTemperature = search_param_temperature
     ? parseFloat(search_param_temperature)
     : selectedAssistant?.tools.some(
@@ -225,6 +227,8 @@ export function ChatPage({
     finalAssistants[0] ||
     availableAssistants[0];
 
+  const noAssistants = liveAssistant == null || liveAssistant == undefined;
+
   useEffect(() => {
     if (!loadedIdSessionRef.current && !currentPersonaId) {
       return;
@@ -1695,6 +1699,9 @@ export function ChatPage({
   };
 
   useEffect(() => {
+    if (noAssistants) {
+      return;
+    }
     const includes = checkAnyAssistantHasSearch(
       messageHistory,
       availableAssistants,
@@ -1704,6 +1711,9 @@ export function ChatPage({
   }, [messageHistory, availableAssistants, liveAssistant]);
 
   const [retrievalEnabled, setRetrievalEnabled] = useState(() => {
+    if (noAssistants) {
+      return false;
+    }
     return checkAnyAssistantHasSearch(
       messageHistory,
       availableAssistants,
@@ -1774,11 +1784,13 @@ export function ChatPage({
     <>
       <HealthCheckBanner />
 
-      {showApiKeyModal && !shouldShowWelcomeModal && (
+      {showApiKeyModal && !shouldShowWelcomeModal ? (
         <ApiKeyModal
           hide={() => setShowApiKeyModal(false)}
           setPopup={setPopup}
         />
+      ) : (
+        noAssistants && <NoAssistantModal isAdmin={isAdmin} />
       )}
 
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 
diff --git a/web/src/components/BasicClickable.tsx b/web/src/components/BasicClickable.tsx
index 650baf7dc90..34132c7bce5 100644
--- a/web/src/components/BasicClickable.tsx
+++ b/web/src/components/BasicClickable.tsx
@@ -3,11 +3,13 @@ export function BasicClickable({
   onClick,
   fullWidth = false,
   inset,
+  className,
 }: {
   children: string | JSX.Element;
   onClick?: () => void;
   inset?: boolean;
   fullWidth?: boolean;
+  className?: string;
 }) {
   return (
     <button
@@ -26,7 +28,9 @@ export function BasicClickable({
         select-none
         overflow-hidden
         hover:bg-hover-light
-        ${fullWidth ? "w-full" : ""}`}
+        ${fullWidth ? "w-full" : ""}
+        ${className ? className : ""}
+        `}
     >
       {children}
     </button>
diff --git a/web/src/components/modals/NoAssistantModal.tsx b/web/src/components/modals/NoAssistantModal.tsx
new file mode 100644
index 00000000000..0eed8876629
--- /dev/null
+++ b/web/src/components/modals/NoAssistantModal.tsx
@@ -0,0 +1,37 @@
+import { ModalWrapper } from "@/components/modals/ModalWrapper";
+
+export const NoAssistantModal = ({ isAdmin }: { isAdmin: boolean }) => {
+  return (
+    <ModalWrapper modalClassName="bg-white max-w-2xl rounded-lg shadow-xl text-center">
+      <>
+        <h2 className="text-3xl font-bold text-gray-800 mb-4">
+          No Assistant Available
+        </h2>
+        <p className="text-gray-600 mb-6">
+          You currently have no assistant configured. To use this feature, you
+          need to take action.
+        </p>
+        {isAdmin ? (
+          <>
+            <p className="text-gray-600 mb-6">
+              As an administrator, you can create a new assistant by visiting
+              the admin panel.
+            </p>
+            <button
+              onClick={() => {
+                window.location.href = "/admin/assistants";
+              }}
+              className="inline-flex flex justify-center items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-background-800 text-center focus:outline-none focus:ring-2 focus:ring-offset-2 "
+            >
+              Go to Admin Panel
+            </button>
+          </>
+        ) : (
+          <p className="text-gray-600 mb-2">
+            Please contact your administrator to configure an assistant for you.
+          </p>
+        )}
+      </>
+    </ModalWrapper>
+  );
+};

From 35d32ea3b0be0ee7687e9c1c06329b8f33214f0a Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Thu, 10 Oct 2024 21:24:34 -0700
Subject: [PATCH 093/376] Fix indexing model server port for warmup (#2767)

---
 backend/danswer/background/update.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index f7a00687c43..58c31236eae 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -48,8 +48,8 @@
 from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
+from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
 from shared_configs.configs import LOG_LEVEL
-from shared_configs.configs import MODEL_SERVER_PORT
 
 logger = setup_logger()
 
@@ -494,7 +494,7 @@ def update_loop(
                                 embedding_model = EmbeddingModel.from_db_model(
                                     search_settings=search_settings,
                                     server_host=INDEXING_MODEL_SERVER_HOST,
-                                    server_port=MODEL_SERVER_PORT,
+                                    server_port=INDEXING_MODEL_SERVER_PORT,
                                 )
                                 warm_up_bi_encoder(embedding_model=embedding_model)
                                 logger.notice("First inference complete.")

From d892203821b480db66d7af9e2fb1a97b5d11b4a5 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 11 Oct 2024 17:39:09 -0700
Subject: [PATCH 094/376] fix typo (#2768)

---
 backend/danswer/tools/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/tools/utils.py b/backend/danswer/tools/utils.py
index 157d4bb6ec9..52d60feb912 100644
--- a/backend/danswer/tools/utils.py
+++ b/backend/danswer/tools/utils.py
@@ -37,7 +37,7 @@ def compute_all_tool_tokens(tools: list[Tool], llm_tokenizer: BaseTokenizer) ->
 def is_image_generation_available(db_session: Session) -> bool:
     providers = db_session.query(LLMProvider).all()
     for provider in providers:
-        if provider.name == "OpenAI":
+        if provider.provider == "openai":
             return True
 
     return bool(AZURE_DALLE_API_KEY)

From d25de6e1cba58e72c8d882d46b7920af18497ed4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 11 Oct 2024 17:41:15 -0700
Subject: [PATCH 095/376] fix web connector (#2769)

---
 web/src/lib/connectors/connectors.tsx | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index 959d981dd96..96e21aa6799 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -125,17 +125,7 @@ export const connectorConfigs: Record<
         ],
       },
     ],
-    advanced_values: [
-      {
-        type: "number",
-        query: "Enter the maximum depth to crawl:",
-        label: "Max Depth",
-        name: "max_depth",
-        optional: true,
-        description:
-          "The maximum depth to crawl from the base URL. Default is 2.",
-      },
-    ],
+    advanced_values: [],
     overrideDefaultFreq: 60 * 60 * 24,
   },
   github: {

From b75b8334a6d993810477b6b825e5c1652f6155d2 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 11 Oct 2024 20:04:48 -0700
Subject: [PATCH 096/376] k (#2771)

---
 .../app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx  | 2 +-
 web/src/app/admin/connector/[ccPairId]/page.tsx               | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index 4bb532981b4..649ef1a97fd 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -163,7 +163,7 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
   // This updates the page number and manages the URL
   const updatePage = (newPage: number) => {
     setPage(newPage);
-    router.push(`/admin/connector/${ccPair.id}?page=${newPage}`, {
+    router.replace(`/admin/connector/${ccPair.id}?page=${newPage}`, {
       scroll: false,
     });
     window.scrollTo({
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index 530b11332b2..2c738c03f9b 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -140,7 +140,9 @@ function Main({ ccPairId }: { ccPairId: number }) {
   return (
     <>
       {popup}
-      <BackButton />
+      <BackButton
+        behaviorOverride={() => router.push("/admin/indexing/status")}
+      />
       <div className="pb-1 flex mt-1">
         <div className="mr-2 my-auto">
           <SourceIcon iconSize={24} sourceType={ccPair.connector.source} />

From 301032f59e5f5ce59d51000f0902c075e709935c Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 11 Oct 2024 20:10:09 -0700
Subject: [PATCH 097/376] k (#2772)

---
 web/src/app/chat/ChatPage.tsx | 20 ++++++++++++++++++--
 web/src/app/chat/lib.tsx      |  2 ++
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 8c7eccc0677..54432c40e3b 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -777,7 +777,11 @@ export function ChatPage({
 
   const handleInputResize = () => {
     setTimeout(() => {
-      if (inputRef.current && lastMessageRef.current) {
+      if (
+        inputRef.current &&
+        lastMessageRef.current &&
+        !waitForScrollRef.current
+      ) {
         const newHeight: number =
           inputRef.current?.getBoundingClientRect().height!;
         const heightDifference = newHeight - previousHeight.current;
@@ -806,8 +810,11 @@ export function ChatPage({
   };
 
   const clientScrollToBottom = (fast?: boolean) => {
+    waitForScrollRef.current = true;
+
     setTimeout(() => {
       if (!endDivRef.current || !scrollableDivRef.current) {
+        console.error("endDivRef or scrollableDivRef not found");
         return;
       }
 
@@ -818,6 +825,7 @@ export function ChatPage({
 
       // Check if all messages are currently rendered
       if (currentVisibleRange.end < messageHistory.length) {
+        console.log("Updating visible range");
         // Update visible range to include the last messages
         updateCurrentVisibleRange({
           start: Math.max(
@@ -835,8 +843,9 @@ export function ChatPage({
             behavior: fast ? "auto" : "smooth",
           });
           setHasPerformedInitialScroll(true);
-        }, 0);
+        }, 100);
       } else {
+        console.log("All messages are already rendered, scrolling immediately");
         // If all messages are already rendered, scroll immediately
         endDivRef.current.scrollIntoView({
           behavior: fast ? "auto" : "smooth",
@@ -844,6 +853,11 @@ export function ChatPage({
         setHasPerformedInitialScroll(true);
       }
     }, 50);
+
+    // Reset waitForScrollRef after 1.5 seconds
+    setTimeout(() => {
+      waitForScrollRef.current = false;
+    }, 1500);
   };
 
   const distance = 500; // distance that should "engage" the scroll
@@ -1553,6 +1567,7 @@ export function ChatPage({
     toggle(false);
   };
 
+  const waitForScrollRef = useRef(false);
   const sidebarElementRef = useRef<HTMLDivElement>(null);
 
   useSidebarVisibility({
@@ -1571,6 +1586,7 @@ export function ChatPage({
     endDivRef,
     distance,
     debounceNumber,
+    waitForScrollRef,
   });
 
   // Virtualization + Scrolling related effects and functions
diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx
index 73ddd63c6ee..7ce86ac2c61 100644
--- a/web/src/app/chat/lib.tsx
+++ b/web/src/app/chat/lib.tsx
@@ -641,9 +641,11 @@ export async function useScrollonStream({
   endDivRef,
   distance,
   debounceNumber,
+  waitForScrollRef,
 }: {
   chatState: ChatState;
   scrollableDivRef: RefObject<HTMLDivElement>;
+  waitForScrollRef: RefObject<boolean>;
   scrollDist: MutableRefObject<number>;
   endDivRef: RefObject<HTMLDivElement>;
   distance: number;

From 7eafdae17f710edeec823e92b772360b5f65d097 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Sat, 12 Oct 2024 16:40:20 -0700
Subject: [PATCH 098/376] update several github actions to silence github
 deprecation warnings (#2730)

---
 .github/workflows/pr-Integration-tests.yml      | 2 +-
 .github/workflows/pr-python-checks.yml          | 4 ++--
 .github/workflows/pr-python-connector-tests.yml | 2 +-
 .github/workflows/pr-python-model-tests.yml     | 2 +-
 .github/workflows/pr-python-tests.yml           | 2 +-
 .github/workflows/pr-quality-checks.yml         | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pr-Integration-tests.yml b/.github/workflows/pr-Integration-tests.yml
index 98fa8be16e6..0e4856f7194 100644
--- a/.github/workflows/pr-Integration-tests.yml
+++ b/.github/workflows/pr-Integration-tests.yml
@@ -167,7 +167,7 @@ jobs:
       
       - name: Upload logs
         if: success() || failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: docker-logs
           path: ${{ github.workspace }}/docker-compose.log
diff --git a/.github/workflows/pr-python-checks.yml b/.github/workflows/pr-python-checks.yml
index 0a9e9f96a63..db16848bd2f 100644
--- a/.github/workflows/pr-python-checks.yml
+++ b/.github/workflows/pr-python-checks.yml
@@ -14,10 +14,10 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'
         cache: 'pip'
diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml
index 642618000d2..108012100b3 100644
--- a/.github/workflows/pr-python-connector-tests.yml
+++ b/.github/workflows/pr-python-connector-tests.yml
@@ -32,7 +32,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: "pip"
diff --git a/.github/workflows/pr-python-model-tests.yml b/.github/workflows/pr-python-model-tests.yml
index f55178281a4..11d1d70d99d 100644
--- a/.github/workflows/pr-python-model-tests.yml
+++ b/.github/workflows/pr-python-model-tests.yml
@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
           cache: "pip"
diff --git a/.github/workflows/pr-python-tests.yml b/.github/workflows/pr-python-tests.yml
index ce57a7a5814..5637300615b 100644
--- a/.github/workflows/pr-python-tests.yml
+++ b/.github/workflows/pr-python-tests.yml
@@ -21,7 +21,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.11'
         cache: 'pip'
diff --git a/.github/workflows/pr-quality-checks.yml b/.github/workflows/pr-quality-checks.yml
index 128317a79ce..3ba206669a6 100644
--- a/.github/workflows/pr-quality-checks.yml
+++ b/.github/workflows/pr-quality-checks.yml
@@ -18,6 +18,6 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - uses: pre-commit/action@v3.0.0
+      - uses: pre-commit/action@v3.0.1
         with:
           extra_args: ${{ github.event_name == 'pull_request' && format('--from-ref {0} --to-ref {1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) || '' }}

From 20df20ae514d24c66d84d6e3b3033b8b41f477f9 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 12 Oct 2024 16:53:11 -0700
Subject: [PATCH 099/376] Multi tenant vespa (#2762)

* add vespa multi tenancy

* k

* formatting

* Billing (#2667)

* k

* data -> control

* nit

* nit: error handling

* auth + app

* nit: color standardization

* nit

* nit: typing

* k

* k

* feat: functional upgrading

* feat: add block for downgrading to seats < active users

* add auth

* remove accomplished todo + prints

* nit

* tiny nit

* nit: centralize security

* add tenant expulsion/gating + invite user -> increment billing seat no.

* add cloud configs

* k

* k

* nit: update

* k

* k

* k

* k

* nit
---
 backend/Dockerfile                            |   1 +
 backend/danswer/auth/users.py                 |  33 +-
 backend/danswer/background/update.py          |  10 +-
 backend/danswer/configs/app_configs.py        |  24 +-
 backend/danswer/db/auth.py                    |  16 +-
 backend/danswer/db/search_settings.py         |   4 +-
 backend/danswer/db/swap_index.py              |  11 +-
 backend/danswer/document_index/interfaces.py  |  11 +
 .../vespa/app_config/schemas/danswer_chunk.sd |   1 +
 backend/danswer/document_index/vespa/index.py | 262 ++++++++-
 .../document_index/vespa/indexing_utils.py    |  11 +-
 .../shared_utils/vespa_request_builders.py    |   4 +
 .../danswer/document_index/vespa_constants.py |   9 +-
 backend/danswer/main.py                       |   5 +-
 backend/danswer/search/models.py              |   1 +
 .../search/preprocessing/preprocessing.py     |   3 +
 backend/danswer/server/auth_check.py          |   2 +-
 backend/danswer/server/manage/users.py        |  67 ++-
 backend/danswer/server/settings/models.py     |   7 +
 backend/danswer/server/utils.py               |  22 +-
 backend/danswer/setup.py                      |  38 ++
 backend/ee/danswer/configs/app_configs.py     |   4 +
 backend/ee/danswer/main.py                    |   6 +-
 backend/ee/danswer/server/tenants/access.py   |  53 ++
 backend/ee/danswer/server/tenants/api.py      |  78 ++-
 backend/ee/danswer/server/tenants/billing.py  |  69 +++
 backend/ee/danswer/server/tenants/models.py   |  23 +
 backend/requirements/default.txt              |   1 +
 backend/shared_configs/configs.py             |  73 +++
 backend/shared_configs/model_server_models.py |   6 +
 .../docker-compose.prod-cloud.yml             | 243 ++++++++
 web/package-lock.json                         | 554 ++----------------
 web/package.json                              |   5 +-
 web/src/app/admin/settings/interfaces.ts      |   7 +
 .../cloud-settings/BillingInformationPage.tsx | 222 +++++++
 web/src/app/ee/admin/cloud-settings/page.tsx  |  23 +
 web/src/app/ee/admin/cloud-settings/utils.ts  |  46 ++
 web/src/app/layout.tsx                        |  57 +-
 web/src/components/admin/ClientLayout.tsx     |  19 +
 web/src/components/admin/Layout.tsx           |   6 +-
 web/src/components/admin/Title.tsx            |   1 +
 web/src/components/settings/lib.ts            |   2 +
 web/src/lib/constants.ts                      |   6 +-
 web/src/middleware.ts                         |  10 +-
 44 files changed, 1456 insertions(+), 600 deletions(-)
 create mode 100644 backend/ee/danswer/server/tenants/billing.py
 create mode 100644 deployment/docker_compose/docker-compose.prod-cloud.yml
 create mode 100644 web/src/app/ee/admin/cloud-settings/BillingInformationPage.tsx
 create mode 100644 web/src/app/ee/admin/cloud-settings/page.tsx
 create mode 100644 web/src/app/ee/admin/cloud-settings/utils.ts

diff --git a/backend/Dockerfile b/backend/Dockerfile
index 17f93979955..f7ea1e3e1d4 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -92,6 +92,7 @@ COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 COPY ./danswer /app/danswer
 COPY ./shared_configs /app/shared_configs
 COPY ./alembic /app/alembic
+COPY ./alembic_tenants /app/alembic_tenants
 COPY ./alembic.ini /app/alembic.ini
 COPY supervisord.conf /usr/etc/supervisord.conf
 
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 3fc117b31a0..983e17182b0 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -10,6 +10,7 @@
 
 import jwt
 from email_validator import EmailNotValidError
+from email_validator import EmailUndeliverableError
 from email_validator import validate_email
 from fastapi import APIRouter
 from fastapi import Depends
@@ -41,10 +42,8 @@
 from danswer.auth.schemas import UserRole
 from danswer.auth.schemas import UserUpdate
 from danswer.configs.app_configs import AUTH_TYPE
-from danswer.configs.app_configs import DATA_PLANE_SECRET
 from danswer.configs.app_configs import DISABLE_AUTH
 from danswer.configs.app_configs import EMAIL_FROM
-from danswer.configs.app_configs import EXPECTED_API_KEY
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
 from danswer.configs.app_configs import SECRET_JWT_KEY
@@ -129,7 +128,10 @@ def verify_email_is_invited(email: str) -> None:
     if not email:
         raise PermissionError("Email must be specified")
 
-    email_info = validate_email(email)  # can raise EmailNotValidError
+    try:
+        email_info = validate_email(email)
+    except EmailUndeliverableError:
+        raise PermissionError("Email is not valid")
 
     for email_whitelist in whitelist:
         try:
@@ -652,28 +654,3 @@ async def current_admin_user(user: User | None = Depends(current_user)) -> User
 def get_default_admin_user_emails_() -> list[str]:
     # No default seeding available for Danswer MIT
     return []
-
-
-async def control_plane_dep(request: Request) -> None:
-    api_key = request.headers.get("X-API-KEY")
-    if api_key != EXPECTED_API_KEY:
-        logger.warning("Invalid API key")
-        raise HTTPException(status_code=401, detail="Invalid API key")
-
-    auth_header = request.headers.get("Authorization")
-    if not auth_header or not auth_header.startswith("Bearer "):
-        logger.warning("Invalid authorization header")
-        raise HTTPException(status_code=401, detail="Invalid authorization header")
-
-    token = auth_header.split(" ")[1]
-    try:
-        payload = jwt.decode(token, DATA_PLANE_SECRET, algorithms=["HS256"])
-        if payload.get("scope") != "tenant:create":
-            logger.warning("Insufficient permissions")
-            raise HTTPException(status_code=403, detail="Insufficient permissions")
-    except jwt.ExpiredSignatureError:
-        logger.warning("Token has expired")
-        raise HTTPException(status_code=401, detail="Token has expired")
-    except jwt.InvalidTokenError:
-        logger.warning("Invalid token")
-        raise HTTPException(status_code=401, detail="Invalid token")
diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index 58c31236eae..075035c46d7 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -42,6 +42,7 @@
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.search_settings import get_secondary_search_settings
 from danswer.db.swap_index import check_index_swap
+from danswer.document_index.vespa.index import VespaIndex
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
 from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.utils.logger import setup_logger
@@ -484,7 +485,14 @@ def update_loop(
                         f"Processing {'index attempts' if tenant_id is None else f'tenant {tenant_id}'}"
                     )
                     with get_session_with_tenant(tenant_id) as db_session:
-                        check_index_swap(db_session=db_session)
+                        index_to_expire = check_index_swap(db_session=db_session)
+
+                        if index_to_expire and tenant_id and MULTI_TENANT:
+                            VespaIndex.delete_entries_by_tenant_id(
+                                tenant_id=tenant_id,
+                                index_name=index_to_expire.index_name,
+                            )
+
                         if not MULTI_TENANT:
                             search_settings = get_current_search_settings(db_session)
                             if search_settings.provider_type is None:
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 04925262196..a061b79019f 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -423,11 +423,27 @@
 AZURE_DALLE_DEPLOYMENT_NAME = os.environ.get("AZURE_DALLE_DEPLOYMENT_NAME")
 
 
+# Cloud configuration
+
+# Multi-tenancy configuration
 MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
-SECRET_JWT_KEY = os.environ.get("SECRET_JWT_KEY", "")
+ENABLE_EMAIL_INVITES = os.environ.get("ENABLE_EMAIL_INVITES", "").lower() == "true"
 
+# Security and authentication
+SECRET_JWT_KEY = os.environ.get(
+    "SECRET_JWT_KEY", ""
+)  # Used for encryption of the JWT token for user's tenant context
+DATA_PLANE_SECRET = os.environ.get(
+    "DATA_PLANE_SECRET", ""
+)  # Used for secure communication between the control and data plane
+EXPECTED_API_KEY = os.environ.get(
+    "EXPECTED_API_KEY", ""
+)  # Additional security check for the control plane API
 
-DATA_PLANE_SECRET = os.environ.get("DATA_PLANE_SECRET", "")
-EXPECTED_API_KEY = os.environ.get("EXPECTED_API_KEY", "")
+# API configuration
+CONTROL_PLANE_API_BASE_URL = os.environ.get(
+    "CONTROL_PLANE_API_BASE_URL", "http://localhost:8082"
+)
 
-ENABLE_EMAIL_INVITES = os.environ.get("ENABLE_EMAIL_INVITES", "").lower() == "true"
+# JWT configuration
+JWT_ALGORITHM = "HS256"
diff --git a/backend/danswer/db/auth.py b/backend/danswer/db/auth.py
index dc3f5a837bd..9eba3806df5 100644
--- a/backend/danswer/db/auth.py
+++ b/backend/danswer/db/auth.py
@@ -10,7 +10,9 @@
 from sqlalchemy import func
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
+from sqlalchemy.orm import Session
 
+from danswer.auth.invited_users import get_invited_users
 from danswer.auth.schemas import UserRole
 from danswer.db.engine import get_async_session
 from danswer.db.engine import get_async_session_with_tenant
@@ -33,10 +35,20 @@ def get_default_admin_user_emails() -> list[str]:
     return get_default_admin_user_emails_fn()
 
 
+def get_total_users(db_session: Session) -> int:
+    """
+    Returns the total number of users in the system.
+    This is the sum of users and invited users.
+    """
+    user_count = db_session.query(User).count()
+    invited_users = len(get_invited_users())
+    return user_count + invited_users
+
+
 async def get_user_count() -> int:
-    async with get_async_session_with_tenant() as asession:
+    async with get_async_session_with_tenant() as session:
         stmt = select(func.count(User.id))
-        result = await asession.execute(stmt)
+        result = await session.execute(stmt)
         user_count = result.scalar()
         if user_count is None:
             raise RuntimeError("Was not able to fetch the user count.")
diff --git a/backend/danswer/db/search_settings.py b/backend/danswer/db/search_settings.py
index e3f35e31007..5392ec23411 100644
--- a/backend/danswer/db/search_settings.py
+++ b/backend/danswer/db/search_settings.py
@@ -12,7 +12,7 @@
 from danswer.configs.model_configs import OLD_DEFAULT_DOCUMENT_ENCODER_MODEL
 from danswer.configs.model_configs import OLD_DEFAULT_MODEL_DOC_EMBEDDING_DIM
 from danswer.configs.model_configs import OLD_DEFAULT_MODEL_NORMALIZE_EMBEDDINGS
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.llm import fetch_embedding_provider
 from danswer.db.models import CloudEmbeddingProvider
 from danswer.db.models import IndexAttempt
@@ -152,7 +152,7 @@ def get_all_search_settings(db_session: Session) -> list[SearchSettings]:
 
 def get_multilingual_expansion(db_session: Session | None = None) -> list[str]:
     if db_session is None:
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant() as db_session:
             search_settings = get_current_search_settings(db_session)
     else:
         search_settings = get_current_search_settings(db_session)
diff --git a/backend/danswer/db/swap_index.py b/backend/danswer/db/swap_index.py
index a11db4dd693..415ade5df00 100644
--- a/backend/danswer/db/swap_index.py
+++ b/backend/danswer/db/swap_index.py
@@ -1,5 +1,6 @@
 from sqlalchemy.orm import Session
 
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.connector_credential_pair import resync_cc_pair
@@ -8,16 +9,18 @@
 from danswer.db.index_attempt import (
     count_unique_cc_pairs_with_successful_index_attempts,
 )
+from danswer.db.models import SearchSettings
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.search_settings import get_secondary_search_settings
 from danswer.db.search_settings import update_search_settings_status
 from danswer.key_value_store.factory import get_kv_store
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 
-def check_index_swap(db_session: Session) -> None:
+def check_index_swap(db_session: Session) -> SearchSettings | None:
     """Get count of cc-pairs and count of successful index_attempts for the
     new model grouped by connector + credential, if it's the same, then assume
     new index is done building. If so, swap the indices and expire the old one."""
@@ -27,7 +30,7 @@ def check_index_swap(db_session: Session) -> None:
     search_settings = get_secondary_search_settings(db_session)
 
     if not search_settings:
-        return
+        return None
 
     unique_cc_indexings = count_unique_cc_pairs_with_successful_index_attempts(
         search_settings_id=search_settings.id, db_session=db_session
@@ -63,3 +66,7 @@ def check_index_swap(db_session: Session) -> None:
             # Recount aggregates
             for cc_pair in all_cc_pairs:
                 resync_cc_pair(cc_pair, db_session=db_session)
+
+            if MULTI_TENANT:
+                return now_old_search_settings
+    return None
diff --git a/backend/danswer/document_index/interfaces.py b/backend/danswer/document_index/interfaces.py
index 42d763d0c98..07c1b24ab2e 100644
--- a/backend/danswer/document_index/interfaces.py
+++ b/backend/danswer/document_index/interfaces.py
@@ -127,6 +127,17 @@ def ensure_indices_exist(
         """
         raise NotImplementedError
 
+    @staticmethod
+    @abc.abstractmethod
+    def register_multitenant_indices(
+        indices: list[str],
+        embedding_dims: list[int],
+    ) -> None:
+        """
+        Register multitenant indices with the document index.
+        """
+        raise NotImplementedError
+
 
 class Indexable(abc.ABC):
     """
diff --git a/backend/danswer/document_index/vespa/app_config/schemas/danswer_chunk.sd b/backend/danswer/document_index/vespa/app_config/schemas/danswer_chunk.sd
index be279f6a611..b98c9343f3f 100644
--- a/backend/danswer/document_index/vespa/app_config/schemas/danswer_chunk.sd
+++ b/backend/danswer/document_index/vespa/app_config/schemas/danswer_chunk.sd
@@ -1,5 +1,6 @@
 schema DANSWER_CHUNK_NAME {
     document DANSWER_CHUNK_NAME {
+        TENANT_ID_REPLACEMENT
         # Not to be confused with the UUID generated for this chunk which is called documentid by default
         field document_id type string {
             indexing: summary | attribute
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 44a5918d756..fd546307674 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -4,17 +4,20 @@
 import os
 import re
 import time
+import urllib
 import zipfile
 from dataclasses import dataclass
 from datetime import datetime
 from datetime import timedelta
 from typing import BinaryIO
 from typing import cast
+from typing import List
 
-import httpx
-import requests
+import httpx  # type: ignore
+import requests  # type: ignore
 
 from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import VESPA_REQUEST_TIMEOUT
 from danswer.configs.chat_configs import DOC_TIME_DECAY
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
@@ -58,6 +61,8 @@
 from danswer.document_index.vespa_constants import HIDDEN
 from danswer.document_index.vespa_constants import NUM_THREADS
 from danswer.document_index.vespa_constants import SEARCH_THREAD_NUMBER_PAT
+from danswer.document_index.vespa_constants import TENANT_ID_PAT
+from danswer.document_index.vespa_constants import TENANT_ID_REPLACEMENT
 from danswer.document_index.vespa_constants import VESPA_APPLICATION_ENDPOINT
 from danswer.document_index.vespa_constants import VESPA_DIM_REPLACEMENT_PAT
 from danswer.document_index.vespa_constants import VESPA_TIMEOUT
@@ -70,6 +75,7 @@
 from danswer.utils.logger import setup_logger
 from shared_configs.model_server_models import Embedding
 
+
 logger = setup_logger()
 
 # Set the logging level to WARNING to ignore INFO and DEBUG logs
@@ -93,7 +99,7 @@ def in_memory_zip_from_file_bytes(file_contents: dict[str, bytes]) -> BinaryIO:
     return zip_buffer
 
 
-def _create_document_xml_lines(doc_names: list[str | None]) -> str:
+def _create_document_xml_lines(doc_names: list[str | None] | list[str]) -> str:
     doc_lines = [
         f'<document type="{doc_name}" mode="index" />'
         for doc_name in doc_names
@@ -127,6 +133,12 @@ def ensure_indices_exist(
         index_embedding_dim: int,
         secondary_index_embedding_dim: int | None,
     ) -> None:
+        if MULTI_TENANT:
+            logger.info(
+                "Skipping Vespa index seup for multitenant (would wipe all indices)"
+            )
+            return None
+
         deploy_url = f"{VESPA_APPLICATION_ENDPOINT}/tenant/default/prepareandactivate"
         logger.info(f"Deploying Vespa application package to {deploy_url}")
 
@@ -174,10 +186,14 @@ def ensure_indices_exist(
 
         with open(schema_file, "r") as schema_f:
             schema_template = schema_f.read()
+        schema_template = schema_template.replace(TENANT_ID_PAT, "")
+
         schema = schema_template.replace(
             DANSWER_CHUNK_REPLACEMENT_PAT, self.index_name
         ).replace(VESPA_DIM_REPLACEMENT_PAT, str(index_embedding_dim))
+
         schema = add_ngrams_to_schema(schema) if needs_reindexing else schema
+        schema = schema.replace(TENANT_ID_PAT, "")
         zip_dict[f"schemas/{schema_names[0]}.sd"] = schema.encode("utf-8")
 
         if self.secondary_index_name:
@@ -195,6 +211,91 @@ def ensure_indices_exist(
                 f"Failed to prepare Vespa Danswer Index. Response: {response.text}"
             )
 
+    @staticmethod
+    def register_multitenant_indices(
+        indices: list[str],
+        embedding_dims: list[int],
+    ) -> None:
+        if not MULTI_TENANT:
+            raise ValueError("Multi-tenant is not enabled")
+
+        deploy_url = f"{VESPA_APPLICATION_ENDPOINT}/tenant/default/prepareandactivate"
+        logger.info(f"Deploying Vespa application package to {deploy_url}")
+
+        vespa_schema_path = os.path.join(
+            os.getcwd(), "danswer", "document_index", "vespa", "app_config"
+        )
+        schema_file = os.path.join(vespa_schema_path, "schemas", "danswer_chunk.sd")
+        services_file = os.path.join(vespa_schema_path, "services.xml")
+        overrides_file = os.path.join(vespa_schema_path, "validation-overrides.xml")
+
+        with open(services_file, "r") as services_f:
+            services_template = services_f.read()
+
+        # Generate schema names from index settings
+        schema_names = [index_name for index_name in indices]
+
+        full_schemas = schema_names
+
+        doc_lines = _create_document_xml_lines(full_schemas)
+
+        services = services_template.replace(DOCUMENT_REPLACEMENT_PAT, doc_lines)
+        services = services.replace(
+            SEARCH_THREAD_NUMBER_PAT, str(VESPA_SEARCHER_THREADS)
+        )
+
+        kv_store = get_kv_store()
+
+        needs_reindexing = False
+        try:
+            needs_reindexing = cast(bool, kv_store.load(KV_REINDEX_KEY))
+        except Exception:
+            logger.debug("Could not load the reindexing flag. Using ngrams")
+
+        with open(overrides_file, "r") as overrides_f:
+            overrides_template = overrides_f.read()
+
+        # Vespa requires an override to erase data including the indices we're no longer using
+        # It also has a 30 day cap from current so we set it to 7 dynamically
+        now = datetime.now()
+        date_in_7_days = now + timedelta(days=7)
+        formatted_date = date_in_7_days.strftime("%Y-%m-%d")
+
+        overrides = overrides_template.replace(DATE_REPLACEMENT, formatted_date)
+
+        zip_dict = {
+            "services.xml": services.encode("utf-8"),
+            "validation-overrides.xml": overrides.encode("utf-8"),
+        }
+
+        with open(schema_file, "r") as schema_f:
+            schema_template = schema_f.read()
+
+        for i, index_name in enumerate(indices):
+            embedding_dim = embedding_dims[i]
+            logger.info(
+                f"Creating index: {index_name} with embedding dimension: {embedding_dim}"
+            )
+
+            schema = schema_template.replace(
+                DANSWER_CHUNK_REPLACEMENT_PAT, index_name
+            ).replace(VESPA_DIM_REPLACEMENT_PAT, str(embedding_dim))
+            schema = schema.replace(
+                TENANT_ID_PAT, TENANT_ID_REPLACEMENT if MULTI_TENANT else ""
+            )
+            schema = add_ngrams_to_schema(schema) if needs_reindexing else schema
+            zip_dict[f"schemas/{index_name}.sd"] = schema.encode("utf-8")
+
+        zip_file = in_memory_zip_from_file_bytes(zip_dict)
+
+        headers = {"Content-Type": "application/zip"}
+        response = requests.post(deploy_url, headers=headers, data=zip_file)
+
+        if response.status_code != 200:
+            raise RuntimeError(
+                f"Failed to prepare Vespa Danswer Indexes. Response: {response.text}"
+            )
+
     def index(
         self,
         chunks: list[DocMetadataAwareIndexChunk],
@@ -644,3 +745,158 @@ def admin_retrieval(
         }
 
         return query_vespa(params)
+
+    @classmethod
+    def delete_entries_by_tenant_id(cls, tenant_id: str, index_name: str) -> None:
+        """
+        Deletes all entries in the specified index with the given tenant_id.
+
+        Parameters:
+            tenant_id (str): The tenant ID whose documents are to be deleted.
+            index_name (str): The name of the index from which to delete documents.
+        """
+        logger.info(
+            f"Deleting entries with tenant_id: {tenant_id} from index: {index_name}"
+        )
+
+        # Step 1: Retrieve all document IDs with the given tenant_id
+        document_ids = cls._get_all_document_ids_by_tenant_id(tenant_id, index_name)
+
+        if not document_ids:
+            logger.info(
+                f"No documents found with tenant_id: {tenant_id} in index: {index_name}"
+            )
+            return
+
+        # Step 2: Delete documents in batches
+        delete_requests = [
+            _VespaDeleteRequest(document_id=doc_id, index_name=index_name)
+            for doc_id in document_ids
+        ]
+
+        cls._apply_deletes_batched(delete_requests)
+
+    @classmethod
+    def _get_all_document_ids_by_tenant_id(
+        cls, tenant_id: str, index_name: str
+    ) -> List[str]:
+        """
+        Retrieves all document IDs with the specified tenant_id, handling pagination.
+
+        Parameters:
+            tenant_id (str): The tenant ID to search for.
+            index_name (str): The name of the index to search in.
+
+        Returns:
+            List[str]: A list of document IDs matching the tenant_id.
+        """
+        offset = 0
+        limit = 1000  # Vespa's maximum hits per query
+        document_ids = []
+
+        logger.debug(
+            f"Starting document ID retrieval for tenant_id: {tenant_id} in index: {index_name}"
+        )
+
+        while True:
+            # Construct the query to fetch document IDs
+            query_params = {
+                "yql": f'select id from sources * where tenant_id contains "{tenant_id}";',
+                "offset": str(offset),
+                "hits": str(limit),
+                "timeout": "10s",
+                "format": "json",
+                "summary": "id",
+            }
+
+            url = f"{VESPA_APPLICATION_ENDPOINT}/search/"
+
+            logger.debug(
+                f"Querying for document IDs with tenant_id: {tenant_id}, offset: {offset}"
+            )
+
+            with httpx.Client(http2=True) as http_client:
+                response = http_client.get(url, params=query_params)
+                response.raise_for_status()
+
+                search_result = response.json()
+                hits = search_result.get("root", {}).get("children", [])
+
+                if not hits:
+                    break
+
+                for hit in hits:
+                    doc_id = hit.get("id")
+                    if doc_id:
+                        document_ids.append(doc_id)
+
+                offset += limit  # Move to the next page
+
+        logger.debug(
+            f"Retrieved {len(document_ids)} document IDs for tenant_id: {tenant_id}"
+        )
+        return document_ids
+
+    @classmethod
+    def _apply_deletes_batched(
+        cls,
+        delete_requests: List["_VespaDeleteRequest"],
+        batch_size: int = BATCH_SIZE,
+    ) -> None:
+        """
+        Deletes documents in batches using multiple threads.
+
+        Parameters:
+            delete_requests (List[_VespaDeleteRequest]): The list of delete requests.
+            batch_size (int): The number of documents to delete in each batch.
+        """
+
+        def _delete_document(
+            delete_request: "_VespaDeleteRequest", http_client: httpx.Client
+        ) -> None:
+            logger.debug(f"Deleting document with ID {delete_request.document_id}")
+            response = http_client.delete(
+                delete_request.url,
+                headers={"Content-Type": "application/json"},
+            )
+            response.raise_for_status()
+
+        logger.debug(f"Starting batch deletion for {len(delete_requests)} documents")
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
+            with httpx.Client(http2=True) as http_client:
+                for batch_start in range(0, len(delete_requests), batch_size):
+                    batch = delete_requests[batch_start : batch_start + batch_size]
+
+                    future_to_document_id = {
+                        executor.submit(
+                            _delete_document,
+                            delete_request,
+                            http_client,
+                        ): delete_request.document_id
+                        for delete_request in batch
+                    }
+
+                    for future in concurrent.futures.as_completed(
+                        future_to_document_id
+                    ):
+                        doc_id = future_to_document_id[future]
+                        try:
+                            future.result()
+                            logger.debug(f"Successfully deleted document: {doc_id}")
+                        except httpx.HTTPError as e:
+                            logger.error(f"Failed to delete document {doc_id}: {e}")
+                            # Optionally, implement retry logic or error handling here
+
+        logger.info("Batch deletion completed")
+
+
+class _VespaDeleteRequest:
+    def __init__(self, document_id: str, index_name: str) -> None:
+        self.document_id = document_id
+        # Encode the document ID to ensure it's safe for use in the URL
+        encoded_doc_id = urllib.parse.quote_plus(self.document_id)
+        self.url = (
+            f"{VESPA_APPLICATION_ENDPOINT}/document/v1/"
+            f"{index_name}/{index_name}/docid/{encoded_doc_id}"
+        )
diff --git a/backend/danswer/document_index/vespa/indexing_utils.py b/backend/danswer/document_index/vespa/indexing_utils.py
index 6b6ba8709d5..de0d5fdaf19 100644
--- a/backend/danswer/document_index/vespa/indexing_utils.py
+++ b/backend/danswer/document_index/vespa/indexing_utils.py
@@ -37,6 +37,7 @@
 from danswer.document_index.vespa_constants import SKIP_TITLE_EMBEDDING
 from danswer.document_index.vespa_constants import SOURCE_LINKS
 from danswer.document_index.vespa_constants import SOURCE_TYPE
+from danswer.document_index.vespa_constants import TENANT_ID
 from danswer.document_index.vespa_constants import TITLE
 from danswer.document_index.vespa_constants import TITLE_EMBEDDING
 from danswer.indexing.models import DocMetadataAwareIndexChunk
@@ -65,6 +66,8 @@ def _does_document_exist(
         raise RuntimeError(
             f"Unexpected fetch document by ID value from Vespa "
             f"with error {doc_fetch_response.status_code}"
+            f"Index name: {index_name}"
+            f"Doc chunk id: {doc_chunk_id}"
         )
     return True
 
@@ -117,7 +120,9 @@ def get_existing_documents_from_chunks(
 
 @retry(tries=3, delay=1, backoff=2)
 def _index_vespa_chunk(
-    chunk: DocMetadataAwareIndexChunk, index_name: str, http_client: httpx.Client
+    chunk: DocMetadataAwareIndexChunk,
+    index_name: str,
+    http_client: httpx.Client,
 ) -> None:
     json_header = {
         "Content-Type": "application/json",
@@ -174,8 +179,10 @@ def _index_vespa_chunk(
         BOOST: chunk.boost,
     }
 
+    if chunk.tenant_id:
+        vespa_document_fields[TENANT_ID] = chunk.tenant_id
+
     vespa_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{vespa_chunk_id}"
-    logger.debug(f'Indexing to URL "{vespa_url}"')
     res = http_client.post(
         vespa_url, headers=json_header, json={"fields": vespa_document_fields}
     )
diff --git a/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py b/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py
index 65752aa09c1..e7b778f1c84 100644
--- a/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py
+++ b/backend/danswer/document_index/vespa/shared_utils/vespa_request_builders.py
@@ -12,6 +12,7 @@
 from danswer.document_index.vespa_constants import HIDDEN
 from danswer.document_index.vespa_constants import METADATA_LIST
 from danswer.document_index.vespa_constants import SOURCE_TYPE
+from danswer.document_index.vespa_constants import TENANT_ID
 from danswer.search.models import IndexFilters
 from danswer.utils.logger import setup_logger
 
@@ -53,6 +54,9 @@ def _build_time_filter(
 
     filter_str = f"!({HIDDEN}=true) and " if not include_hidden else ""
 
+    if filters.tenant_id:
+        filter_str += f'({TENANT_ID} contains "{filters.tenant_id}") and '
+
     # CAREFUL touching this one, currently there is no second ACL double-check post retrieval
     if filters.access_control_list is not None:
         filter_str += _build_or_filters(
diff --git a/backend/danswer/document_index/vespa_constants.py b/backend/danswer/document_index/vespa_constants.py
index 8409efe1dea..a4d6aa52e2f 100644
--- a/backend/danswer/document_index/vespa_constants.py
+++ b/backend/danswer/document_index/vespa_constants.py
@@ -9,7 +9,14 @@
 DOCUMENT_REPLACEMENT_PAT = "DOCUMENT_REPLACEMENT"
 SEARCH_THREAD_NUMBER_PAT = "SEARCH_THREAD_NUMBER"
 DATE_REPLACEMENT = "DATE_REPLACEMENT"
+SEARCH_THREAD_NUMBER_PAT = "SEARCH_THREAD_NUMBER"
+TENANT_ID_PAT = "TENANT_ID_REPLACEMENT"
 
+TENANT_ID_REPLACEMENT = """field tenant_id type string {
+            indexing: summary | attribute
+            rank: filter
+            attribute: fast-search
+        }"""
 # config server
 VESPA_CONFIG_SERVER_URL = f"http://{VESPA_CONFIG_SERVER_HOST}:{VESPA_TENANT_PORT}"
 VESPA_APPLICATION_ENDPOINT = f"{VESPA_CONFIG_SERVER_URL}/application/v2"
@@ -35,7 +42,7 @@
 VESPA_TIMEOUT = "3s"
 BATCH_SIZE = 128  # Specific to Vespa
 
-
+TENANT_ID = "tenant_id"
 DOCUMENT_ID = "document_id"
 CHUNK_ID = "chunk_id"
 BLURB = "blurb"
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index cd0c5c195a6..151f852486c 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -81,6 +81,7 @@
     router as token_rate_limit_settings_router,
 )
 from danswer.setup import setup_danswer
+from danswer.setup import setup_multitenant_danswer
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import get_or_generate_uuid
 from danswer.utils.telemetry import optional_telemetry
@@ -175,10 +176,12 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
         # We cache this at the beginning so there is no delay in the first telemetry
         get_or_generate_uuid()
 
-        # If we are multi-tenant, we need to only set up initial public tables
         with Session(engine) as db_session:
             setup_danswer(db_session)
 
+    else:
+        setup_multitenant_danswer()
+
     optional_telemetry(record_type=RecordType.VERSION, data={"version": __version__})
     yield
 
diff --git a/backend/danswer/search/models.py b/backend/danswer/search/models.py
index 503b07653ef..815fa9d885f 100644
--- a/backend/danswer/search/models.py
+++ b/backend/danswer/search/models.py
@@ -102,6 +102,7 @@ class BaseFilters(BaseModel):
 
 class IndexFilters(BaseFilters):
     access_control_list: list[str] | None
+    tenant_id: str | None = None
 
 
 class ChunkMetric(BaseModel):
diff --git a/backend/danswer/search/preprocessing/preprocessing.py b/backend/danswer/search/preprocessing/preprocessing.py
index 37fb254884a..aa3124617e5 100644
--- a/backend/danswer/search/preprocessing/preprocessing.py
+++ b/backend/danswer/search/preprocessing/preprocessing.py
@@ -1,5 +1,6 @@
 from sqlalchemy.orm import Session
 
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.chat_configs import BASE_RECENCY_DECAY
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_BELOW
@@ -9,6 +10,7 @@
 from danswer.configs.chat_configs import HYBRID_ALPHA_KEYWORD
 from danswer.configs.chat_configs import NUM_POSTPROCESSED_RESULTS
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
+from danswer.db.engine import current_tenant_id
 from danswer.db.models import User
 from danswer.db.search_settings import get_current_search_settings
 from danswer.llm.interfaces import LLM
@@ -160,6 +162,7 @@ def retrieval_preprocessing(
         time_cutoff=time_filter or predicted_time_cutoff,
         tags=preset_filters.tags,  # Tags are never auto-extracted
         access_control_list=user_acl_filters,
+        tenant_id=current_tenant_id.get() if MULTI_TENANT else None,
     )
 
     llm_evaluation_type = LLMEvaluationType.BASIC
diff --git a/backend/danswer/server/auth_check.py b/backend/danswer/server/auth_check.py
index c79b9ad0967..69aede4241f 100644
--- a/backend/danswer/server/auth_check.py
+++ b/backend/danswer/server/auth_check.py
@@ -4,13 +4,13 @@
 from fastapi.dependencies.models import Dependant
 from starlette.routing import BaseRoute
 
-from danswer.auth.users import control_plane_dep
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
 from danswer.auth.users import current_user_with_expired_token
 from danswer.configs.app_configs import APP_API_PREFIX
 from danswer.server.danswer_api.ingestion import api_key_dep
+from ee.danswer.server.tenants.access import control_plane_dep
 
 
 PUBLIC_ENDPOINT_SPECS = [
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index 0614a4beb85..ae2ab8c6e8c 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -3,6 +3,8 @@
 from datetime import timezone
 
 import jwt
+from email_validator import EmailNotValidError
+from email_validator import EmailUndeliverableError
 from email_validator import validate_email
 from fastapi import APIRouter
 from fastapi import Body
@@ -35,6 +37,7 @@
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
+from danswer.db.auth import get_total_users
 from danswer.db.engine import current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.models import AccessToken
@@ -60,6 +63,7 @@
 from ee.danswer.db.api_key import is_api_key_email_address
 from ee.danswer.db.external_perm import delete_user__ext_group_for_user__no_commit
 from ee.danswer.db.user_group import remove_curator_status__no_commit
+from ee.danswer.server.tenants.billing import register_tenant_users
 from ee.danswer.server.tenants.provisioning import add_users_to_tenant
 from ee.danswer.server.tenants.provisioning import remove_users_from_tenant
 
@@ -174,19 +178,29 @@ def list_all_users(
 def bulk_invite_users(
     emails: list[str] = Body(..., embed=True),
     current_user: User | None = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
 ) -> int:
     """emails are string validated. If any email fails validation, no emails are
     invited and an exception is raised."""
+
     if current_user is None:
         raise HTTPException(
             status_code=400, detail="Auth is disabled, cannot invite users"
         )
+
     tenant_id = current_tenant_id.get()
 
     normalized_emails = []
-    for email in emails:
-        email_info = validate_email(email)  # can raise EmailNotValidError
-        normalized_emails.append(email_info.normalized)  # type: ignore
+    try:
+        for email in emails:
+            email_info = validate_email(email)
+            normalized_emails.append(email_info.normalized)  # type: ignore
+
+    except (EmailUndeliverableError, EmailNotValidError):
+        raise HTTPException(
+            status_code=400,
+            detail="One or more emails in the list are invalid",
+        )
 
     if MULTI_TENANT:
         try:
@@ -199,30 +213,58 @@ def bulk_invite_users(
                 )
             raise
 
-    all_emails = list(set(normalized_emails) | set(get_invited_users()))
+    initial_invited_users = get_invited_users()
 
-    if MULTI_TENANT and ENABLE_EMAIL_INVITES:
-        try:
-            for email in all_emails:
-                send_user_email_invite(email, current_user)
-        except Exception as e:
-            logger.error(f"Error sending email invite to invited users: {e}")
+    all_emails = list(set(normalized_emails) | set(initial_invited_users))
+    number_of_invited_users = write_invited_users(all_emails)
 
-    return write_invited_users(all_emails)
+    if not MULTI_TENANT:
+        return number_of_invited_users
+    try:
+        logger.info("Registering tenant users")
+        register_tenant_users(current_tenant_id.get(), get_total_users(db_session))
+        if ENABLE_EMAIL_INVITES:
+            try:
+                for email in all_emails:
+                    send_user_email_invite(email, current_user)
+            except Exception as e:
+                logger.error(f"Error sending email invite to invited users: {e}")
+
+        return number_of_invited_users
+    except Exception as e:
+        logger.error(f"Failed to register tenant users: {str(e)}")
+        logger.info(
+            "Reverting changes: removing users from tenant and resetting invited users"
+        )
+        write_invited_users(initial_invited_users)  # Reset to original state
+        remove_users_from_tenant(normalized_emails, tenant_id)
+        raise e
 
 
 @router.patch("/manage/admin/remove-invited-user")
 def remove_invited_user(
     user_email: UserByEmail,
     _: User | None = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
 ) -> int:
     user_emails = get_invited_users()
     remaining_users = [user for user in user_emails if user != user_email.user_email]
 
     tenant_id = current_tenant_id.get()
     remove_users_from_tenant([user_email.user_email], tenant_id)
+    number_of_invited_users = write_invited_users(remaining_users)
+
+    try:
+        if MULTI_TENANT:
+            register_tenant_users(current_tenant_id.get(), get_total_users(db_session))
+    except Exception:
+        logger.error(
+            "Request to update number of seats taken in control plane failed. "
+            "This may cause synchronization issues/out of date enforcement of seat limits."
+        )
+        raise
 
-    return write_invited_users(remaining_users)
+    return number_of_invited_users
 
 
 @router.patch("/manage/admin/deactivate-user")
@@ -421,7 +463,6 @@ def get_current_token_creation(
 
 @router.get("/me")
 def verify_user_logged_in(
-    request: Request,
     user: User | None = Depends(optional_user),
     db_session: Session = Depends(get_session),
 ) -> UserInfo:
diff --git a/backend/danswer/server/settings/models.py b/backend/danswer/server/settings/models.py
index ae7e7236c8d..6713f7f67e8 100644
--- a/backend/danswer/server/settings/models.py
+++ b/backend/danswer/server/settings/models.py
@@ -12,6 +12,12 @@ class PageType(str, Enum):
     SEARCH = "search"
 
 
+class GatingType(str, Enum):
+    FULL = "full"  # Complete restriction of access to the product or service
+    PARTIAL = "partial"  # Full access but warning (no credit card on file)
+    NONE = "none"  # No restrictions, full access to all features
+
+
 class Notification(BaseModel):
     id: int
     notif_type: NotificationType
@@ -38,6 +44,7 @@ class Settings(BaseModel):
     default_page: PageType = PageType.SEARCH
     maximum_chat_retention_days: int | None = None
     gpu_enabled: bool | None = None
+    product_gating: GatingType = GatingType.NONE
 
     def check_validity(self) -> None:
         chat_page_enabled = self.chat_page_enabled
diff --git a/backend/danswer/server/utils.py b/backend/danswer/server/utils.py
index 70404537f70..68e6dc8d0b8 100644
--- a/backend/danswer/server/utils.py
+++ b/backend/danswer/server/utils.py
@@ -3,6 +3,7 @@
 from datetime import datetime
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
+from textwrap import dedent
 from typing import Any
 
 from danswer.configs.app_configs import SMTP_PASS
@@ -58,22 +59,25 @@ def mask_credential_dict(credential_dict: dict[str, Any]) -> dict[str, str]:
 def send_user_email_invite(user_email: str, current_user: User) -> None:
     msg = MIMEMultipart()
     msg["Subject"] = "Invitation to Join Danswer Workspace"
-    msg["To"] = user_email
     msg["From"] = current_user.email
+    msg["To"] = user_email
 
-    email_body = f"""
-Hello,
+    email_body = dedent(
+        f"""\
+        Hello,
 
-You have been invited to join a workspace on Danswer.
+        You have been invited to join a workspace on Danswer.
 
-To join the workspace, please do so at the following link:
-{WEB_DOMAIN}/auth/login
+        To join the workspace, please visit the following link:
 
-Best regards,
-The Danswer Team"""
+        {WEB_DOMAIN}/auth/login
 
-    msg.attach(MIMEText(email_body, "plain"))
+        Best regards,
+        The Danswer Team
+    """
+    )
 
+    msg.attach(MIMEText(email_body, "plain"))
     with smtplib.SMTP(SMTP_SERVER, SMTP_PORT) as smtp_server:
         smtp_server.starttls()
         smtp_server.login(SMTP_USER, SMTP_PASS)
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index 443ab501d6b..84f0382f9ef 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -30,6 +30,7 @@
 from danswer.db.swap_index import check_index_swap
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import DocumentIndex
+from danswer.document_index.vespa.index import VespaIndex
 from danswer.indexing.models import IndexingSetting
 from danswer.key_value_store.factory import get_kv_store
 from danswer.key_value_store.interface import KvKeyNotFoundError
@@ -46,8 +47,11 @@
 from danswer.tools.built_in_tools import refresh_built_in_tools_cache
 from danswer.utils.gpu_utils import gpu_status_request
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import ALT_INDEX_SUFFIX
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
+from shared_configs.configs import SUPPORTED_EMBEDDING_MODELS
+from shared_configs.model_server_models import SupportedEmbeddingModel
 
 logger = setup_logger()
 
@@ -303,3 +307,37 @@ def update_default_multipass_indexing(db_session: Session) -> None:
         logger.debug(
             "Existing docs or connectors found. Skipping multipass indexing update."
         )
+
+
+def setup_multitenant_danswer() -> None:
+    setup_vespa_multitenant(SUPPORTED_EMBEDDING_MODELS)
+
+
+def setup_vespa_multitenant(supported_indices: list[SupportedEmbeddingModel]) -> bool:
+    WAIT_SECONDS = 5
+    VESPA_ATTEMPTS = 5
+    for x in range(VESPA_ATTEMPTS):
+        try:
+            logger.notice(f"Setting up Vespa (attempt {x+1}/{VESPA_ATTEMPTS})...")
+            VespaIndex.register_multitenant_indices(
+                indices=[index.index_name for index in supported_indices]
+                + [
+                    f"{index.index_name}{ALT_INDEX_SUFFIX}"
+                    for index in supported_indices
+                ],
+                embedding_dims=[index.dim for index in supported_indices]
+                + [index.dim for index in supported_indices],
+            )
+
+            logger.notice("Vespa setup complete.")
+            return True
+        except Exception:
+            logger.notice(
+                f"Vespa setup did not succeed. The Vespa service may not be ready yet. Retrying in {WAIT_SECONDS} seconds."
+            )
+            time.sleep(WAIT_SECONDS)
+
+    logger.error(
+        f"Vespa setup did not succeed. Attempt limit reached. ({VESPA_ATTEMPTS})"
+    )
+    return False
diff --git a/backend/ee/danswer/configs/app_configs.py b/backend/ee/danswer/configs/app_configs.py
index 1430a499136..2c782241076 100644
--- a/backend/ee/danswer/configs/app_configs.py
+++ b/backend/ee/danswer/configs/app_configs.py
@@ -21,3 +21,7 @@
 # Auto Permission Sync
 #####
 NUM_PERMISSION_WORKERS = int(os.environ.get("NUM_PERMISSION_WORKERS") or 2)
+
+
+STRIPE_SECRET_KEY = os.environ.get("STRIPE_SECRET_KEY")
+STRIPE_PRICE_ID = os.environ.get("STRIPE_PRICE")
diff --git a/backend/ee/danswer/main.py b/backend/ee/danswer/main.py
index e6483f75ae1..4584e06a00b 100644
--- a/backend/ee/danswer/main.py
+++ b/backend/ee/danswer/main.py
@@ -85,8 +85,6 @@ def get_application() -> FastAPI:
 
     # RBAC / group access control
     include_router_with_global_prefix_prepended(application, user_group_router)
-    # Tenant management
-    include_router_with_global_prefix_prepended(application, tenants_router)
     # Analytics endpoints
     include_router_with_global_prefix_prepended(application, analytics_router)
     include_router_with_global_prefix_prepended(application, query_history_router)
@@ -107,6 +105,10 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, enterprise_settings_router)
     include_router_with_global_prefix_prepended(application, usage_export_router)
 
+    if MULTI_TENANT:
+        # Tenant management
+        include_router_with_global_prefix_prepended(application, tenants_router)
+
     # Ensure all routes have auth enabled or are explicitly marked as public
     check_ee_router_auth(application)
 
diff --git a/backend/ee/danswer/server/tenants/access.py b/backend/ee/danswer/server/tenants/access.py
index e69de29bb2d..255e6c0ea94 100644
--- a/backend/ee/danswer/server/tenants/access.py
+++ b/backend/ee/danswer/server/tenants/access.py
@@ -0,0 +1,53 @@
+from datetime import datetime
+from datetime import timedelta
+
+import jwt
+from fastapi import HTTPException
+from fastapi import Request
+
+from danswer.configs.app_configs import DATA_PLANE_SECRET
+from danswer.configs.app_configs import EXPECTED_API_KEY
+from danswer.configs.app_configs import JWT_ALGORITHM
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def generate_data_plane_token() -> str:
+    if DATA_PLANE_SECRET is None:
+        raise ValueError("DATA_PLANE_SECRET is not set")
+
+    payload = {
+        "iss": "data_plane",
+        "exp": datetime.utcnow() + timedelta(minutes=5),
+        "iat": datetime.utcnow(),
+        "scope": "api_access",
+    }
+
+    token = jwt.encode(payload, DATA_PLANE_SECRET, algorithm=JWT_ALGORITHM)
+    return token
+
+
+async def control_plane_dep(request: Request) -> None:
+    api_key = request.headers.get("X-API-KEY")
+    if api_key != EXPECTED_API_KEY:
+        logger.warning("Invalid API key")
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    auth_header = request.headers.get("Authorization")
+    if not auth_header or not auth_header.startswith("Bearer "):
+        logger.warning("Invalid authorization header")
+        raise HTTPException(status_code=401, detail="Invalid authorization header")
+
+    token = auth_header.split(" ")[1]
+    try:
+        payload = jwt.decode(token, DATA_PLANE_SECRET, algorithms=[JWT_ALGORITHM])
+        if payload.get("scope") != "tenant:create":
+            logger.warning("Insufficient permissions")
+            raise HTTPException(status_code=403, detail="Insufficient permissions")
+    except jwt.ExpiredSignatureError:
+        logger.warning("Token has expired")
+        raise HTTPException(status_code=401, detail="Token has expired")
+    except jwt.InvalidTokenError:
+        logger.warning("Invalid token")
+        raise HTTPException(status_code=401, detail="Invalid token")
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index b522112ae06..0438772486a 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -1,19 +1,33 @@
+import stripe
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
 
-from danswer.auth.users import control_plane_dep
+from danswer.auth.users import current_admin_user
+from danswer.auth.users import User
 from danswer.configs.app_configs import MULTI_TENANT
+from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.db.engine import get_session_with_tenant
+from danswer.server.settings.store import load_settings
+from danswer.server.settings.store import store_settings
 from danswer.setup import setup_danswer
 from danswer.utils.logger import setup_logger
+from ee.danswer.configs.app_configs import STRIPE_SECRET_KEY
+from ee.danswer.server.tenants.access import control_plane_dep
+from ee.danswer.server.tenants.billing import fetch_billing_information
+from ee.danswer.server.tenants.billing import fetch_tenant_stripe_information
+from ee.danswer.server.tenants.models import BillingInformation
 from ee.danswer.server.tenants.models import CreateTenantRequest
+from ee.danswer.server.tenants.models import ProductGatingRequest
 from ee.danswer.server.tenants.provisioning import add_users_to_tenant
 from ee.danswer.server.tenants.provisioning import ensure_schema_exists
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
 from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
 from shared_configs.configs import current_tenant_id
 
+
+stripe.api_key = STRIPE_SECRET_KEY
+
 logger = setup_logger()
 router = APIRouter(prefix="/tenants")
 
@@ -22,30 +36,30 @@
 def create_tenant(
     create_tenant_request: CreateTenantRequest, _: None = Depends(control_plane_dep)
 ) -> dict[str, str]:
+    if not MULTI_TENANT:
+        raise HTTPException(status_code=403, detail="Multi-tenancy is not enabled")
+
     tenant_id = create_tenant_request.tenant_id
     email = create_tenant_request.initial_admin_email
     token = None
+
     if user_owns_a_tenant(email):
         raise HTTPException(
             status_code=409, detail="User already belongs to an organization"
         )
 
     try:
-        if not MULTI_TENANT:
-            raise HTTPException(status_code=403, detail="Multi-tenancy is not enabled")
-
         if not ensure_schema_exists(tenant_id):
             logger.info(f"Created schema for tenant {tenant_id}")
         else:
             logger.info(f"Schema already exists for tenant {tenant_id}")
 
-        run_alembic_migrations(tenant_id)
         token = current_tenant_id.set(tenant_id)
-        print("getting session", tenant_id)
+        run_alembic_migrations(tenant_id)
+
         with get_session_with_tenant(tenant_id) as db_session:
             setup_danswer(db_session)
 
-        logger.info(f"Tenant {tenant_id} created successfully")
         add_users_to_tenant([email], tenant_id)
 
         return {
@@ -60,3 +74,53 @@ def create_tenant(
     finally:
         if token is not None:
             current_tenant_id.reset(token)
+
+
+@router.post("/product-gating")
+def gate_product(
+    product_gating_request: ProductGatingRequest, _: None = Depends(control_plane_dep)
+) -> None:
+    """
+    Gating the product means that the product is not available to the tenant.
+    They will be directed to the billing page.
+    We gate the product when
+    1) User has ended free trial without adding payment method
+    2) User's card has declined
+    """
+    token = current_tenant_id.set(current_tenant_id.get())
+
+    settings = load_settings()
+    settings.product_gating = product_gating_request.product_gating
+    store_settings(settings)
+
+    if token is not None:
+        current_tenant_id.reset(token)
+
+
+@router.get("/billing-information", response_model=BillingInformation)
+async def billing_information(
+    _: User = Depends(current_admin_user),
+) -> BillingInformation:
+    logger.info("Fetching billing information")
+    return BillingInformation(**fetch_billing_information(current_tenant_id.get()))
+
+
+@router.post("/create-customer-portal-session")
+async def create_customer_portal_session(_: User = Depends(current_admin_user)) -> dict:
+    try:
+        # Fetch tenant_id and current tenant's information
+        tenant_id = current_tenant_id.get()
+        stripe_info = fetch_tenant_stripe_information(tenant_id)
+        stripe_customer_id = stripe_info.get("stripe_customer_id")
+        if not stripe_customer_id:
+            raise HTTPException(status_code=400, detail="Stripe customer ID not found")
+        logger.info(stripe_customer_id)
+        portal_session = stripe.billing_portal.Session.create(
+            customer=stripe_customer_id,
+            return_url=f"{WEB_DOMAIN}/admin/cloud-settings",
+        )
+        logger.info(portal_session)
+        return {"url": portal_session.url}
+    except Exception as e:
+        logger.exception("Failed to create customer portal session")
+        raise HTTPException(status_code=500, detail=str(e))
diff --git a/backend/ee/danswer/server/tenants/billing.py b/backend/ee/danswer/server/tenants/billing.py
new file mode 100644
index 00000000000..5dcd96713de
--- /dev/null
+++ b/backend/ee/danswer/server/tenants/billing.py
@@ -0,0 +1,69 @@
+from typing import cast
+
+import requests
+import stripe
+
+from danswer.configs.app_configs import CONTROL_PLANE_API_BASE_URL
+from danswer.utils.logger import setup_logger
+from ee.danswer.configs.app_configs import STRIPE_PRICE_ID
+from ee.danswer.configs.app_configs import STRIPE_SECRET_KEY
+from ee.danswer.server.tenants.access import generate_data_plane_token
+from shared_configs.configs import current_tenant_id
+
+stripe.api_key = STRIPE_SECRET_KEY
+
+logger = setup_logger()
+
+
+def fetch_tenant_stripe_information(tenant_id: str) -> dict:
+    token = generate_data_plane_token()
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+    }
+    url = f"{CONTROL_PLANE_API_BASE_URL}/tenant-stripe-information"
+    params = {"tenant_id": tenant_id}
+    response = requests.get(url, headers=headers, params=params)
+    response.raise_for_status()
+    return response.json()
+
+
+def fetch_billing_information(tenant_id: str) -> dict:
+    logger.info("Fetching billing information")
+    token = generate_data_plane_token()
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Content-Type": "application/json",
+    }
+    url = f"{CONTROL_PLANE_API_BASE_URL}/billing-information"
+    params = {"tenant_id": tenant_id}
+    response = requests.get(url, headers=headers, params=params)
+    response.raise_for_status()
+    billing_info = response.json()
+    return billing_info
+
+
+def register_tenant_users(tenant_id: str, number_of_users: int) -> stripe.Subscription:
+    """
+    Send a request to the control service to register the number of users for a tenant.
+    """
+    if not STRIPE_PRICE_ID:
+        raise Exception("STRIPE_PRICE_ID is not set")
+
+    tenant_id = current_tenant_id.get()
+    response = fetch_tenant_stripe_information(tenant_id)
+    stripe_subscription_id = cast(str, response.get("stripe_subscription_id"))
+
+    subscription = stripe.Subscription.retrieve(stripe_subscription_id)
+    updated_subscription = stripe.Subscription.modify(
+        stripe_subscription_id,
+        items=[
+            {
+                "id": subscription["items"]["data"][0].id,
+                "price": STRIPE_PRICE_ID,
+                "quantity": number_of_users,
+            }
+        ],
+        metadata={"tenant_id": str(tenant_id)},
+    )
+    return updated_subscription
diff --git a/backend/ee/danswer/server/tenants/models.py b/backend/ee/danswer/server/tenants/models.py
index 833650c42a6..32642ecfcda 100644
--- a/backend/ee/danswer/server/tenants/models.py
+++ b/backend/ee/danswer/server/tenants/models.py
@@ -1,6 +1,29 @@
 from pydantic import BaseModel
 
+from danswer.server.settings.models import GatingType
+
+
+class CheckoutSessionCreationRequest(BaseModel):
+    quantity: int
+
 
 class CreateTenantRequest(BaseModel):
     tenant_id: str
     initial_admin_email: str
+
+
+class ProductGatingRequest(BaseModel):
+    tenant_id: str
+    product_gating: GatingType
+
+
+class BillingInformation(BaseModel):
+    seats: int
+    subscription_status: str
+    billing_start: str
+    billing_end: str
+    payment_method_enabled: bool
+
+
+class CheckoutSessionCreationResponse(BaseModel):
+    id: str
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 20305c1a3d0..3820d63b066 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -77,3 +77,4 @@ zenpy==2.0.41
 dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
 ultimate_sitemap_parser==0.5
+stripe==10.12.0
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index 898c01509aa..c5044b8b89c 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -3,6 +3,8 @@
 from typing import List
 from urllib.parse import urlparse
 
+from shared_configs.model_server_models import SupportedEmbeddingModel
+
 # Used for logging
 SLACK_CHANNEL_ID = "channel_id"
 
@@ -112,3 +114,74 @@ def validate_cors_origin(origin: str) -> None:
     CORS_ALLOWED_ORIGIN = ["*"]
 
 current_tenant_id = contextvars.ContextVar("current_tenant_id", default="public")
+
+
+SUPPORTED_EMBEDDING_MODELS = [
+    # Cloud-based models
+    SupportedEmbeddingModel(
+        name="cohere/embed-english-v3.0",
+        dim=1024,
+        index_name="danswer_chunk_cohere_embed_english_v3_0",
+    ),
+    SupportedEmbeddingModel(
+        name="cohere/embed-english-light-v3.0",
+        dim=384,
+        index_name="danswer_chunk_cohere_embed_english_light_v3_0",
+    ),
+    SupportedEmbeddingModel(
+        name="openai/text-embedding-3-large",
+        dim=3072,
+        index_name="danswer_chunk_openai_text_embedding_3_large",
+    ),
+    SupportedEmbeddingModel(
+        name="openai/text-embedding-3-small",
+        dim=1536,
+        index_name="danswer_chunk_openai_text_embedding_3_small",
+    ),
+    SupportedEmbeddingModel(
+        name="google/text-embedding-004",
+        dim=768,
+        index_name="danswer_chunk_google_text_embedding_004",
+    ),
+    SupportedEmbeddingModel(
+        name="google/textembedding-gecko@003",
+        dim=768,
+        index_name="danswer_chunk_google_textembedding_gecko_003",
+    ),
+    SupportedEmbeddingModel(
+        name="voyage/voyage-large-2-instruct",
+        dim=1024,
+        index_name="danswer_chunk_voyage_large_2_instruct",
+    ),
+    SupportedEmbeddingModel(
+        name="voyage/voyage-light-2-instruct",
+        dim=384,
+        index_name="danswer_chunk_voyage_light_2_instruct",
+    ),
+    # Self-hosted models
+    SupportedEmbeddingModel(
+        name="nomic-ai/nomic-embed-text-v1",
+        dim=768,
+        index_name="danswer_chunk_nomic_ai_nomic_embed_text_v1",
+    ),
+    SupportedEmbeddingModel(
+        name="intfloat/e5-base-v2",
+        dim=768,
+        index_name="danswer_chunk_intfloat_e5_base_v2",
+    ),
+    SupportedEmbeddingModel(
+        name="intfloat/e5-small-v2",
+        dim=384,
+        index_name="danswer_chunk_intfloat_e5_small_v2",
+    ),
+    SupportedEmbeddingModel(
+        name="intfloat/multilingual-e5-base",
+        dim=768,
+        index_name="danswer_chunk_intfloat_multilingual_e5_base",
+    ),
+    SupportedEmbeddingModel(
+        name="intfloat/multilingual-e5-small",
+        dim=384,
+        index_name="danswer_chunk_intfloat_multilingual_e5_small",
+    ),
+]
diff --git a/backend/shared_configs/model_server_models.py b/backend/shared_configs/model_server_models.py
index dd846ed6bad..737867b7fc8 100644
--- a/backend/shared_configs/model_server_models.py
+++ b/backend/shared_configs/model_server_models.py
@@ -64,3 +64,9 @@ class IntentRequest(BaseModel):
 class IntentResponse(BaseModel):
     is_keyword: bool
     keywords: list[str]
+
+
+class SupportedEmbeddingModel(BaseModel):
+    name: str
+    dim: int
+    index_name: str
diff --git a/deployment/docker_compose/docker-compose.prod-cloud.yml b/deployment/docker_compose/docker-compose.prod-cloud.yml
new file mode 100644
index 00000000000..392d7c67ad4
--- /dev/null
+++ b/deployment/docker_compose/docker-compose.prod-cloud.yml
@@ -0,0 +1,243 @@
+services:
+  api_server:
+    image: danswer/danswer-backend:${IMAGE_TAG:-latest}
+    build:
+      context: ../../backend
+      dockerfile: Dockerfile.cloud
+    command: >
+      /bin/sh -c "alembic -n schema_private upgrade head &&
+      echo \"Starting Danswer Api Server\" &&
+      uvicorn danswer.main:app --host 0.0.0.0 --port 8080"
+    depends_on:
+      - relational_db
+      - index
+      - cache
+      - inference_model_server
+    restart: always
+    env_file:
+      - .env
+    environment:
+      - AUTH_TYPE=${AUTH_TYPE:-oidc}
+      - POSTGRES_HOST=relational_db
+      - VESPA_HOST=index
+      - REDIS_HOST=cache
+      - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+
+  background:
+    image: danswer/danswer-backend:${IMAGE_TAG:-latest}
+    build:
+      context: ../../backend
+      dockerfile: Dockerfile
+    command: /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+    depends_on:
+      - relational_db
+      - index
+      - cache
+      - inference_model_server
+      - indexing_model_server
+    restart: always
+    env_file:
+      - .env
+    environment:
+      - AUTH_TYPE=${AUTH_TYPE:-oidc}
+      - POSTGRES_HOST=relational_db
+      - VESPA_HOST=index
+      - REDIS_HOST=cache
+      - MODEL_SERVER_HOST=${MODEL_SERVER_HOST:-inference_model_server}
+      - INDEXING_MODEL_SERVER_HOST=${INDEXING_MODEL_SERVER_HOST:-indexing_model_server}
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+  web_server:
+    image: danswer/danswer-web-server:${IMAGE_TAG:-latest}
+    build:
+      context: ../../web
+      dockerfile: Dockerfile
+      args:
+        - NEXT_PUBLIC_DISABLE_STREAMING=${NEXT_PUBLIC_DISABLE_STREAMING:-false}
+        - NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA=${NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA:-false}
+        - NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS=${NEXT_PUBLIC_POSITIVE_PREDEFINED_FEEDBACK_OPTIONS:-}
+        - NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS=${NEXT_PUBLIC_NEGATIVE_PREDEFINED_FEEDBACK_OPTIONS:-}
+        - NEXT_PUBLIC_DISABLE_LOGOUT=${NEXT_PUBLIC_DISABLE_LOGOUT:-}
+        - NEXT_PUBLIC_THEME=${NEXT_PUBLIC_THEME:-}
+    depends_on:
+      - api_server
+    restart: always
+    env_file:
+      - .env
+    environment:
+      - INTERNAL_URL=http://api_server:8080
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+
+  relational_db:
+    image: postgres:15.2-alpine
+    command: -c 'max_connections=250'
+    restart: always
+    # POSTGRES_USER and POSTGRES_PASSWORD should be set in .env file
+    env_file:
+      - .env
+    volumes:
+      - db_volume:/var/lib/postgresql/data
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+
+  inference_model_server:
+    image: danswer/danswer-model-server:${IMAGE_TAG:-latest}
+    build:
+      context: ../../backend
+      dockerfile: Dockerfile.model_server
+    command: >
+      /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-false}\" = \"True\" ]; then
+        echo 'Skipping service...';
+        exit 0;
+      else
+        exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000;
+      fi"
+    restart: on-failure
+    environment:
+      - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-}
+      # Set to debug to get more fine-grained logs
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+    volumes:
+      # Not necessary, this is just to reduce download time during startup
+      - model_cache_huggingface:/root/.cache/huggingface/
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+
+  indexing_model_server:
+    image: danswer/danswer-model-server:${IMAGE_TAG:-latest}
+    build:
+      context: ../../backend
+      dockerfile: Dockerfile.model_server
+    command: >
+      /bin/sh -c "if [ \"${DISABLE_MODEL_SERVER:-false}\" = \"True\" ]; then
+        echo 'Skipping service...';
+        exit 0;
+      else
+        exec uvicorn model_server.main:app --host 0.0.0.0 --port 9000;
+      fi"
+    restart: on-failure
+    environment:
+      - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-}
+      - INDEXING_ONLY=True
+      # Set to debug to get more fine-grained logs
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - VESPA_SEARCHER_THREADS=${VESPA_SEARCHER_THREADS:-1}
+    volumes:
+      # Not necessary, this is just to reduce download time during startup
+      - indexing_huggingface_model_cache:/root/.cache/huggingface/
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+
+  # This container name cannot have an underscore in it due to Vespa expectations of the URL
+  index:
+    image: vespaengine/vespa:8.277.17
+    restart: always
+    ports:
+      - "19071:19071"
+      - "8081:8081"
+    volumes:
+      - vespa_volume:/opt/vespa/var
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+
+
+  nginx:
+    image: nginx:1.23.4-alpine
+    restart: always
+    # nginx will immediately crash with `nginx: [emerg] host not found in upstream`
+    # if api_server / web_server are not up 
+    depends_on:
+      - api_server
+      - web_server
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ../data/nginx:/etc/nginx/conf.d
+      - ../data/certbot/conf:/etc/letsencrypt
+      - ../data/certbot/www:/var/www/certbot
+    # sleep a little bit to allow the web_server / api_server to start up. 
+    # Without this we've seen issues where nginx shows no error logs but 
+    # does not recieve any traffic
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+    # The specified script waits for the api_server to start up. 
+    # Without this we've seen issues where nginx shows no error logs but 
+    # does not recieve any traffic  
+    # NOTE: we have to use dos2unix to remove Carriage Return chars from the file
+    # in order to make this work on both Unix-like systems and windows
+    command: > 
+      /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
+      && /etc/nginx/conf.d/run-nginx.sh app.conf.template"
+    env_file:
+      - .env.nginx
+
+
+  # follows https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71
+  certbot:
+    image: certbot/certbot
+    restart: always
+    volumes:
+      - ../data/certbot/conf:/etc/letsencrypt
+      - ../data/certbot/www:/var/www/certbot
+    logging:
+      driver: json-file
+      options:
+        max-size: "50m"
+        max-file: "6"
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
+
+
+  cache:
+    image: redis:7.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    # docker silently mounts /data even without an explicit volume mount, which enables
+    # persistence. explicitly setting save and appendonly forces ephemeral behavior.
+    command: redis-server --save "" --appendonly no
+
+
+volumes:
+  db_volume:
+  vespa_volume:
+  # Created by the container itself
+  model_cache_huggingface:
+  indexing_huggingface_model_cache:
diff --git a/web/package-lock.json b/web/package-lock.json
index 338cf0a9f0f..36a76cbe05c 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -15,6 +15,7 @@
         "@radix-ui/react-dialog": "^1.0.5",
         "@radix-ui/react-popover": "^1.0.7",
         "@radix-ui/react-tooltip": "^1.0.7",
+        "@stripe/stripe-js": "^4.6.0",
         "@tremor/react": "^3.9.2",
         "@types/js-cookie": "^3.0.3",
         "@types/lodash": "^4.17.0",
@@ -42,7 +43,7 @@
         "rehype-prism-plus": "^2.0.0",
         "remark-gfm": "^4.0.0",
         "semver": "^7.5.4",
-        "sharp": "^0.32.6",
+        "stripe": "^17.0.0",
         "swr": "^2.1.5",
         "tailwindcss": "^3.3.1",
         "typescript": "5.0.3",
@@ -1670,6 +1671,14 @@
       "integrity": "sha512-qC/xYId4NMebE6w/V33Fh9gWxLgURiNYgVNObbJl2LZv0GUUItCcCqC5axQSwRaAgaxl2mELq1rMzlswaQ0Zxg==",
       "dev": true
     },
+    "node_modules/@stripe/stripe-js": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/@stripe/stripe-js/-/stripe-js-4.6.0.tgz",
+      "integrity": "sha512-ZoK0dMFnVH0J5XUWGqsta8S8xm980qEwJKAIgZcLQxaSsbGRB9CsVvfOjwQFE1JC1q3rPwb/b+gQAmzIESnHnA==",
+      "engines": {
+        "node": ">=12.16"
+      }
+    },
     "node_modules/@swc/counter": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz",
@@ -2415,11 +2424,6 @@
         "dequal": "^2.0.3"
       }
     },
-    "node_modules/b4a": {
-      "version": "1.6.6",
-      "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz",
-      "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg=="
-    },
     "node_modules/babel-plugin-macros": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz",
@@ -2463,66 +2467,6 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
-    "node_modules/bare-events": {
-      "version": "2.2.2",
-      "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.2.2.tgz",
-      "integrity": "sha512-h7z00dWdG0PYOQEvChhOSWvOfkIKsdZGkWr083FgN/HyoQuebSew/cgirYqh9SCuy/hRvxc5Vy6Fw8xAmYHLkQ==",
-      "optional": true
-    },
-    "node_modules/bare-fs": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-2.3.0.tgz",
-      "integrity": "sha512-TNFqa1B4N99pds2a5NYHR15o0ZpdNKbAeKTE/+G6ED/UeOavv8RY3dr/Fu99HW3zU3pXpo2kDNO8Sjsm2esfOw==",
-      "optional": true,
-      "dependencies": {
-        "bare-events": "^2.0.0",
-        "bare-path": "^2.0.0",
-        "bare-stream": "^1.0.0"
-      }
-    },
-    "node_modules/bare-os": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-2.3.0.tgz",
-      "integrity": "sha512-oPb8oMM1xZbhRQBngTgpcQ5gXw6kjOaRsSWsIeNyRxGed2w/ARyP7ScBYpWR1qfX2E5rS3gBw6OWcSQo+s+kUg==",
-      "optional": true
-    },
-    "node_modules/bare-path": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-2.1.2.tgz",
-      "integrity": "sha512-o7KSt4prEphWUHa3QUwCxUI00R86VdjiuxmJK0iNVDHYPGo+HsDaVCnqCmPbf/MiW1ok8F4p3m8RTHlWk8K2ig==",
-      "optional": true,
-      "dependencies": {
-        "bare-os": "^2.1.0"
-      }
-    },
-    "node_modules/bare-stream": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-1.0.0.tgz",
-      "integrity": "sha512-KhNUoDL40iP4gFaLSsoGE479t0jHijfYdIcxRn/XtezA2BaUD0NRf/JGRpsMq6dMNM+SrCrB0YSSo/5wBY4rOQ==",
-      "optional": true,
-      "dependencies": {
-        "streamx": "^2.16.1"
-      }
-    },
-    "node_modules/base64-js": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
     "node_modules/binary-extensions": {
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
@@ -2534,16 +2478,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/bl": {
-      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
-      "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
-      "dependencies": {
-        "buffer": "^5.5.0",
-        "inherits": "^2.0.4",
-        "readable-stream": "^3.4.0"
-      }
-    },
     "node_modules/brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -2596,29 +2530,6 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
-    "node_modules/buffer": {
-      "version": "5.7.1",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
-      "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.1.13"
-      }
-    },
     "node_modules/busboy": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@@ -2634,7 +2545,6 @@
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.7.tgz",
       "integrity": "sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==",
-      "dev": true,
       "dependencies": {
         "es-define-property": "^1.0.0",
         "es-errors": "^1.3.0",
@@ -2787,11 +2697,6 @@
         "node": ">= 6"
       }
     },
-    "node_modules/chownr": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
-    },
     "node_modules/client-only": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
@@ -2805,18 +2710,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/color": {
-      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
-      "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
-      "dependencies": {
-        "color-convert": "^2.0.1",
-        "color-string": "^1.9.0"
-      },
-      "engines": {
-        "node": ">=12.5.0"
-      }
-    },
     "node_modules/color-convert": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -2833,15 +2726,6 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
-    "node_modules/color-string": {
-      "version": "1.9.1",
-      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
-      "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
-      "dependencies": {
-        "color-name": "^1.0.0",
-        "simple-swizzle": "^0.2.2"
-      }
-    },
     "node_modules/comma-separated-tokens": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz",
@@ -3150,28 +3034,6 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/decompress-response": {
-      "version": "6.0.0",
-      "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
-      "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
-      "dependencies": {
-        "mimic-response": "^3.1.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
-    "node_modules/deep-extend": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
-      "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
-      "engines": {
-        "node": ">=4.0.0"
-      }
-    },
     "node_modules/deep-is": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
@@ -3190,7 +3052,6 @@
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz",
       "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==",
-      "dev": true,
       "dependencies": {
         "es-define-property": "^1.0.0",
         "es-errors": "^1.3.0",
@@ -3228,14 +3089,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/detect-libc": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
-      "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/detect-node-es": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
@@ -3311,14 +3164,6 @@
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
     },
-    "node_modules/end-of-stream": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-      "dependencies": {
-        "once": "^1.4.0"
-      }
-    },
     "node_modules/enhanced-resolve": {
       "version": "5.16.1",
       "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.16.1.tgz",
@@ -3420,7 +3265,6 @@
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz",
       "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==",
-      "dev": true,
       "dependencies": {
         "get-intrinsic": "^1.2.4"
       },
@@ -3432,7 +3276,6 @@
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz",
       "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==",
-      "dev": true,
       "engines": {
         "node": ">= 0.4"
       }
@@ -3959,14 +3802,6 @@
       "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
       "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
     },
-    "node_modules/expand-template": {
-      "version": "2.0.3",
-      "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
-      "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -3986,11 +3821,6 @@
         "node": ">=6.0.0"
       }
     },
-    "node_modules/fast-fifo": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz",
-      "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ=="
-    },
     "node_modules/fast-glob": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz",
@@ -4172,11 +4002,6 @@
         "url": "https://github.com/sponsors/rawify"
       }
     },
-    "node_modules/fs-constants": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
-    },
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@@ -4244,7 +4069,6 @@
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
       "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==",
-      "dev": true,
       "dependencies": {
         "es-errors": "^1.3.0",
         "function-bind": "^1.1.2",
@@ -4296,11 +4120,6 @@
         "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
       }
     },
-    "node_modules/github-from-package": {
-      "version": "0.0.0",
-      "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
-      "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
-    },
     "node_modules/glob": {
       "version": "10.3.10",
       "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
@@ -4410,7 +4229,6 @@
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz",
       "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==",
-      "dev": true,
       "dependencies": {
         "get-intrinsic": "^1.1.3"
       },
@@ -4451,7 +4269,6 @@
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
       "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==",
-      "dev": true,
       "dependencies": {
         "es-define-property": "^1.0.0"
       },
@@ -4463,7 +4280,6 @@
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz",
       "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==",
-      "dev": true,
       "engines": {
         "node": ">= 0.4"
       },
@@ -4475,7 +4291,6 @@
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz",
       "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==",
-      "dev": true,
       "engines": {
         "node": ">= 0.4"
       },
@@ -4694,25 +4509,6 @@
         "url": "https://opencollective.com/unified"
       }
     },
-    "node_modules/ieee754": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
     "node_modules/ignore": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
@@ -4759,12 +4555,8 @@
     "node_modules/inherits": {
       "version": "2.0.4",
       "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
-      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
-    },
-    "node_modules/ini": {
-      "version": "1.3.8",
-      "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
-      "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
+      "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
+      "dev": true
     },
     "node_modules/inline-style-parser": {
       "version": "0.2.3",
@@ -4839,11 +4631,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/is-arrayish": {
-      "version": "0.3.2",
-      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
-      "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
-    },
     "node_modules/is-async-function": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz",
@@ -6312,17 +6099,6 @@
         "node": ">=8.6"
       }
     },
-    "node_modules/mimic-response": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
-      "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -6339,6 +6115,7 @@
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
       "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
+      "dev": true,
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -6351,11 +6128,6 @@
         "node": ">=16 || 14 >=14.17"
       }
     },
-    "node_modules/mkdirp-classic": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
-    },
     "node_modules/ms": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
@@ -6388,11 +6160,6 @@
         "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1"
       }
     },
-    "node_modules/napi-build-utils": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz",
-      "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
-    },
     "node_modules/natural-compare": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -6475,22 +6242,6 @@
         "node": "^10 || ^12 || >=14"
       }
     },
-    "node_modules/node-abi": {
-      "version": "3.62.0",
-      "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.62.0.tgz",
-      "integrity": "sha512-CPMcGa+y33xuL1E0TcNIu4YyaZCxnnvkVaEXrsosR3FxN+fV8xvb7Mzpb7IgKler10qeMkE6+Dp8qJhpzdq35g==",
-      "dependencies": {
-        "semver": "^7.3.5"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/node-addon-api": {
-      "version": "6.1.0",
-      "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz",
-      "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA=="
-    },
     "node_modules/node-releases": {
       "version": "2.0.14",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",
@@ -8926,7 +8677,6 @@
       "version": "1.13.1",
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz",
       "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==",
-      "dev": true,
       "funding": {
         "url": "https://github.com/sponsors/ljharb"
       }
@@ -9042,6 +8792,7 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
+      "dev": true,
       "dependencies": {
         "wrappy": "1"
       }
@@ -9410,57 +9161,6 @@
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
     },
-    "node_modules/prebuild-install": {
-      "version": "7.1.2",
-      "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz",
-      "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==",
-      "dependencies": {
-        "detect-libc": "^2.0.0",
-        "expand-template": "^2.0.3",
-        "github-from-package": "0.0.0",
-        "minimist": "^1.2.3",
-        "mkdirp-classic": "^0.5.3",
-        "napi-build-utils": "^1.0.1",
-        "node-abi": "^3.3.0",
-        "pump": "^3.0.0",
-        "rc": "^1.2.7",
-        "simple-get": "^4.0.0",
-        "tar-fs": "^2.0.0",
-        "tunnel-agent": "^0.6.0"
-      },
-      "bin": {
-        "prebuild-install": "bin.js"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/prebuild-install/node_modules/tar-fs": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz",
-      "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
-      "dependencies": {
-        "chownr": "^1.1.1",
-        "mkdirp-classic": "^0.5.2",
-        "pump": "^3.0.0",
-        "tar-stream": "^2.1.4"
-      }
-    },
-    "node_modules/prebuild-install/node_modules/tar-stream": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
-      "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
-      "dependencies": {
-        "bl": "^4.0.3",
-        "end-of-stream": "^1.4.1",
-        "fs-constants": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^3.1.1"
-      },
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -9517,15 +9217,6 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
-    "node_modules/pump": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-      "dependencies": {
-        "end-of-stream": "^1.1.0",
-        "once": "^1.3.1"
-      }
-    },
     "node_modules/punycode": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
@@ -9535,6 +9226,20 @@
         "node": ">=6"
       }
     },
+    "node_modules/qs": {
+      "version": "6.13.0",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz",
+      "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==",
+      "dependencies": {
+        "side-channel": "^1.0.6"
+      },
+      "engines": {
+        "node": ">=0.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/queue-microtask": {
       "version": "1.2.3",
       "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@@ -9554,33 +9259,6 @@
         }
       ]
     },
-    "node_modules/queue-tick": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz",
-      "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag=="
-    },
-    "node_modules/rc": {
-      "version": "1.2.8",
-      "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
-      "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
-      "dependencies": {
-        "deep-extend": "^0.6.0",
-        "ini": "~1.3.0",
-        "minimist": "^1.2.0",
-        "strip-json-comments": "~2.0.1"
-      },
-      "bin": {
-        "rc": "cli.js"
-      }
-    },
-    "node_modules/rc/node_modules/strip-json-comments": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
-      "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
-      "engines": {
-        "node": ">=0.10.0"
-      }
-    },
     "node_modules/react": {
       "version": "18.3.1",
       "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
@@ -9828,19 +9506,6 @@
         "pify": "^2.3.0"
       }
     },
-    "node_modules/readable-stream": {
-      "version": "3.6.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
-      "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
-      "dependencies": {
-        "inherits": "^2.0.3",
-        "string_decoder": "^1.1.1",
-        "util-deprecate": "^1.0.1"
-      },
-      "engines": {
-        "node": ">= 6"
-      }
-    },
     "node_modules/readdirp": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
@@ -10160,25 +9825,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/safe-buffer": {
-      "version": "5.2.1",
-      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
-      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
     "node_modules/safe-regex-test": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz",
@@ -10219,7 +9865,6 @@
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
       "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==",
-      "dev": true,
       "dependencies": {
         "define-data-property": "^1.1.4",
         "es-errors": "^1.3.0",
@@ -10252,28 +9897,6 @@
       "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
       "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ=="
     },
-    "node_modules/sharp": {
-      "version": "0.32.6",
-      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz",
-      "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==",
-      "hasInstallScript": true,
-      "dependencies": {
-        "color": "^4.2.3",
-        "detect-libc": "^2.0.2",
-        "node-addon-api": "^6.1.0",
-        "prebuild-install": "^7.1.1",
-        "semver": "^7.5.4",
-        "simple-get": "^4.0.1",
-        "tar-fs": "^3.0.4",
-        "tunnel-agent": "^0.6.0"
-      },
-      "engines": {
-        "node": ">=14.15.0"
-      },
-      "funding": {
-        "url": "https://opencollective.com/libvips"
-      }
-    },
     "node_modules/shebang-command": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -10297,7 +9920,6 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
       "integrity": "sha512-fDW/EZ6Q9RiO8eFG8Hj+7u/oW+XrPTIChwCOM2+th2A6OblDtYYIpve9m+KvI9Z4C9qSEXlaGR6bTEYHReuglA==",
-      "dev": true,
       "dependencies": {
         "call-bind": "^1.0.7",
         "es-errors": "^1.3.0",
@@ -10322,57 +9944,6 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
-    "node_modules/simple-concat": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
-      "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
-    },
-    "node_modules/simple-get": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
-      "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "decompress-response": "^6.0.0",
-        "once": "^1.3.1",
-        "simple-concat": "^1.0.0"
-      }
-    },
-    "node_modules/simple-swizzle": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
-      "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
-      "dependencies": {
-        "is-arrayish": "^0.3.1"
-      }
-    },
     "node_modules/slash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
@@ -10415,26 +9986,6 @@
         "node": ">=10.0.0"
       }
     },
-    "node_modules/streamx": {
-      "version": "2.16.1",
-      "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.16.1.tgz",
-      "integrity": "sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ==",
-      "dependencies": {
-        "fast-fifo": "^1.1.0",
-        "queue-tick": "^1.0.1"
-      },
-      "optionalDependencies": {
-        "bare-events": "^2.2.0"
-      }
-    },
-    "node_modules/string_decoder": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
-      "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
-      "dependencies": {
-        "safe-buffer": "~5.2.0"
-      }
-    },
     "node_modules/string-width": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
@@ -10627,6 +10178,18 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/stripe": {
+      "version": "17.0.0",
+      "resolved": "https://registry.npmjs.org/stripe/-/stripe-17.0.0.tgz",
+      "integrity": "sha512-URKpnjH2O+OWxhvXLIaEIaAkp2fQvqITm/3zJS0a3nGCREjH3qJYxmGowngA46Qu1x2MumNL3Y/OdY6uzIhpCQ==",
+      "dependencies": {
+        "@types/node": ">=8.1.0",
+        "qs": "^6.11.0"
+      },
+      "engines": {
+        "node": ">=12.*"
+      }
+    },
     "node_modules/style-to-object": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.6.tgz",
@@ -10842,29 +10405,6 @@
         "node": ">=6"
       }
     },
-    "node_modules/tar-fs": {
-      "version": "3.0.6",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz",
-      "integrity": "sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w==",
-      "dependencies": {
-        "pump": "^3.0.0",
-        "tar-stream": "^3.1.5"
-      },
-      "optionalDependencies": {
-        "bare-fs": "^2.1.1",
-        "bare-path": "^2.1.0"
-      }
-    },
-    "node_modules/tar-stream": {
-      "version": "3.1.7",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz",
-      "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==",
-      "dependencies": {
-        "b4a": "^1.6.4",
-        "fast-fifo": "^1.2.0",
-        "streamx": "^2.15.0"
-      }
-    },
     "node_modules/text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -10993,17 +10533,6 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz",
       "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q=="
     },
-    "node_modules/tunnel-agent": {
-      "version": "0.6.0",
-      "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
-      "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
-      "dependencies": {
-        "safe-buffer": "^5.0.1"
-      },
-      "engines": {
-        "node": "*"
-      }
-    },
     "node_modules/type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@@ -11611,7 +11140,8 @@
     "node_modules/wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
+      "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
+      "dev": true
     },
     "node_modules/yallist": {
       "version": "3.1.1",
diff --git a/web/package.json b/web/package.json
index 1e55fec590e..39d4b316e1e 100644
--- a/web/package.json
+++ b/web/package.json
@@ -16,6 +16,7 @@
     "@radix-ui/react-dialog": "^1.0.5",
     "@radix-ui/react-popover": "^1.0.7",
     "@radix-ui/react-tooltip": "^1.0.7",
+    "@stripe/stripe-js": "^4.6.0",
     "@tremor/react": "^3.9.2",
     "@types/js-cookie": "^3.0.3",
     "@types/lodash": "^4.17.0",
@@ -43,7 +44,7 @@
     "rehype-prism-plus": "^2.0.0",
     "remark-gfm": "^4.0.0",
     "semver": "^7.5.4",
-    "sharp": "^0.32.6",
+    "stripe": "^17.0.0",
     "swr": "^2.1.5",
     "tailwindcss": "^3.3.1",
     "typescript": "5.0.3",
@@ -56,4 +57,4 @@
     "eslint-config-next": "^14.1.0",
     "prettier": "2.8.8"
   }
-}
\ No newline at end of file
+}
diff --git a/web/src/app/admin/settings/interfaces.ts b/web/src/app/admin/settings/interfaces.ts
index 8327d69d448..2df8b5c26b2 100644
--- a/web/src/app/admin/settings/interfaces.ts
+++ b/web/src/app/admin/settings/interfaces.ts
@@ -1,3 +1,9 @@
+export enum GatingType {
+  FULL = "full",
+  PARTIAL = "partial",
+  NONE = "none",
+}
+
 export interface Settings {
   chat_page_enabled: boolean;
   search_page_enabled: boolean;
@@ -6,6 +12,7 @@ export interface Settings {
   notifications: Notification[];
   needs_reindexing: boolean;
   gpu_enabled: boolean;
+  product_gating: GatingType;
 }
 
 export interface Notification {
diff --git a/web/src/app/ee/admin/cloud-settings/BillingInformationPage.tsx b/web/src/app/ee/admin/cloud-settings/BillingInformationPage.tsx
new file mode 100644
index 00000000000..de2e4142947
--- /dev/null
+++ b/web/src/app/ee/admin/cloud-settings/BillingInformationPage.tsx
@@ -0,0 +1,222 @@
+"use client";
+
+import { CreditCard, ArrowFatUp } from "@phosphor-icons/react";
+import { useState } from "react";
+import { useRouter } from "next/navigation";
+import { loadStripe } from "@stripe/stripe-js";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { SettingsIcon } from "@/components/icons/icons";
+import {
+  updateSubscriptionQuantity,
+  fetchCustomerPortal,
+  statusToDisplay,
+  useBillingInformation,
+} from "./utils";
+import { useEffect } from "react";
+
+export default function BillingInformationPage() {
+  const router = useRouter();
+  const { popup, setPopup } = usePopup();
+  const stripePromise = loadStripe(
+    process.env.NEXT_PUBLIC_STRIPE_PUBLISHABLE_KEY!
+  );
+
+  const {
+    data: billingInformation,
+    error,
+    isLoading,
+    refreshBillingInformation,
+  } = useBillingInformation();
+
+  const [seats, setSeats] = useState<number>(1);
+
+  useEffect(() => {
+    if (billingInformation?.seats) {
+      setSeats(billingInformation.seats);
+    }
+  }, [billingInformation?.seats]);
+
+  if (error) {
+    console.error("Failed to fetch billing information:", error);
+  }
+  useEffect(() => {
+    const url = new URL(window.location.href);
+    if (url.searchParams.has("session_id")) {
+      setPopup({
+        message:
+          "Congratulations! Your subscription has been updated successfully.",
+        type: "success",
+      });
+      // Remove the session_id from the URL
+      url.searchParams.delete("session_id");
+      window.history.replaceState({}, "", url.toString());
+      // You might want to refresh the billing information here
+      // by calling an API endpoint to get the latest data
+    }
+  }, [setPopup]);
+
+  if (isLoading) {
+    return <div>Loading...</div>;
+  }
+
+  const handleManageSubscription = async () => {
+    try {
+      const response = await fetchCustomerPortal();
+
+      if (!response.ok) {
+        const errorData = await response.json();
+        throw new Error(
+          `Failed to create customer portal session: ${errorData.message || response.statusText}`
+        );
+      }
+
+      const { url } = await response.json();
+
+      if (!url) {
+        throw new Error("No portal URL returned from the server");
+      }
+
+      router.push(url);
+    } catch (error) {
+      console.error("Error creating customer portal session:", error);
+      setPopup({
+        message: "Error creating customer portal session",
+        type: "error",
+      });
+    }
+  };
+  if (!billingInformation) {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <div className="space-y-8">
+      <div className="bg-gray-50 rounded-lg p-8 border border-gray-200">
+        {popup}
+
+        <h2 className="text-2xl font-bold mb-6 text-gray-800 flex items-center">
+          <CreditCard className="mr-4 text-gray-600" size={24} />
+          Billing Information
+        </h2>
+
+        <div className="space-y-4">
+          <div className="bg-white p-5 rounded-lg shadow-sm transition-all duration-300 hover:shadow-md">
+            <div className="flex justify-between items-center">
+              <div>
+                <p className="text-lg font-medium text-gray-700">Seats</p>
+                <p className="text-sm text-gray-500">
+                  Number of licensed users
+                </p>
+              </div>
+              <p className="text-xl font-semibold text-gray-900">
+                {billingInformation.seats}
+              </p>
+            </div>
+          </div>
+
+          <div className="bg-white p-5 rounded-lg shadow-sm transition-all duration-300 hover:shadow-md">
+            <div className="flex justify-between items-center">
+              <div>
+                <p className="text-lg font-medium text-gray-700">
+                  Subscription Status
+                </p>
+                <p className="text-sm text-gray-500">
+                  Current state of your subscription
+                </p>
+              </div>
+              <p className="text-xl font-semibold text-gray-900">
+                {statusToDisplay(billingInformation.subscription_status)}
+              </p>
+            </div>
+          </div>
+
+          <div className="bg-white p-5 rounded-lg shadow-sm transition-all duration-300 hover:shadow-md">
+            <div className="flex justify-between items-center">
+              <div>
+                <p className="text-lg font-medium text-gray-700">
+                  Billing Start
+                </p>
+                <p className="text-sm text-gray-500">
+                  Start date of current billing cycle
+                </p>
+              </div>
+              <p className="text-xl font-semibold text-gray-900">
+                {new Date(
+                  billingInformation.billing_start
+                ).toLocaleDateString()}
+              </p>
+            </div>
+          </div>
+
+          <div className="bg-white p-5 rounded-lg shadow-sm transition-all duration-300 hover:shadow-md">
+            <div className="flex justify-between items-center">
+              <div>
+                <p className="text-lg font-medium text-gray-700">Billing End</p>
+                <p className="text-sm text-gray-500">
+                  End date of current billing cycle
+                </p>
+              </div>
+              <p className="text-xl font-semibold text-gray-900">
+                {new Date(billingInformation.billing_end).toLocaleDateString()}
+              </p>
+            </div>
+          </div>
+        </div>
+
+        {!billingInformation.payment_method_enabled && (
+          <div className="mt-4 p-4 bg-yellow-100 border-l-4 border-yellow-500 text-yellow-700">
+            <p className="font-bold">Notice:</p>
+            <p>
+              You&apos;ll need to add a payment method before your trial ends to
+              continue using the service.
+            </p>
+          </div>
+        )}
+
+        {billingInformation.subscription_status === "trialing" ? (
+          <div className="bg-white p-5 rounded-lg shadow-sm transition-all duration-300 hover:shadow-md mt-8">
+            <p className="text-lg font-medium text-gray-700">
+              No cap on users during trial
+            </p>
+          </div>
+        ) : (
+          <div className="flex items-center space-x-4 mt-8">
+            <div className="flex items-center space-x-4">
+              <p className="text-lg font-medium text-gray-700">
+                Current Seats:
+              </p>
+              <p className="text-xl font-semibold text-gray-900">
+                {billingInformation.seats}
+              </p>
+            </div>
+            <p className="text-sm text-gray-500">
+              Seats automatically update based on adding, removing, or inviting
+              users.
+            </p>
+          </div>
+        )}
+      </div>
+
+      <div className="bg-white p-5 rounded-lg shadow-sm transition-all duration-300 hover:shadow-md">
+        <div className="flex justify-between items-center mb-4">
+          <div>
+            <p className="text-lg font-medium text-gray-700">
+              Manage Subscription
+            </p>
+            <p className="text-sm text-gray-500">
+              View your plan, update payment, or change subscription
+            </p>
+          </div>
+          <SettingsIcon className="text-gray-600" size={20} />
+        </div>
+        <button
+          onClick={handleManageSubscription}
+          className="bg-gray-600 text-white px-4 py-2 rounded-md hover:bg-gray-700 transition duration-300 ease-in-out focus:outline-none focus:ring-2 focus:ring-gray-500 focus:ring-opacity-50 font-medium shadow-sm text-sm flex items-center justify-center"
+        >
+          <ArrowFatUp className="mr-2" size={16} />
+          Manage Subscription
+        </button>
+      </div>
+    </div>
+  );
+}
diff --git a/web/src/app/ee/admin/cloud-settings/page.tsx b/web/src/app/ee/admin/cloud-settings/page.tsx
new file mode 100644
index 00000000000..6566e069ba7
--- /dev/null
+++ b/web/src/app/ee/admin/cloud-settings/page.tsx
@@ -0,0 +1,23 @@
+import { AdminPageTitle } from "@/components/admin/Title";
+import BillingInformationPage from "./BillingInformationPage";
+import { FaCloud } from "react-icons/fa";
+
+export interface BillingInformation {
+  seats: number;
+  subscription_status: string;
+  billing_start: Date;
+  billing_end: Date;
+  payment_method_enabled: boolean;
+}
+
+export default function page() {
+  return (
+    <div className="container max-w-4xl">
+      <AdminPageTitle
+        title="Cloud Settings"
+        icon={<FaCloud size={32} className="my-auto" />}
+      />
+      <BillingInformationPage />
+    </div>
+  );
+}
diff --git a/web/src/app/ee/admin/cloud-settings/utils.ts b/web/src/app/ee/admin/cloud-settings/utils.ts
new file mode 100644
index 00000000000..1f2aaa8e8eb
--- /dev/null
+++ b/web/src/app/ee/admin/cloud-settings/utils.ts
@@ -0,0 +1,46 @@
+import { BillingInformation } from "./page";
+import useSWR, { mutate } from "swr";
+
+export const updateSubscriptionQuantity = async (seats: number) => {
+  return await fetch("/api/tenants/update-subscription-quantity", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ quantity: seats }),
+  });
+};
+
+export const fetchCustomerPortal = async () => {
+  return await fetch("/api/tenants/create-customer-portal-session", {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+};
+
+export const statusToDisplay = (status: string) => {
+  switch (status) {
+    case "trialing":
+      return "Trialing";
+    case "active":
+      return "Active";
+    case "canceled":
+      return "Canceled";
+    default:
+      return "Unknown";
+  }
+};
+
+export const useBillingInformation = () => {
+  const url = "/api/tenants/billing-information";
+  const swrResponse = useSWR<BillingInformation>(url, (url: string) =>
+    fetch(url).then((res) => res.json())
+  );
+
+  return {
+    ...swrResponse,
+    refreshBillingInformation: () => mutate(url),
+  };
+};
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index f49864aac75..41611266f61 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -13,14 +13,12 @@ import { Metadata } from "next";
 import { buildClientUrl } from "@/lib/utilsSS";
 import { Inter } from "next/font/google";
 import Head from "next/head";
-import { EnterpriseSettings } from "./admin/settings/interfaces";
+import { EnterpriseSettings, GatingType } from "./admin/settings/interfaces";
 import { Card } from "@tremor/react";
 import { HeaderTitle } from "@/components/header/HeaderTitle";
 import { Logo } from "@/components/Logo";
 import { UserProvider } from "@/components/user/UserProvider";
 import { ProviderContextProvider } from "@/components/chat_search/ProviderContext";
-import { redirect } from "next/navigation";
-import { headers } from "next/headers";
 
 const inter = Inter({
   subsets: ["latin"],
@@ -57,6 +55,9 @@ export default async function RootLayout({
 }) {
   const combinedSettings = await fetchSettingsSS();
 
+  const productGating =
+    combinedSettings?.settings.product_gating ?? GatingType.NONE;
+
   if (!combinedSettings) {
     return (
       <html lang="en" className={`${inter.variable} font-sans`}>
@@ -109,6 +110,42 @@ export default async function RootLayout({
       </html>
     );
   }
+  if (productGating === GatingType.FULL) {
+    return (
+      <html lang="en" className={`${inter.variable} font-sans`}>
+        <Head>
+          <title>Access Restricted | Danswer</title>
+        </Head>
+        <body className="bg-background text-default">
+          <div className="flex flex-col items-center justify-center min-h-screen">
+            <div className="mb-2 flex items-center max-w-[175px]">
+              <HeaderTitle>Danswer</HeaderTitle>
+              <Logo height={40} width={40} />
+            </div>
+            <Card className="p-8 max-w-md">
+              <h1 className="text-2xl font-bold mb-4 text-error">
+                Access Restricted
+              </h1>
+              <p className="text-text-500 mb-4">
+                We regret to inform you that your access to Danswer has been
+                temporarily suspended due to a lapse in your subscription.
+              </p>
+              <p className="text-text-500 mb-4">
+                To reinstate your access and continue benefiting from
+                Danswer&apos;s powerful features, please update your payment
+                information.
+              </p>
+              <p className="text-text-500">
+                If you&apos;re an admin, you can resolve this by visiting the
+                billing section. For other users, please reach out to your
+                administrator to address this matter.
+              </p>
+            </Card>
+          </div>
+        </body>
+      </html>
+    );
+  }
 
   return (
     <html lang="en">
@@ -137,6 +174,20 @@ export default async function RootLayout({
             process.env.THEME_IS_DARK?.toLowerCase() === "true" ? "dark" : ""
           }`}
         >
+          {productGating === GatingType.PARTIAL && (
+            <div className="fixed top-0 left-0 right-0 z-50 bg-warning-100 text-warning-900 p-2 text-center">
+              <p className="text-sm font-medium">
+                Your account is pending payment!{" "}
+                <a
+                  href="/admin/cloud-settings"
+                  className="font-bold underline hover:text-warning-700 transition-colors"
+                >
+                  Update your billing information
+                </a>{" "}
+                or access will be suspended soon.
+              </p>
+            </div>
+          )}
           <UserProvider>
             <ProviderContextProvider>
               <SettingsProvider settings={combinedSettings}>
diff --git a/web/src/components/admin/ClientLayout.tsx b/web/src/components/admin/ClientLayout.tsx
index e0415f84544..68154c90387 100644
--- a/web/src/components/admin/ClientLayout.tsx
+++ b/web/src/components/admin/ClientLayout.tsx
@@ -30,15 +30,18 @@ import { User } from "@/lib/types";
 import { usePathname } from "next/navigation";
 import { SettingsContext } from "../settings/SettingsProvider";
 import { useContext } from "react";
+import { Cloud } from "@phosphor-icons/react";
 
 export function ClientLayout({
   user,
   children,
   enableEnterprise,
+  enableCloud,
 }: {
   user: User | null;
   children: React.ReactNode;
   enableEnterprise: boolean;
+  enableCloud: boolean;
 }) {
   const isCurator =
     user?.role === UserRole.CURATOR || user?.role === UserRole.GLOBAL_CURATOR;
@@ -390,6 +393,22 @@ export function ClientLayout({
                               },
                             ]
                           : []),
+                        ...(enableCloud
+                          ? [
+                              {
+                                name: (
+                                  <div className="flex">
+                                    <Cloud
+                                      className="text-icon-settings-sidebar"
+                                      size={18}
+                                    />
+                                    <div className="ml-1">Cloud Settings</div>
+                                  </div>
+                                ),
+                                link: "/admin/cloud-settings",
+                              },
+                            ]
+                          : []),
                       ],
                     },
                   ]
diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx
index 145d8d34786..2a4fbaa3de9 100644
--- a/web/src/components/admin/Layout.tsx
+++ b/web/src/components/admin/Layout.tsx
@@ -6,7 +6,10 @@ import {
 } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 import { ClientLayout } from "./ClientLayout";
-import { SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED } from "@/lib/constants";
+import {
+  SERVER_SIDE_ONLY__CLOUD_ENABLED,
+  SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED,
+} from "@/lib/constants";
 import { AnnouncementBanner } from "../header/AnnouncementBanner";
 
 export async function Layout({ children }: { children: React.ReactNode }) {
@@ -43,6 +46,7 @@ export async function Layout({ children }: { children: React.ReactNode }) {
   return (
     <ClientLayout
       enableEnterprise={SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED}
+      enableCloud={SERVER_SIDE_ONLY__CLOUD_ENABLED}
       user={user}
     >
       <AnnouncementBanner />
diff --git a/web/src/components/admin/Title.tsx b/web/src/components/admin/Title.tsx
index 9309cb3b997..0511a39dfed 100644
--- a/web/src/components/admin/Title.tsx
+++ b/web/src/components/admin/Title.tsx
@@ -1,3 +1,4 @@
+"use client";
 import { HealthCheckBanner } from "../health/healthcheck";
 import { Divider } from "@tremor/react";
 
diff --git a/web/src/components/settings/lib.ts b/web/src/components/settings/lib.ts
index 1c1ec9249f1..28a1133be52 100644
--- a/web/src/components/settings/lib.ts
+++ b/web/src/components/settings/lib.ts
@@ -1,6 +1,7 @@
 import {
   CombinedSettings,
   EnterpriseSettings,
+  GatingType,
   Settings,
 } from "@/app/admin/settings/interfaces";
 import {
@@ -42,6 +43,7 @@ export async function fetchSettingsSS(): Promise<CombinedSettings | null> {
     if (!results[0].ok) {
       if (results[0].status === 403 || results[0].status === 401) {
         settings = {
+          product_gating: GatingType.NONE,
           gpu_enabled: false,
           chat_page_enabled: true,
           search_page_enabled: true,
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index 4fe1d616dcd..3b209cfb364 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -56,6 +56,10 @@ export const CUSTOM_ANALYTICS_ENABLED = process.env.CUSTOM_ANALYTICS_SECRET_KEY
 export const DISABLE_LLM_DOC_RELEVANCE =
   process.env.DISABLE_LLM_DOC_RELEVANCE?.toLowerCase() === "true";
 
-export const CLOUD_ENABLED = process.env.NEXT_PUBLIC_CLOUD_ENABLED;
+export const CLOUD_ENABLED =
+  process.env.NEXT_PUBLIC_CLOUD_ENABLED?.toLowerCase() === "true";
 export const REGISTRATION_URL =
   process.env.INTERNAL_URL || "http://127.0.0.1:3001";
+
+export const SERVER_SIDE_ONLY__CLOUD_ENABLED = true;
+// process.env.NEXT_PUBLIC_CLOUD_ENABLED?.toLowerCase() === "true";
diff --git a/web/src/middleware.ts b/web/src/middleware.ts
index b887bec8bbc..06c106fffc4 100644
--- a/web/src/middleware.ts
+++ b/web/src/middleware.ts
@@ -12,12 +12,16 @@ const eePaths = [
   "/admin/whitelabeling/:path*",
   "/admin/performance/custom-analytics/:path*",
   "/admin/standard-answer/:path*",
+  ...(process.env.NEXT_PUBLIC_CLOUD_ENABLED
+    ? ["/admin/cloud-settings/:path*"]
+    : []),
 ];
 
 // removes the "/:path*" from the end
-const strippedEEPaths = eePaths.map((path) =>
-  path.replace(/(.*):\path\*$/, "$1").replace(/\/$/, "")
-);
+const stripPath = (path: string) =>
+  path.replace(/(.*):\path\*$/, "$1").replace(/\/$/, "");
+
+const strippedEEPaths = eePaths.map(stripPath);
 
 export async function middleware(request: NextRequest) {
   if (SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED) {

From 40dc4708d2ec8b5fe7baaf684554e8c4109198ed Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 12 Oct 2024 18:44:28 -0700
Subject: [PATCH 100/376] slightly cleaner loading (#2776)

---
 .../configuration/search/UpgradingPage.tsx     | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/web/src/app/admin/configuration/search/UpgradingPage.tsx b/web/src/app/admin/configuration/search/UpgradingPage.tsx
index 2d9415e10e8..60177661c9d 100644
--- a/web/src/app/admin/configuration/search/UpgradingPage.tsx
+++ b/web/src/app/admin/configuration/search/UpgradingPage.tsx
@@ -27,11 +27,11 @@ export default function UpgradingPage({
   const [isCancelling, setIsCancelling] = useState<boolean>(false);
 
   const { setPopup, popup } = usePopup();
-  const { data: connectors } = useSWR<Connector<any>[]>(
-    "/api/manage/connector",
-    errorHandlingFetcher,
-    { refreshInterval: 5000 } // 5 seconds
-  );
+  const { data: connectors, isLoading: isLoadingConnectors } = useSWR<
+    Connector<any>[]
+  >("/api/manage/connector", errorHandlingFetcher, {
+    refreshInterval: 5000, // 5 seconds
+  });
 
   const {
     data: ongoingReIndexingStatus,
@@ -90,6 +90,10 @@ export default function UpgradingPage({
     });
   }, [ongoingReIndexingStatus]);
 
+  if (isLoadingConnectors || isLoadingOngoingReIndexingStatus) {
+    return <ThreeDotsLoader />;
+  }
+
   return (
     <>
       {popup}
@@ -150,9 +154,7 @@ export default function UpgradingPage({
                   downtime is necessary during this transition.
                 </Text>
 
-                {isLoadingOngoingReIndexingStatus ? (
-                  <ThreeDotsLoader />
-                ) : sortedReindexingProgress ? (
+                {sortedReindexingProgress ? (
                   <ReindexingProgressTable
                     reindexingProgress={sortedReindexingProgress}
                   />

From 3365e0b16e372605489651702195cd3e525aed10 Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Sat, 12 Oct 2024 19:15:59 -0700
Subject: [PATCH 101/376] Fix tag background

---
 web/src/components/search/filtering/TagFilter.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/components/search/filtering/TagFilter.tsx b/web/src/components/search/filtering/TagFilter.tsx
index 17600383583..80a6ee78922 100644
--- a/web/src/components/search/filtering/TagFilter.tsx
+++ b/web/src/components/search/filtering/TagFilter.tsx
@@ -83,7 +83,7 @@ export function TagFilter({
       />
       {selectedTags.length > 0 && (
         <div className="mt-2">
-          <div className="mt-1 flex bg-black flex-wrap gap-x-1 gap-y-1">
+          <div className="mt-1 flex flex-wrap gap-x-1 gap-y-1">
             {selectedTags.map((tag) => (
               <div
                 key={tag.tag_key + tag.tag_value}

From 26bdb41e8f6988ef44a5881cb4a73b9f3e177fdf Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Sat, 12 Oct 2024 20:29:18 -0700
Subject: [PATCH 102/376] Fix parallel tool calls (#2779)

* Fix parallel tool calls

* remove comments
---
 backend/danswer/llm/chat_llm.py               |   5 +-
 .../tests/unit/danswer/llm/test_chat_llm.py   | 290 ++++++++++++++++++
 2 files changed, 293 insertions(+), 2 deletions(-)
 create mode 100644 backend/tests/unit/danswer/llm/test_chat_llm.py

diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
index 90136a76bd8..d50f8253182 100644
--- a/backend/danswer/llm/chat_llm.py
+++ b/backend/danswer/llm/chat_llm.py
@@ -109,7 +109,7 @@ def _convert_message_to_dict(message: BaseMessage) -> dict:
                         "arguments": json.dumps(tool_call["args"]),
                     },
                     "type": "function",
-                    "index": 0,  # only support a single tool call atm
+                    "index": tool_call.get("index", 0),
                 }
                 for tool_call in message.tool_calls
             ]
@@ -158,12 +158,13 @@ def _convert_delta_to_message_chunk(
         if tool_calls:
             tool_call = tool_calls[0]
             tool_name = tool_call.function.name or (curr_msg and curr_msg.name) or ""
+            idx = tool_call.index
 
             tool_call_chunk = ToolCallChunk(
                 name=tool_name,
                 id=tool_call.id,
                 args=tool_call.function.arguments,
-                index=0,  # only support a single tool call atm
+                index=idx,
             )
 
             return AIMessageChunk(
diff --git a/backend/tests/unit/danswer/llm/test_chat_llm.py b/backend/tests/unit/danswer/llm/test_chat_llm.py
new file mode 100644
index 00000000000..efe0281f53c
--- /dev/null
+++ b/backend/tests/unit/danswer/llm/test_chat_llm.py
@@ -0,0 +1,290 @@
+from unittest.mock import patch
+
+import litellm
+import pytest
+from langchain_core.messages import AIMessage
+from langchain_core.messages import AIMessageChunk
+from langchain_core.messages import HumanMessage
+from litellm.types.utils import ChatCompletionDeltaToolCall
+from litellm.types.utils import Delta
+from litellm.types.utils import Function as LiteLLMFunction
+
+from danswer.llm.chat_llm import DefaultMultiLLM
+
+
+def _create_delta(
+    role: str | None = None,
+    content: str | None = None,
+    tool_calls: list[ChatCompletionDeltaToolCall] | None = None,
+) -> Delta:
+    delta = Delta(role=role, content=content)
+    # NOTE: for some reason, if you pass tool_calls to the constructor, it doesn't actually
+    # get set, so we have to do it this way
+    delta.tool_calls = tool_calls
+    return delta
+
+
+@pytest.fixture
+def default_multi_llm() -> DefaultMultiLLM:
+    return DefaultMultiLLM(
+        api_key="test_key",
+        timeout=30,
+        model_provider="openai",
+        model_name="gpt-3.5-turbo",
+    )
+
+
+def test_multiple_tool_calls(default_multi_llm: DefaultMultiLLM) -> None:
+    # Mock the litellm.completion function
+    with patch("danswer.llm.chat_llm.litellm.completion") as mock_completion:
+        # Create a mock response with multiple tool calls using litellm objects
+        mock_response = litellm.ModelResponse(
+            id="chatcmpl-123",
+            choices=[
+                litellm.Choices(
+                    finish_reason="tool_calls",
+                    index=0,
+                    message=litellm.Message(
+                        content=None,
+                        role="assistant",
+                        tool_calls=[
+                            litellm.ChatCompletionMessageToolCall(
+                                id="call_1",
+                                function=LiteLLMFunction(
+                                    name="get_weather",
+                                    arguments='{"location": "New York"}',
+                                ),
+                                type="function",
+                            ),
+                            litellm.ChatCompletionMessageToolCall(
+                                id="call_2",
+                                function=LiteLLMFunction(
+                                    name="get_time", arguments='{"timezone": "EST"}'
+                                ),
+                                type="function",
+                            ),
+                        ],
+                    ),
+                )
+            ],
+            model="gpt-3.5-turbo",
+            usage=litellm.Usage(
+                prompt_tokens=50, completion_tokens=30, total_tokens=80
+            ),
+        )
+        mock_completion.return_value = mock_response
+
+        # Define input messages
+        messages = [HumanMessage(content="What's the weather and time in New York?")]
+
+        # Define available tools
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get the current weather for a location",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"location": {"type": "string"}},
+                        "required": ["location"],
+                    },
+                },
+            },
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_time",
+                    "description": "Get the current time for a timezone",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"timezone": {"type": "string"}},
+                        "required": ["timezone"],
+                    },
+                },
+            },
+        ]
+
+        # Call the _invoke_implementation method
+        result = default_multi_llm.invoke(messages, tools)
+
+        # Assert that the result is an AIMessage
+        assert isinstance(result, AIMessage)
+
+        # Assert that the content is None (as per the mock response)
+        assert result.content == ""
+
+        # Assert that there are two tool calls
+        assert len(result.tool_calls) == 2
+
+        # Assert the details of the first tool call
+        assert result.tool_calls[0]["id"] == "call_1"
+        assert result.tool_calls[0]["name"] == "get_weather"
+        assert result.tool_calls[0]["args"] == {"location": "New York"}
+
+        # Assert the details of the second tool call
+        assert result.tool_calls[1]["id"] == "call_2"
+        assert result.tool_calls[1]["name"] == "get_time"
+        assert result.tool_calls[1]["args"] == {"timezone": "EST"}
+
+        # Verify that litellm.completion was called with the correct arguments
+        mock_completion.assert_called_once_with(
+            model="openai/gpt-3.5-turbo",
+            api_key="test_key",
+            base_url=None,
+            api_version=None,
+            custom_llm_provider=None,
+            messages=[
+                {"role": "user", "content": "What's the weather and time in New York?"}
+            ],
+            tools=tools,
+            tool_choice=None,
+            stream=False,
+            temperature=0.0,  # Default value from GEN_AI_TEMPERATURE
+            timeout=30,
+            parallel_tool_calls=False,
+        )
+
+
+def test_multiple_tool_calls_streaming(default_multi_llm: DefaultMultiLLM) -> None:
+    # Mock the litellm.completion function
+    with patch("danswer.llm.chat_llm.litellm.completion") as mock_completion:
+        # Create a mock response with multiple tool calls using litellm objects
+        mock_response = [
+            litellm.ModelResponse(
+                id="chatcmpl-123",
+                choices=[
+                    litellm.Choices(
+                        delta=_create_delta(
+                            role="assistant",
+                            tool_calls=[
+                                ChatCompletionDeltaToolCall(
+                                    id="call_1",
+                                    function=LiteLLMFunction(
+                                        name="get_weather", arguments='{"location": '
+                                    ),
+                                    type="function",
+                                    index=0,
+                                )
+                            ],
+                        ),
+                        finish_reason=None,
+                        index=0,
+                    )
+                ],
+                model="gpt-3.5-turbo",
+            ),
+            litellm.ModelResponse(
+                id="chatcmpl-123",
+                choices=[
+                    litellm.Choices(
+                        delta=_create_delta(
+                            tool_calls=[
+                                ChatCompletionDeltaToolCall(
+                                    id="",
+                                    function=LiteLLMFunction(arguments='"New York"}'),
+                                    type="function",
+                                    index=0,
+                                )
+                            ]
+                        ),
+                        finish_reason=None,
+                        index=0,
+                    )
+                ],
+                model="gpt-3.5-turbo",
+            ),
+            litellm.ModelResponse(
+                id="chatcmpl-123",
+                choices=[
+                    litellm.Choices(
+                        delta=_create_delta(
+                            tool_calls=[
+                                ChatCompletionDeltaToolCall(
+                                    id="call_2",
+                                    function=LiteLLMFunction(
+                                        name="get_time", arguments='{"timezone": "EST"}'
+                                    ),
+                                    type="function",
+                                    index=1,
+                                )
+                            ]
+                        ),
+                        finish_reason="tool_calls",
+                        index=0,
+                    )
+                ],
+                model="gpt-3.5-turbo",
+            ),
+        ]
+        mock_completion.return_value = mock_response
+
+        # Define input messages and tools (same as in the non-streaming test)
+        messages = [HumanMessage(content="What's the weather and time in New York?")]
+
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get the current weather for a location",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"location": {"type": "string"}},
+                        "required": ["location"],
+                    },
+                },
+            },
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_time",
+                    "description": "Get the current time for a timezone",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"timezone": {"type": "string"}},
+                        "required": ["timezone"],
+                    },
+                },
+            },
+        ]
+
+        # Call the stream method
+        stream_result = list(default_multi_llm.stream(messages, tools))
+
+        # Assert that we received the correct number of chunks
+        assert len(stream_result) == 3
+
+        # Combine all chunks into a single AIMessage
+        combined_result: AIMessage = AIMessageChunk(content="")
+        for chunk in stream_result:
+            combined_result += chunk  # type: ignore
+
+        # Assert that the combined result matches our expectations
+        assert isinstance(combined_result, AIMessage)
+        assert combined_result.content == ""
+        assert len(combined_result.tool_calls) == 2
+        assert combined_result.tool_calls[0]["id"] == "call_1"
+        assert combined_result.tool_calls[0]["name"] == "get_weather"
+        assert combined_result.tool_calls[0]["args"] == {"location": "New York"}
+        assert combined_result.tool_calls[1]["id"] == "call_2"
+        assert combined_result.tool_calls[1]["name"] == "get_time"
+        assert combined_result.tool_calls[1]["args"] == {"timezone": "EST"}
+
+        # Verify that litellm.completion was called with the correct arguments
+        mock_completion.assert_called_once_with(
+            model="openai/gpt-3.5-turbo",
+            api_key="test_key",
+            base_url=None,
+            api_version=None,
+            custom_llm_provider=None,
+            messages=[
+                {"role": "user", "content": "What's the weather and time in New York?"}
+            ],
+            tools=tools,
+            tool_choice=None,
+            stream=True,
+            temperature=0.0,  # Default value from GEN_AI_TEMPERATURE
+            timeout=30,
+            parallel_tool_calls=False,
+        )

From b393af676c8e76a862861d50ebaf808be8fab5d8 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 13 Oct 2024 14:35:56 -0700
Subject: [PATCH 103/376] Mypy (#2785)

---
 backend/pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 993b46e2c30..032c665196d 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -5,7 +5,7 @@ explicit_package_bases = true
 disallow_untyped_defs = true
 
 [[tool.mypy.overrides]]
-module = "alembic.versions.*"
+module = "alembic*.versions.*"
 disable_error_code = ["var-annotated"]
 
 [tool.ruff]

From 86ecf8e0fc3e5e99ef8dccff2908931f1d10e11c Mon Sep 17 00:00:00 2001
From: OMKAR MAKHARE <omimakhare11@gmail.com>
Date: Sun, 13 Oct 2024 11:12:53 +0530
Subject: [PATCH 104/376] Update README.md

- Corrected misspelling of Noteable to Notable.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index aff3cd57d5a..087b0df531f 100644
--- a/README.md
+++ b/README.md
@@ -74,7 +74,7 @@ We also have built-in support for deployment on Kubernetes. Files for that can b
 * Organizational understanding and ability to locate and suggest experts from your team.
 
 
-## Other Noteable Benefits of Danswer
+## Other Notable Benefits of Danswer
 * User Authentication with document level access management.
 * Best in class Hybrid Search across all sources (BM-25 + prefix aware embedding models).
 * Admin Dashboard to configure connectors, document-sets, access, etc.

From ded42e20364541eb293cd94208e1c8221f2404c4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 13 Oct 2024 18:22:14 -0700
Subject: [PATCH 105/376] nit (#2787)

---
 backend/danswer/document_index/vespa/indexing_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/document_index/vespa/indexing_utils.py b/backend/danswer/document_index/vespa/indexing_utils.py
index de0d5fdaf19..35ebd52430f 100644
--- a/backend/danswer/document_index/vespa/indexing_utils.py
+++ b/backend/danswer/document_index/vespa/indexing_utils.py
@@ -183,6 +183,7 @@ def _index_vespa_chunk(
         vespa_document_fields[TENANT_ID] = chunk.tenant_id
 
     vespa_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{vespa_chunk_id}"
+    logger.debug(f'Indexing to URL "{vespa_url}"')
     res = http_client.post(
         vespa_url, headers=json_header, json={"fields": vespa_document_fields}
     )

From a9bcc89a2cc45ecdf53dd7c84aae6d42913abb54 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 13 Oct 2024 19:09:17 -0700
Subject: [PATCH 106/376] Add cursor to cql confluence (#2775)

* add cursor to cql confluence

* k

* k

* fixed space indexing issue

* fixed .get

---------

Co-authored-by: hagen-danswer <hagen@danswer.ai>
---
 .../connectors/confluence/connector.py        | 479 +++++++-----------
 web/src/lib/connectors/connectors.tsx         |   2 +-
 2 files changed, 184 insertions(+), 297 deletions(-)

diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index 623e742ef0a..82174c8b931 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -1,13 +1,13 @@
 import io
 import os
-import re
 from collections.abc import Callable
 from collections.abc import Collection
 from datetime import datetime
 from datetime import timezone
 from functools import lru_cache
 from typing import Any
-from typing import cast
+from urllib.parse import parse_qs
+from urllib.parse import urlparse
 
 import bs4
 from atlassian import Confluence  # type:ignore
@@ -33,7 +33,6 @@
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
-from danswer.connectors.interfaces import SecondsSinceUnixEpoch
 from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import ConnectorMissingCredentialError
 from danswer.connectors.models import Document
@@ -70,86 +69,25 @@ def danswer_cql(
         self,
         cql: str,
         expand: str | None = None,
-        start: int = 0,
+        cursor: str | None = None,
         limit: int = 500,
         include_archived_spaces: bool = False,
-    ) -> list[dict[str, Any]]:
-        # Performs the query expansion and start/limit url additions
+    ) -> dict[str, Any]:
         url_suffix = f"rest/api/content/search?cql={cql}"
         if expand:
             url_suffix += f"&expand={expand}"
-        url_suffix += f"&start={start}&limit={limit}"
+        if cursor:
+            url_suffix += f"&cursor={cursor}"
+        url_suffix += f"&limit={limit}"
         if include_archived_spaces:
             url_suffix += "&includeArchivedSpaces=true"
         try:
             response = self.get(url_suffix)
-            return response.get("results", [])
+            return response
         except Exception as e:
             raise e
 
 
-def _replace_cql_time_filter(
-    cql_query: str, start_time: datetime, end_time: datetime
-) -> str:
-    """
-    This function replaces the lastmodified filter in the CQL query with the start and end times.
-    This selects the more restrictive time range.
-    """
-    # Extract existing lastmodified >= and <= filters
-    existing_start_match = re.search(
-        r'lastmodified\s*>=\s*["\']?(\d{4}-\d{2}-\d{2}(?:\s+\d{2}:\d{2})?)["\']?',
-        cql_query,
-        flags=re.IGNORECASE,
-    )
-    existing_end_match = re.search(
-        r'lastmodified\s*<=\s*["\']?(\d{4}-\d{2}-\d{2}(?:\s+\d{2}:\d{2})?)["\']?',
-        cql_query,
-        flags=re.IGNORECASE,
-    )
-
-    # Remove all existing lastmodified and updated filters
-    cql_query = re.sub(
-        r'\s*AND\s+(lastmodified|updated)\s*[<>=]+\s*["\']?[\d-]+(?:\s+[\d:]+)?["\']?',
-        "",
-        cql_query,
-        flags=re.IGNORECASE,
-    )
-
-    # Determine the start time to use
-    if existing_start_match:
-        existing_start_str = existing_start_match.group(1)
-        existing_start = datetime.strptime(
-            existing_start_str,
-            "%Y-%m-%d %H:%M" if " " in existing_start_str else "%Y-%m-%d",
-        )
-        existing_start = existing_start.replace(
-            tzinfo=timezone.utc
-        )  # Make offset-aware
-        start_time_to_use = max(start_time.astimezone(timezone.utc), existing_start)
-    else:
-        start_time_to_use = start_time.astimezone(timezone.utc)
-
-    # Determine the end time to use
-    if existing_end_match:
-        existing_end_str = existing_end_match.group(1)
-        existing_end = datetime.strptime(
-            existing_end_str,
-            "%Y-%m-%d %H:%M" if " " in existing_end_str else "%Y-%m-%d",
-        )
-        existing_end = existing_end.replace(tzinfo=timezone.utc)  # Make offset-aware
-        end_time_to_use = min(end_time.astimezone(timezone.utc), existing_end)
-    else:
-        end_time_to_use = end_time.astimezone(timezone.utc)
-
-    # Add new time filters
-    cql_query += (
-        f" and lastmodified >= '{start_time_to_use.strftime('%Y-%m-%d %H:%M')}'"
-    )
-    cql_query += f" and lastmodified <= '{end_time_to_use.strftime('%Y-%m-%d %H:%M')}'"
-
-    return cql_query.strip()
-
-
 @lru_cache()
 def _get_user(user_id: str, confluence_client: DanswerConfluence) -> str:
     """Get Confluence Display Name based on the account-id or userkey value
@@ -253,126 +191,86 @@ class RecursiveIndexer:
     def __init__(
         self,
         batch_size: int,
-        confluence_client: DanswerConfluence,
+        confluence_client: Confluence,
         index_recursively: bool,
         origin_page_id: str,
     ) -> None:
-        self.batch_size = 1
-        # batch_size
+        self.batch_size = batch_size
         self.confluence_client = confluence_client
         self.index_recursively = index_recursively
         self.origin_page_id = origin_page_id
-        self.pages = self.recurse_children_pages(0, self.origin_page_id)
+        self.pages = self.recurse_children_pages(self.origin_page_id)
 
     def get_origin_page(self) -> list[dict[str, Any]]:
         return [self._fetch_origin_page()]
 
-    def get_pages(self, ind: int, size: int) -> list[dict]:
-        if ind * size > len(self.pages):
-            return []
-        return self.pages[ind * size : (ind + 1) * size]
+    def get_pages(self) -> list[dict[str, Any]]:
+        return self.pages
 
-    def _fetch_origin_page(
-        self,
-    ) -> dict[str, Any]:
+    def _fetch_origin_page(self) -> dict[str, Any]:
         get_page_by_id = make_confluence_call_handle_rate_limit(
             self.confluence_client.get_page_by_id
         )
         try:
             origin_page = get_page_by_id(
-                self.origin_page_id, expand="body.storage.value,version"
+                self.origin_page_id, expand="body.storage.value,version,space"
             )
             return origin_page
         except Exception as e:
             logger.warning(
-                f"Appending orgin page with id {self.origin_page_id} failed: {e}"
+                f"Appending origin page with id {self.origin_page_id} failed: {e}"
             )
             return {}
 
     def recurse_children_pages(
         self,
-        start_ind: int,
         page_id: str,
     ) -> list[dict[str, Any]]:
         pages: list[dict[str, Any]] = []
-        current_level_pages: list[dict[str, Any]] = []
-        next_level_pages: list[dict[str, Any]] = []
-
-        # Initial fetch of first level children
-        index = start_ind
-        while batch := self._fetch_single_depth_child_pages(
-            index, self.batch_size, page_id
-        ):
-            current_level_pages.extend(batch)
-            index += len(batch)
-
-        pages.extend(current_level_pages)
-
-        # Recursively index children and children's children, etc.
-        while current_level_pages:
-            for child in current_level_pages:
-                child_index = 0
-                while child_batch := self._fetch_single_depth_child_pages(
-                    child_index, self.batch_size, child["id"]
-                ):
-                    next_level_pages.extend(child_batch)
-                    child_index += len(child_batch)
-
-            pages.extend(next_level_pages)
-            current_level_pages = next_level_pages
-            next_level_pages = []
-
-        try:
-            origin_page = self._fetch_origin_page()
-            pages.append(origin_page)
-        except Exception as e:
-            logger.warning(f"Appending origin page with id {page_id} failed: {e}")
-
-        return pages
-
-    def _fetch_single_depth_child_pages(
-        self, start_ind: int, batch_size: int, page_id: str
-    ) -> list[dict[str, Any]]:
-        child_pages: list[dict[str, Any]] = []
+        queue: list[str] = [page_id]
+        visited_pages: set[str] = set()
 
         get_page_child_by_type = make_confluence_call_handle_rate_limit(
             self.confluence_client.get_page_child_by_type
         )
 
-        try:
-            child_page = get_page_child_by_type(
-                page_id,
-                type="page",
-                start=start_ind,
-                limit=batch_size,
-                expand="body.storage.value,version",
-            )
+        while queue:
+            current_page_id = queue.pop(0)
+            if current_page_id in visited_pages:
+                continue
+            visited_pages.add(current_page_id)
 
-            child_pages.extend(child_page)
-            return child_pages
+            try:
+                # Fetch the page itself
+                page = self.confluence_client.get_page_by_id(
+                    current_page_id, expand="body.storage.value,version,space"
+                )
+                pages.append(page)
+            except Exception as e:
+                logger.warning(f"Failed to fetch page {current_page_id}: {e}")
+                continue
 
-        except Exception:
-            logger.warning(
-                f"Batch failed with page {page_id} at offset {start_ind} "
-                f"with size {batch_size}, processing pages individually..."
-            )
+            if not self.index_recursively:
+                continue
 
-            for i in range(batch_size):
-                ind = start_ind + i
-                try:
-                    child_page = get_page_child_by_type(
-                        page_id,
-                        type="page",
-                        start=ind,
-                        limit=1,
-                        expand="body.storage.value,version",
-                    )
-                    child_pages.extend(child_page)
-                except Exception as e:
-                    logger.warning(f"Page {page_id} at offset {ind} failed: {e}")
-                    raise e
+            # Fetch child pages
+            start = 0
+            while True:
+                child_pages_response = get_page_child_by_type(
+                    current_page_id,
+                    type="page",
+                    start=start,
+                    limit=self.batch_size,
+                    expand="",
+                )
+                if not child_pages_response:
+                    break
+                for child_page in child_pages_response:
+                    child_page_id = child_page["id"]
+                    queue.append(child_page_id)
+                start += len(child_pages_response)
 
-            return child_pages
+        return pages
 
 
 class ConfluenceConnector(LoadConnector, PollConnector):
@@ -399,7 +297,6 @@ def __init__(
 
         # Remove trailing slash from wiki_base if present
         self.wiki_base = wiki_base.rstrip("/")
-        self.space = space
         self.page_id = "" if cql_query else page_id
         self.space_level_scan = bool(not self.page_id)
 
@@ -409,16 +306,16 @@ def __init__(
 
         # if a cql_query is provided, we will use it to fetch the pages
         # if no cql_query is provided, we will use the space to fetch the pages
-        # if no space is provided, we will default to fetching all pages, regardless of space
+        # if no space is provided and no cql_query, we will default to fetching all pages, regardless of space
         if cql_query:
             self.cql_query = cql_query
-        elif self.space:
-            self.cql_query = f"type=page and space={self.space}"
+        elif space:
+            self.cql_query = f"type=page and space='{space}'"
         else:
             self.cql_query = "type=page"
 
         logger.info(
-            f"wiki_base: {self.wiki_base}, space: {self.space}, page_id: {self.page_id},"
+            f"wiki_base: {self.wiki_base}, space: {space}, page_id: {self.page_id},"
             + f" space_level_scan: {self.space_level_scan}, index_recursively: {self.index_recursively},"
             + f" cql_query: {self.cql_query}"
         )
@@ -428,7 +325,6 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
         access_token = credentials["confluence_access_token"]
         self.confluence_client = DanswerConfluence(
             url=self.wiki_base,
-            # passing in username causes issues for Confluence data center
             username=username if self.is_cloud else None,
             password=access_token if self.is_cloud else None,
             token=access_token if not self.is_cloud else None,
@@ -437,12 +333,16 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
 
     def _fetch_pages(
         self,
-        start_ind: int,
-    ) -> list[dict[str, Any]]:
-        def _fetch_space(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
-            if self.confluence_client is None:
-                raise ConnectorMissingCredentialError("Confluence")
+        cursor: str | None,
+    ) -> tuple[list[dict[str, Any]], str | None]:
+        if self.confluence_client is None:
+            raise Exception("Confluence client is not initialized")
 
+        def _fetch_space(
+            cursor: str | None, batch_size: int
+        ) -> tuple[list[dict[str, Any]], str | None]:
+            if not self.confluence_client:
+                raise Exception("Confluence client is not initialized")
             get_all_pages = make_confluence_call_handle_rate_limit(
                 self.confluence_client.danswer_cql
             )
@@ -454,53 +354,84 @@ def _fetch_space(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
             )
 
             try:
-                return get_all_pages(
+                response = get_all_pages(
                     cql=self.cql_query,
-                    start=start_ind,
+                    cursor=cursor,
                     limit=batch_size,
-                    expand="body.storage.value,version",
+                    expand="body.storage.value,version,space",
                     include_archived_spaces=include_archived_spaces,
                 )
+                pages = response.get("results", [])
+                next_cursor = None
+                if "_links" in response and "next" in response["_links"]:
+                    next_link = response["_links"]["next"]
+                    parsed_url = urlparse(next_link)
+                    query_params = parse_qs(parsed_url.query)
+                    cursor_list = query_params.get("cursor", [])
+                    if cursor_list:
+                        next_cursor = cursor_list[0]
+                return pages, next_cursor
             except Exception:
                 logger.warning(
-                    f"Batch failed with cql {self.cql_query} at offset {start_ind} "
-                    f"with size {batch_size}, processing pages individually..."
+                    f"Batch failed with cql {self.cql_query} with cursor {cursor} "
+                    f"and size {batch_size}, processing pages individually..."
                 )
 
                 view_pages: list[dict[str, Any]] = []
-                for i in range(self.batch_size):
+                for _ in range(self.batch_size):
                     try:
-                        # Could be that one of the pages here failed due to this bug:
-                        # https://jira.atlassian.com/browse/CONFCLOUD-76433
-                        view_pages.extend(
-                            get_all_pages(
-                                cql=self.cql_query,
-                                start=start_ind + i,
-                                limit=1,
-                                expand="body.storage.value,version",
-                                include_archived_spaces=include_archived_spaces,
-                            )
+                        response = get_all_pages(
+                            cql=self.cql_query,
+                            cursor=cursor,
+                            limit=1,
+                            expand="body.view.value,version,space",
+                            include_archived_spaces=include_archived_spaces,
                         )
+                        pages = response.get("results", [])
+                        view_pages.extend(pages)
+                        if "_links" in response and "next" in response["_links"]:
+                            next_link = response["_links"]["next"]
+                            parsed_url = urlparse(next_link)
+                            query_params = parse_qs(parsed_url.query)
+                            cursor_list = query_params.get("cursor", [])
+                            if cursor_list:
+                                cursor = cursor_list[0]
+                            else:
+                                cursor = None
+                        else:
+                            cursor = None
+                            break
                     except HTTPError as e:
                         logger.warning(
-                            f"Page failed with cql {self.cql_query} at offset {start_ind + i}, "
+                            f"Page failed with cql {self.cql_query} with cursor {cursor}, "
                             f"trying alternative expand option: {e}"
                         )
-                        # Use view instead, which captures most info but is less complete
-                        view_pages.extend(
-                            get_all_pages(
-                                cql=self.cql_query,
-                                start=start_ind + i,
-                                limit=1,
-                                expand="body.view.value,version",
-                            )
+                        response = get_all_pages(
+                            cql=self.cql_query,
+                            cursor=cursor,
+                            limit=1,
+                            expand="body.view.value,version,space",
                         )
-
-                return view_pages
-
-        def _fetch_page(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
+                        pages = response.get("results", [])
+                        view_pages.extend(pages)
+                        if "_links" in response and "next" in response["_links"]:
+                            next_link = response["_links"]["next"]
+                            parsed_url = urlparse(next_link)
+                            query_params = parse_qs(parsed_url.query)
+                            cursor_list = query_params.get("cursor", [])
+                            if cursor_list:
+                                cursor = cursor_list[0]
+                            else:
+                                cursor = None
+                        else:
+                            cursor = None
+                            break
+
+                return view_pages, cursor
+
+        def _fetch_page() -> tuple[list[dict[str, Any]], str | None]:
             if self.confluence_client is None:
-                raise ConnectorMissingCredentialError("Confluence")
+                raise Exception("Confluence client is not initialized")
 
             if self.recursive_indexer is None:
                 self.recursive_indexer = RecursiveIndexer(
@@ -510,59 +441,37 @@ def _fetch_page(start_ind: int, batch_size: int) -> list[dict[str, Any]]:
                     index_recursively=self.index_recursively,
                 )
 
-            if self.index_recursively:
-                return self.recursive_indexer.get_pages(start_ind, batch_size)
-            else:
-                return self.recursive_indexer.get_origin_page()
-
-        pages: list[dict[str, Any]] = []
+            pages = self.recursive_indexer.get_pages()
+            return pages, None  # Since we fetched all pages, no cursor
 
         try:
-            pages = (
-                _fetch_space(start_ind, self.batch_size)
+            pages, next_cursor = (
+                _fetch_space(cursor, self.batch_size)
                 if self.space_level_scan
-                else _fetch_page(start_ind, self.batch_size)
+                else _fetch_page()
             )
-            return pages
-
+            return pages, next_cursor
         except Exception as e:
             if not self.continue_on_failure:
                 raise e
 
-        # error checking phase, only reachable if `self.continue_on_failure=True`
-        for _ in range(self.batch_size):
-            try:
-                pages = (
-                    _fetch_space(start_ind, self.batch_size)
-                    if self.space_level_scan
-                    else _fetch_page(start_ind, self.batch_size)
-                )
-                return pages
+            logger.exception("Ran into exception when fetching pages from Confluence")
+            return [], None
 
-            except Exception:
-                logger.exception(
-                    "Ran into exception when fetching pages from Confluence"
-                )
-
-        return pages
-
-    def _fetch_comments(
-        self, confluence_client: DanswerConfluence, page_id: str
-    ) -> str:
+    def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
         get_page_child_by_type = make_confluence_call_handle_rate_limit(
             confluence_client.get_page_child_by_type
         )
 
         try:
-            comment_pages = cast(
-                Collection[dict[str, Any]],
+            comment_pages = list(
                 get_page_child_by_type(
                     page_id,
                     type="comment",
                     start=None,
                     limit=None,
                     expand="body.storage.value",
-                ),
+                )
             )
             return _comment_dfs("", comment_pages, confluence_client)
         except Exception as e:
@@ -574,9 +483,7 @@ def _fetch_comments(
             )
             return ""
 
-    def _fetch_labels(
-        self, confluence_client: DanswerConfluence, page_id: str
-    ) -> list[str]:
+    def _fetch_labels(self, confluence_client: Confluence, page_id: str) -> list[str]:
         get_page_labels = make_confluence_call_handle_rate_limit(
             confluence_client.get_page_labels
         )
@@ -647,22 +554,22 @@ def _attachment_to_content(
         return extracted_text
 
     def _fetch_attachments(
-        self, confluence_client: Confluence, page_id: str, files_in_used: list[str]
+        self, confluence_client: Confluence, page_id: str, files_in_use: list[str]
     ) -> tuple[str, list[dict[str, Any]]]:
-        unused_attachments: list = []
+        unused_attachments: list[dict[str, Any]] = []
+        files_attachment_content: list[str] = []
 
         get_attachments_from_content = make_confluence_call_handle_rate_limit(
             confluence_client.get_attachments_from_content
         )
-        files_attachment_content: list = []
 
         try:
             expand = "history.lastUpdated,metadata.labels"
             attachments_container = get_attachments_from_content(
-                page_id, start=0, limit=500, expand=expand
+                page_id, start=None, limit=None, expand=expand
             )
-            for attachment in attachments_container["results"]:
-                if attachment["title"] not in files_in_used:
+            for attachment in attachments_container.get("results", []):
+                if attachment["title"] not in files_in_use:
                     unused_attachments.append(attachment)
                     continue
 
@@ -680,7 +587,6 @@ def _fetch_attachments(
                     f"User does not have access to attachments on page '{page_id}'"
                 )
                 return "", []
-
             if not self.continue_on_failure:
                 raise e
             logger.exception(
@@ -690,24 +596,26 @@ def _fetch_attachments(
         return "\n".join(files_attachment_content), unused_attachments
 
     def _get_doc_batch(
-        self, start_ind: int
-    ) -> tuple[list[Document], list[dict[str, Any]], int]:
+        self, cursor: str | None
+    ) -> tuple[list[Any], str | None, list[dict[str, Any]]]:
         if self.confluence_client is None:
-            raise ConnectorMissingCredentialError("Confluence")
+            raise Exception("Confluence client is not initialized")
 
-        doc_batch: list[Document] = []
+        doc_batch: list[Any] = []
         unused_attachments: list[dict[str, Any]] = []
 
-        batch = self._fetch_pages(start_ind)
+        batch, next_cursor = self._fetch_pages(cursor)
 
         for page in batch:
             last_modified = _datetime_from_string(page["version"]["when"])
-            author = cast(str | None, page["version"].get("by", {}).get("email"))
+            author = page["version"].get("by", {}).get("email")
 
             page_id = page["id"]
 
             if self.labels_to_skip or not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING:
                 page_labels = self._fetch_labels(self.confluence_client, page_id)
+            else:
+                page_labels = []
 
             # check disallowed labels
             if self.labels_to_skip:
@@ -717,7 +625,6 @@ def _get_doc_batch(
                         f"Page with ID '{page_id}' has a label which has been "
                         f"designated as disallowed: {label_intersection}. Skipping."
                     )
-
                     continue
 
             page_html = (
@@ -732,16 +639,18 @@ def _get_doc_batch(
                 continue
             page_text = parse_html_page(page_html, self.confluence_client)
 
-            files_in_used = get_used_attachments(page_html)
+            files_in_use = get_used_attachments(page_html)
             attachment_text, unused_page_attachments = self._fetch_attachments(
-                self.confluence_client, page_id, files_in_used
+                self.confluence_client, page_id, files_in_use
             )
             unused_attachments.extend(unused_page_attachments)
 
             page_text += "\n" + attachment_text if attachment_text else ""
             comments_text = self._fetch_comments(self.confluence_client, page_id)
             page_text += comments_text
-            doc_metadata: dict[str, str | list[str]] = {"Wiki Space Name": self.space}
+            doc_metadata: dict[str, str | list[str]] = {
+                "Wiki Space Name": page["space"]["name"]
+            }
             if not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING and page_labels:
                 doc_metadata["labels"] = page_labels
 
@@ -760,8 +669,8 @@ def _get_doc_batch(
             )
         return (
             doc_batch,
+            next_cursor,
             unused_attachments,
-            len(batch),
         )
 
     def _get_attachment_batch(
@@ -769,8 +678,8 @@ def _get_attachment_batch(
         start_ind: int,
         attachments: list[dict[str, Any]],
         time_filter: Callable[[datetime], bool] | None = None,
-    ) -> tuple[list[Document], int]:
-        doc_batch: list[Document] = []
+    ) -> tuple[list[Any], int]:
+        doc_batch: list[Any] = []
 
         if self.confluence_client is None:
             raise ConnectorMissingCredentialError("Confluence")
@@ -798,7 +707,7 @@ def _get_attachment_batch(
             creator_email = attachment["history"]["createdBy"].get("email")
 
             comment = attachment["metadata"].get("comment", "")
-            doc_metadata: dict[str, str | list[str]] = {"comment": comment}
+            doc_metadata: dict[str, Any] = {"comment": comment}
 
             attachment_labels: list[str] = []
             if not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING:
@@ -825,25 +734,36 @@ def _get_attachment_batch(
 
         return doc_batch, end_ind - start_ind
 
-    def load_from_state(self) -> GenerateDocumentsOutput:
-        unused_attachments: list[dict[str, Any]] = []
+    def _handle_batch_retrieval(
+        self,
+        start: float | None = None,
+        end: float | None = None,
+    ) -> GenerateDocumentsOutput:
+        start_time = datetime.fromtimestamp(start, tz=timezone.utc) if start else None
+        end_time = datetime.fromtimestamp(end, tz=timezone.utc) if end else None
 
-        start_ind = 0
+        unused_attachments: list[dict[str, Any]] = []
+        cursor = None
         while True:
-            doc_batch, unused_attachments, num_pages = self._get_doc_batch(start_ind)
-            unused_attachments.extend(unused_attachments)
-            start_ind += num_pages
+            doc_batch, cursor, new_unused_attachments = self._get_doc_batch(cursor)
+            unused_attachments.extend(new_unused_attachments)
             if doc_batch:
                 yield doc_batch
 
-            if num_pages < self.batch_size:
+            if not cursor:
                 break
 
+        # Process attachments if any
         start_ind = 0
         while True:
             attachment_batch, num_attachments = self._get_attachment_batch(
-                start_ind, unused_attachments
+                start_ind=start_ind,
+                attachments=unused_attachments,
+                time_filter=(lambda t: start_time <= t <= end_time)
+                if start_time and end_time
+                else None,
             )
+
             start_ind += num_attachments
             if attachment_batch:
                 yield attachment_batch
@@ -851,44 +771,11 @@ def load_from_state(self) -> GenerateDocumentsOutput:
             if num_attachments < self.batch_size:
                 break
 
-    def poll_source(
-        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
-    ) -> GenerateDocumentsOutput:
-        unused_attachments: list[dict[str, Any]] = []
-
-        if self.confluence_client is None:
-            raise ConnectorMissingCredentialError("Confluence")
-
-        start_time = datetime.fromtimestamp(start, tz=timezone.utc)
-        end_time = datetime.fromtimestamp(end, tz=timezone.utc)
-
-        self.cql_query = _replace_cql_time_filter(self.cql_query, start_time, end_time)
-
-        start_ind = 0
-        while True:
-            doc_batch, unused_attachments, num_pages = self._get_doc_batch(start_ind)
-            unused_attachments.extend(unused_attachments)
-
-            start_ind += num_pages
-            if doc_batch:
-                yield doc_batch
-
-            if num_pages < self.batch_size:
-                break
-
-        start_ind = 0
-        while True:
-            attachment_batch, num_attachments = self._get_attachment_batch(
-                start_ind,
-                unused_attachments,
-                time_filter=lambda t: start_time <= t <= end_time,
-            )
-            start_ind += num_attachments
-            if attachment_batch:
-                yield attachment_batch
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self._handle_batch_retrieval()
 
-            if num_attachments < self.batch_size:
-                break
+    def poll_source(self, start: float, end: float) -> GenerateDocumentsOutput:
+        return self._handle_batch_retrieval(start=start, end=end)
 
 
 if __name__ == "__main__":
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index 96e21aa6799..d722fcf9848 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -306,7 +306,7 @@ export const connectorConfigs: Record<
         name: "cql_query",
         optional: true,
         description:
-          "IMPORTANT: This will overwrite all other selected connector settings (besides Wiki Base URL). We currently only support CQL queries that return objects of type 'page'. This means all CQL queries must contain 'type=page' as the only type filter. We will still get all attachments and comments for the pages returned by the CQL query. Any 'lastmodified' filters will be overwritten. See https://developer.atlassian.com/server/confluence/advanced-searching-using-cql/ for more details.",
+          "IMPORTANT: This will overwrite all other selected connector settings (besides Wiki Base URL). We currently only support CQL queries that return objects of type 'page'. This means all CQL queries must contain 'type=page' as the only type filter. It is also important that no filters for 'lastModified' are used as it will cause issues with our connector polling logic. We will still get all attachments and comments for the pages returned by the CQL query. Any 'lastmodified' filters will be overwritten. See https://developer.atlassian.com/server/confluence/advanced-searching-using-cql/ for more details.",
       },
     ],
   },

From ba712d447dc597231c7a2533b6e57e1322c7c09f Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 13 Oct 2024 23:01:17 -0700
Subject: [PATCH 107/376] Notion Connector Improvements (#2789)

---
 .../danswer/connectors/notion/connector.py    | 171 ++++++++++++++----
 backend/pyproject.toml                        |   6 +-
 backend/requirements/dev.txt                  |   1 +
 3 files changed, 143 insertions(+), 35 deletions(-)

diff --git a/backend/danswer/connectors/notion/connector.py b/backend/danswer/connectors/notion/connector.py
index 7878434da04..e856e9970bc 100644
--- a/backend/danswer/connectors/notion/connector.py
+++ b/backend/danswer/connectors/notion/connector.py
@@ -29,6 +29,9 @@
 _NOTION_CALL_TIMEOUT = 30  # 30 seconds
 
 
+# TODO: Tables need to be ingested, Pages need to have their metadata ingested
+
+
 @dataclass
 class NotionPage:
     """Represents a Notion Page object"""
@@ -40,6 +43,8 @@ class NotionPage:
     properties: dict[str, Any]
     url: str
 
+    database_name: str | None  # Only applicable to the database type page (wiki)
+
     def __init__(self, **kwargs: dict[str, Any]) -> None:
         names = set([f.name for f in fields(self)])
         for k, v in kwargs.items():
@@ -47,6 +52,17 @@ def __init__(self, **kwargs: dict[str, Any]) -> None:
                 setattr(self, k, v)
 
 
+@dataclass
+class NotionBlock:
+    """Represents a Notion Block object"""
+
+    id: str  # Used for the URL
+    text: str
+    # In a plaintext representation of the page, how this block should be joined
+    # with the existing text up to this point, separated out from text for clarity
+    prefix: str
+
+
 @dataclass
 class NotionSearchResponse:
     """Represents the response from the Notion Search API"""
@@ -62,7 +78,6 @@ def __init__(self, **kwargs: dict[str, Any]) -> None:
                 setattr(self, k, v)
 
 
-# TODO - Add the ability to optionally limit to specific Notion databases
 class NotionConnector(LoadConnector, PollConnector):
     """Notion Page connector that reads all Notion pages
     this integration has been granted access to.
@@ -126,21 +141,47 @@ def _fetch_child_blocks(
 
     @retry(tries=3, delay=1, backoff=2)
     def _fetch_page(self, page_id: str) -> NotionPage:
-        """Fetch a page from it's ID via the Notion API."""
+        """Fetch a page from its ID via the Notion API, retry with database if page fetch fails."""
         logger.debug(f"Fetching page for ID '{page_id}'")
-        block_url = f"https://api.notion.com/v1/pages/{page_id}"
+        page_url = f"https://api.notion.com/v1/pages/{page_id}"
         res = rl_requests.get(
-            block_url,
+            page_url,
             headers=self.headers,
             timeout=_NOTION_CALL_TIMEOUT,
         )
         try:
             res.raise_for_status()
         except Exception as e:
-            logger.exception(f"Error fetching page - {res.json()}")
-            raise e
+            logger.warning(
+                f"Failed to fetch page, trying database for ID '{page_id}'. Exception: {e}"
+            )
+            # Try fetching as a database if page fetch fails, this happens if the page is set to a wiki
+            # it becomes a database from the notion perspective
+            return self._fetch_database_as_page(page_id)
         return NotionPage(**res.json())
 
+    @retry(tries=3, delay=1, backoff=2)
+    def _fetch_database_as_page(self, database_id: str) -> NotionPage:
+        """Attempt to fetch a database as a page."""
+        logger.debug(f"Fetching database for ID '{database_id}' as a page")
+        database_url = f"https://api.notion.com/v1/databases/{database_id}"
+        res = rl_requests.get(
+            database_url,
+            headers=self.headers,
+            timeout=_NOTION_CALL_TIMEOUT,
+        )
+        try:
+            res.raise_for_status()
+        except Exception as e:
+            logger.exception(f"Error fetching database as page - {res.json()}")
+            raise e
+        database_name = res.json().get("title")
+        database_name = (
+            database_name[0].get("text", {}).get("content") if database_name else None
+        )
+
+        return NotionPage(**res.json(), database_name=database_name)
+
     @retry(tries=3, delay=1, backoff=2)
     def _fetch_database(
         self, database_id: str, cursor: str | None = None
@@ -171,8 +212,51 @@ def _fetch_database(
             raise e
         return res.json()
 
-    def _read_pages_from_database(self, database_id: str) -> list[str]:
-        """Returns a list of all page IDs in the database"""
+    @staticmethod
+    def _properties_to_str(properties: dict[str, Any]) -> str:
+        """Converts Notion properties to a string"""
+
+        def _recurse_properties(inner_dict: dict[str, Any]) -> str:
+            while "type" in inner_dict:
+                type_name = inner_dict["type"]
+                inner_dict = inner_dict[type_name]
+                if isinstance(inner_dict, list):
+                    return ", ".join([_recurse_properties(item) for item in inner_dict])
+            # TODO there may be more types to handle here
+            if "name" in inner_dict:
+                return inner_dict["name"]
+            if "content" in inner_dict:
+                return inner_dict["content"]
+            start = inner_dict.get("start")
+            end = inner_dict.get("end")
+            if start is not None:
+                if end is not None:
+                    return f"{start} - {end}"
+                return start
+            elif end is not None:
+                return f"Until {end}"
+
+            if "id" in inner_dict:
+                logger.debug("Skipping Notion Id field")
+                return "Unreadable Property"
+
+            logger.debug(f"Unreadable property from innermost prop: {inner_dict}")
+            return "Unreadable Property"
+
+        result = ""
+        for prop_name, prop in properties.items():
+            inner_value = _recurse_properties(prop)
+            # Not a perfect way to format Notion database tables but there's no perfect representation
+            # since this must be represented as plaintext
+            result += f"{prop_name}: {inner_value}\t"
+
+        return result
+
+    def _read_pages_from_database(
+        self, database_id: str
+    ) -> tuple[list[NotionBlock], list[str]]:
+        """Returns a list of top level blocks and all page IDs in the database"""
+        result_blocks: list[NotionBlock] = []
         result_pages: list[str] = []
         cursor = None
         while True:
@@ -181,29 +265,33 @@ def _read_pages_from_database(self, database_id: str) -> list[str]:
             for result in data["results"]:
                 obj_id = result["id"]
                 obj_type = result["object"]
+                text = self._properties_to_str(result.get("properties", {}))
+                if text:
+                    result_blocks.append(NotionBlock(id=obj_id, text=text, prefix="\n"))
                 if obj_type == "page":
                     logger.debug(
                         f"Found page with ID '{obj_id}' in database '{database_id}'"
                     )
                     result_pages.append(result["id"])
                 elif obj_type == "database":
+                    # TODO add block for database
                     logger.debug(
                         f"Found database with ID '{obj_id}' in database '{database_id}'"
                     )
-                    result_pages.extend(self._read_pages_from_database(obj_id))
+                    # The inner contents are ignored at this level
+                    _, child_pages = self._read_pages_from_database(obj_id)
+                    result_pages.extend(child_pages)
 
             if data["next_cursor"] is None:
                 break
 
             cursor = data["next_cursor"]
 
-        return result_pages
+        return result_blocks, result_pages
 
-    def _read_blocks(
-        self, base_block_id: str
-    ) -> tuple[list[tuple[str, str]], list[str]]:
-        """Reads all child blocks for the specified block"""
-        result_lines: list[tuple[str, str]] = []
+    def _read_blocks(self, base_block_id: str) -> tuple[list[NotionBlock], list[str]]:
+        """Reads all child blocks for the specified block, returns a list of blocks and child page ids"""
+        result_blocks: list[NotionBlock] = []
         child_pages: list[str] = []
         cursor = None
         while True:
@@ -211,7 +299,7 @@ def _read_blocks(
 
             # this happens when a block is not shared with the integration
             if data is None:
-                return result_lines, child_pages
+                return result_blocks, child_pages
 
             for result in data["results"]:
                 logger.debug(
@@ -255,39 +343,49 @@ def _read_blocks(
 
                 if result["has_children"]:
                     if result_type == "child_page":
+                        # Child pages will not be included at this top level, it will be a separate document
                         child_pages.append(result_block_id)
                     else:
                         logger.debug(f"Entering sub-block: {result_block_id}")
-                        subblock_result_lines, subblock_child_pages = self._read_blocks(
+                        subblocks, subblock_child_pages = self._read_blocks(
                             result_block_id
                         )
                         logger.debug(f"Finished sub-block: {result_block_id}")
-                        result_lines.extend(subblock_result_lines)
+                        result_blocks.extend(subblocks)
                         child_pages.extend(subblock_child_pages)
 
                 if result_type == "child_database" and self.recursive_index_enabled:
-                    child_pages.extend(self._read_pages_from_database(result_block_id))
-
-                cur_result_text = "\n".join(cur_result_text_arr)
-                if cur_result_text:
-                    result_lines.append((cur_result_text, result_block_id))
+                    inner_blocks, inner_child_pages = self._read_pages_from_database(
+                        result_block_id
+                    )
+                    result_blocks.extend(inner_blocks)
+                    child_pages.extend(inner_child_pages)
+
+                if cur_result_text_arr:
+                    new_block = NotionBlock(
+                        id=result_block_id,
+                        text="\n".join(cur_result_text_arr),
+                        prefix="\n",
+                    )
+                    result_blocks.append(new_block)
 
             if data["next_cursor"] is None:
                 break
 
             cursor = data["next_cursor"]
 
-        return result_lines, child_pages
+        return result_blocks, child_pages
 
-    def _read_page_title(self, page: NotionPage) -> str:
+    def _read_page_title(self, page: NotionPage) -> str | None:
         """Extracts the title from a Notion page"""
         page_title = None
+        if hasattr(page, "database_name") and page.database_name:
+            return page.database_name
         for _, prop in page.properties.items():
             if prop["type"] == "title" and len(prop["title"]) > 0:
                 page_title = " ".join([t["plain_text"] for t in prop["title"]]).strip()
                 break
-        if page_title is None:
-            page_title = f"Untitled Page [{page.id}]"
+
         return page_title
 
     def _read_pages(
@@ -304,18 +402,23 @@ def _read_pages(
             logger.info(f"Reading page with ID '{page.id}', with url {page.url}")
             page_blocks, child_page_ids = self._read_blocks(page.id)
             all_child_page_ids.extend(child_page_ids)
-            page_title = self._read_page_title(page)
+
+            if not page_blocks:
+                continue
+
+            page_title = (
+                self._read_page_title(page) or f"Untitled Page with ID {page.id}"
+            )
+
             yield (
                 Document(
                     id=page.id,
-                    # Will add title to the first section later in processing
-                    sections=[Section(link=page.url, text="")]
-                    + [
+                    sections=[
                         Section(
-                            link=f"{page.url}#{block_id.replace('-', '')}",
-                            text=block_text,
+                            link=f"{page.url}#{block.id.replace('-', '')}",
+                            text=block.prefix + block.text,
                         )
-                        for block_text, block_id in page_blocks
+                        for block in page_blocks
                     ],
                     source=DocumentSource.NOTION,
                     semantic_identifier=page_title,
diff --git a/backend/pyproject.toml b/backend/pyproject.toml
index 032c665196d..d32255d9f65 100644
--- a/backend/pyproject.toml
+++ b/backend/pyproject.toml
@@ -5,7 +5,11 @@ explicit_package_bases = true
 disallow_untyped_defs = true
 
 [[tool.mypy.overrides]]
-module = "alembic*.versions.*"
+module = "alembic.versions.*"
+disable_error_code = ["var-annotated"]
+
+[[tool.mypy.overrides]]
+module = "alembic_tenants.versions.*"
 disable_error_code = ["var-annotated"]
 
 [tool.ruff]
diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt
index 881920af7f2..691157f732a 100644
--- a/backend/requirements/dev.txt
+++ b/backend/requirements/dev.txt
@@ -11,6 +11,7 @@ types-beautifulsoup4==4.12.0.3
 types-html5lib==1.1.11.13
 types-oauthlib==3.2.0.9
 types-setuptools==68.0.0.3
+types-Pillow==10.2.0.20240822
 types-passlib==1.7.7.20240106
 types-psutil==5.9.5.17
 types-psycopg2==2.9.21.10

From 3ccd951307ea97ff20ddcba1e897e1e956592edd Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Sun, 13 Oct 2024 19:32:44 -0700
Subject: [PATCH 108/376] Fix stopping of indexing runs when pausing a
 connector

---
 backend/danswer/background/indexing/run_indexing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index d5e14675c65..69c8a739389 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -200,7 +200,7 @@ def _run_indexing(
                 # index being built. We want to populate it even for paused connectors
                 # Often paused connectors are sources that aren't updated frequently but the
                 # contents still need to be initially pulled.
-                db_session.refresh(db_connector)
+                db_session.refresh(db_cc_pair)
                 if (
                     (
                         db_cc_pair.status == ConnectorCredentialPairStatus.PAUSED

From 9537a2581eef01fe861186b691b726c863301aad Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Mon, 14 Oct 2024 08:25:39 -0700
Subject: [PATCH 109/376] Handle 'cannotExportFile' + fix forms

---
 .../connectors/google_drive/connector.py      | 20 ++++++++++++++++++-
 .../pages/ConnectorInput/ListInput.tsx        |  5 ++---
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index 48b514e80b5..a50d632edb3 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -462,8 +462,26 @@ def _fetch_docs_from_drive(
                             for permission in file["permissions"]
                         ):
                             continue
+                    try:
+                        text_contents = extract_text(file, service) or ""
+                    except HttpError as e:
+                        reason = (
+                            e.error_details[0]["reason"]
+                            if e.error_details
+                            else e.reason
+                        )
+                        message = (
+                            e.error_details[0]["message"]
+                            if e.error_details
+                            else e.reason
+                        )
+                        if e.status_code == 403 and reason == "cannotExportFile":
+                            logger.warning(
+                                f"Could not export file '{file['name']}' due to '{message}', skipping..."
+                            )
+                            continue
 
-                    text_contents = extract_text(file, service) or ""
+                        raise
 
                     doc_batch.append(
                         Document(
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
index edb057a3e3e..956e0c24597 100644
--- a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
@@ -8,13 +8,12 @@ interface ListInputProps {
 }
 
 const ListInput: React.FC<ListInputProps> = ({ field }) => {
-  const { values } = useFormikContext();
-
+  const { values } = useFormikContext<any>();
   return (
     <TextArrayField
       name={field.name}
       label={field.label}
-      values={Array.isArray(values) ? values : []}
+      values={values}
       subtext={field.description}
       placeholder={`Enter ${field.label.toLowerCase()}`}
     />

From 89eaa8bc30d9e32b77d3eb49fba29cec02333af6 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 14 Oct 2024 11:39:41 -0700
Subject: [PATCH 110/376] nit (#2795)

---
 web/src/lib/constants.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index 3b209cfb364..c0b916ebadf 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -61,5 +61,5 @@ export const CLOUD_ENABLED =
 export const REGISTRATION_URL =
   process.env.INTERNAL_URL || "http://127.0.0.1:3001";
 
-export const SERVER_SIDE_ONLY__CLOUD_ENABLED = true;
-// process.env.NEXT_PUBLIC_CLOUD_ENABLED?.toLowerCase() === "true";
+export const SERVER_SIDE_ONLY__CLOUD_ENABLED =
+  process.env.NEXT_PUBLIC_CLOUD_ENABLED?.toLowerCase() === "true";

From 494fda906d94f5039df7719203a2d96282586c51 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Mon, 14 Oct 2024 13:52:57 -0700
Subject: [PATCH 111/376] Confluence permission sync fix for server deployment
 (#2784)

* initial commit

* Made perm sync with with cql

* filter fix

* undo connector changes

* fixed everything

* whoops
---
 .../confluence/confluence_sync_utils.py       |  18 --
 .../confluence/doc_sync.py                    | 288 ++++++++++++------
 .../confluence/group_sync.py                  |  37 ++-
 .../confluence/sync_utils.py                  |  40 +++
 4 files changed, 256 insertions(+), 127 deletions(-)
 delete mode 100644 backend/ee/danswer/external_permissions/confluence/confluence_sync_utils.py
 create mode 100644 backend/ee/danswer/external_permissions/confluence/sync_utils.py

diff --git a/backend/ee/danswer/external_permissions/confluence/confluence_sync_utils.py b/backend/ee/danswer/external_permissions/confluence/confluence_sync_utils.py
deleted file mode 100644
index e911e2649ba..00000000000
--- a/backend/ee/danswer/external_permissions/confluence/confluence_sync_utils.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from typing import Any
-
-from atlassian import Confluence  # type:ignore
-
-
-def build_confluence_client(
-    connector_specific_config: dict[str, Any], raw_credentials_json: dict[str, Any]
-) -> Confluence:
-    is_cloud = connector_specific_config.get("is_cloud", False)
-    return Confluence(
-        api_version="cloud" if is_cloud else "latest",
-        # Remove trailing slash from wiki_base if present
-        url=connector_specific_config["wiki_base"].rstrip("/"),
-        # passing in username causes issues for Confluence data center
-        username=raw_credentials_json["confluence_username"] if is_cloud else None,
-        password=raw_credentials_json["confluence_access_token"] if is_cloud else None,
-        token=raw_credentials_json["confluence_access_token"] if not is_cloud else None,
-    )
diff --git a/backend/ee/danswer/external_permissions/confluence/doc_sync.py b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
index e87dcf1d79f..1330655ec61 100644
--- a/backend/ee/danswer/external_permissions/confluence/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
@@ -1,12 +1,18 @@
+"""
+Rules defined here:
+https://confluence.atlassian.com/conf85/check-who-can-view-a-page-1283360557.html
+"""
 from typing import Any
+from urllib.parse import parse_qs
+from urllib.parse import urlparse
 
-from atlassian import Confluence  # type:ignore
 from sqlalchemy.orm import Session
 
 from danswer.access.models import ExternalAccess
 from danswer.connectors.confluence.confluence_utils import (
     build_confluence_document_id,
 )
+from danswer.connectors.confluence.connector import DanswerConfluence
 from danswer.connectors.confluence.rate_limit_handler import (
     make_confluence_call_handle_rate_limit,
 )
@@ -14,36 +20,136 @@
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.document import upsert_document_external_perms__no_commit
-from ee.danswer.external_permissions.confluence.confluence_sync_utils import (
+from ee.danswer.external_permissions.confluence.sync_utils import (
     build_confluence_client,
 )
-
+from ee.danswer.external_permissions.confluence.sync_utils import (
+    get_user_email_from_username__server,
+)
 
 logger = setup_logger()
 
+_VIEWSPACE_PERMISSION_TYPE = "VIEWSPACE"
 _REQUEST_PAGINATION_LIMIT = 100
 
 
-def _extract_user_email(subjects: dict[str, Any]) -> str | None:
-    # If the subject is a user, then return the user's email
-    user = subjects.get("user", {})
-    result = user.get("results", [{}])[0]
-    return result.get("email")
+def _get_server_space_permissions(
+    confluence_client: DanswerConfluence, space_key: str
+) -> ExternalAccess:
+    get_space_permissions = make_confluence_call_handle_rate_limit(
+        confluence_client.get_space_permissions
+    )
+
+    permissions = get_space_permissions(space_key)
+
+    viewspace_permissions = []
+    for permission_category in permissions:
+        if permission_category.get("type") == _VIEWSPACE_PERMISSION_TYPE:
+            viewspace_permissions.extend(
+                permission_category.get("spacePermissions", [])
+            )
+
+    user_names = set()
+    group_names = set()
+    for permission in viewspace_permissions:
+        if user_name := permission.get("userName"):
+            user_names.add(user_name)
+        if group_name := permission.get("groupName"):
+            group_names.add(group_name)
+
+    user_emails = set()
+    for user_name in user_names:
+        user_email = get_user_email_from_username__server(confluence_client, user_name)
+        if user_email:
+            user_emails.add(user_email)
+        else:
+            logger.warning(f"Email for user {user_name} not found in Confluence")
+
+    return ExternalAccess(
+        external_user_emails=user_emails,
+        external_user_group_ids=group_names,
+        # TODO: Check if the space is publicly accessible
+        # Currently, we assume the space is not public
+        # We need to check if anonymous access is turned on for the site and space
+        # This information is paywalled so it remains unimplemented
+        is_public=False,
+    )
+
+
+def _get_cloud_space_permissions(
+    confluence_client: DanswerConfluence, space_key: str
+) -> ExternalAccess:
+    get_space_permissions = make_confluence_call_handle_rate_limit(
+        confluence_client.get_space
+    )
+    space_permissions_result = get_space_permissions(
+        space_key=space_key, expand="permissions"
+    )
+    space_permissions = space_permissions_result.get("permissions", [])
+
+    user_emails = set()
+    group_names = set()
+    is_externally_public = False
+    for permission in space_permissions:
+        subs = permission.get("subjects")
+        if subs:
+            # If there are subjects, then there are explicit users or groups with access
+            if email := subs.get("user", {}).get("results", [{}])[0].get("email"):
+                user_emails.add(email)
+            if group_name := subs.get("group", {}).get("results", [{}])[0].get("name"):
+                group_names.add(group_name)
+        else:
+            # If there are no subjects, then the permission is for everyone
+            if permission.get("operation", {}).get(
+                "operation"
+            ) == "read" and permission.get("anonymousAccess", False):
+                # If the permission specifies read access for anonymous users, then
+                # the space is publicly accessible
+                is_externally_public = True
+
+    return ExternalAccess(
+        external_user_emails=user_emails,
+        external_user_group_ids=group_names,
+        is_public=is_externally_public,
+    )
+
+
+def _get_space_permissions(
+    confluence_client: DanswerConfluence,
+    is_cloud: bool,
+) -> dict[str, ExternalAccess]:
+    # Gets all the spaces in the Confluence instance
+    get_all_spaces = make_confluence_call_handle_rate_limit(
+        confluence_client.get_all_spaces
+    )
+    all_space_keys = []
+    start = 0
+    while True:
+        spaces_batch = get_all_spaces(start=start, limit=_REQUEST_PAGINATION_LIMIT)
+        for space in spaces_batch.get("results", []):
+            all_space_keys.append(space.get("key"))
+
+        if len(spaces_batch.get("results", [])) < _REQUEST_PAGINATION_LIMIT:
+            break
 
+        start += len(spaces_batch.get("results", []))
 
-def _extract_group_name(subjects: dict[str, Any]) -> str | None:
-    # If the subject is a group, then return the group's name
-    group = subjects.get("group", {})
-    result = group.get("results", [{}])[0]
-    return result.get("name")
+    # Gets the permissions for each space
+    space_permissions_by_space_key: dict[str, ExternalAccess] = {}
+    for space_key in all_space_keys:
+        if is_cloud:
+            space_permissions = _get_cloud_space_permissions(
+                confluence_client=confluence_client, space_key=space_key
+            )
+        else:
+            space_permissions = _get_server_space_permissions(
+                confluence_client=confluence_client, space_key=space_key
+            )
 
+        # Stores the permissions for each space
+        space_permissions_by_space_key[space_key] = space_permissions
 
-def _is_public_read_permission(permission: dict[str, Any]) -> bool:
-    # If the permission is a public read permission, then return True
-    operation = permission.get("operation", {})
-    operation_value = operation.get("operation")
-    anonymous_access = permission.get("anonymousAccess", False)
-    return operation_value == "read" and anonymous_access
+    return space_permissions_by_space_key
 
 
 def _extract_read_access_restrictions(
@@ -75,50 +181,7 @@ def _extract_read_access_restrictions(
     return read_access_user_emails, read_access_group_names
 
 
-def _get_space_permissions(
-    db_session: Session,
-    confluence_client: Confluence,
-    space_id: str,
-) -> ExternalAccess:
-    get_space_permissions = make_confluence_call_handle_rate_limit(
-        confluence_client.get_space_permissions
-    )
-
-    space_permissions_result = get_space_permissions(space_id)
-    logger.debug(f"space_permissions_result: {space_permissions_result}")
-
-    space_permissions = space_permissions_result.get("permissions", [])
-    user_emails = set()
-    # Confluence enforces that group names are unique
-    group_names = set()
-    is_externally_public = False
-    for permission in space_permissions:
-        subjects = permission.get("subjects")
-        if subjects:
-            # If there are subjects, then there are explicit users or groups with access
-            if email := _extract_user_email(subjects):
-                user_emails.add(email)
-            if group_name := _extract_group_name(subjects):
-                group_names.add(group_name)
-        else:
-            # If there are no subjects, then the permission is for everyone
-            if _is_public_read_permission(permission):
-                # If the permission specifies read access for anonymous users, then
-                # the space is publicly accessible
-                is_externally_public = True
-
-    batch_add_non_web_user_if_not_exists__no_commit(
-        db_session=db_session, emails=list(user_emails)
-    )
-    return ExternalAccess(
-        external_user_emails=user_emails,
-        external_user_group_ids=group_names,
-        is_public=is_externally_public,
-    )
-
-
 def _get_page_specific_restrictions(
-    db_session: Session,
     page: dict[str, Any],
 ) -> ExternalAccess | None:
     user_emails, group_names = _extract_read_access_restrictions(
@@ -131,9 +194,6 @@ def _get_page_specific_restrictions(
     if is_space_public:
         return None
 
-    batch_add_non_web_user_if_not_exists__no_commit(
-        db_session=db_session, emails=list(user_emails)
-    )
     return ExternalAccess(
         external_user_emails=set(user_emails),
         external_user_group_ids=set(group_names),
@@ -143,7 +203,7 @@ def _get_page_specific_restrictions(
 
 
 def _fetch_attachment_document_ids_for_page_paginated(
-    confluence_client: Confluence, page: dict[str, Any]
+    confluence_client: DanswerConfluence, page: dict[str, Any]
 ) -> list[str]:
     """
     Starts by just extracting the first page of attachments from
@@ -184,11 +244,11 @@ def _fetch_attachment_document_ids_for_page_paginated(
 
 
 def _fetch_all_pages_paginated(
-    confluence_client: Confluence,
-    space_id: str,
+    confluence_client: DanswerConfluence,
+    cql_query: str,
 ) -> list[dict[str, Any]]:
-    get_all_pages_from_space = make_confluence_call_handle_rate_limit(
-        confluence_client.get_all_pages_from_space
+    get_all_pages = make_confluence_call_handle_rate_limit(
+        confluence_client.danswer_cql
     )
 
     # For each page, this fetches the page's attachments and restrictions.
@@ -196,39 +256,43 @@ def _fetch_all_pages_paginated(
         "children.attachment",
         "restrictions.read.restrictions.user",
         "restrictions.read.restrictions.group",
+        "space",
     ]
     expansion_string = ",".join(expansion_strings)
 
-    all_pages = []
-    start = 0
+    all_pages: list[dict[str, Any]] = []
+    cursor = None
     while True:
-        pages_dict = get_all_pages_from_space(
-            space=space_id,
-            start=start,
-            limit=_REQUEST_PAGINATION_LIMIT,
+        response = get_all_pages(
+            cql=cql_query,
             expand=expansion_string,
+            cursor=cursor,
+            limit=_REQUEST_PAGINATION_LIMIT,
         )
-        all_pages.extend(pages_dict)
 
-        response_size = len(pages_dict)
-        if response_size < _REQUEST_PAGINATION_LIMIT:
+        all_pages.extend(response.get("results", []))
+
+        # Handle pagination
+        next_cursor = response.get("_links", {}).get("next", "")
+        cursor = parse_qs(urlparse(next_cursor).query).get("cursor", [None])[0]
+
+        if not cursor:
             break
-        start += response_size
 
     return all_pages
 
 
 def _fetch_all_page_restrictions_for_space(
-    db_session: Session,
-    confluence_client: Confluence,
-    space_id: str,
-) -> dict[str, ExternalAccess | None]:
+    confluence_client: DanswerConfluence,
+    cql_query: str,
+    space_permissions_by_space_key: dict[str, ExternalAccess],
+) -> dict[str, ExternalAccess]:
     all_pages = _fetch_all_pages_paginated(
         confluence_client=confluence_client,
-        space_id=space_id,
+        cql_query=cql_query,
     )
 
-    document_restrictions: dict[str, ExternalAccess | None] = {}
+    document_restrictions: dict[str, ExternalAccess] = {}
     for page in all_pages:
         """
         This assigns the same permissions to all attachments of a page and
@@ -257,10 +321,17 @@ def _fetch_all_page_restrictions_for_space(
 
         # Get the page's specific restrictions
         page_permissions = _get_page_specific_restrictions(
-            db_session=db_session,
             page=page,
         )
 
+        if not page_permissions:
+            # If there are no page specific restrictions,
+            # the page inherits the space's restrictions
+            page_permissions = space_permissions_by_space_key.get(page["space"]["key"])
+            if not page_permissions:
+                # If nothing is in the dict, then the space has not been indexed, so we move on
+                continue
+
         # Apply the page's specific restrictions to the page and its attachments
         for document_id in document_ids:
             document_restrictions[document_id] = page_permissions
@@ -268,6 +339,19 @@ def _fetch_all_page_restrictions_for_space(
     return document_restrictions
 
 
+def _build_cql_query_from_connector_config(
+    cc_pair: ConnectorCredentialPair,
+) -> str:
+    cql_query = cc_pair.connector.connector_specific_config.get("cql_query")
+    if cql_query:
+        return cql_query
+
+    space_id = cc_pair.connector.connector_specific_config.get("space")
+    if space_id:
+        return f"type=page and space='{space_id}'"
+    return "type=page"
+
+
 def confluence_doc_sync(
     db_session: Session,
     cc_pair: ConnectorCredentialPair,
@@ -279,26 +363,32 @@ def confluence_doc_sync(
     already populated
     """
     confluence_client = build_confluence_client(
-        cc_pair.connector.connector_specific_config, cc_pair.credential.credential_json
+        connector_specific_config=cc_pair.connector.connector_specific_config,
+        credentials_json=cc_pair.credential.credential_json,
     )
-    space_permissions = _get_space_permissions(
-        db_session=db_session,
+
+    cql_query = _build_cql_query_from_connector_config(cc_pair)
+    is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
+
+    space_permissions_by_space_key = _get_space_permissions(
         confluence_client=confluence_client,
-        space_id=cc_pair.connector.connector_specific_config["space"],
+        is_cloud=is_cloud,
     )
-    fresh_doc_permissions = _fetch_all_page_restrictions_for_space(
-        db_session=db_session,
+
+    permissions_by_doc_id = _fetch_all_page_restrictions_for_space(
         confluence_client=confluence_client,
-        space_id=cc_pair.connector.connector_specific_config["space"],
+        cql_query=cql_query,
+        space_permissions_by_space_key=space_permissions_by_space_key,
     )
-    for doc_id, page_specific_access in fresh_doc_permissions.items():
-        # If there are no page specific restrictions, then
-        # the page inherits the space's restrictions
-        page_access = page_specific_access or space_permissions
 
+    all_emails = set()
+    for doc_id, page_specific_access in permissions_by_doc_id.items():
         upsert_document_external_perms__no_commit(
             db_session=db_session,
             doc_id=doc_id,
-            external_access=page_access,
+            external_access=page_specific_access,
             source_type=cc_pair.connector.source,
         )
+        all_emails.update(page_specific_access.external_user_emails)
+
+    batch_add_non_web_user_if_not_exists__no_commit(db_session, list(all_emails))
diff --git a/backend/ee/danswer/external_permissions/confluence/group_sync.py b/backend/ee/danswer/external_permissions/confluence/group_sync.py
index 33bc60cc6d5..9ef376ba641 100644
--- a/backend/ee/danswer/external_permissions/confluence/group_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/group_sync.py
@@ -12,9 +12,12 @@
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.external_perm import ExternalUserGroup
 from ee.danswer.db.external_perm import replace_user__ext_group_for_cc_pair__no_commit
-from ee.danswer.external_permissions.confluence.confluence_sync_utils import (
+from ee.danswer.external_permissions.confluence.sync_utils import (
     build_confluence_client,
 )
+from ee.danswer.external_permissions.confluence.sync_utils import (
+    get_user_email_from_username__server,
+)
 
 
 logger = setup_logger()
@@ -50,11 +53,12 @@ def _get_confluence_group_names_paginated(
 def _get_group_members_email_paginated(
     confluence_client: Confluence,
     group_name: str,
-) -> list[str]:
+    is_cloud: bool,
+) -> set[str]:
     get_group_members = make_confluence_call_handle_rate_limit(
         confluence_client.get_group_members
     )
-    group_member_emails: list[str] = []
+    group_member_emails: set[str] = set()
     start = 0
     while True:
         try:
@@ -66,12 +70,22 @@ def _get_group_members_email_paginated(
                 return group_member_emails
             raise e
 
-        group_member_emails.extend(
-            [member.get("email") for member in members if member.get("email")]
-        )
+        for member in members:
+            if is_cloud:
+                email = member.get("email")
+            elif user_name := member.get("username"):
+                email = get_user_email_from_username__server(
+                    confluence_client, user_name
+                )
+
+            if email:
+                group_member_emails.add(email)
+
         if len(members) < _PAGE_SIZE:
             break
+
         start += _PAGE_SIZE
+
     return group_member_emails
 
 
@@ -80,22 +94,25 @@ def confluence_group_sync(
     cc_pair: ConnectorCredentialPair,
 ) -> None:
     confluence_client = build_confluence_client(
-        cc_pair.connector.connector_specific_config, cc_pair.credential.credential_json
+        connector_specific_config=cc_pair.connector.connector_specific_config,
+        credentials_json=cc_pair.credential.credential_json,
     )
 
     danswer_groups: list[ExternalUserGroup] = []
+    is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
     # Confluence enforces that group names are unique
     for group_name in _get_confluence_group_names_paginated(confluence_client):
         group_member_emails = _get_group_members_email_paginated(
-            confluence_client, group_name
+            confluence_client, group_name, is_cloud
         )
         group_members = batch_add_non_web_user_if_not_exists__no_commit(
-            db_session=db_session, emails=group_member_emails
+            db_session=db_session, emails=list(group_member_emails)
         )
         if group_members:
             danswer_groups.append(
                 ExternalUserGroup(
-                    id=group_name, user_ids=[user.id for user in group_members]
+                    id=group_name,
+                    user_ids=[user.id for user in group_members],
                 )
             )
 
diff --git a/backend/ee/danswer/external_permissions/confluence/sync_utils.py b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
new file mode 100644
index 00000000000..183e390595e
--- /dev/null
+++ b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
@@ -0,0 +1,40 @@
+from typing import Any
+
+from danswer.connectors.confluence.connector import DanswerConfluence
+from danswer.connectors.confluence.rate_limit_handler import (
+    make_confluence_call_handle_rate_limit,
+)
+
+_USER_EMAIL_CACHE: dict[str, str | None] = {}
+
+
+def build_confluence_client(
+    connector_specific_config: dict[str, Any], credentials_json: dict[str, Any]
+) -> DanswerConfluence:
+    is_cloud = connector_specific_config.get("is_cloud", False)
+    return DanswerConfluence(
+        api_version="cloud" if is_cloud else "latest",
+        # Remove trailing slash from wiki_base if present
+        url=connector_specific_config["wiki_base"].rstrip("/"),
+        # passing in username causes issues for Confluence data center
+        username=credentials_json["confluence_username"] if is_cloud else None,
+        password=credentials_json["confluence_access_token"] if is_cloud else None,
+        token=credentials_json["confluence_access_token"] if not is_cloud else None,
+    )
+
+
+def get_user_email_from_username__server(
+    confluence_client: DanswerConfluence, user_name: str
+) -> str | None:
+    global _USER_EMAIL_CACHE
+    get_user_info = make_confluence_call_handle_rate_limit(
+        confluence_client.get_mobile_parameters
+    )
+    if _USER_EMAIL_CACHE.get(user_name) is None:
+        try:
+            response = get_user_info(user_name)
+            email = response.get("email")
+        except Exception:
+            email = None
+        _USER_EMAIL_CACHE[user_name] = email
+    return _USER_EMAIL_CACHE[user_name]

From f8a7749b46239e9b2f082178b6bbabe58905a64b Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Mon, 14 Oct 2024 14:47:36 -0700
Subject: [PATCH 112/376] Fix file too large error (#2799)

* Fix file too large error

* Add cannotDownloadFile
---
 backend/danswer/connectors/google_drive/connector.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index a50d632edb3..9f8c6fbfda8 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -475,7 +475,15 @@ def _fetch_docs_from_drive(
                             if e.error_details
                             else e.reason
                         )
-                        if e.status_code == 403 and reason == "cannotExportFile":
+
+                        # these errors don't represent a failure in the connector, but simply files
+                        # that can't / shouldn't be indexed
+                        ERRORS_TO_CONTINUE_ON = [
+                            "cannotExportFile",
+                            "exportSizeLimitExceeded",
+                            "cannotDownloadFile",
+                        ]
+                        if e.status_code == 403 and reason in ERRORS_TO_CONTINUE_ON:
                             logger.warning(
                                 f"Could not export file '{file['name']}' due to '{message}', skipping..."
                             )

From dee197570d64762ebe1b04d7edf0b0e643b00721 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 14 Oct 2024 15:48:06 -0700
Subject: [PATCH 113/376] Bugfix/mediawiki (#2800)

* fix formatting

* fix poorly structured doc id, fix empty page id, fix family_class_dispatch invalid name (no spaces), fix setting id with int pageid

* fix mediawiki test
---
 .../danswer/connectors/mediawiki/family.py    |  3 +--
 backend/danswer/connectors/mediawiki/wiki.py  | 22 +++++++++++++++----
 .../danswer/connectors/mediawiki/test_wiki.py |  2 +-
 3 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/backend/danswer/connectors/mediawiki/family.py b/backend/danswer/connectors/mediawiki/family.py
index 0d953066700..163bca2ef6b 100644
--- a/backend/danswer/connectors/mediawiki/family.py
+++ b/backend/danswer/connectors/mediawiki/family.py
@@ -45,8 +45,7 @@ def __init__(
 
         if any(x not in generate_family_file.NAME_CHARACTERS for x in name):
             raise ValueError(
-                'ERROR: Name of family "{}" must be ASCII letters and digits [a-zA-Z0-9]',
-                name,
+                f'ERROR: Name of family "{name}" must be ASCII letters and digits [a-zA-Z0-9]',
             )
 
         if isinstance(dointerwiki, bool):
diff --git a/backend/danswer/connectors/mediawiki/wiki.py b/backend/danswer/connectors/mediawiki/wiki.py
index f4ec1e02311..a3de963982b 100644
--- a/backend/danswer/connectors/mediawiki/wiki.py
+++ b/backend/danswer/connectors/mediawiki/wiki.py
@@ -3,6 +3,7 @@
 import datetime
 import itertools
 from collections.abc import Generator
+from collections.abc import Iterator
 from typing import Any
 from typing import ClassVar
 
@@ -19,6 +20,9 @@
 from danswer.connectors.mediawiki.family import family_class_dispatch
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
 
 
 def pywikibot_timestamp_to_utc_datetime(
@@ -74,7 +78,7 @@ def get_doc_from_page(
         sections=sections,
         semantic_identifier=page.title(),
         metadata={"categories": [category.title() for category in page.categories()]},
-        id=page.pageid,
+        id=f"MEDIAWIKI_{page.pageid}_{page.full_url()}",
     )
 
 
@@ -117,13 +121,18 @@ def __init__(
 
         # short names can only have ascii letters and digits
 
-        self.family = family_class_dispatch(hostname, "Wikipedia Connector")()
+        self.family = family_class_dispatch(hostname, "WikipediaConnector")()
         self.site = pywikibot.Site(fam=self.family, code=language_code)
         self.categories = [
             pywikibot.Category(self.site, f"Category:{category.replace(' ', '_')}")
             for category in categories
         ]
-        self.pages = [pywikibot.Page(self.site, page) for page in pages]
+
+        self.pages = []
+        for page in pages:
+            if not page:
+                continue
+            self.pages.append(pywikibot.Page(self.site, page))
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         """Load credentials for a MediaWiki site.
@@ -169,8 +178,13 @@ def _get_doc_batch(
         ]
 
         # Since we can specify both individual pages and categories, we need to iterate over all of them.
-        all_pages = itertools.chain(self.pages, *category_pages)
+        all_pages: Iterator[pywikibot.Page] = itertools.chain(
+            self.pages, *category_pages
+        )
         for page in all_pages:
+            logger.info(
+                f"MediaWikiConnector: title='{page.title()}' url={page.full_url()}"
+            )
             doc_batch.append(
                 get_doc_from_page(page, self.site, self.document_source_type)
             )
diff --git a/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py b/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
index 2a2c841a466..9aaacfc1eae 100644
--- a/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
+++ b/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
@@ -100,7 +100,7 @@ def test_get_doc_from_page(site: pywikibot.Site) -> None:
     assert doc.metadata == {
         "categories": [category.title() for category in test_page.categories()]
     }
-    assert doc.id == test_page.pageid
+    assert doc.id == f"MEDIAWIKI_{test_page.pageid}_{test_page.full_url()}"
 
 
 def test_mediawiki_connector_recurse_depth() -> None:

From 6f9740d0267d6256deeb1b8e0f17f43bfc2e48c5 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 14 Oct 2024 15:53:02 -0700
Subject: [PATCH 114/376] Ensure warmup occurs once (#2777)

* ensure shared chats are shared

* ensure warm occurs once

* nit

* ensure warmup occurs once

* Revert "ensure shared chats are shared"

This reverts commit 8be887f3ee651e64b7f2eb3d45f7c36b53471a20.
---
 backend/danswer/background/update.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index 075035c46d7..b981a90e315 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -27,6 +27,7 @@
 from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
 from danswer.db.engine import get_db_current_time
 from danswer.db.engine import get_session_with_tenant
+from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.engine import SqlEngine
 from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import get_index_attempt
@@ -441,6 +442,30 @@ def update_loop(
     num_workers: int = NUM_INDEXING_WORKERS,
     num_secondary_workers: int = NUM_SECONDARY_INDEXING_WORKERS,
 ) -> None:
+    if not MULTI_TENANT:
+        # We can use this function as we are certain only the public schema exists
+        # (explicitly for the non-`MULTI_TENANT` case)
+        engine = get_sqlalchemy_engine()
+        with Session(engine) as db_session:
+            check_index_swap(db_session=db_session)
+
+            search_settings = get_current_search_settings(db_session)
+
+            # So that the first time users aren't surprised by really slow speed of first
+            # batch of documents indexed
+            if search_settings.provider_type is None:
+                logger.notice("Running a first inference to warm up embedding model")
+                embedding_model = EmbeddingModel.from_db_model(
+                    search_settings=search_settings,
+                    server_host=INDEXING_MODEL_SERVER_HOST,
+                    server_port=INDEXING_MODEL_SERVER_PORT,
+                )
+
+                warm_up_bi_encoder(
+                    embedding_model=embedding_model,
+                )
+                logger.notice("First inference complete.")
+
     client_primary: Client | SimpleJobClient
     client_secondary: Client | SimpleJobClient
     if DASK_JOB_CLIENT_ENABLED:

From efe2e79f275a2c9ab4c004e98649d8c4db6ff739 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 14 Oct 2024 16:51:24 -0700
Subject: [PATCH 115/376] Rate limiting confluence through redis (#2798)

* try rate limiting through redis

* fix circular import issue

* fix bad formatting of family string

* Revert "fix bad formatting of family string"

This reverts commit be688899e5b4dd189dc13d9fec1d0f3ade07ad4f.

* redis usage optional

* disable test that doesn't match with new design
---
 .../confluence/rate_limit_handler.py          | 62 +++++++++++++++--
 backend/danswer/connectors/interfaces.py      |  2 +
 .../confluence/test_rate_limit_handler.py     | 66 ++++++++++---------
 3 files changed, 94 insertions(+), 36 deletions(-)

diff --git a/backend/danswer/connectors/confluence/rate_limit_handler.py b/backend/danswer/connectors/confluence/rate_limit_handler.py
index ea0e46800ff..c05754bb105 100644
--- a/backend/danswer/connectors/confluence/rate_limit_handler.py
+++ b/backend/danswer/connectors/confluence/rate_limit_handler.py
@@ -1,11 +1,15 @@
+import math
 import time
 from collections.abc import Callable
 from typing import Any
 from typing import cast
 from typing import TypeVar
 
+from redis.exceptions import ConnectionError
 from requests import HTTPError
 
+from danswer.connectors.interfaces import BaseConnector
+from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -21,15 +25,46 @@ class ConfluenceRateLimitError(Exception):
     pass
 
 
+# https://developer.atlassian.com/cloud/confluence/rate-limiting/
 def make_confluence_call_handle_rate_limit(confluence_call: F) -> F:
     def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
         max_retries = 5
         starting_delay = 5
         backoff = 2
-        max_delay = 600
+
+        # max_delay is used when the server doesn't hand back "Retry-After"
+        # and we have to decide the retry delay ourselves
+        max_delay = 30  # Atlassian uses max_delay = 30 in their examples
+
+        # max_retry_after is used when we do get a "Retry-After" header
+        max_retry_after = 300  # should we really cap the maximum retry delay?
+
+        NEXT_RETRY_KEY = BaseConnector.REDIS_KEY_PREFIX + "confluence_next_retry"
+
+        # for testing purposes, rate limiting is written to fall back to a simpler
+        # rate limiting approach when redis is not available
+        r = get_redis_client()
 
         for attempt in range(max_retries):
             try:
+                # if multiple connectors are waiting for the next attempt, there could be an issue
+                # where many connectors are "released" onto the server at the same time.
+                # That's not ideal ... but coming up with a mechanism for queueing
+                # all of these connectors is a bigger problem that we want to take on
+                # right now
+                try:
+                    next_attempt = r.get(NEXT_RETRY_KEY)
+                    if next_attempt is None:
+                        next_attempt = 0
+                    else:
+                        next_attempt = int(cast(int, next_attempt))
+
+                    # TODO: all connectors need to be interruptible moving forward
+                    while time.monotonic() < next_attempt:
+                        time.sleep(1)
+                except ConnectionError:
+                    pass
+
                 return confluence_call(*args, **kwargs)
             except HTTPError as e:
                 # Check if the response or headers are None to avoid potential AttributeError
@@ -50,7 +85,7 @@ def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
                             pass
 
                     if retry_after is not None:
-                        if retry_after > 600:
+                        if retry_after > max_retry_after:
                             logger.warning(
                                 f"Clamping retry_after from {retry_after} to {max_delay} seconds..."
                             )
@@ -59,13 +94,25 @@ def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
                         logger.warning(
                             f"Rate limit hit. Retrying after {retry_after} seconds..."
                         )
-                        time.sleep(retry_after)
+                        try:
+                            r.set(
+                                NEXT_RETRY_KEY,
+                                math.ceil(time.monotonic() + retry_after),
+                            )
+                        except ConnectionError:
+                            pass
                     else:
                         logger.warning(
                             "Rate limit hit. Retrying with exponential backoff..."
                         )
                         delay = min(starting_delay * (backoff**attempt), max_delay)
-                        time.sleep(delay)
+                        delay_until = math.ceil(time.monotonic() + delay)
+
+                        try:
+                            r.set(NEXT_RETRY_KEY, delay_until)
+                        except ConnectionError:
+                            while time.monotonic() < delay_until:
+                                time.sleep(1)
                 else:
                     # re-raise, let caller handle
                     raise
@@ -74,7 +121,12 @@ def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
                 # Users reported it to be intermittent, so just retry
                 logger.warning(f"Confluence Internal Error, retrying... {e}")
                 delay = min(starting_delay * (backoff**attempt), max_delay)
-                time.sleep(delay)
+                delay_until = math.ceil(time.monotonic() + delay)
+                try:
+                    r.set(NEXT_RETRY_KEY, delay_until)
+                except ConnectionError:
+                    while time.monotonic() < delay_until:
+                        time.sleep(1)
 
                 if attempt == max_retries - 1:
                     raise e
diff --git a/backend/danswer/connectors/interfaces.py b/backend/danswer/connectors/interfaces.py
index 3bd99792cce..c5b4850d9b0 100644
--- a/backend/danswer/connectors/interfaces.py
+++ b/backend/danswer/connectors/interfaces.py
@@ -11,6 +11,8 @@
 
 
 class BaseConnector(abc.ABC):
+    REDIS_KEY_PREFIX = "da_connector_data:"
+
     @abc.abstractmethod
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         raise NotImplementedError
diff --git a/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py b/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py
index 92bccaa050d..1779e8b1c33 100644
--- a/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py
+++ b/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py
@@ -1,5 +1,4 @@
 from unittest.mock import Mock
-from unittest.mock import patch
 
 import pytest
 from requests import HTTPError
@@ -14,36 +13,41 @@ def mock_confluence_call() -> Mock:
     return Mock()
 
 
-@pytest.mark.parametrize(
-    "status_code,text,retry_after",
-    [
-        (429, "Rate limit exceeded", "5"),
-        (200, "Rate limit exceeded", None),
-        (429, "Some other error", "5"),
-    ],
-)
-def test_rate_limit_handling(
-    mock_confluence_call: Mock, status_code: int, text: str, retry_after: str | None
-) -> None:
-    with patch("time.sleep") as mock_sleep:
-        mock_confluence_call.side_effect = [
-            HTTPError(
-                response=Mock(
-                    status_code=status_code,
-                    text=text,
-                    headers={"Retry-After": retry_after} if retry_after else {},
-                )
-            ),
-        ] * 2 + ["Success"]
-
-        handled_call = make_confluence_call_handle_rate_limit(mock_confluence_call)
-        result = handled_call()
-
-        assert result == "Success"
-        assert mock_confluence_call.call_count == 3
-        assert mock_sleep.call_count == 2
-        if retry_after:
-            mock_sleep.assert_called_with(int(retry_after))
+# ***** Checking call count to sleep() won't correctly reflect test correctness
+# especially since we really need to sleep multiple times and check for
+# abort signals moving forward. Disabling this test for now until we come up with
+# a better way forward.
+
+# @pytest.mark.parametrize(
+#     "status_code,text,retry_after",
+#     [
+#         (429, "Rate limit exceeded", "5"),
+#         (200, "Rate limit exceeded", None),
+#         (429, "Some other error", "5"),
+#     ],
+# )
+# def test_rate_limit_handling(
+#     mock_confluence_call: Mock, status_code: int, text: str, retry_after: str | None
+# ) -> None:
+#     with patch("time.sleep") as mock_sleep:
+#         mock_confluence_call.side_effect = [
+#             HTTPError(
+#                 response=Mock(
+#                     status_code=status_code,
+#                     text=text,
+#                     headers={"Retry-After": retry_after} if retry_after else {},
+#                 )
+#             ),
+#         ] * 2 + ["Success"]
+
+#         handled_call = make_confluence_call_handle_rate_limit(mock_confluence_call)
+#         result = handled_call()
+
+#         assert result == "Success"
+#         assert mock_confluence_call.call_count == 3
+#         assert mock_sleep.call_count == 2
+#         if retry_after:
+#             mock_sleep.assert_called_with(int(retry_after))
 
 
 def test_non_rate_limit_error(mock_confluence_call: Mock) -> None:

From aa5be37f97e94b304339c739d6308f5c594595cd Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 14 Oct 2024 19:59:33 -0700
Subject: [PATCH 116/376] fix index attempt refreshing automatically (#2791)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 .../connector/[ccPairId]/IndexingAttemptsTable.tsx    | 11 ++++++++++-
 web/src/app/admin/connector/[ccPairId]/page.tsx       |  2 +-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index 649ef1a97fd..51b7f8881ee 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -145,7 +145,7 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
     if (!cachedBatches[0]) {
       fetchBatchData(0);
     }
-  }, [ccPair.id, page, cachedBatches, totalPages]);
+  }, [ccPair.id, page, cachedBatches, totalPages, fetchBatchData]);
 
   // This updates the data on the current page
   useEffect(() => {
@@ -160,6 +160,15 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
     }
   }, [page, cachedBatches]);
 
+  useEffect(() => {
+    const interval = setInterval(() => {
+      const batchNum = Math.floor((page - 1) / BATCH_SIZE);
+      fetchBatchData(batchNum); // Re-fetch the current batch data
+    }, 5000); // Refresh every 5 seconds
+
+    return () => clearInterval(interval); // Cleanup on unmount
+  }, [page, fetchBatchData]); // Dependencies to ensure correct batch is fetched
+
   // This updates the page number and manages the URL
   const updatePage = (newPage: number) => {
     setPage(newPage);
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index 2c738c03f9b..9cdf7c83ec2 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -74,7 +74,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
     ) {
       finishConnectorDeletion();
     }
-  }, [isLoading, ccPair, error, hasLoadedOnce]);
+  }, [isLoading, ccPair, error, hasLoadedOnce, finishConnectorDeletion]);
 
   const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
     setEditableName(e.target.value);

From da46f611237eeae13725a9b31af62411f969b702 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 10:09:13 -0700
Subject: [PATCH 117/376] Ensure regenerate has dropdown too (#2797)

* ensure regenerate has dropdown too

* ensure applied to all

* nit
---
 web/src/app/chat/RegenerateOption.tsx        | 14 +++++-
 web/src/app/chat/message/Messages.tsx        | 46 +++++++++++++++-----
 web/src/components/tooltip/CustomTooltip.tsx |  3 ++
 3 files changed, 49 insertions(+), 14 deletions(-)

diff --git a/web/src/app/chat/RegenerateOption.tsx b/web/src/app/chat/RegenerateOption.tsx
index f28a83b03fa..8a9234b0a23 100644
--- a/web/src/app/chat/RegenerateOption.tsx
+++ b/web/src/app/chat/RegenerateOption.tsx
@@ -24,6 +24,7 @@ export function RegenerateDropdown({
   side,
   maxHeight,
   alternate,
+  onDropdownVisibleChange,
 }: {
   alternate?: string;
   options: StringOrNumberOption[];
@@ -32,9 +33,15 @@ export function RegenerateDropdown({
   includeDefault?: boolean;
   side?: "top" | "right" | "bottom" | "left";
   maxHeight?: string;
+  onDropdownVisibleChange: (isVisible: boolean) => void;
 }) {
   const [isOpen, setIsOpen] = useState(false);
 
+  const toggleDropdownVisible = (isVisible: boolean) => {
+    setIsOpen(isVisible);
+    onDropdownVisibleChange(isVisible);
+  };
+
   const Dropdown = (
     <div
       className={`
@@ -81,9 +88,9 @@ export function RegenerateDropdown({
   return (
     <Popover
       open={isOpen}
-      onOpenChange={(open) => setIsOpen(open)}
+      onOpenChange={toggleDropdownVisible}
       content={
-        <div onClick={() => setIsOpen(!isOpen)}>
+        <div onClick={() => toggleDropdownVisible(!isOpen)}>
           {!alternate ? (
             <Hoverable size={16} icon={StarFeedback as IconType} />
           ) : (
@@ -109,11 +116,13 @@ export default function RegenerateOption({
   regenerate,
   overriddenModel,
   onHoverChange,
+  onDropdownVisibleChange,
 }: {
   selectedAssistant: Persona;
   regenerate: (modelOverRide: LlmOverride) => Promise<void>;
   overriddenModel?: string;
   onHoverChange: (isHovered: boolean) => void;
+  onDropdownVisibleChange: (isVisible: boolean) => void;
 }) {
   const llmOverrideManager = useLlmOverride();
 
@@ -164,6 +173,7 @@ export default function RegenerateOption({
       onMouseLeave={() => onHoverChange(false)}
     >
       <RegenerateDropdown
+        onDropdownVisibleChange={onDropdownVisibleChange}
         alternate={overriddenModel}
         options={llmOptions}
         selected={currentModelName}
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 7a8a47ced54..56203cb06fc 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -195,6 +195,8 @@ export const AIMessage = ({
   const finalContent = processContent(content as string);
 
   const [isRegenerateHovered, setIsRegenerateHovered] = useState(false);
+  const [isRegenerateDropdownVisible, setIsRegenerateDropdownVisible] =
+    useState(false);
   const { isHovering, trackedElementRef, hoverElementRef } = useMouseTracking();
 
   const settings = useContext(SettingsContext);
@@ -522,12 +524,22 @@ export const AIMessage = ({
                             />
                           </CustomTooltip>
                           {regenerate && (
-                            <RegenerateOption
-                              onHoverChange={setIsRegenerateHovered}
-                              selectedAssistant={currentPersona!}
-                              regenerate={regenerate}
-                              overriddenModel={overriddenModel}
-                            />
+                            <CustomTooltip
+                              disabled={isRegenerateDropdownVisible}
+                              showTick
+                              line
+                              content="Regenerate!"
+                            >
+                              <RegenerateOption
+                                onDropdownVisibleChange={
+                                  setIsRegenerateDropdownVisible
+                                }
+                                onHoverChange={setIsRegenerateHovered}
+                                selectedAssistant={currentPersona!}
+                                regenerate={regenerate}
+                                overriddenModel={overriddenModel}
+                              />
+                            </CustomTooltip>
                           )}
                         </TooltipGroup>
                       </div>
@@ -587,12 +599,22 @@ export const AIMessage = ({
                             />
                           </CustomTooltip>
                           {regenerate && (
-                            <RegenerateOption
-                              selectedAssistant={currentPersona!}
-                              regenerate={regenerate}
-                              overriddenModel={overriddenModel}
-                              onHoverChange={setIsRegenerateHovered}
-                            />
+                            <CustomTooltip
+                              disabled={isRegenerateDropdownVisible}
+                              showTick
+                              line
+                              content="Regenerate!"
+                            >
+                              <RegenerateOption
+                                selectedAssistant={currentPersona!}
+                                onDropdownVisibleChange={
+                                  setIsRegenerateDropdownVisible
+                                }
+                                regenerate={regenerate}
+                                overriddenModel={overriddenModel}
+                                onHoverChange={setIsRegenerateHovered}
+                              />
+                            </CustomTooltip>
                           )}
                         </TooltipGroup>
                       </div>
diff --git a/web/src/components/tooltip/CustomTooltip.tsx b/web/src/components/tooltip/CustomTooltip.tsx
index 2f4ca2d1254..52a98d8aad6 100644
--- a/web/src/components/tooltip/CustomTooltip.tsx
+++ b/web/src/components/tooltip/CustomTooltip.tsx
@@ -46,6 +46,7 @@ export const CustomTooltip = ({
   showTick = false,
   delay = 500,
   position = "bottom",
+  disabled = false,
 }: {
   medium?: boolean;
   content: string | ReactNode;
@@ -58,6 +59,7 @@ export const CustomTooltip = ({
   wrap?: boolean;
   citation?: boolean;
   position?: "top" | "bottom";
+  disabled?: boolean;
 }) => {
   const [isVisible, setIsVisible] = useState(false);
   const [tooltipPosition, setTooltipPosition] = useState({ top: 0, left: 0 });
@@ -119,6 +121,7 @@ export const CustomTooltip = ({
         {children}
       </span>
       {isVisible &&
+        !disabled &&
         createPortal(
           <div
             className={`min-w-8 fixed z-[1000] ${citation ? "max-w-[350px]" : "w-40"} ${

From 98e88e271509ab161b1bda580e53fc5a13eb8d0b Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 10:26:01 -0700
Subject: [PATCH 118/376] ensure shared chats are shared (#2801)

* ensure shared chats are shared

* k

* k

* nit

* k
---
 web/src/app/admin/assistants/lib.ts           | 25 ++++++++++++++++++-
 web/src/app/chat/ChatPage.tsx                 |  7 ------
 web/src/app/chat/message/Messages.tsx         |  4 ---
 .../shared/[chatId]/SharedChatDisplay.tsx     | 12 +++------
 web/src/app/chat/shared/[chatId]/page.tsx     | 13 ++++++----
 .../chat_search/ProviderContext.tsx           |  1 +
 6 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/web/src/app/admin/assistants/lib.ts b/web/src/app/admin/assistants/lib.ts
index 9b79ea7ef6c..9be6dbae834 100644
--- a/web/src/app/admin/assistants/lib.ts
+++ b/web/src/app/admin/assistants/lib.ts
@@ -1,5 +1,5 @@
 import { FullLLMProvider } from "../configuration/llm/interfaces";
-import { Persona, Prompt, StarterMessage } from "./interfaces";
+import { Persona, StarterMessage } from "./interfaces";
 
 interface PersonaCreationRequest {
   name: string;
@@ -377,3 +377,26 @@ export function providersContainImageGeneratingSupport(
 ) {
   return providers.some((provider) => provider.provider === "openai");
 }
+
+// Default fallback persona for when we must display a persona
+// but assistant has access to none
+export const defaultPersona: Persona = {
+  id: 0,
+  name: "Default Assistant",
+  description: "A default assistant",
+  is_visible: true,
+  is_public: true,
+  builtin_persona: false,
+  is_default_persona: true,
+  users: [],
+  groups: [],
+  document_sets: [],
+  prompts: [],
+  tools: [],
+  starter_messages: null,
+  display_priority: null,
+  search_start_date: null,
+  owner: null,
+  icon_shape: 50910,
+  icon_color: "#FF6F6F",
+};
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 54432c40e3b..f218f7fab95 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -2185,7 +2185,6 @@ export function ChatPage({
                                       query={
                                         messageHistory[i]?.query || undefined
                                       }
-                                      personaName={liveAssistant.name}
                                       citedDocuments={getCitedDocumentsFromMessage(
                                         message
                                       )}
@@ -2247,9 +2246,6 @@ export function ChatPage({
                                             }
                                           : undefined
                                       }
-                                      isCurrentlyShowingRetrieved={
-                                        isShowingRetrieved
-                                      }
                                       handleShowRetrieved={(messageNumber) => {
                                         if (isShowingRetrieved) {
                                           setSelectedMessageForDocDisplay(null);
@@ -2299,7 +2295,6 @@ export function ChatPage({
                                     <AIMessage
                                       currentPersona={liveAssistant}
                                       messageId={message.messageId}
-                                      personaName={liveAssistant.name}
                                       content={
                                         <p className="text-red-700 text-sm my-auto">
                                           {message.message}
@@ -2347,7 +2342,6 @@ export function ChatPage({
                                     alternativeAssistant
                                   }
                                   messageId={null}
-                                  personaName={liveAssistant.name}
                                   content={
                                     <div
                                       key={"Generating"}
@@ -2367,7 +2361,6 @@ export function ChatPage({
                                 <AIMessage
                                   currentPersona={liveAssistant}
                                   messageId={-1}
-                                  personaName={liveAssistant.name}
                                   content={
                                     <p className="text-red-700 text-sm my-auto">
                                       {loadingError}
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 56203cb06fc..56b917164df 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -124,13 +124,11 @@ export const AIMessage = ({
   files,
   selectedDocuments,
   query,
-  personaName,
   citedDocuments,
   toolCall,
   isComplete,
   hasDocs,
   handleFeedback,
-  isCurrentlyShowingRetrieved,
   handleShowRetrieved,
   handleSearchQueryEdit,
   handleForceSearch,
@@ -153,13 +151,11 @@ export const AIMessage = ({
   content: string | JSX.Element;
   files?: FileDescriptor[];
   query?: string;
-  personaName?: string;
   citedDocuments?: [string, DanswerDocument][] | null;
   toolCall?: ToolCallMetadata;
   isComplete?: boolean;
   hasDocs?: boolean;
   handleFeedback?: (feedbackType: FeedbackType) => void;
-  isCurrentlyShowingRetrieved?: boolean;
   handleShowRetrieved?: (messageNumber: number | null) => void;
   handleSearchQueryEdit?: (query: string) => void;
   handleForceSearch?: () => void;
diff --git a/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx b/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
index 489163aa3c8..de2d443963f 100644
--- a/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
+++ b/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
@@ -11,10 +11,10 @@ import {
 import { AIMessage, HumanMessage } from "../../message/Messages";
 import { Button, Callout, Divider } from "@tremor/react";
 import { useRouter } from "next/navigation";
-import { Persona } from "@/app/admin/assistants/interfaces";
 import { useContext, useEffect, useState } from "react";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { DanswerInitializingLoader } from "@/components/DanswerInitializingLoader";
+import { Persona } from "@/app/admin/assistants/interfaces";
 
 function BackToDanswerButton() {
   const router = useRouter();
@@ -34,10 +34,10 @@ function BackToDanswerButton() {
 
 export function SharedChatDisplay({
   chatSession,
-  availableAssistants,
+  persona,
 }: {
   chatSession: BackendChatSession | null;
-  availableAssistants: Persona[];
+  persona: Persona;
 }) {
   const [isReady, setIsReady] = useState(false);
   useEffect(() => {
@@ -56,9 +56,6 @@ export function SharedChatDisplay({
       </div>
     );
   }
-  const currentPersona = availableAssistants.find(
-    (persona) => persona.id === chatSession.persona_id
-  );
 
   const messages = buildLatestMessageChain(
     processRawChatHistory(chatSession.messages)
@@ -96,12 +93,11 @@ export function SharedChatDisplay({
                     return (
                       <AIMessage
                         shared
-                        currentPersona={currentPersona!}
+                        currentPersona={persona}
                         key={message.messageId}
                         messageId={message.messageId}
                         content={message.message}
                         files={message.files || []}
-                        personaName={chatSession.persona_name}
                         citedDocuments={getCitedDocumentsFromMessage(message)}
                         isComplete
                       />
diff --git a/web/src/app/chat/shared/[chatId]/page.tsx b/web/src/app/chat/shared/[chatId]/page.tsx
index 9e8ce58432e..d012b4a0d77 100644
--- a/web/src/app/chat/shared/[chatId]/page.tsx
+++ b/web/src/app/chat/shared/[chatId]/page.tsx
@@ -11,6 +11,7 @@ import { SharedChatDisplay } from "./SharedChatDisplay";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { fetchAssistantsSS } from "@/lib/assistants/fetchAssistantsSS";
 import FunctionalHeader from "@/components/chat_search/Header";
+import { defaultPersona } from "@/app/admin/assistants/lib";
 
 async function getSharedChat(chatId: string) {
   const response = await fetchSS(
@@ -43,7 +44,7 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   const authTypeMetadata = results[0] as AuthTypeMetadata | null;
   const user = results[1] as User | null;
   const chatSession = results[2] as BackendChatSession | null;
-  const [availableAssistants, _] = results[3] as [Persona[], string | null];
+  const availableAssistants = results[3] as Persona[] | null;
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
@@ -53,6 +54,11 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   if (user && !user.is_verified && authTypeMetadata?.requiresVerification) {
     return redirect("/auth/waiting-on-verification");
   }
+  const persona = chatSession?.persona_id
+    ? (availableAssistants?.find((p) => p.id === chatSession.persona_id) ??
+      availableAssistants?.[0] ??
+      null)
+    : (availableAssistants?.[0] ?? defaultPersona);
 
   return (
     <div>
@@ -61,10 +67,7 @@ export default async function Page({ params }: { params: { chatId: string } }) {
       </div>
 
       <div className="flex relative bg-background text-default overflow-hidden pt-16 h-screen">
-        <SharedChatDisplay
-          chatSession={chatSession}
-          availableAssistants={availableAssistants}
-        />
+        <SharedChatDisplay chatSession={chatSession} persona={persona!} />
       </div>
     </div>
   );
diff --git a/web/src/components/chat_search/ProviderContext.tsx b/web/src/components/chat_search/ProviderContext.tsx
index 609713f874b..f0839373b8d 100644
--- a/web/src/components/chat_search/ProviderContext.tsx
+++ b/web/src/components/chat_search/ProviderContext.tsx
@@ -37,6 +37,7 @@ export function ProviderContextProvider({
   const fetchProviderInfo = useCallback(async () => {
     const { providers, options, defaultCheckSuccessful } =
       await checkLlmProvider(user);
+
     setValidProviderExists(providers.length > 0 && defaultCheckSuccessful);
     setProviderOptions(options);
   }, [user, setValidProviderExists, setProviderOptions]);

From 0e6c2f0b51435bdf023c56160e2bc90419ff3f7a Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 12:23:04 -0700
Subject: [PATCH 119/376] add ca option (#2774)

---
 backend/Dockerfile | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/backend/Dockerfile b/backend/Dockerfile
index f7ea1e3e1d4..9bcd71952d7 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -12,6 +12,8 @@ ARG DANSWER_VERSION=0.3-dev
 ENV DANSWER_VERSION=${DANSWER_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true"
 
+ARG CA_CERT_CONTENT=""
+
 RUN echo "DANSWER_VERSION: ${DANSWER_VERSION}"
 # Install system dependencies
 # cmake needed for psycopg (postgres)
@@ -36,6 +38,17 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get clean
 
+
+# Conditionally write the CA certificate and update certificates
+RUN if [ -n "$CA_CERT_CONTENT" ]; then \
+    echo "Adding custom CA certificate"; \
+    echo "$CA_CERT_CONTENT" > /usr/local/share/ca-certificates/my-ca.crt && \
+    chmod 644 /usr/local/share/ca-certificates/my-ca.crt && \
+    update-ca-certificates; \
+else \
+    echo "No custom CA certificate provided"; \
+fi
+
 # Install Python dependencies
 # Remove py which is pulled in by retry, py is not needed and is a CVE
 COPY ./requirements/default.txt /tmp/requirements.txt

From bfe963988e158d077df14d4639f48b31c30c2269 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 13:10:57 -0700
Subject: [PATCH 120/376] various multi tenant improvements (#2803)

* various multi tenant improvements

* nit

* ensure consistent db session operations

* minor robustification
---
 .../celery/tasks/connector_deletion/tasks.py  |  4 +--
 .../background/celery/tasks/periodic/tasks.py |  6 ++--
 .../background/celery/tasks/vespa/tasks.py    |  9 ++---
 .../background/indexing/run_indexing.py       |  1 +
 backend/danswer/configs/constants.py          |  3 ++
 backend/danswer/connectors/factory.py         |  6 ++++
 backend/danswer/connectors/file/connector.py  |  7 ++--
 backend/danswer/db/index_attempt.py           |  5 +--
 backend/danswer/server/documents/cc_pair.py   |  4 ++-
 .../danswer/server/manage/search_settings.py  |  1 +
 .../server/query_and_chat/token_limit.py      | 12 ++++---
 .../server/query_and_chat/token_limit.py      | 22 ++++++------
 .../scripts/force_delete_connector_by_id.py   |  2 ++
 web/src/app/auth/logout/route.ts              | 35 +++++++++++++++++--
 web/src/lib/constants.ts                      |  1 +
 15 files changed, 84 insertions(+), 34 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index 6a4c4da8243..b13daff61fc 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -12,7 +12,7 @@
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.models import ConnectorCredentialPair
 from danswer.redis.redis_pool import get_redis_client
@@ -36,7 +36,7 @@ def check_for_connector_deletion_task(tenant_id: str | None) -> None:
         if not lock_beat.acquire(blocking=False):
             return
 
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = get_connector_credential_pairs(db_session)
             for cc_pair in cc_pairs:
                 try_generate_document_cc_pair_cleanup_tasks(
diff --git a/backend/danswer/background/celery/tasks/periodic/tasks.py b/backend/danswer/background/celery/tasks/periodic/tasks.py
index 99b1cab7e77..d8da5ba9ca9 100644
--- a/backend/danswer/background/celery/tasks/periodic/tasks.py
+++ b/backend/danswer/background/celery/tasks/periodic/tasks.py
@@ -14,7 +14,7 @@
 from danswer.background.celery.celery_app import task_logger
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import PostgresAdvisoryLocks
-from danswer.db.engine import get_sqlalchemy_engine  # type: ignore
+from danswer.db.engine import get_session_with_tenant
 
 
 @shared_task(
@@ -23,7 +23,7 @@
     bind=True,
     base=AbortableTask,
 )
-def kombu_message_cleanup_task(self: Any) -> int:
+def kombu_message_cleanup_task(self: Any, tenant_id: str | None) -> int:
     """Runs periodically to clean up the kombu_message table"""
 
     # we will select messages older than this amount to clean up
@@ -35,7 +35,7 @@ def kombu_message_cleanup_task(self: Any) -> int:
     ctx["deleted"] = 0
     ctx["cleanup_age"] = KOMBU_MESSAGE_CLEANUP_AGE
     ctx["page_limit"] = KOMBU_MESSAGE_CLEANUP_PAGE_LIMIT
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         # Exit the task if we can't take the advisory lock
         result = db_session.execute(
             text("SELECT pg_try_advisory_lock(:id)"),
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index e6a017b7ac7..c43de3a85f8 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -39,7 +39,6 @@
 from danswer.db.document_set import get_document_set_by_id
 from danswer.db.document_set import mark_document_set_as_synced
 from danswer.db.engine import get_session_with_tenant
-from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.index_attempt import delete_index_attempts
 from danswer.db.models import DocumentSet
 from danswer.db.models import UserGroup
@@ -341,7 +340,9 @@ def monitor_document_set_taskset(
     r.delete(rds.fence_key)
 
 
-def monitor_connector_deletion_taskset(key_bytes: bytes, r: Redis) -> None:
+def monitor_connector_deletion_taskset(
+    key_bytes: bytes, r: Redis, tenant_id: str | None
+) -> None:
     fence_key = key_bytes.decode("utf-8")
     cc_pair_id = RedisConnectorDeletion.get_id_from_fence_key(fence_key)
     if cc_pair_id is None:
@@ -367,7 +368,7 @@ def monitor_connector_deletion_taskset(key_bytes: bytes, r: Redis) -> None:
     if count > 0:
         return
 
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session)
         if not cc_pair:
             task_logger.warning(
@@ -529,7 +530,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
 
         lock_beat.reacquire()
         for key_bytes in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
-            monitor_connector_deletion_taskset(key_bytes, r)
+            monitor_connector_deletion_taskset(key_bytes, r, tenant_id)
 
         with get_session_with_tenant(tenant_id) as db_session:
             lock_beat.reacquire()
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index 69c8a739389..c48d07ffd85 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -65,6 +65,7 @@ def _get_connector_runner(
             input_type=task,
             connector_specific_config=attempt.connector_credential_pair.connector.connector_specific_config,
             credential=attempt.connector_credential_pair.credential,
+            tenant_id=tenant_id,
         )
     except Exception as e:
         logger.exception(f"Unable to instantiate connector due to {e}")
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index e4aeb88c279..6b167246a66 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -118,6 +118,9 @@ class DocumentSource(str, Enum):
     NOT_APPLICABLE = "not_applicable"
 
 
+DocumentSourceRequiringTenantContext: list[DocumentSource] = [DocumentSource.FILE]
+
+
 class NotificationType(str, Enum):
     REINDEX = "reindex"
 
diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py
index 75e0d9bb238..52fb0194aa6 100644
--- a/backend/danswer/connectors/factory.py
+++ b/backend/danswer/connectors/factory.py
@@ -4,6 +4,7 @@
 from sqlalchemy.orm import Session
 
 from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import DocumentSourceRequiringTenantContext
 from danswer.connectors.asana.connector import AsanaConnector
 from danswer.connectors.axero.connector import AxeroConnector
 from danswer.connectors.blob.connector import BlobStorageConnector
@@ -134,8 +135,13 @@ def instantiate_connector(
     input_type: InputType,
     connector_specific_config: dict[str, Any],
     credential: Credential,
+    tenant_id: str | None = None,
 ) -> BaseConnector:
     connector_class = identify_connector_class(source, input_type)
+
+    if source in DocumentSourceRequiringTenantContext:
+        connector_specific_config["tenant_id"] = tenant_id
+
     connector = connector_class(**connector_specific_config)
     new_credentials = connector.load_credentials(credential.credential_json)
 
diff --git a/backend/danswer/connectors/file/connector.py b/backend/danswer/connectors/file/connector.py
index 8ef98716c91..106fed8b2af 100644
--- a/backend/danswer/connectors/file/connector.py
+++ b/backend/danswer/connectors/file/connector.py
@@ -10,13 +10,14 @@
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.file_processing.extract_file_text import check_file_ext_is_valid
 from danswer.file_processing.extract_file_text import detect_encoding
 from danswer.file_processing.extract_file_text import extract_file_text
@@ -159,10 +160,12 @@ class LocalFileConnector(LoadConnector):
     def __init__(
         self,
         file_locations: list[Path | str],
+        tenant_id: str = POSTGRES_DEFAULT_SCHEMA,
         batch_size: int = INDEX_BATCH_SIZE,
     ) -> None:
         self.file_locations = [Path(file_location) for file_location in file_locations]
         self.batch_size = batch_size
+        self.tenant_id = tenant_id
         self.pdf_pass: str | None = None
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
@@ -171,7 +174,7 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
 
     def load_from_state(self) -> GenerateDocumentsOutput:
         documents: list[Document] = []
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(self.tenant_id) as db_session:
             for file_path in self.file_locations:
                 current_datetime = datetime.now(timezone.utc)
                 files = _read_files_and_metadata(
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index 32e20d065c0..d9b1569e427 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -435,14 +435,13 @@ def cancel_indexing_attempts_for_ccpair(
 
     db_session.execute(stmt)
 
-    db_session.commit()
-
 
 def cancel_indexing_attempts_past_model(
     db_session: Session,
 ) -> None:
     """Stops all indexing attempts that are in progress or not started for
     any embedding model that not present/future"""
+
     db_session.execute(
         update(IndexAttempt)
         .where(
@@ -455,8 +454,6 @@ def cancel_indexing_attempts_past_model(
         .values(status=IndexingStatus.FAILED)
     )
 
-    db_session.commit()
-
 
 def count_unique_cc_pairs_with_successful_index_attempts(
     search_settings_id: int | None,
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index ea513b5c21d..d835a25e26e 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -154,6 +154,7 @@ def update_cc_pair_status(
         user=user,
         get_editable=True,
     )
+
     if not cc_pair:
         raise HTTPException(
             status_code=400,
@@ -163,7 +164,6 @@ def update_cc_pair_status(
     if status_update_request.status == ConnectorCredentialPairStatus.PAUSED:
         cancel_indexing_attempts_for_ccpair(cc_pair_id, db_session)
 
-        # Just for good measure
         cancel_indexing_attempts_past_model(db_session)
 
     update_connector_credential_pair_from_id(
@@ -172,6 +172,8 @@ def update_cc_pair_status(
         status=status_update_request.status,
     )
 
+    db_session.commit()
+
 
 @router.put("/admin/cc-pair/{cc_pair_id}/name")
 def update_cc_pair_name(
diff --git a/backend/danswer/server/manage/search_settings.py b/backend/danswer/server/manage/search_settings.py
index 6436a0bd8c0..79f690e5db6 100644
--- a/backend/danswer/server/manage/search_settings.py
+++ b/backend/danswer/server/manage/search_settings.py
@@ -115,6 +115,7 @@ def set_new_search_settings(
         for cc_pair in get_connector_credential_pairs(db_session):
             resync_cc_pair(cc_pair, db_session=db_session)
 
+    db_session.commit()
     return IdReturn(id=new_search_settings.id)
 
 
diff --git a/backend/danswer/server/query_and_chat/token_limit.py b/backend/danswer/server/query_and_chat/token_limit.py
index 3f5d76bac7f..6221eae3346 100644
--- a/backend/danswer/server/query_and_chat/token_limit.py
+++ b/backend/danswer/server/query_and_chat/token_limit.py
@@ -13,6 +13,7 @@
 
 from danswer.auth.users import current_user
 from danswer.db.engine import get_session_context_manager
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.models import ChatMessage
 from danswer.db.models import ChatSession
 from danswer.db.models import TokenRateLimit
@@ -20,6 +21,7 @@
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from ee.danswer.db.token_limit import fetch_all_global_token_rate_limits
+from shared_configs.configs import current_tenant_id
 
 
 logger = setup_logger()
@@ -39,11 +41,11 @@ def check_token_rate_limits(
     versioned_rate_limit_strategy = fetch_versioned_implementation(
         "danswer.server.query_and_chat.token_limit", "_check_token_rate_limits"
     )
-    return versioned_rate_limit_strategy(user)
+    return versioned_rate_limit_strategy(user, current_tenant_id.get())
 
 
-def _check_token_rate_limits(_: User | None) -> None:
-    _user_is_rate_limited_by_global()
+def _check_token_rate_limits(_: User | None, tenant_id: str | None) -> None:
+    _user_is_rate_limited_by_global(tenant_id)
 
 
 """
@@ -51,8 +53,8 @@ def _check_token_rate_limits(_: User | None) -> None:
 """
 
 
-def _user_is_rate_limited_by_global() -> None:
-    with get_session_context_manager() as db_session:
+def _user_is_rate_limited_by_global(tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         global_rate_limits = fetch_all_global_token_rate_limits(
             db_session=db_session, enabled_only=True, ordered=False
         )
diff --git a/backend/ee/danswer/server/query_and_chat/token_limit.py b/backend/ee/danswer/server/query_and_chat/token_limit.py
index 538458fb63f..b4c588dc416 100644
--- a/backend/ee/danswer/server/query_and_chat/token_limit.py
+++ b/backend/ee/danswer/server/query_and_chat/token_limit.py
@@ -12,7 +12,7 @@
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 
-from danswer.db.engine import get_session_context_manager
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.models import ChatMessage
 from danswer.db.models import ChatSession
 from danswer.db.models import TokenRateLimit
@@ -28,21 +28,21 @@
 from ee.danswer.db.token_limit import fetch_all_user_token_rate_limits
 
 
-def _check_token_rate_limits(user: User | None) -> None:
+def _check_token_rate_limits(user: User | None, tenant_id: str | None) -> None:
     if user is None:
         # Unauthenticated users are only rate limited by global settings
-        _user_is_rate_limited_by_global()
+        _user_is_rate_limited_by_global(tenant_id)
 
     elif is_api_key_email_address(user.email):
         # API keys are only rate limited by global settings
-        _user_is_rate_limited_by_global()
+        _user_is_rate_limited_by_global(tenant_id)
 
     else:
         run_functions_tuples_in_parallel(
             [
-                (_user_is_rate_limited, (user.id,)),
-                (_user_is_rate_limited_by_group, (user.id,)),
-                (_user_is_rate_limited_by_global, ()),
+                (_user_is_rate_limited, (user.id, tenant_id)),
+                (_user_is_rate_limited_by_group, (user.id, tenant_id)),
+                (_user_is_rate_limited_by_global, (tenant_id,)),
             ]
         )
 
@@ -52,8 +52,8 @@ def _check_token_rate_limits(user: User | None) -> None:
 """
 
 
-def _user_is_rate_limited(user_id: UUID) -> None:
-    with get_session_context_manager() as db_session:
+def _user_is_rate_limited(user_id: UUID, tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         user_rate_limits = fetch_all_user_token_rate_limits(
             db_session=db_session, enabled_only=True, ordered=False
         )
@@ -93,8 +93,8 @@ def _fetch_user_usage(
 """
 
 
-def _user_is_rate_limited_by_group(user_id: UUID) -> None:
-    with get_session_context_manager() as db_session:
+def _user_is_rate_limited_by_group(user_id: UUID, tenant_id: str | None) -> None:
+    with get_session_with_tenant(tenant_id) as db_session:
         group_rate_limits = _fetch_all_user_group_rate_limits(user_id, db_session)
 
         if group_rate_limits:
diff --git a/backend/scripts/force_delete_connector_by_id.py b/backend/scripts/force_delete_connector_by_id.py
index 0a9857304c8..241242f4a23 100755
--- a/backend/scripts/force_delete_connector_by_id.py
+++ b/backend/scripts/force_delete_connector_by_id.py
@@ -206,6 +206,8 @@ def _delete_connector(cc_pair_id: int, db_session: Session) -> None:
             logger.notice(f"Deleting file {file_name}")
             file_store.delete_file(file_name)
 
+    db_session.commit()
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Delete a connector by its ID")
diff --git a/web/src/app/auth/logout/route.ts b/web/src/app/auth/logout/route.ts
index 7de902c7acf..e3bae04bb22 100644
--- a/web/src/app/auth/logout/route.ts
+++ b/web/src/app/auth/logout/route.ts
@@ -1,3 +1,4 @@
+import { CLOUD_ENABLED } from "@/lib/constants";
 import { getAuthTypeMetadataSS, logoutSS } from "@/lib/userSS";
 import { NextRequest } from "next/server";
 
@@ -6,8 +7,38 @@ export const POST = async (request: NextRequest) => {
   // Needed since env variables don't work well on the client-side
   const authTypeMetadata = await getAuthTypeMetadataSS();
   const response = await logoutSS(authTypeMetadata.authType, request.headers);
-  if (!response || response.ok) {
+
+  if (response && !response.ok) {
+    return new Response(response.body, { status: response?.status });
+  }
+
+  // Delete cookies only if cloud is enabled (jwt auth)
+  if (CLOUD_ENABLED) {
+    const cookiesToDelete = ["fastapiusersauth", "tenant_details"];
+    const cookieOptions = {
+      path: "/",
+      secure: process.env.NODE_ENV === "production",
+      httpOnly: true,
+      sameSite: "lax" as const,
+    };
+
+    // Logout successful, delete cookies
+    const headers = new Headers();
+
+    cookiesToDelete.forEach((cookieName) => {
+      headers.append(
+        "Set-Cookie",
+        `${cookieName}=; Max-Age=0; ${Object.entries(cookieOptions)
+          .map(([key, value]) => `${key}=${value}`)
+          .join("; ")}`
+      );
+    });
+
+    return new Response(null, {
+      status: 204,
+      headers: headers,
+    });
+  } else {
     return new Response(null, { status: 204 });
   }
-  return new Response(response.body, { status: response?.status });
 };
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index c0b916ebadf..15e5b5cbcf0 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -58,6 +58,7 @@ export const DISABLE_LLM_DOC_RELEVANCE =
 
 export const CLOUD_ENABLED =
   process.env.NEXT_PUBLIC_CLOUD_ENABLED?.toLowerCase() === "true";
+
 export const REGISTRATION_URL =
   process.env.INTERNAL_URL || "http://127.0.0.1:3001";
 

From 02cc211e9113ba95ce267555adb49098429cedfe Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 16:22:40 -0700
Subject: [PATCH 121/376] improved code block copying (#2802)

* improved code block copying

* k
---
 web/src/app/chat/message/Messages.tsx |  2 +-
 web/src/app/chat/message/codeUtils.ts | 42 +++++++++++----------------
 2 files changed, 18 insertions(+), 26 deletions(-)

diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 56b917164df..181e00845ee 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -256,7 +256,7 @@ export const AIMessage = ({
     () => ({
       a: MemoizedLink,
       p: MemoizedParagraph,
-      code: ({ node, inline, className, children, ...props }: any) => {
+      code: ({ node, className, children, ...props }: any) => {
         const codeText = extractCodeText(
           node,
           finalContent as string,
diff --git a/web/src/app/chat/message/codeUtils.ts b/web/src/app/chat/message/codeUtils.ts
index 2aaae71bc82..d2aad299044 100644
--- a/web/src/app/chat/message/codeUtils.ts
+++ b/web/src/app/chat/message/codeUtils.ts
@@ -4,40 +4,32 @@ export function extractCodeText(
   children: React.ReactNode
 ): string {
   let codeText: string | null = null;
+
   if (
     node?.position?.start?.offset != null &&
     node?.position?.end?.offset != null
   ) {
-    codeText = content.slice(
-      node.position.start.offset,
-      node.position.end.offset
-    );
-    codeText = codeText.trim();
+    codeText = content
+      .slice(node.position.start.offset, node.position.end.offset)
+      .trim();
 
-    // Find the last occurrence of closing backticks
-    const lastBackticksIndex = codeText.lastIndexOf("```");
-    if (lastBackticksIndex !== -1) {
-      codeText = codeText.slice(0, lastBackticksIndex + 3);
+    // Match code block with optional language declaration
+    const codeBlockMatch = codeText.match(/^```[^\n]*\n([\s\S]*?)\n?```$/);
+    if (codeBlockMatch) {
+      codeText = codeBlockMatch[1];
     }
 
-    // Remove the language declaration and trailing backticks
+    // Normalize indentation
     const codeLines = codeText.split("\n");
-    if (codeLines.length > 1 && codeLines[0].trim().startsWith("```")) {
-      codeLines.shift(); // Remove the first line with the language declaration
-      if (codeLines[codeLines.length - 1]?.trim() === "```") {
-        codeLines.pop(); // Remove the last line with the trailing backticks
-      }
-
-      const minIndent = codeLines
-        .filter((line) => line.trim().length > 0)
-        .reduce((min, line) => {
-          const match = line.match(/^\s*/);
-          return Math.min(min, match ? match[0].length : 0);
-        }, Infinity);
+    const minIndent = codeLines
+      .filter((line) => line.trim().length > 0)
+      .reduce((min, line) => {
+        const match = line.match(/^\s*/);
+        return Math.min(min, match ? match[0].length : min);
+      }, Infinity);
 
-      const formattedCodeLines = codeLines.map((line) => line.slice(minIndent));
-      codeText = formattedCodeLines.join("\n");
-    }
+    const formattedCodeLines = codeLines.map((line) => line.slice(minIndent));
+    codeText = formattedCodeLines.join("\n").trim();
   } else {
     // Fallback if position offsets are not available
     codeText = children?.toString() || null;

From e022e77b6d0d8442e425769f93c920fa46fb1df4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 16:23:11 -0700
Subject: [PATCH 122/376] Simpler azure embedding (#2751)

* functional but janky

* nit

* adapt for azure

* nit

* minor updates

* nits

* nit

* nit

* ensure access to litellm

* k
---
 ...add_api_version_and_deployment_name_to_.py |  30 +++
 backend/danswer/configs/app_configs.py        |   1 -
 backend/danswer/db/models.py                  |  18 ++
 backend/danswer/indexing/embedder.py          |  12 +
 .../search_nlp_models.py                      |   8 +
 .../danswer/server/manage/embedding/api.py    |   2 +
 .../danswer/server/manage/embedding/models.py |   8 +
 backend/model_server/encoders.py              |  49 +++-
 backend/requirements/model_server.txt         |   1 +
 backend/shared_configs/enums.py               |   1 +
 backend/shared_configs/model_server_models.py |   3 +-
 .../EmbeddingModelSelectionForm.tsx           |  60 ++---
 .../modals/ChangeCredentialsModal.tsx         | 231 +++++++++---------
 .../modals/ProviderCreationModal.tsx          |  89 +++++--
 .../embeddings/pages/CloudEmbeddingPage.tsx   | 162 ++++++++++--
 .../embeddings/pages/EmbeddingFormPage.tsx    |  15 +-
 web/src/components/admin/Layout.tsx           |   2 +-
 ...lForm.tsx => CustomEmbeddingModelForm.tsx} |  22 +-
 .../components/embedding/EmbeddingSidebar.tsx |   2 +-
 web/src/components/embedding/interfaces.tsx   |  18 ++
 20 files changed, 522 insertions(+), 212 deletions(-)
 create mode 100644 backend/alembic/versions/5d12a446f5c0_add_api_version_and_deployment_name_to_.py
 rename web/src/components/embedding/{LiteLLMModelForm.tsx => CustomEmbeddingModelForm.tsx} (81%)

diff --git a/backend/alembic/versions/5d12a446f5c0_add_api_version_and_deployment_name_to_.py b/backend/alembic/versions/5d12a446f5c0_add_api_version_and_deployment_name_to_.py
new file mode 100644
index 00000000000..85b5431ecc3
--- /dev/null
+++ b/backend/alembic/versions/5d12a446f5c0_add_api_version_and_deployment_name_to_.py
@@ -0,0 +1,30 @@
+"""add api_version and deployment_name to search settings
+
+Revision ID: 5d12a446f5c0
+Revises: e4334d5b33ba
+Create Date: 2024-10-08 15:56:07.975636
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "5d12a446f5c0"
+down_revision = "e4334d5b33ba"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "embedding_provider", sa.Column("api_version", sa.String(), nullable=True)
+    )
+    op.add_column(
+        "embedding_provider", sa.Column("deployment_name", sa.String(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("embedding_provider", "deployment_name")
+    op.drop_column("embedding_provider", "api_version")
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index a061b79019f..337a5361d8b 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -53,7 +53,6 @@
     os.environ.get("MASK_CREDENTIAL_PREFIX", "True").lower() != "false"
 )
 
-
 SESSION_EXPIRE_TIME_SECONDS = int(
     os.environ.get("SESSION_EXPIRE_TIME_SECONDS") or 86400 * 7
 )  # 7 days
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index fc7dad7793f..c7cbbe9e8d0 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -615,6 +615,7 @@ class SearchSettings(Base):
     normalize: Mapped[bool] = mapped_column(Boolean)
     query_prefix: Mapped[str | None] = mapped_column(String, nullable=True)
     passage_prefix: Mapped[str | None] = mapped_column(String, nullable=True)
+
     status: Mapped[IndexModelStatus] = mapped_column(
         Enum(IndexModelStatus, native_enum=False)
     )
@@ -670,6 +671,20 @@ def __repr__(self) -> str:
         return f"<EmbeddingModel(model_name='{self.model_name}', status='{self.status}',\
           cloud_provider='{self.cloud_provider.provider_type if self.cloud_provider else 'None'}')>"
 
+    @property
+    def api_version(self) -> str | None:
+        return (
+            self.cloud_provider.api_version if self.cloud_provider is not None else None
+        )
+
+    @property
+    def deployment_name(self) -> str | None:
+        return (
+            self.cloud_provider.deployment_name
+            if self.cloud_provider is not None
+            else None
+        )
+
     @property
     def api_url(self) -> str | None:
         return self.cloud_provider.api_url if self.cloud_provider is not None else None
@@ -1164,6 +1179,9 @@ class CloudEmbeddingProvider(Base):
     )
     api_url: Mapped[str | None] = mapped_column(String, nullable=True)
     api_key: Mapped[str | None] = mapped_column(EncryptedString())
+    api_version: Mapped[str | None] = mapped_column(String, nullable=True)
+    deployment_name: Mapped[str | None] = mapped_column(String, nullable=True)
+
     search_settings: Mapped[list["SearchSettings"]] = relationship(
         "SearchSettings",
         back_populates="cloud_provider",
diff --git a/backend/danswer/indexing/embedder.py b/backend/danswer/indexing/embedder.py
index 259bebd3fd9..3e9383cb828 100644
--- a/backend/danswer/indexing/embedder.py
+++ b/backend/danswer/indexing/embedder.py
@@ -32,6 +32,8 @@ def __init__(
         provider_type: EmbeddingProvider | None,
         api_key: str | None,
         api_url: str | None,
+        api_version: str | None,
+        deployment_name: str | None,
         heartbeat: Heartbeat | None,
     ):
         self.model_name = model_name
@@ -41,6 +43,8 @@ def __init__(
         self.provider_type = provider_type
         self.api_key = api_key
         self.api_url = api_url
+        self.api_version = api_version
+        self.deployment_name = deployment_name
 
         self.embedding_model = EmbeddingModel(
             model_name=model_name,
@@ -50,6 +54,8 @@ def __init__(
             api_key=api_key,
             provider_type=provider_type,
             api_url=api_url,
+            api_version=api_version,
+            deployment_name=deployment_name,
             # The below are globally set, this flow always uses the indexing one
             server_host=INDEXING_MODEL_SERVER_HOST,
             server_port=INDEXING_MODEL_SERVER_PORT,
@@ -75,6 +81,8 @@ def __init__(
         provider_type: EmbeddingProvider | None = None,
         api_key: str | None = None,
         api_url: str | None = None,
+        api_version: str | None = None,
+        deployment_name: str | None = None,
         heartbeat: Heartbeat | None = None,
     ):
         super().__init__(
@@ -85,6 +93,8 @@ def __init__(
             provider_type,
             api_key,
             api_url,
+            api_version,
+            deployment_name,
             heartbeat,
         )
 
@@ -193,5 +203,7 @@ def from_db_search_settings(
             provider_type=search_settings.provider_type,
             api_key=search_settings.api_key,
             api_url=search_settings.api_url,
+            api_version=search_settings.api_version,
+            deployment_name=search_settings.deployment_name,
             heartbeat=heartbeat,
         )
diff --git a/backend/danswer/natural_language_processing/search_nlp_models.py b/backend/danswer/natural_language_processing/search_nlp_models.py
index 2fbf94a5be2..700c8c08cf0 100644
--- a/backend/danswer/natural_language_processing/search_nlp_models.py
+++ b/backend/danswer/natural_language_processing/search_nlp_models.py
@@ -97,6 +97,8 @@ def __init__(
         provider_type: EmbeddingProvider | None,
         retrim_content: bool = False,
         heartbeat: Heartbeat | None = None,
+        api_version: str | None = None,
+        deployment_name: str | None = None,
     ) -> None:
         self.api_key = api_key
         self.provider_type = provider_type
@@ -106,6 +108,8 @@ def __init__(
         self.model_name = model_name
         self.retrim_content = retrim_content
         self.api_url = api_url
+        self.api_version = api_version
+        self.deployment_name = deployment_name
         self.tokenizer = get_tokenizer(
             model_name=model_name, provider_type=provider_type
         )
@@ -157,6 +161,8 @@ def _batch_encode_texts(
             embed_request = EmbedRequest(
                 model_name=self.model_name,
                 texts=text_batch,
+                api_version=self.api_version,
+                deployment_name=self.deployment_name,
                 max_context_length=max_seq_length,
                 normalize_embeddings=self.normalize,
                 api_key=self.api_key,
@@ -239,6 +245,8 @@ def from_db_model(
             provider_type=search_settings.provider_type,
             api_url=search_settings.api_url,
             retrim_content=retrim_content,
+            api_version=search_settings.api_version,
+            deployment_name=search_settings.deployment_name,
         )
 
 
diff --git a/backend/danswer/server/manage/embedding/api.py b/backend/danswer/server/manage/embedding/api.py
index eac872810ef..5d6e55e7a6d 100644
--- a/backend/danswer/server/manage/embedding/api.py
+++ b/backend/danswer/server/manage/embedding/api.py
@@ -43,6 +43,8 @@ def test_embedding_configuration(
             api_url=test_llm_request.api_url,
             provider_type=test_llm_request.provider_type,
             model_name=test_llm_request.model_name,
+            api_version=test_llm_request.api_version,
+            deployment_name=test_llm_request.deployment_name,
             normalize=False,
             query_prefix=None,
             passage_prefix=None,
diff --git a/backend/danswer/server/manage/embedding/models.py b/backend/danswer/server/manage/embedding/models.py
index d6210118df5..a7e7cc8e1ac 100644
--- a/backend/danswer/server/manage/embedding/models.py
+++ b/backend/danswer/server/manage/embedding/models.py
@@ -17,6 +17,8 @@ class TestEmbeddingRequest(BaseModel):
     api_key: str | None = None
     api_url: str | None = None
     model_name: str | None = None
+    api_version: str | None = None
+    deployment_name: str | None = None
 
     # This disables the "model_" protected namespace for pydantic
     model_config = {"protected_namespaces": ()}
@@ -26,6 +28,8 @@ class CloudEmbeddingProvider(BaseModel):
     provider_type: EmbeddingProvider
     api_key: str | None = None
     api_url: str | None = None
+    api_version: str | None = None
+    deployment_name: str | None = None
 
     @classmethod
     def from_request(
@@ -35,6 +39,8 @@ def from_request(
             provider_type=cloud_provider_model.provider_type,
             api_key=cloud_provider_model.api_key,
             api_url=cloud_provider_model.api_url,
+            api_version=cloud_provider_model.api_version,
+            deployment_name=cloud_provider_model.deployment_name,
         )
 
 
@@ -42,3 +48,5 @@ class CloudEmbeddingProviderCreationRequest(BaseModel):
     provider_type: EmbeddingProvider
     api_key: str | None = None
     api_url: str | None = None
+    api_version: str | None = None
+    deployment_name: str | None = None
diff --git a/backend/model_server/encoders.py b/backend/model_server/encoders.py
index e2e167520ba..f252b29d172 100644
--- a/backend/model_server/encoders.py
+++ b/backend/model_server/encoders.py
@@ -10,6 +10,7 @@
 from fastapi import APIRouter
 from fastapi import HTTPException
 from google.oauth2 import service_account  # type: ignore
+from litellm import embedding
 from retry import retry
 from sentence_transformers import CrossEncoder  # type: ignore
 from sentence_transformers import SentenceTransformer  # type: ignore
@@ -54,7 +55,11 @@
 
 
 def _initialize_client(
-    api_key: str, provider: EmbeddingProvider, model: str | None = None
+    api_key: str,
+    provider: EmbeddingProvider,
+    model: str | None = None,
+    api_url: str | None = None,
+    api_version: str | None = None,
 ) -> Any:
     if provider == EmbeddingProvider.OPENAI:
         return openai.OpenAI(api_key=api_key, timeout=OPENAI_EMBEDDING_TIMEOUT)
@@ -69,6 +74,8 @@ def _initialize_client(
         project_id = json.loads(api_key)["project_id"]
         vertexai.init(project=project_id, credentials=credentials)
         return TextEmbeddingModel.from_pretrained(model or DEFAULT_VERTEX_MODEL)
+    elif provider == EmbeddingProvider.AZURE:
+        return {"api_key": api_key, "api_url": api_url, "api_version": api_version}
     else:
         raise ValueError(f"Unsupported provider: {provider}")
 
@@ -78,11 +85,15 @@ def __init__(
         self,
         api_key: str,
         provider: EmbeddingProvider,
+        api_url: str | None = None,
+        api_version: str | None = None,
         # Only for Google as is needed on client setup
         model: str | None = None,
     ) -> None:
         self.provider = provider
-        self.client = _initialize_client(api_key, self.provider, model)
+        self.client = _initialize_client(
+            api_key, self.provider, model, api_url, api_version
+        )
 
     def _embed_openai(self, texts: list[str], model: str | None) -> list[Embedding]:
         if not model:
@@ -144,6 +155,18 @@ def _embed_voyage(
         )
         return response.embeddings
 
+    def _embed_azure(self, texts: list[str], model: str | None) -> list[Embedding]:
+        response = embedding(
+            model=model,
+            input=texts,
+            api_key=self.client["api_key"],
+            api_base=self.client["api_url"],
+            api_version=self.client["api_version"],
+        )
+        embeddings = [embedding["embedding"] for embedding in response.data]
+
+        return embeddings
+
     def _embed_vertex(
         self, texts: list[str], model: str | None, embedding_type: str
     ) -> list[Embedding]:
@@ -169,10 +192,13 @@ def embed(
         texts: list[str],
         text_type: EmbedTextType,
         model_name: str | None = None,
+        deployment_name: str | None = None,
     ) -> list[Embedding]:
         try:
             if self.provider == EmbeddingProvider.OPENAI:
                 return self._embed_openai(texts, model_name)
+            elif self.provider == EmbeddingProvider.AZURE:
+                return self._embed_azure(texts, f"azure/{deployment_name}")
             embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
             if self.provider == EmbeddingProvider.COHERE:
                 return self._embed_cohere(texts, model_name, embedding_type)
@@ -190,10 +216,14 @@ def embed(
 
     @staticmethod
     def create(
-        api_key: str, provider: EmbeddingProvider, model: str | None = None
+        api_key: str,
+        provider: EmbeddingProvider,
+        model: str | None = None,
+        api_url: str | None = None,
+        api_version: str | None = None,
     ) -> "CloudEmbedding":
         logger.debug(f"Creating Embedding instance for provider: {provider}")
-        return CloudEmbedding(api_key, provider, model)
+        return CloudEmbedding(api_key, provider, model, api_url, api_version)
 
 
 def get_embedding_model(
@@ -260,12 +290,14 @@ def embed_text(
     texts: list[str],
     text_type: EmbedTextType,
     model_name: str | None,
+    deployment_name: str | None,
     max_context_length: int,
     normalize_embeddings: bool,
     api_key: str | None,
     provider_type: EmbeddingProvider | None,
     prefix: str | None,
     api_url: str | None,
+    api_version: str | None,
 ) -> list[Embedding]:
     logger.info(f"Embedding {len(texts)} texts with provider: {provider_type}")
 
@@ -307,11 +339,16 @@ def embed_text(
             )
 
         cloud_model = CloudEmbedding(
-            api_key=api_key, provider=provider_type, model=model_name
+            api_key=api_key,
+            provider=provider_type,
+            model=model_name,
+            api_url=api_url,
+            api_version=api_version,
         )
         embeddings = cloud_model.embed(
             texts=texts,
             model_name=model_name,
+            deployment_name=deployment_name,
             text_type=text_type,
         )
 
@@ -405,12 +442,14 @@ async def process_embed_request(
         embeddings = embed_text(
             texts=embed_request.texts,
             model_name=embed_request.model_name,
+            deployment_name=embed_request.deployment_name,
             max_context_length=embed_request.max_context_length,
             normalize_embeddings=embed_request.normalize_embeddings,
             api_key=embed_request.api_key,
             provider_type=embed_request.provider_type,
             text_type=embed_request.text_type,
             api_url=embed_request.api_url,
+            api_version=embed_request.api_version,
             prefix=prefix,
         )
         return EmbedResponse(embeddings=embeddings)
diff --git a/backend/requirements/model_server.txt b/backend/requirements/model_server.txt
index 1e7baa415ee..694a41a1ce6 100644
--- a/backend/requirements/model_server.txt
+++ b/backend/requirements/model_server.txt
@@ -12,3 +12,4 @@ torch==2.2.0
 transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
+litellm==1.48.7
diff --git a/backend/shared_configs/enums.py b/backend/shared_configs/enums.py
index b58ac0a8928..3fe1cd0bd01 100644
--- a/backend/shared_configs/enums.py
+++ b/backend/shared_configs/enums.py
@@ -7,6 +7,7 @@ class EmbeddingProvider(str, Enum):
     VOYAGE = "voyage"
     GOOGLE = "google"
     LITELLM = "litellm"
+    AZURE = "azure"
 
 
 class RerankerProvider(str, Enum):
diff --git a/backend/shared_configs/model_server_models.py b/backend/shared_configs/model_server_models.py
index 737867b7fc8..9f7e853d26a 100644
--- a/backend/shared_configs/model_server_models.py
+++ b/backend/shared_configs/model_server_models.py
@@ -20,6 +20,7 @@ class EmbedRequest(BaseModel):
     texts: list[str]
     # Can be none for cloud embedding model requests, error handling logic exists for other cases
     model_name: str | None = None
+    deployment_name: str | None = None
     max_context_length: int
     normalize_embeddings: bool
     api_key: str | None = None
@@ -28,7 +29,7 @@ class EmbedRequest(BaseModel):
     manual_query_prefix: str | None = None
     manual_passage_prefix: str | None = None
     api_url: str | None = None
-
+    api_version: str | None = None
     # This disables the "model_" protected namespace for pydantic
     model_config = {"protected_namespaces": ()}
 
diff --git a/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx b/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
index 54d180f6498..f9d0368e128 100644
--- a/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
+++ b/web/src/app/admin/embeddings/EmbeddingModelSelectionForm.tsx
@@ -27,10 +27,13 @@ import {
   EMBEDDING_MODELS_ADMIN_URL,
   EMBEDDING_PROVIDERS_ADMIN_URL,
 } from "../configuration/llm/constants";
+import { AdvancedSearchConfiguration } from "./interfaces";
 
 export interface EmbeddingDetails {
   api_key?: string;
   api_url?: string;
+  api_version?: string;
+  deployment_name?: string;
   custom_config: any;
   provider_type: EmbeddingProvider;
 }
@@ -41,6 +44,8 @@ export function EmbeddingModelSelection({
   updateSelectedProvider,
   modelTab,
   setModelTab,
+  updateCurrentModel,
+  advancedEmbeddingDetails,
 }: {
   modelTab: "open" | "cloud" | null;
   setModelTab: Dispatch<SetStateAction<"open" | "cloud" | null>>;
@@ -49,6 +54,11 @@ export function EmbeddingModelSelection({
   updateSelectedProvider: (
     model: CloudEmbeddingModel | HostedEmbeddingModel
   ) => void;
+  updateCurrentModel: (
+    newModel: string,
+    provider_type: EmbeddingProvider
+  ) => void;
+  advancedEmbeddingDetails: AdvancedSearchConfiguration;
 }) {
   // Cloud Provider based modals
   const [showTentativeProvider, setShowTentativeProvider] =
@@ -72,12 +82,6 @@ export function EmbeddingModelSelection({
   const [showTentativeOpenProvider, setShowTentativeOpenProvider] =
     useState<HostedEmbeddingModel | null>(null);
 
-  // Enabled / unenabled providers
-  const [newEnabledProviders, setNewEnabledProviders] = useState<string[]>([]);
-  const [newUnenabledProviders, setNewUnenabledProviders] = useState<string[]>(
-    []
-  );
-
   const [showDeleteCredentialsModal, setShowDeleteCredentialsModal] =
     useState<boolean>(false);
 
@@ -90,7 +94,10 @@ export function EmbeddingModelSelection({
     { refreshInterval: 5000 } // 5 seconds
   );
 
-  const { data: embeddingProviderDetails } = useSWR<EmbeddingDetails[]>(
+  const {
+    data: embeddingProviderDetails,
+    mutate: mutateEmbeddingProviderDetails,
+  } = useSWR<EmbeddingDetails[]>(
     EMBEDDING_PROVIDERS_ADMIN_URL,
     errorHandlingFetcher,
     { refreshInterval: 5000 } // 5 seconds
@@ -132,32 +139,6 @@ export function EmbeddingModelSelection({
     }
   };
 
-  const clientsideAddProvider = (provider: CloudEmbeddingProvider) => {
-    const providerType = provider.provider_type;
-    setNewEnabledProviders((newEnabledProviders) => [
-      ...newEnabledProviders,
-      providerType,
-    ]);
-    setNewUnenabledProviders((newUnenabledProviders) =>
-      newUnenabledProviders.filter(
-        (givenProviderType) => givenProviderType != providerType
-      )
-    );
-  };
-
-  const clientsideRemoveProvider = (provider: CloudEmbeddingProvider) => {
-    const providerType = provider.provider_type;
-    setNewEnabledProviders((newEnabledProviders) =>
-      newEnabledProviders.filter(
-        (givenProviderType) => givenProviderType != providerType
-      )
-    );
-    setNewUnenabledProviders((newUnenabledProviders) => [
-      ...newUnenabledProviders,
-      providerType,
-    ]);
-  };
-
   return (
     <div className="p-2">
       {alreadySelectedModel && (
@@ -186,14 +167,16 @@ export function EmbeddingModelSelection({
 
       {showTentativeProvider && (
         <ProviderCreationModal
+          updateCurrentModel={updateCurrentModel}
           isProxy={showTentativeProvider.provider_type == "LiteLLM"}
+          isAzure={showTentativeProvider.provider_type == "Azure"}
           selectedProvider={showTentativeProvider}
           onConfirm={() => {
             setShowTentativeProvider(showUnconfiguredProvider);
-            clientsideAddProvider(showTentativeProvider);
             if (showModelInQueue) {
               setShowTentativeModel(showModelInQueue);
             }
+            mutateEmbeddingProviderDetails();
           }}
           onCancel={() => {
             setShowModelInQueue(null);
@@ -205,10 +188,11 @@ export function EmbeddingModelSelection({
       {changeCredentialsProvider && (
         <ChangeCredentialsModal
           isProxy={changeCredentialsProvider.provider_type == "LiteLLM"}
+          isAzure={changeCredentialsProvider.provider_type == "Azure"}
           useFileUpload={changeCredentialsProvider.provider_type == "Google"}
           onDeleted={() => {
-            clientsideRemoveProvider(changeCredentialsProvider);
             setChangeCredentialsProvider(null);
+            mutateEmbeddingProviderDetails();
           }}
           provider={changeCredentialsProvider}
           onConfirm={() => setChangeCredentialsProvider(null)}
@@ -236,12 +220,13 @@ export function EmbeddingModelSelection({
           modelProvider={showTentativeProvider!}
           onConfirm={() => {
             setShowDeleteCredentialsModal(false);
+            mutateEmbeddingProviderDetails();
           }}
           onCancel={() => setShowDeleteCredentialsModal(false)}
         />
       )}
 
-      <p className="t mb-4">
+      <p className="mb-4">
         Select from cloud, self-hosted models, or continue with your current
         embedding model.
       </p>
@@ -291,14 +276,13 @@ export function EmbeddingModelSelection({
 
       {modelTab == "cloud" && (
         <CloudEmbeddingPage
+          advancedEmbeddingDetails={advancedEmbeddingDetails}
           embeddingModelDetails={embeddingModelDetails}
           setShowModelInQueue={setShowModelInQueue}
           setShowTentativeModel={setShowTentativeModel}
           currentModel={selectedProvider || currentEmbeddingModel}
           setAlreadySelectedModel={setAlreadySelectedModel}
           embeddingProviderDetails={embeddingProviderDetails}
-          newEnabledProviders={newEnabledProviders}
-          newUnenabledProviders={newUnenabledProviders}
           setShowTentativeProvider={setShowTentativeProvider}
           setChangeCredentialsProvider={setChangeCredentialsProvider}
         />
diff --git a/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx b/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
index 912bd569a98..f769099e77d 100644
--- a/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
@@ -16,6 +16,7 @@ export function ChangeCredentialsModal({
   onDeleted,
   useFileUpload,
   isProxy = false,
+  isAzure = false,
 }: {
   provider: CloudEmbeddingProvider;
   onConfirm: () => void;
@@ -23,6 +24,7 @@ export function ChangeCredentialsModal({
   onDeleted: () => void;
   useFileUpload: boolean;
   isProxy?: boolean;
+  isAzure?: boolean;
 }) {
   const [apiKey, setApiKey] = useState("");
   const [apiUrl, setApiUrl] = useState("");
@@ -151,7 +153,6 @@ export function ChangeCredentialsModal({
       );
     }
   };
-
   return (
     <Modal
       width="max-w-3xl"
@@ -160,133 +161,131 @@ export function ChangeCredentialsModal({
       onOutsideClick={onCancel}
     >
       <>
-        <p className="mb-4">
-          You can modify your configuration by providing a new API key
-          {isProxy ? " or API URL." : "."}
-        </p>
+        {!isAzure && (
+          <>
+            <p className="mb-4">
+              You can modify your configuration by providing a new API key
+              {isProxy ? " or API URL." : "."}
+            </p>
 
-        <div className="mb-4 flex flex-col gap-y-2">
-          <Label className="mt-2">API Key</Label>
-          {useFileUpload ? (
-            <>
-              <Label className="mt-2">Upload JSON File</Label>
-              <input
-                ref={fileInputRef}
-                type="file"
-                accept=".json"
-                onChange={handleFileUpload}
-                className="text-lg w-full p-1"
-              />
-              {fileName && <p>Uploaded file: {fileName}</p>}
-            </>
-          ) : (
-            <>
-              <input
-                className={`
-                    border 
-                    border-border 
-                    rounded 
-                    w-full 
-                    py-2 
-                    px-3 
-                    bg-background-emphasis
-                `}
-                value={apiKey}
-                onChange={(e: any) => setApiKey(e.target.value)}
-                placeholder="Paste your API key here"
-              />
-            </>
-          )}
+            <div className="mb-4 flex flex-col gap-y-2">
+              <Label className="mt-2">API Key</Label>
+              {useFileUpload ? (
+                <>
+                  <Label className="mt-2">Upload JSON File</Label>
+                  <input
+                    ref={fileInputRef}
+                    type="file"
+                    accept=".json"
+                    onChange={handleFileUpload}
+                    className="text-lg w-full p-1"
+                  />
+                  {fileName && <p>Uploaded file: {fileName}</p>}
+                </>
+              ) : (
+                <>
+                  <input
+                    className={`
+                        border 
+                        border-border 
+                        rounded 
+                        w-full 
+                        py-2 
+                        px-3 
+                        bg-background-emphasis
+                    `}
+                    value={apiKey}
+                    onChange={(e: any) => setApiKey(e.target.value)}
+                    placeholder="Paste your API key here"
+                  />
+                </>
+              )}
 
-          {isProxy && (
-            <>
-              <Label className="mt-2">API URL</Label>
+              {isProxy && (
+                <>
+                  <Label className="mt-2">API URL</Label>
 
-              <input
-                className={`
-                    border 
-                    border-border 
-                    rounded 
-                    w-full 
-                    py-2 
-                    px-3 
-                    bg-background-emphasis
-                `}
-                value={apiUrl}
-                onChange={(e: any) => setApiUrl(e.target.value)}
-                placeholder="Paste your API URL here"
-              />
+                  <input
+                    className={`
+                        border 
+                        border-border 
+                        rounded 
+                        w-full 
+                        py-2 
+                        px-3 
+                        bg-background-emphasis
+                    `}
+                    value={apiUrl}
+                    onChange={(e: any) => setApiUrl(e.target.value)}
+                    placeholder="Paste your API URL here"
+                  />
 
-              {deletionError && (
-                <Callout title="Error" color="red" className="mt-4">
-                  {deletionError}
-                </Callout>
-              )}
+                  {deletionError && (
+                    <Callout title="Error" color="red" className="mt-4">
+                      {deletionError}
+                    </Callout>
+                  )}
 
-              <div>
-                <Label className="mt-2">Test Model</Label>
-                <p>
-                  Since you are using a liteLLM proxy, we&apos;ll need a model
-                  name to test the connection with.
-                </p>
-              </div>
-              <input
-                className={`
-                 border 
-                 border-border 
-                 rounded 
-                 w-full 
-                 py-2 
-                 px-3 
-                 bg-background-emphasis
-             `}
-                value={modelName}
-                onChange={(e: any) => setModelName(e.target.value)}
-                placeholder="Paste your API URL here"
-              />
+                  <div>
+                    <Label className="mt-2">Test Model</Label>
+                    <p>
+                      Since you are using a liteLLM proxy, we&apos;ll need a
+                      model name to test the connection with.
+                    </p>
+                  </div>
+                  <input
+                    className={`
+                     border 
+                     border-border 
+                     rounded 
+                     w-full 
+                     py-2 
+                     px-3 
+                     bg-background-emphasis
+                 `}
+                    value={modelName}
+                    onChange={(e: any) => setModelName(e.target.value)}
+                    placeholder="Paste your model name here"
+                  />
+                </>
+              )}
 
-              {deletionError && (
-                <Callout title="Error" color="red" className="mt-4">
-                  {deletionError}
+              {testError && (
+                <Callout title="Error" color="red" className="my-4">
+                  {testError}
                 </Callout>
               )}
-            </>
-          )}
-
-          {testError && (
-            <Callout title="Error" color="red" className="my-4">
-              {testError}
-            </Callout>
-          )}
 
-          <Button
-            className="mr-auto mt-4"
-            color="blue"
-            onClick={() => handleSubmit()}
-            disabled={!apiKey}
-          >
-            Update Configuration
-          </Button>
+              <Button
+                className="mr-auto mt-4"
+                color="blue"
+                onClick={() => handleSubmit()}
+                disabled={!apiKey}
+              >
+                Update Configuration
+              </Button>
 
-          <Divider />
+              <Divider />
+            </div>
+          </>
+        )}
 
-          <Subtitle className="mt-4 font-bold text-lg mb-2">
-            You can also delete your configuration.
-          </Subtitle>
-          <Text className="mb-2">
-            This is only possible if you have already switched to a different
-            embedding type!
-          </Text>
+        <Subtitle className="mt-4 font-bold text-lg mb-2">
+          You can delete your configuration.
+        </Subtitle>
+        <Text className="mb-2">
+          This is only possible if you have already switched to a different
+          embedding type!
+        </Text>
 
-          <Button className="mr-auto" onClick={handleDelete} color="red">
-            Delete Configuration
-          </Button>
-          {deletionError && (
-            <Callout title="Error" color="red" className="mt-4">
-              {deletionError}
-            </Callout>
-          )}
-        </div>
+        <Button className="mr-auto" onClick={handleDelete} color="red">
+          Delete Configuration
+        </Button>
+        {deletionError && (
+          <Callout title="Error" color="red" className="mt-4">
+            {deletionError}
+          </Callout>
+        )}
       </>
     </Modal>
   );
diff --git a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
index 7f127d8ef47..cf9d5844397 100644
--- a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
@@ -4,7 +4,10 @@ import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { Label, TextFormField } from "@/components/admin/connectors/Field";
 import { LoadingAnimation } from "@/components/Loading";
-import { CloudEmbeddingProvider } from "../../../../components/embedding/interfaces";
+import {
+  CloudEmbeddingProvider,
+  EmbeddingProvider,
+} from "../../../../components/embedding/interfaces";
 import { EMBEDDING_PROVIDERS_ADMIN_URL } from "../../configuration/llm/constants";
 import { Modal } from "@/components/Modal";
 
@@ -14,12 +17,19 @@ export function ProviderCreationModal({
   onCancel,
   existingProvider,
   isProxy,
+  isAzure,
+  updateCurrentModel,
 }: {
+  updateCurrentModel: (
+    newModel: string,
+    provider_type: EmbeddingProvider
+  ) => void;
   selectedProvider: CloudEmbeddingProvider;
   onConfirm: () => void;
   onCancel: () => void;
   existingProvider?: CloudEmbeddingProvider;
   isProxy?: boolean;
+  isAzure?: boolean;
 }) {
   const useFileUpload = selectedProvider.provider_type == "Google";
 
@@ -41,16 +51,24 @@ export function ProviderCreationModal({
 
   const validationSchema = Yup.object({
     provider_type: Yup.string().required("Provider type is required"),
-    api_key: isProxy
-      ? Yup.string()
-      : useFileUpload
+    api_key:
+      isProxy || isAzure
         ? Yup.string()
-        : Yup.string().required("API Key is required"),
+        : useFileUpload
+          ? Yup.string()
+          : Yup.string().required("API Key is required"),
     model_name: isProxy
       ? Yup.string().required("Model name is required")
       : Yup.string().nullable(),
-    api_url: isProxy
-      ? Yup.string().required("API URL is required")
+    api_url:
+      isProxy || isAzure
+        ? Yup.string().required("API URL is required")
+        : Yup.string(),
+    deployment_name: isAzure
+      ? Yup.string().required("Deployment name is required")
+      : Yup.string(),
+    api_version: isAzure
+      ? Yup.string().required("API Version is required")
       : Yup.string(),
     custom_config: Yup.array().of(Yup.array().of(Yup.string()).length(2)),
   });
@@ -101,6 +119,8 @@ export function ProviderCreationModal({
             api_key: values.api_key,
             api_url: values.api_url,
             model_name: values.model_name,
+            api_version: values.api_version,
+            deployment_name: values.deployment_name,
           }),
         }
       );
@@ -118,6 +138,8 @@ export function ProviderCreationModal({
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({
           ...values,
+          api_version: values.api_version,
+          deployment_name: values.deployment_name,
           provider_type: values.provider_type.toLowerCase().split(" ")[0],
           custom_config: customConfig,
           is_default_provider: false,
@@ -125,6 +147,10 @@ export function ProviderCreationModal({
         }),
       });
 
+      if (isAzure) {
+        updateCurrentModel(values.model_name, EmbeddingProvider.AZURE);
+      }
+
       if (!response.ok) {
         const errorData = await response.json();
         throw new Error(
@@ -178,26 +204,45 @@ export function ProviderCreationModal({
                   href={selectedProvider.apiLink}
                   rel="noreferrer"
                 >
-                  {isProxy ? "API URL" : "API KEY"}
+                  {isProxy || isAzure ? "API URL" : "API KEY"}
                 </a>
               </Text>
 
               <div className="flex w-full flex-col gap-y-6">
+                {(isProxy || isAzure) && (
+                  <TextFormField
+                    name="api_url"
+                    label="API URL"
+                    placeholder="API URL"
+                    type="text"
+                  />
+                )}
+
                 {isProxy && (
-                  <>
-                    <TextFormField
-                      name="api_url"
-                      label="API URL"
-                      placeholder="API URL"
-                      type="text"
-                    />
-                    <TextFormField
-                      name="model_name"
-                      label="Model Name (for testing)"
-                      placeholder="Model Name"
-                      type="text"
-                    />
-                  </>
+                  <TextFormField
+                    name="model_name"
+                    label={`Model Name ${isProxy ? "(for testing)" : ""}`}
+                    placeholder="Model Name"
+                    type="text"
+                  />
+                )}
+
+                {isAzure && (
+                  <TextFormField
+                    name="deployment_name"
+                    label="Deployment Name"
+                    placeholder="Deployment Name"
+                    type="text"
+                  />
+                )}
+
+                {isAzure && (
+                  <TextFormField
+                    name="api_version"
+                    label="API Version"
+                    placeholder="API Version"
+                    type="text"
+                  />
                 )}
 
                 {useFileUpload ? (
diff --git a/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx b/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
index dd202dc4f4b..6818101de43 100644
--- a/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
+++ b/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
@@ -10,42 +10,42 @@ import {
   EmbeddingModelDescriptor,
   EmbeddingProvider,
   LITELLM_CLOUD_PROVIDER,
+  AZURE_CLOUD_PROVIDER,
 } from "../../../../components/embedding/interfaces";
 import { EmbeddingDetails } from "../EmbeddingModelSelectionForm";
 import { FiExternalLink, FiInfo, FiTrash } from "react-icons/fi";
 import { HoverPopup } from "@/components/HoverPopup";
 import { Dispatch, SetStateAction, useEffect, useState } from "react";
-import { LiteLLMModelForm } from "@/components/embedding/LiteLLMModelForm";
+import { CustomEmbeddingModelForm } from "@/components/embedding/CustomEmbeddingModelForm";
 import { deleteSearchSettings } from "./utils";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
+import { AdvancedSearchConfiguration } from "../interfaces";
 
 export default function CloudEmbeddingPage({
   currentModel,
   embeddingProviderDetails,
   embeddingModelDetails,
-  newEnabledProviders,
-  newUnenabledProviders,
   setShowTentativeProvider,
   setChangeCredentialsProvider,
   setAlreadySelectedModel,
   setShowTentativeModel,
   setShowModelInQueue,
+  advancedEmbeddingDetails,
 }: {
   setShowModelInQueue: Dispatch<SetStateAction<CloudEmbeddingModel | null>>;
   setShowTentativeModel: Dispatch<SetStateAction<CloudEmbeddingModel | null>>;
   currentModel: EmbeddingModelDescriptor | CloudEmbeddingModel;
   setAlreadySelectedModel: Dispatch<SetStateAction<CloudEmbeddingModel | null>>;
-  newUnenabledProviders: string[];
   embeddingModelDetails?: CloudEmbeddingModel[];
   embeddingProviderDetails?: EmbeddingDetails[];
-  newEnabledProviders: string[];
   setShowTentativeProvider: React.Dispatch<
     React.SetStateAction<CloudEmbeddingProvider | null>
   >;
   setChangeCredentialsProvider: React.Dispatch<
     React.SetStateAction<CloudEmbeddingProvider | null>
   >;
+  advancedEmbeddingDetails: AdvancedSearchConfiguration;
 }) {
   function hasProviderTypeinArray(
     arr: Array<{ provider_type: string }>,
@@ -60,27 +60,38 @@ export default function CloudEmbeddingPage({
     (model) => ({
       ...model,
       configured:
-        !newUnenabledProviders.includes(model.provider_type) &&
-        (newEnabledProviders.includes(model.provider_type) ||
-          (embeddingProviderDetails &&
-            hasProviderTypeinArray(
-              embeddingProviderDetails,
-              model.provider_type
-            ))!),
+        embeddingProviderDetails &&
+        hasProviderTypeinArray(embeddingProviderDetails, model.provider_type),
     })
   );
   const [liteLLMProvider, setLiteLLMProvider] = useState<
     EmbeddingDetails | undefined
   >(undefined);
 
+  const [azureProvider, setAzureProvider] = useState<
+    EmbeddingDetails | undefined
+  >(undefined);
+
   useEffect(() => {
-    const foundProvider = embeddingProviderDetails?.find(
+    const liteLLMProvider = embeddingProviderDetails?.find(
       (provider) =>
         provider.provider_type === EmbeddingProvider.LITELLM.toLowerCase()
     );
-    setLiteLLMProvider(foundProvider);
+    setLiteLLMProvider(liteLLMProvider);
+    const azureProvider = embeddingProviderDetails?.find(
+      (provider) =>
+        provider.provider_type === EmbeddingProvider.AZURE.toLowerCase()
+    );
+    setAzureProvider(azureProvider);
   }, [embeddingProviderDetails]);
 
+  const isAzureConfigured = azureProvider !== undefined;
+
+  // Get details of the configured Azure provider
+  const azureProviderDetails = embeddingProviderDetails?.find(
+    (provider) => provider.provider_type.toLowerCase() === "azure"
+  );
+
   return (
     <div>
       <Title className="mt-8">
@@ -248,7 +259,8 @@ export default function CloudEmbeddingPage({
                       : ""
                   }`}
                 >
-                  <LiteLLMModelForm
+                  <CustomEmbeddingModelForm
+                    embeddingType={EmbeddingProvider.LITELLM}
                     provider={liteLLMProvider}
                     currentValues={
                       currentModel.provider_type === EmbeddingProvider.LITELLM
@@ -262,6 +274,126 @@ export default function CloudEmbeddingPage({
             )}
           </div>
         </div>
+
+        <Text className="mt-6">
+          You can also use Azure OpenAI models for embeddings. Azure requires
+          separate configuration for each model.
+        </Text>
+
+        <div key={AZURE_CLOUD_PROVIDER.provider_type} className="mt-4 w-full">
+          <div className="flex items-center mb-2">
+            {AZURE_CLOUD_PROVIDER.icon({ size: 40 })}
+            <h2 className="ml-2  mt-2 text-xl font-bold">
+              {AZURE_CLOUD_PROVIDER.provider_type}{" "}
+            </h2>
+            <HoverPopup
+              mainContent={
+                <FiInfo className="ml-2 mt-2 cursor-pointer" size={18} />
+              }
+              popupContent={
+                <div className="text-sm text-text-800 w-52">
+                  <div className="my-auto">
+                    {AZURE_CLOUD_PROVIDER.description}
+                  </div>
+                </div>
+              }
+              style="dark"
+            />
+          </div>
+        </div>
+
+        <div className="w-full flex flex-col items-start">
+          {!isAzureConfigured ? (
+            <>
+              <button
+                onClick={() => setShowTentativeProvider(AZURE_CLOUD_PROVIDER)}
+                className="mb-2 px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600 text-sm cursor-pointer"
+              >
+                Configure Azure OpenAI
+              </button>
+              <div className="mt-2 w-full max-w-4xl">
+                <Card className="p-4 border border-gray-200 rounded-lg shadow-sm">
+                  <Text className="text-base font-medium mb-2">
+                    Configure Azure OpenAI for Embeddings
+                  </Text>
+                  <Text className="text-sm text-gray-600 mb-3">
+                    Click &quot;Configure Azure OpenAI&quot; to set up Azure
+                    OpenAI for embeddings.
+                  </Text>
+                  <div className="flex items-center text-sm text-gray-700">
+                    <FiInfo className="text-gray-400 mr-2" size={16} />
+                    <Text>
+                      You&apos;ll need: API version, base URL, API key, model
+                      name, and deployment name.
+                    </Text>
+                  </div>
+                </Card>
+              </div>
+            </>
+          ) : (
+            <>
+              <div className="mb-6 w-full">
+                <Text className="text-lg font-semibold mb-3">
+                  Current Azure Configuration
+                </Text>
+
+                {azureProviderDetails ? (
+                  <Card className="bg-white shadow-sm border border-gray-200 rounded-lg">
+                    <div className="p-4 space-y-3">
+                      <div className="flex justify-between">
+                        <span className="font-medium">API Version:</span>
+                        <span>{azureProviderDetails.api_version}</span>
+                      </div>
+                      <div className="flex justify-between">
+                        <span className="font-medium">Base URL:</span>
+                        <span>{azureProviderDetails.api_url}</span>
+                      </div>
+                      <div className="flex justify-between">
+                        <span className="font-medium">Deployment Name:</span>
+                        <span>{azureProviderDetails.deployment_name}</span>
+                      </div>
+                    </div>
+                    <button
+                      onClick={() =>
+                        setChangeCredentialsProvider(AZURE_CLOUD_PROVIDER)
+                      }
+                      className="mt-2 px-4 py-2 bg-red-500 text-white rounded hover:bg-red-600 text-sm"
+                    >
+                      Delete Current Azure Provider
+                    </button>
+                  </Card>
+                ) : (
+                  <Card className="bg-gray-50 border border-gray-200 rounded-lg">
+                    <div className="p-4 text-gray-500 text-center">
+                      No Azure provider has been configured yet.
+                    </div>
+                  </Card>
+                )}
+              </div>
+
+              <Card
+                className={`mt-2 w-full max-w-4xl ${
+                  currentModel.provider_type === EmbeddingProvider.AZURE
+                    ? "border-2 border-blue-500"
+                    : ""
+                }`}
+              >
+                {azureProvider && (
+                  <CustomEmbeddingModelForm
+                    embeddingType={EmbeddingProvider.AZURE}
+                    provider={azureProvider}
+                    currentValues={
+                      currentModel.provider_type === EmbeddingProvider.AZURE
+                        ? (currentModel as CloudEmbeddingModel)
+                        : null
+                    }
+                    setShowTentativeModel={setShowTentativeModel}
+                  />
+                )}
+              </Card>
+            </>
+          )}
+        </div>
       </div>
     </div>
   );
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index 6060868e85c..1b4852ed4d8 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -152,11 +152,6 @@ export default function EmbeddingForm() {
     }
   }, [currentEmbeddingModel]);
 
-  useEffect(() => {
-    if (currentEmbeddingModel) {
-      setSelectedProvider(currentEmbeddingModel);
-    }
-  }, [currentEmbeddingModel]);
   if (!selectedProvider) {
     return <ThreeDotsLoader />;
   }
@@ -164,10 +159,18 @@ export default function EmbeddingForm() {
     return <ErrorCallout errorTitle="Failed to fetch embedding model status" />;
   }
 
+  const updateCurrentModel = (newModel: string) => {
+    setAdvancedEmbeddingDetails((values) => ({
+      ...values,
+      model_name: newModel,
+    }));
+  };
+
   const updateSearch = async () => {
     const values: SavedSearchSettings = {
       ...rerankingDetails,
       ...advancedEmbeddingDetails,
+      ...selectedProvider,
       provider_type:
         selectedProvider.provider_type?.toLowerCase() as EmbeddingProvider | null,
     };
@@ -311,11 +314,13 @@ export default function EmbeddingForm() {
             </Text>
             <Card>
               <EmbeddingModelSelection
+                updateCurrentModel={updateCurrentModel}
                 setModelTab={setModelTab}
                 modelTab={modelTab}
                 selectedProvider={selectedProvider}
                 currentEmbeddingModel={currentEmbeddingModel}
                 updateSelectedProvider={updateSelectedProvider}
+                advancedEmbeddingDetails={advancedEmbeddingDetails}
               />
             </Card>
             <div className="mt-4 flex w-full justify-end">
diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx
index 2a4fbaa3de9..97fc6343285 100644
--- a/web/src/components/admin/Layout.tsx
+++ b/web/src/components/admin/Layout.tsx
@@ -27,7 +27,7 @@ export async function Layout({ children }: { children: React.ReactNode }) {
 
   const authTypeMetadata = results[0] as AuthTypeMetadata | null;
   const user = results[1] as User | null;
-
+  console.log("authTypeMetadata", authTypeMetadata);
   const authDisabled = authTypeMetadata?.authType === "disabled";
   const requiresVerification = authTypeMetadata?.requiresVerification;
 
diff --git a/web/src/components/embedding/LiteLLMModelForm.tsx b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
similarity index 81%
rename from web/src/components/embedding/LiteLLMModelForm.tsx
rename to web/src/components/embedding/CustomEmbeddingModelForm.tsx
index b84db4f9067..8dcd34f0ba6 100644
--- a/web/src/components/embedding/LiteLLMModelForm.tsx
+++ b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
@@ -1,4 +1,4 @@
-import { CloudEmbeddingModel, CloudEmbeddingProvider } from "./interfaces";
+import { CloudEmbeddingModel, EmbeddingProvider } from "./interfaces";
 import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { TextFormField, BooleanFormField } from "../admin/connectors/Field";
@@ -6,14 +6,16 @@ import { Dispatch, SetStateAction } from "react";
 import { Button, Text } from "@tremor/react";
 import { EmbeddingDetails } from "@/app/admin/embeddings/EmbeddingModelSelectionForm";
 
-export function LiteLLMModelForm({
+export function CustomEmbeddingModelForm({
   setShowTentativeModel,
   currentValues,
   provider,
+  embeddingType,
 }: {
   setShowTentativeModel: Dispatch<SetStateAction<CloudEmbeddingModel | null>>;
   currentValues: CloudEmbeddingModel | null;
   provider: EmbeddingDetails;
+  embeddingType: EmbeddingProvider;
 }) {
   return (
     <div>
@@ -25,7 +27,7 @@ export function LiteLLMModelForm({
             normalize: false,
             query_prefix: "",
             passage_prefix: "",
-            provider_type: "LiteLLM",
+            provider_type: embeddingType,
             api_key: "",
             enabled: true,
             api_url: provider.api_url,
@@ -55,18 +57,21 @@ export function LiteLLMModelForm({
           max_tokens: Yup.number(),
         })}
         onSubmit={async (values) => {
+          console.log(values);
           setShowTentativeModel(values as CloudEmbeddingModel);
         }}
       >
-        {({ isSubmitting }) => (
+        {({ isSubmitting, submitForm, errors }) => (
           <Form>
             <Text className="text-xl text-text-900 font-bold mb-4">
-              Add a new model to LiteLLM proxy at {provider.api_url}
+              Specify details for your{" "}
+              {embeddingType === EmbeddingProvider.AZURE ? "Azure" : "LiteLLM"}{" "}
+              Provider&apos;s model
             </Text>
             <TextFormField
               name="model_name"
               label="Model Name:"
-              subtext="The name of the LiteLLM model"
+              subtext={`The name of the ${embeddingType === EmbeddingProvider.AZURE ? "Azure" : "LiteLLM"} model`}
               placeholder="e.g. 'all-MiniLM-L6-v2'"
               autoCompleteDisabled={true}
             />
@@ -103,10 +108,13 @@ export function LiteLLMModelForm({
 
             <Button
               type="submit"
+              onClick={() => console.log(errors)}
               disabled={isSubmitting}
               className="w-64 mx-auto"
             >
-              Configure LiteLLM Model
+              Configure{" "}
+              {embeddingType === EmbeddingProvider.AZURE ? "Azure" : "LiteLLM"}{" "}
+              Model
             </Button>
           </Form>
         )}
diff --git a/web/src/components/embedding/EmbeddingSidebar.tsx b/web/src/components/embedding/EmbeddingSidebar.tsx
index 57659f2131f..27894c6001e 100644
--- a/web/src/components/embedding/EmbeddingSidebar.tsx
+++ b/web/src/components/embedding/EmbeddingSidebar.tsx
@@ -31,7 +31,7 @@ export default function EmbeddingSidebar() {
                   w-[250px]
                   `}
       >
-        <div className="fixed h-full left-0 top-0 w-[250px]">
+        <div className="fixed h-full left-0 top-0 bg-background-100 w-[250px]">
           <div className="ml-4 mr-3 flex flex gap-x-1 items-center mt-2 my-auto text-text-700 text-xl">
             <div className="mr-1 my-auto h-6 w-6">
               <Logo height={24} width={24} />
diff --git a/web/src/components/embedding/interfaces.tsx b/web/src/components/embedding/interfaces.tsx
index daa56128c3d..ff80ffd6ace 100644
--- a/web/src/components/embedding/interfaces.tsx
+++ b/web/src/components/embedding/interfaces.tsx
@@ -1,4 +1,5 @@
 import {
+  AzureIcon,
   CohereIcon,
   GoogleIcon,
   IconProps,
@@ -16,6 +17,7 @@ export enum EmbeddingProvider {
   VOYAGE = "Voyage",
   GOOGLE = "Google",
   LITELLM = "LiteLLM",
+  AZURE = "Azure",
 }
 
 export interface CloudEmbeddingProvider {
@@ -49,6 +51,8 @@ export interface EmbeddingModelDescriptor {
   description: string;
   api_key: string | null;
   api_url: string | null;
+  api_version?: string | null;
+  deployment_name?: string | null;
   index_name: string | null;
 }
 
@@ -161,6 +165,20 @@ export const LITELLM_CLOUD_PROVIDER: CloudEmbeddingProvider = {
   embedding_models: [], // No default embedding models
 };
 
+export const AZURE_CLOUD_PROVIDER: CloudEmbeddingProvider = {
+  provider_type: EmbeddingProvider.AZURE,
+  website:
+    "https://azure.microsoft.com/en-us/products/cognitive-services/openai/",
+  icon: AzureIcon,
+  description:
+    "Azure OpenAI is a cloud-based AI service that provides access to OpenAI models.",
+  apiLink:
+    "https://docs.microsoft.com/en-us/azure/ai-services/openai/how-to/create-resource",
+  costslink:
+    "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai/",
+  embedding_models: [], // No default embedding models
+};
+
 export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
   {
     provider_type: EmbeddingProvider.COHERE,

From f23a89ccfd9d024efee5f08ce40a2cec781df991 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Tue, 15 Oct 2024 21:52:00 -0700
Subject: [PATCH 123/376] Notion Empty Property Fix (#2817)

---
 .../danswer/connectors/notion/connector.py    | 59 +++++++++++++------
 1 file changed, 42 insertions(+), 17 deletions(-)

diff --git a/backend/danswer/connectors/notion/connector.py b/backend/danswer/connectors/notion/connector.py
index e856e9970bc..6e2da4a9c65 100644
--- a/backend/danswer/connectors/notion/connector.py
+++ b/backend/danswer/connectors/notion/connector.py
@@ -217,11 +217,18 @@ def _properties_to_str(properties: dict[str, Any]) -> str:
         """Converts Notion properties to a string"""
 
         def _recurse_properties(inner_dict: dict[str, Any]) -> str:
+            if not inner_dict:
+                # Edge case handling, should not happen
+                return "N/A"
+
             while "type" in inner_dict:
                 type_name = inner_dict["type"]
                 inner_dict = inner_dict[type_name]
                 if isinstance(inner_dict, list):
-                    return ", ".join([_recurse_properties(item) for item in inner_dict])
+                    return ", ".join(
+                        [_recurse_properties(item) for item in inner_dict if item]
+                    )
+
             # TODO there may be more types to handle here
             if "name" in inner_dict:
                 return inner_dict["name"]
@@ -245,6 +252,9 @@ def _recurse_properties(inner_dict: dict[str, Any]) -> str:
 
         result = ""
         for prop_name, prop in properties.items():
+            if not prop:
+                continue
+
             inner_value = _recurse_properties(prop)
             # Not a perfect way to format Notion database tables but there's no perfect representation
             # since this must be represented as plaintext
@@ -268,19 +278,20 @@ def _read_pages_from_database(
                 text = self._properties_to_str(result.get("properties", {}))
                 if text:
                     result_blocks.append(NotionBlock(id=obj_id, text=text, prefix="\n"))
-                if obj_type == "page":
-                    logger.debug(
-                        f"Found page with ID '{obj_id}' in database '{database_id}'"
-                    )
-                    result_pages.append(result["id"])
-                elif obj_type == "database":
-                    # TODO add block for database
-                    logger.debug(
-                        f"Found database with ID '{obj_id}' in database '{database_id}'"
-                    )
-                    # The inner contents are ignored at this level
-                    _, child_pages = self._read_pages_from_database(obj_id)
-                    result_pages.extend(child_pages)
+
+                if self.recursive_index_enabled:
+                    if obj_type == "page":
+                        logger.debug(
+                            f"Found page with ID '{obj_id}' in database '{database_id}'"
+                        )
+                        result_pages.append(result["id"])
+                    elif obj_type == "database":
+                        logger.debug(
+                            f"Found database with ID '{obj_id}' in database '{database_id}'"
+                        )
+                        # The inner contents are ignored at this level
+                        _, child_pages = self._read_pages_from_database(obj_id)
+                        result_pages.extend(child_pages)
 
             if data["next_cursor"] is None:
                 break
@@ -354,12 +365,16 @@ def _read_blocks(self, base_block_id: str) -> tuple[list[NotionBlock], list[str]
                         result_blocks.extend(subblocks)
                         child_pages.extend(subblock_child_pages)
 
-                if result_type == "child_database" and self.recursive_index_enabled:
+                if result_type == "child_database":
                     inner_blocks, inner_child_pages = self._read_pages_from_database(
                         result_block_id
                     )
+                    # A database on a page often looks like a table, we need to include it for the contents
+                    # of the page but the children (cells) should be processed as other Documents
                     result_blocks.extend(inner_blocks)
-                    child_pages.extend(inner_child_pages)
+
+                    if self.recursive_index_enabled:
+                        child_pages.extend(inner_child_pages)
 
                 if cur_result_text_arr:
                     new_block = NotionBlock(
@@ -392,7 +407,17 @@ def _read_pages(
         self,
         pages: list[NotionPage],
     ) -> Generator[Document, None, None]:
-        """Reads pages for rich text content and generates Documents"""
+        """Reads pages for rich text content and generates Documents
+
+        Note that a page which is turned into a "wiki" becomes a database but both top level pages and top level databases
+        do not seem to have any properties associated with them.
+
+        Pages that are part of a database can have properties which are like the values of the row in the "database" table
+        in which they exist
+
+        This is not clearly outlined in the Notion API docs but it is observable empirically.
+        https://developers.notion.com/docs/working-with-page-content
+        """
         all_child_page_ids: list[str] = []
         for page in pages:
             if page.id in self.indexed_pages:

From 11372aac8fc9aeb90707d36da808361d1eea8fa5 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 15 Oct 2024 21:37:00 -0700
Subject: [PATCH 124/376] Add custom tool headers (#2773)

* add custom tool headers

* simplify

* k

* k

* k

* nit
---
 backend/danswer/chat/chat_utils.py            | 29 +++++++++++++++++++
 backend/danswer/chat/process_message.py       |  3 ++
 backend/danswer/configs/model_configs.py      | 16 ++++++++++
 backend/danswer/llm/headers.py                | 22 --------------
 .../server/query_and_chat/chat_backend.py     | 15 +++++++---
 backend/danswer/tools/custom/custom_tool.py   | 13 +++++----
 6 files changed, 66 insertions(+), 32 deletions(-)

diff --git a/backend/danswer/chat/chat_utils.py b/backend/danswer/chat/chat_utils.py
index b1e4132779b..cf0fa28c145 100644
--- a/backend/danswer/chat/chat_utils.py
+++ b/backend/danswer/chat/chat_utils.py
@@ -1,6 +1,7 @@
 import re
 from typing import cast
 
+from fastapi.datastructures import Headers
 from sqlalchemy.orm import Session
 
 from danswer.chat.models import CitationInfo
@@ -166,3 +167,31 @@ def slack_link_format(match: re.Match) -> str:
             new_citation_info[citation.citation_num] = citation
 
     return new_answer, list(new_citation_info.values())
+
+
+def extract_headers(
+    headers: dict[str, str] | Headers, pass_through_headers: list[str] | None
+) -> dict[str, str]:
+    """
+    Extract headers specified in pass_through_headers from input headers.
+    Handles both dict and FastAPI Headers objects, accounting for lowercase keys.
+
+    Args:
+        headers: Input headers as dict or Headers object.
+
+    Returns:
+        dict: Filtered headers based on pass_through_headers.
+    """
+    if not pass_through_headers:
+        return {}
+
+    extracted_headers: dict[str, str] = {}
+    for key in pass_through_headers:
+        if key in headers:
+            extracted_headers[key] = headers[key]
+        else:
+            # fastapi makes all header keys lowercase, handling that here
+            lowercase_key = key.lower()
+            if lowercase_key in headers:
+                extracted_headers[lowercase_key] = headers[lowercase_key]
+    return extracted_headers
diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index 19787545e4a..244aeb2b70f 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -276,6 +276,7 @@ def stream_chat_message_objects(
     # on the `new_msg_req.message`. Currently, requires a state where the last message is a
     use_existing_user_message: bool = False,
     litellm_additional_headers: dict[str, str] | None = None,
+    tool_additional_headers: dict[str, str] | None = None,
     is_connected: Callable[[], bool] | None = None,
     enforce_chat_session_id_for_search_docs: bool = True,
 ) -> ChatPacketStream:
@@ -862,6 +863,7 @@ def stream_chat_message(
     user: User | None,
     use_existing_user_message: bool = False,
     litellm_additional_headers: dict[str, str] | None = None,
+    tool_additional_headers: dict[str, str] | None = None,
     is_connected: Callable[[], bool] | None = None,
 ) -> Iterator[str]:
     with get_session_context_manager() as db_session:
@@ -871,6 +873,7 @@ def stream_chat_message(
             db_session=db_session,
             use_existing_user_message=use_existing_user_message,
             litellm_additional_headers=litellm_additional_headers,
+            tool_additional_headers=tool_additional_headers,
             is_connected=is_connected,
         )
         for obj in objects:
diff --git a/backend/danswer/configs/model_configs.py b/backend/danswer/configs/model_configs.py
index c9668cd8136..4454eee159b 100644
--- a/backend/danswer/configs/model_configs.py
+++ b/backend/danswer/configs/model_configs.py
@@ -119,3 +119,19 @@
         logger.error(
             "Failed to parse LITELLM_PASS_THROUGH_HEADERS, must be a valid JSON object"
         )
+
+
+# List of headers to pass through to tool calls (e.g., API requests made by tools)
+# This allows for dynamic configuration of tool behavior based on incoming request headers
+TOOL_PASS_THROUGH_HEADERS: list[str] | None = None
+_TOOL_PASS_THROUGH_HEADERS_RAW = os.environ.get("TOOL_PASS_THROUGH_HEADERS")
+if _TOOL_PASS_THROUGH_HEADERS_RAW:
+    try:
+        TOOL_PASS_THROUGH_HEADERS = json.loads(_TOOL_PASS_THROUGH_HEADERS_RAW)
+    except Exception:
+        from danswer.utils.logger import setup_logger
+
+        logger = setup_logger()
+        logger.error(
+            "Failed to parse TOOL_PASS_THROUGH_HEADERS, must be a valid JSON object"
+        )
diff --git a/backend/danswer/llm/headers.py b/backend/danswer/llm/headers.py
index b43c83e141e..13622167d99 100644
--- a/backend/danswer/llm/headers.py
+++ b/backend/danswer/llm/headers.py
@@ -1,26 +1,4 @@
-from fastapi.datastructures import Headers
-
 from danswer.configs.model_configs import LITELLM_EXTRA_HEADERS
-from danswer.configs.model_configs import LITELLM_PASS_THROUGH_HEADERS
-
-
-def get_litellm_additional_request_headers(
-    headers: dict[str, str] | Headers
-) -> dict[str, str]:
-    if not LITELLM_PASS_THROUGH_HEADERS:
-        return {}
-
-    pass_through_headers: dict[str, str] = {}
-    for key in LITELLM_PASS_THROUGH_HEADERS:
-        if key in headers:
-            pass_through_headers[key] = headers[key]
-        else:
-            # fastapi makes all header keys lowercase, handling that here
-            lowercase_key = key.lower()
-            if lowercase_key in headers:
-                pass_through_headers[lowercase_key] = headers[lowercase_key]
-
-    return pass_through_headers
 
 
 def build_llm_extra_headers(
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index 49603fa3971..c26bc9c5c8b 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -18,10 +18,13 @@
 
 from danswer.auth.users import current_user
 from danswer.chat.chat_utils import create_chat_chain
+from danswer.chat.chat_utils import extract_headers
 from danswer.chat.process_message import stream_chat_message
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import MessageType
+from danswer.configs.model_configs import LITELLM_PASS_THROUGH_HEADERS
+from danswer.configs.model_configs import TOOL_PASS_THROUGH_HEADERS
 from danswer.db.chat import create_chat_session
 from danswer.db.chat import create_new_chat_message
 from danswer.db.chat import delete_chat_session
@@ -50,7 +53,6 @@
 from danswer.llm.exceptions import GenAIDisabledException
 from danswer.llm.factory import get_default_llms
 from danswer.llm.factory import get_llms_for_persona
-from danswer.llm.headers import get_litellm_additional_request_headers
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.secondary_llm_flows.chat_session_naming import (
     get_renamed_conversation_name,
@@ -229,7 +231,9 @@ def rename_chat_session(
 
     try:
         llm, _ = get_default_llms(
-            additional_headers=get_litellm_additional_request_headers(request.headers)
+            additional_headers=extract_headers(
+                request.headers, LITELLM_PASS_THROUGH_HEADERS
+            )
         )
     except GenAIDisabledException:
         # This may be longer than what the LLM tends to produce but is the most
@@ -330,8 +334,11 @@ def stream_generator() -> Generator[str, None, None]:
                 new_msg_req=chat_message_req,
                 user=user,
                 use_existing_user_message=chat_message_req.use_existing_user_message,
-                litellm_additional_headers=get_litellm_additional_request_headers(
-                    request.headers
+                litellm_additional_headers=extract_headers(
+                    request.headers, LITELLM_PASS_THROUGH_HEADERS
+                ),
+                tool_additional_headers=extract_headers(
+                    request.headers, TOOL_PASS_THROUGH_HEADERS
                 ),
                 is_connected=is_disconnected_func,
             ):
diff --git a/backend/danswer/tools/custom/custom_tool.py b/backend/danswer/tools/custom/custom_tool.py
index 85830f1ca30..8f4a4b23fa8 100644
--- a/backend/danswer/tools/custom/custom_tool.py
+++ b/backend/danswer/tools/custom/custom_tool.py
@@ -47,6 +47,7 @@ def __init__(
         method_spec: MethodSpec,
         base_url: str,
         custom_headers: list[dict[str, str]] | None = [],
+        tool_additional_headers: dict[str, str] | None = None,
     ) -> None:
         self._base_url = base_url
         self._method_spec = method_spec
@@ -54,11 +55,9 @@ def __init__(
 
         self._name = self._method_spec.name
         self._description = self._method_spec.summary
-        self.headers = (
-            {header["key"]: header["value"] for header in custom_headers}
-            if custom_headers
-            else {}
-        )
+        self.headers = {
+            header["key"]: header["value"] for header in (custom_headers or [])
+        } | (tool_additional_headers or {})
 
     @property
     def name(self) -> str:
@@ -185,6 +184,7 @@ def final_result(self, *args: ToolResponse) -> JSON_ro:
 
 def build_custom_tools_from_openapi_schema_and_headers(
     openapi_schema: dict[str, Any],
+    tool_additional_headers: dict[str, str] | None = None,
     custom_headers: list[dict[str, str]] | None = [],
     dynamic_schema_info: DynamicSchemaInfo | None = None,
 ) -> list[CustomTool]:
@@ -205,7 +205,8 @@ def build_custom_tools_from_openapi_schema_and_headers(
     url = openapi_to_url(openapi_schema)
     method_specs = openapi_to_method_specs(openapi_schema)
     return [
-        CustomTool(method_spec, url, custom_headers) for method_spec in method_specs
+        CustomTool(method_spec, url, custom_headers, tool_additional_headers)
+        for method_spec in method_specs
     ]
 
 

From c148fa5bfae4f38fb5dd10f2158872c54d04ae4f Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Tue, 15 Oct 2024 23:03:37 -0700
Subject: [PATCH 125/376] Notion Recurse Empty Final Field (#2819)

---
 .../danswer/connectors/notion/connector.py    | 36 +++++++++++++------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/backend/danswer/connectors/notion/connector.py b/backend/danswer/connectors/notion/connector.py
index 6e2da4a9c65..13cad489d80 100644
--- a/backend/danswer/connectors/notion/connector.py
+++ b/backend/danswer/connectors/notion/connector.py
@@ -216,17 +216,28 @@ def _fetch_database(
     def _properties_to_str(properties: dict[str, Any]) -> str:
         """Converts Notion properties to a string"""
 
-        def _recurse_properties(inner_dict: dict[str, Any]) -> str:
-            if not inner_dict:
-                # Edge case handling, should not happen
-                return "N/A"
-
+        def _recurse_properties(inner_dict: dict[str, Any]) -> str | None:
             while "type" in inner_dict:
                 type_name = inner_dict["type"]
                 inner_dict = inner_dict[type_name]
+
+                # If the innermost layer is None, the value is not set
+                if not inner_dict:
+                    return None
+
                 if isinstance(inner_dict, list):
-                    return ", ".join(
-                        [_recurse_properties(item) for item in inner_dict if item]
+                    list_properties = [
+                        _recurse_properties(item) for item in inner_dict if item
+                    ]
+                    return (
+                        ", ".join(
+                            [
+                                list_property
+                                for list_property in list_properties
+                                if list_property
+                            ]
+                        )
+                        or None
                     )
 
             # TODO there may be more types to handle here
@@ -244,11 +255,13 @@ def _recurse_properties(inner_dict: dict[str, Any]) -> str:
                 return f"Until {end}"
 
             if "id" in inner_dict:
-                logger.debug("Skipping Notion Id field")
-                return "Unreadable Property"
+                # This is not useful to index, it's a reference to another Notion object
+                # and this ID value in plaintext is useless outside of the Notion context
+                logger.debug("Skipping Notion object id field property")
+                return None
 
             logger.debug(f"Unreadable property from innermost prop: {inner_dict}")
-            return "Unreadable Property"
+            return None
 
         result = ""
         for prop_name, prop in properties.items():
@@ -258,7 +271,8 @@ def _recurse_properties(inner_dict: dict[str, Any]) -> str:
             inner_value = _recurse_properties(prop)
             # Not a perfect way to format Notion database tables but there's no perfect representation
             # since this must be represented as plaintext
-            result += f"{prop_name}: {inner_value}\t"
+            if inner_value:
+                result += f"{prop_name}: {inner_value}\t"
 
         return result
 

From 65573210f1dad138daae9bebfd3c7f5fa6468bca Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 16 Oct 2024 09:00:32 -0700
Subject: [PATCH 126/376] add llama 3.2 (#2812)

---
 web/src/lib/hooks.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/web/src/lib/hooks.ts b/web/src/lib/hooks.ts
index eb9741b7516..1f03decd44a 100644
--- a/web/src/lib/hooks.ts
+++ b/web/src/lib/hooks.ts
@@ -283,6 +283,10 @@ const MODEL_DISPLAY_NAMES: { [key: string]: string } = {
   "meta.llama3-1-70b-instruct-v1:0": "Llama 3.1 70B",
   "meta.llama3-1-8b-instruct-v1:0": "Llama 3.1 8B",
   "meta.llama3-70b-instruct-v1:0": "Llama 3 70B",
+  "meta.llama3-2-1b-instruct-v1:0": "Llama 3.2 1B",
+  "meta.llama3-2-3b-instruct-v1:0": "Llama 3.2 3B",
+  "meta.llama3-2-11b-instruct-v1:0": "Llama 3.2 11B",
+  "meta.llama3-2-90b-instruct-v1:0": "Llama 3.2 90B",
   "meta.llama3-8b-instruct-v1:0": "Llama 3 8B",
   "meta.llama2-70b-chat-v1": "Llama 2 70B",
   "meta.llama2-13b-chat-v1": "Llama 2 13B",

From a385234c0e6abeb1ebd8190cc4ce7709eb7619d4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 16 Oct 2024 09:44:19 -0700
Subject: [PATCH 127/376] Parsing (#2734)

* k

* update chunking limits

* nit

* nit

* clean up types

* nit

* validate

* k
---
 backend/danswer/configs/app_configs.py        |  2 +
 backend/danswer/file_processing/html_utils.py | 36 ++++++++++-
 backend/danswer/indexing/chunker.py           | 60 ++++++++++++++++---
 backend/requirements/default.txt              |  3 +
 backend/requirements/dev.txt                  |  3 +
 backend/shared_configs/configs.py             |  4 ++
 6 files changed, 99 insertions(+), 9 deletions(-)

diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 337a5361d8b..0d8b7da7010 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -404,6 +404,8 @@
 
 SYSTEM_RECURSION_LIMIT = int(os.environ.get("SYSTEM_RECURSION_LIMIT") or "1000")
 
+PARSE_WITH_TRAFILATURA = os.environ.get("PARSE_WITH_TRAFILATURA", "").lower() == "true"
+
 #####
 # Enterprise Edition Configs
 #####
diff --git a/backend/danswer/file_processing/html_utils.py b/backend/danswer/file_processing/html_utils.py
index 48782981f89..d1948d011f5 100644
--- a/backend/danswer/file_processing/html_utils.py
+++ b/backend/danswer/file_processing/html_utils.py
@@ -4,11 +4,17 @@
 from typing import IO
 
 import bs4
+import trafilatura  # type: ignore
+from trafilatura.settings import use_config  # type: ignore
 
 from danswer.configs.app_configs import HTML_BASED_CONNECTOR_TRANSFORM_LINKS_STRATEGY
+from danswer.configs.app_configs import PARSE_WITH_TRAFILATURA
 from danswer.configs.app_configs import WEB_CONNECTOR_IGNORED_CLASSES
 from danswer.configs.app_configs import WEB_CONNECTOR_IGNORED_ELEMENTS
 from danswer.file_processing.enums import HtmlBasedConnectorTransformLinksStrategy
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
 
 MINTLIFY_UNWANTED = ["sticky", "hidden"]
 
@@ -47,6 +53,18 @@ def format_element_text(element_text: str, link_href: str | None) -> str:
     return f"[{element_text_no_newlines}]({link_href})"
 
 
+def parse_html_with_trafilatura(html_content: str) -> str:
+    """Parse HTML content using trafilatura."""
+    config = use_config()
+    config.set("DEFAULT", "include_links", "True")
+    config.set("DEFAULT", "include_tables", "True")
+    config.set("DEFAULT", "include_images", "True")
+    config.set("DEFAULT", "include_formatting", "True")
+
+    extracted_text = trafilatura.extract(html_content, config=config)
+    return strip_excessive_newlines_and_spaces(extracted_text) if extracted_text else ""
+
+
 def format_document_soup(
     document: bs4.BeautifulSoup, table_cell_separator: str = "\t"
 ) -> str:
@@ -183,7 +201,21 @@ def web_html_cleanup(
         for undesired_tag in additional_element_types_to_discard:
             [tag.extract() for tag in soup.find_all(undesired_tag)]
 
+    soup_string = str(soup)
+    page_text = ""
+
+    if PARSE_WITH_TRAFILATURA:
+        try:
+            page_text = parse_html_with_trafilatura(soup_string)
+            if not page_text:
+                raise ValueError("Empty content returned by trafilatura.")
+        except Exception as e:
+            logger.info(f"Trafilatura parsing failed: {e}. Falling back on bs4.")
+            page_text = format_document_soup(soup)
+    else:
+        page_text = format_document_soup(soup)
+
     # 200B is ZeroWidthSpace which we don't care for
-    page_text = format_document_soup(soup).replace("\u200B", "")
+    cleaned_text = page_text.replace("\u200B", "")
 
-    return ParsedHTML(title=title, cleaned_text=page_text)
+    return ParsedHTML(title=title, cleaned_text=cleaned_text)
diff --git a/backend/danswer/indexing/chunker.py b/backend/danswer/indexing/chunker.py
index 9cb4b3e1954..57f05a66e1e 100644
--- a/backend/danswer/indexing/chunker.py
+++ b/backend/danswer/indexing/chunker.py
@@ -15,7 +15,7 @@
 from danswer.natural_language_processing.utils import BaseTokenizer
 from danswer.utils.logger import setup_logger
 from danswer.utils.text_processing import shared_precompare_cleanup
-
+from shared_configs.configs import STRICT_CHUNK_TOKEN_LIMIT
 
 # Not supporting overlaps, we need a clean combination of chunks and it is unclear if overlaps
 # actually help quality at all
@@ -158,6 +158,24 @@ def __init__(
             else None
         )
 
+    def _split_oversized_chunk(self, text: str, content_token_limit: int) -> list[str]:
+        """
+        Splits the text into smaller chunks based on token count to ensure
+        no chunk exceeds the content_token_limit.
+        """
+        tokens = self.tokenizer.tokenize(text)
+        chunks = []
+        start = 0
+        total_tokens = len(tokens)
+        while start < total_tokens:
+            end = min(start + content_token_limit, total_tokens)
+            token_chunk = tokens[start:end]
+            # Join the tokens to reconstruct the text
+            chunk_text = " ".join(token_chunk)
+            chunks.append(chunk_text)
+            start = end
+        return chunks
+
     def _extract_blurb(self, text: str) -> str:
         texts = self.blurb_splitter.split_text(text)
         if not texts:
@@ -218,14 +236,42 @@ def _create_chunk(
                     chunk_text = ""
 
                 split_texts = self.chunk_splitter.split_text(section_text)
+
                 for i, split_text in enumerate(split_texts):
-                    chunks.append(
-                        _create_chunk(
-                            text=split_text,
-                            links={0: section_link_text},
-                            is_continuation=(i != 0),
+                    split_token_count = len(self.tokenizer.tokenize(split_text))
+
+                    if STRICT_CHUNK_TOKEN_LIMIT:
+                        split_token_count = len(self.tokenizer.tokenize(split_text))
+                        if split_token_count > content_token_limit:
+                            # Further split the oversized chunk
+                            smaller_chunks = self._split_oversized_chunk(
+                                split_text, content_token_limit
+                            )
+                            for i, small_chunk in enumerate(smaller_chunks):
+                                chunks.append(
+                                    _create_chunk(
+                                        text=small_chunk,
+                                        links={0: section_link_text},
+                                        is_continuation=(i != 0),
+                                    )
+                                )
+                        else:
+                            chunks.append(
+                                _create_chunk(
+                                    text=split_text,
+                                    links={0: section_link_text},
+                                )
+                            )
+
+                    else:
+                        chunks.append(
+                            _create_chunk(
+                                text=split_text,
+                                links={0: section_link_text},
+                                is_continuation=(i != 0),
+                            )
                         )
-                    )
+
                 continue
 
             current_token_count = len(self.tokenizer.tokenize(chunk_text))
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 3820d63b066..8ecfcdb1528 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -25,10 +25,13 @@ httpx-oauth==0.15.1
 huggingface-hub==0.20.1
 jira==3.5.1
 jsonref==1.1.0
+trafilatura==1.12.2
 langchain==0.1.17
 langchain-core==0.1.50
 langchain-text-splitters==0.0.1
 litellm==1.48.7
+lxml==5.3.0
+lxml_html_clean==0.2.2
 llama-index==0.9.45
 Mako==1.2.4
 msal==1.28.0
diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt
index 691157f732a..fb589509d64 100644
--- a/backend/requirements/dev.txt
+++ b/backend/requirements/dev.txt
@@ -21,4 +21,7 @@ types-regex==2023.3.23.1
 types-requests==2.28.11.17
 types-retry==0.9.9.3
 types-urllib3==1.26.25.11
+trafilatura==1.12.2
+lxml==5.3.0
+lxml_html_clean==0.2.2
 boto3-stubs[s3]==1.34.133
\ No newline at end of file
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index c5044b8b89c..c5b79c4751a 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -66,6 +66,10 @@
 # Only used for OpenAI
 OPENAI_EMBEDDING_TIMEOUT = int(os.environ.get("OPENAI_EMBEDDING_TIMEOUT", "600"))
 
+# Whether or not to strictly enforce token limit for chunking.
+STRICT_CHUNK_TOKEN_LIMIT = (
+    os.environ.get("STRICT_CHUNK_TOKEN_LIMIT", "").lower() == "true"
+)
 
 # Fields which should only be set on new search setting
 PRESERVED_SEARCH_FIELDS = [

From 1a9921f63e0a1dfb5c52cc303947ec2e0b5106e8 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 16 Oct 2024 10:26:44 -0700
Subject: [PATCH 128/376] Redirect with query param (#2811)

* validated

* k

* k

* k

* minor update
---
 backend/danswer/auth/users.py            | 201 ++++++++++++++++++++++-
 backend/danswer/main.py                  |   5 +-
 backend/ee/danswer/main.py               |   3 +-
 web/src/app/auth/login/page.tsx          |   9 +-
 web/src/app/auth/oauth/callback/route.ts |   9 +-
 web/src/app/auth/oidc/callback/route.ts  |  16 +-
 web/src/app/page.tsx                     |   1 -
 web/src/app/search/page.tsx              |  18 +-
 web/src/components/UserDropdown.tsx      |  15 +-
 web/src/lib/chat/fetchChatData.ts        |  13 +-
 web/src/lib/userSS.ts                    |  17 +-
 11 files changed, 281 insertions(+), 26 deletions(-)

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 983e17182b0..f8b07d15b5a 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -5,6 +5,8 @@
 from datetime import timezone
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
+from typing import Dict
+from typing import List
 from typing import Optional
 from typing import Tuple
 
@@ -15,9 +17,11 @@
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Query
 from fastapi import Request
 from fastapi import Response
 from fastapi import status
+from fastapi.responses import RedirectResponse
 from fastapi.security import OAuth2PasswordRequestForm
 from fastapi_users import BaseUserManager
 from fastapi_users import exceptions
@@ -31,8 +35,19 @@
 from fastapi_users.authentication import Strategy
 from fastapi_users.authentication.strategy.db import AccessTokenDatabase
 from fastapi_users.authentication.strategy.db import DatabaseStrategy
+from fastapi_users.exceptions import UserAlreadyExists
+from fastapi_users.jwt import decode_jwt
+from fastapi_users.jwt import generate_jwt
+from fastapi_users.jwt import SecretType
+from fastapi_users.manager import UserManagerDependency
 from fastapi_users.openapi import OpenAPIResponseType
+from fastapi_users.router.common import ErrorCode
+from fastapi_users.router.common import ErrorModel
 from fastapi_users_db_sqlalchemy import SQLAlchemyUserDatabase
+from httpx_oauth.integrations.fastapi import OAuth2AuthorizeCallback
+from httpx_oauth.oauth2 import BaseOAuth2
+from httpx_oauth.oauth2 import OAuth2Token
+from pydantic import BaseModel
 from sqlalchemy import select
 from sqlalchemy.orm import attributes
 from sqlalchemy.orm import Session
@@ -298,7 +313,7 @@ async def oauth_callback(
         token = None
         async with get_async_session_with_tenant(tenant_id) as db_session:
             token = current_tenant_id.set(tenant_id)
-            # Print a list of tables in the current database session
+
             verify_email_in_whitelist(account_email, tenant_id)
             verify_email_domain(account_email)
             if MULTI_TENANT:
@@ -422,7 +437,6 @@ async def authenticate(
         email = credentials.username
 
         # Get tenant_id from mapping table
-
         tenant_id = get_tenant_id_for_email(email)
         if not tenant_id:
             # User not found in mapping
@@ -654,3 +668,186 @@ async def current_admin_user(user: User | None = Depends(current_user)) -> User
 def get_default_admin_user_emails_() -> list[str]:
     # No default seeding available for Danswer MIT
     return []
+
+
+STATE_TOKEN_AUDIENCE = "fastapi-users:oauth-state"
+
+
+class OAuth2AuthorizeResponse(BaseModel):
+    authorization_url: str
+
+
+def generate_state_token(
+    data: Dict[str, str], secret: SecretType, lifetime_seconds: int = 3600
+) -> str:
+    data["aud"] = STATE_TOKEN_AUDIENCE
+
+    return generate_jwt(data, secret, lifetime_seconds)
+
+
+# refer to https://github.com/fastapi-users/fastapi-users/blob/42ddc241b965475390e2bce887b084152ae1a2cd/fastapi_users/fastapi_users.py#L91
+
+
+def create_danswer_oauth_router(
+    oauth_client: BaseOAuth2,
+    backend: AuthenticationBackend,
+    state_secret: SecretType,
+    redirect_url: Optional[str] = None,
+    associate_by_email: bool = False,
+    is_verified_by_default: bool = False,
+) -> APIRouter:
+    return get_oauth_router(
+        oauth_client,
+        backend,
+        get_user_manager,
+        state_secret,
+        redirect_url,
+        associate_by_email,
+        is_verified_by_default,
+    )
+
+
+def get_oauth_router(
+    oauth_client: BaseOAuth2,
+    backend: AuthenticationBackend,
+    get_user_manager: UserManagerDependency[models.UP, models.ID],
+    state_secret: SecretType,
+    redirect_url: Optional[str] = None,
+    associate_by_email: bool = False,
+    is_verified_by_default: bool = False,
+) -> APIRouter:
+    """Generate a router with the OAuth routes."""
+    router = APIRouter()
+    callback_route_name = f"oauth:{oauth_client.name}.{backend.name}.callback"
+
+    if redirect_url is not None:
+        oauth2_authorize_callback = OAuth2AuthorizeCallback(
+            oauth_client,
+            redirect_url=redirect_url,
+        )
+    else:
+        oauth2_authorize_callback = OAuth2AuthorizeCallback(
+            oauth_client,
+            route_name=callback_route_name,
+        )
+
+    @router.get(
+        "/authorize",
+        name=f"oauth:{oauth_client.name}.{backend.name}.authorize",
+        response_model=OAuth2AuthorizeResponse,
+    )
+    async def authorize(
+        request: Request, scopes: List[str] = Query(None)
+    ) -> OAuth2AuthorizeResponse:
+        if redirect_url is not None:
+            authorize_redirect_url = redirect_url
+        else:
+            authorize_redirect_url = str(request.url_for(callback_route_name))
+
+        next_url = request.query_params.get("next", "/")
+        state_data: Dict[str, str] = {"next_url": next_url}
+        state = generate_state_token(state_data, state_secret)
+        authorization_url = await oauth_client.get_authorization_url(
+            authorize_redirect_url,
+            state,
+            scopes,
+        )
+
+        return OAuth2AuthorizeResponse(authorization_url=authorization_url)
+
+    @router.get(
+        "/callback",
+        name=callback_route_name,
+        description="The response varies based on the authentication backend used.",
+        responses={
+            status.HTTP_400_BAD_REQUEST: {
+                "model": ErrorModel,
+                "content": {
+                    "application/json": {
+                        "examples": {
+                            "INVALID_STATE_TOKEN": {
+                                "summary": "Invalid state token.",
+                                "value": None,
+                            },
+                            ErrorCode.LOGIN_BAD_CREDENTIALS: {
+                                "summary": "User is inactive.",
+                                "value": {"detail": ErrorCode.LOGIN_BAD_CREDENTIALS},
+                            },
+                        }
+                    }
+                },
+            },
+        },
+    )
+    async def callback(
+        request: Request,
+        access_token_state: Tuple[OAuth2Token, str] = Depends(
+            oauth2_authorize_callback
+        ),
+        user_manager: BaseUserManager[models.UP, models.ID] = Depends(get_user_manager),
+        strategy: Strategy[models.UP, models.ID] = Depends(backend.get_strategy),
+    ) -> RedirectResponse:
+        token, state = access_token_state
+        account_id, account_email = await oauth_client.get_id_email(
+            token["access_token"]
+        )
+
+        if account_email is None:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ErrorCode.OAUTH_NOT_AVAILABLE_EMAIL,
+            )
+
+        try:
+            state_data = decode_jwt(state, state_secret, [STATE_TOKEN_AUDIENCE])
+        except jwt.DecodeError:
+            raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
+
+        next_url = state_data.get("next_url", "/")
+
+        # Authenticate user
+        try:
+            user = await user_manager.oauth_callback(
+                oauth_client.name,
+                token["access_token"],
+                account_id,
+                account_email,
+                token.get("expires_at"),
+                token.get("refresh_token"),
+                request,
+                associate_by_email=associate_by_email,
+                is_verified_by_default=is_verified_by_default,
+            )
+        except UserAlreadyExists:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ErrorCode.OAUTH_USER_ALREADY_EXISTS,
+            )
+
+        if not user.is_active:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail=ErrorCode.LOGIN_BAD_CREDENTIALS,
+            )
+
+        # Login user
+        response = await backend.login(strategy, user)
+        await user_manager.on_after_login(user, request, response)
+
+        # Prepare redirect response
+        redirect_response = RedirectResponse(next_url, status_code=302)
+
+        # Copy headers and other attributes from 'response' to 'redirect_response'
+        for header_name, header_value in response.headers.items():
+            redirect_response.headers[header_name] = header_value
+
+        if hasattr(response, "body"):
+            redirect_response.body = response.body
+        if hasattr(response, "status_code"):
+            redirect_response.status_code = response.status_code
+        if hasattr(response, "media_type"):
+            redirect_response.media_type = response.media_type
+
+        return redirect_response
+
+    return router
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index 151f852486c..cd0c5c195a6 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -81,7 +81,6 @@
     router as token_rate_limit_settings_router,
 )
 from danswer.setup import setup_danswer
-from danswer.setup import setup_multitenant_danswer
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import get_or_generate_uuid
 from danswer.utils.telemetry import optional_telemetry
@@ -176,12 +175,10 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
         # We cache this at the beginning so there is no delay in the first telemetry
         get_or_generate_uuid()
 
+        # If we are multi-tenant, we need to only set up initial public tables
         with Session(engine) as db_session:
             setup_danswer(db_session)
 
-    else:
-        setup_multitenant_danswer()
-
     optional_telemetry(record_type=RecordType.VERSION, data={"version": __version__})
     yield
 
diff --git a/backend/ee/danswer/main.py b/backend/ee/danswer/main.py
index 4584e06a00b..2d1793b8b27 100644
--- a/backend/ee/danswer/main.py
+++ b/backend/ee/danswer/main.py
@@ -2,6 +2,7 @@
 from httpx_oauth.clients.openid import OpenID
 
 from danswer.auth.users import auth_backend
+from danswer.auth.users import create_danswer_oauth_router
 from danswer.auth.users import fastapi_users
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import MULTI_TENANT
@@ -61,7 +62,7 @@ def get_application() -> FastAPI:
     if AUTH_TYPE == AuthType.OIDC:
         include_router_with_global_prefix_prepended(
             application,
-            fastapi_users.get_oauth_router(
+            create_danswer_oauth_router(
                 OpenID(OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET, OPENID_CONFIG_URL),
                 auth_backend,
                 USER_AUTH_SECRET,
diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index 9ec047d61e2..e47bdb420be 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -11,7 +11,7 @@ import { SignInButton } from "./SignInButton";
 import { EmailPasswordForm } from "./EmailPasswordForm";
 import { Card, Title, Text } from "@tremor/react";
 import Link from "next/link";
-import { Logo } from "@/components/Logo";
+
 import { LoginText } from "./LoginText";
 import { getSecondsUntilExpiration } from "@/lib/time";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
@@ -37,6 +37,10 @@ const Page = async ({
     console.log(`Some fetch failed for the login page - ${e}`);
   }
 
+  const nextUrl = Array.isArray(searchParams?.next)
+    ? searchParams?.next[0]
+    : searchParams?.next || null;
+
   // simply take the user to the home page if Auth is disabled
   if (authTypeMetadata?.authType === "disabled") {
     return redirect("/");
@@ -59,7 +63,7 @@ const Page = async ({
   let authUrl: string | null = null;
   if (authTypeMetadata) {
     try {
-      authUrl = await getAuthUrlSS(authTypeMetadata.authType);
+      authUrl = await getAuthUrlSS(authTypeMetadata.authType, nextUrl!);
     } catch (e) {
       console.log(`Some fetch failed for the login page - ${e}`);
     }
@@ -88,6 +92,7 @@ const Page = async ({
             />
           </>
         )}
+
         {authTypeMetadata?.authType === "basic" && (
           <Card className="mt-4 w-96">
             <div className="flex">
diff --git a/web/src/app/auth/oauth/callback/route.ts b/web/src/app/auth/oauth/callback/route.ts
index 6e8f290a65f..ca5a82743d3 100644
--- a/web/src/app/auth/oauth/callback/route.ts
+++ b/web/src/app/auth/oauth/callback/route.ts
@@ -8,7 +8,8 @@ export const GET = async (request: NextRequest) => {
   const url = new URL(buildUrl("/auth/oauth/callback"));
   url.search = request.nextUrl.search;
 
-  const response = await fetch(url.toString());
+  // Set 'redirect' to 'manual' to prevent automatic redirection
+  const response = await fetch(url.toString(), { redirect: "manual" });
   const setCookieHeader = response.headers.get("set-cookie");
 
   if (response.status === 401) {
@@ -21,9 +22,13 @@ export const GET = async (request: NextRequest) => {
     return NextResponse.redirect(new URL("/auth/error", getDomain(request)));
   }
 
+  // Get the redirect URL from the backend's 'Location' header, or default to '/'
+  const redirectUrl = response.headers.get("location") || "/";
+
   const redirectResponse = NextResponse.redirect(
-    new URL("/", getDomain(request))
+    new URL(redirectUrl, getDomain(request))
   );
+
   redirectResponse.headers.set("set-cookie", setCookieHeader);
   return redirectResponse;
 };
diff --git a/web/src/app/auth/oidc/callback/route.ts b/web/src/app/auth/oidc/callback/route.ts
index 353119409b9..1bdf2b61db1 100644
--- a/web/src/app/auth/oidc/callback/route.ts
+++ b/web/src/app/auth/oidc/callback/route.ts
@@ -7,17 +7,27 @@ export const GET = async (request: NextRequest) => {
   // which adds back a redirect to the main app.
   const url = new URL(buildUrl("/auth/oidc/callback"));
   url.search = request.nextUrl.search;
-
-  const response = await fetch(url.toString());
+  // Set 'redirect' to 'manual' to prevent automatic redirection
+  const response = await fetch(url.toString(), { redirect: "manual" });
   const setCookieHeader = response.headers.get("set-cookie");
 
+  if (response.status === 401) {
+    return NextResponse.redirect(
+      new URL("/auth/create-account", getDomain(request))
+    );
+  }
+
   if (!setCookieHeader) {
     return NextResponse.redirect(new URL("/auth/error", getDomain(request)));
   }
 
+  // Get the redirect URL from the backend's 'Location' header, or default to '/'
+  const redirectUrl = response.headers.get("location") || "/";
+
   const redirectResponse = NextResponse.redirect(
-    new URL("/", getDomain(request))
+    new URL(redirectUrl, getDomain(request))
   );
+
   redirectResponse.headers.set("set-cookie", setCookieHeader);
   return redirectResponse;
 };
diff --git a/web/src/app/page.tsx b/web/src/app/page.tsx
index 9cc0c56c5e2..00776084a59 100644
--- a/web/src/app/page.tsx
+++ b/web/src/app/page.tsx
@@ -3,7 +3,6 @@ import { redirect } from "next/navigation";
 
 export default async function Page() {
   const settings = await fetchSettingsSS();
-
   if (!settings) {
     redirect("/search");
   }
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index 818dfe6b965..3d26053dd41 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -36,8 +36,13 @@ import WrappedSearch from "./WrappedSearch";
 import { SearchProvider } from "@/components/context/SearchContext";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "../admin/configuration/llm/interfaces";
+import { headers } from "next/headers";
 
-export default async function Home() {
+export default async function Home({
+  searchParams,
+}: {
+  searchParams: { [key: string]: string | string[] | undefined };
+}) {
   // Disable caching so we always get the up to date connector / document set / persona info
   // importantly, this prevents users from adding a connector, going back to the main page,
   // and then getting hit with a "No Connectors" popup
@@ -82,8 +87,17 @@ export default async function Home() {
   const llmProviders = (results[7] || []) as LLMProviderDescriptor[];
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
+
   if (!authDisabled && !user) {
-    return redirect("/auth/login");
+    const headersList = headers();
+    const fullUrl = headersList.get("x-url") || "/search";
+    const searchParamsString = new URLSearchParams(
+      searchParams as unknown as Record<string, string>
+    ).toString();
+    const redirectUrl = searchParamsString
+      ? `${fullUrl}?${searchParamsString}`
+      : fullUrl;
+    return redirect(`/auth/login?next=${encodeURIComponent(redirectUrl)}`);
   }
 
   if (user && !user.is_verified && authTypeMetadata?.requiresVerification) {
diff --git a/web/src/components/UserDropdown.tsx b/web/src/components/UserDropdown.tsx
index e59e7291c52..00c3b83c1fa 100644
--- a/web/src/components/UserDropdown.tsx
+++ b/web/src/components/UserDropdown.tsx
@@ -3,7 +3,7 @@
 import { useState, useRef, useContext, useEffect, useMemo } from "react";
 import { FiLogOut } from "react-icons/fi";
 import Link from "next/link";
-import { useRouter } from "next/navigation";
+import { useRouter, usePathname, useSearchParams } from "next/navigation";
 import { User, UserRole } from "@/lib/types";
 import { checkUserIsNoAuthUser, logout } from "@/lib/user";
 import { Popover } from "./popover/Popover";
@@ -65,6 +65,8 @@ export function UserDropdown({
   const [userInfoVisible, setUserInfoVisible] = useState(false);
   const userInfoRef = useRef<HTMLDivElement>(null);
   const router = useRouter();
+  const pathname = usePathname();
+  const searchParams = useSearchParams();
 
   const combinedSettings = useContext(SettingsContext);
   const customNavItems: NavigationItem[] = useMemo(
@@ -87,8 +89,17 @@ export function UserDropdown({
     logout().then((isSuccess) => {
       if (!isSuccess) {
         alert("Failed to logout");
+        return;
       }
-      router.push("/auth/login");
+
+      // Construct the current URL
+      const currentUrl = `${pathname}${searchParams.toString() ? `?${searchParams.toString()}` : ""}`;
+
+      // Encode the current URL to use as a redirect parameter
+      const encodedRedirect = encodeURIComponent(currentUrl);
+
+      // Redirect to login page with the current page as a redirect parameter
+      router.push(`/auth/login?next=${encodedRedirect}`);
     });
   };
 
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index 144a839cd73..c4188720689 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -20,7 +20,7 @@ import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
 import { personaComparator } from "@/app/admin/assistants/lib";
-import { cookies } from "next/headers";
+import { cookies, headers } from "next/headers";
 import {
   SIDEBAR_TOGGLED_COOKIE_NAME,
   DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME,
@@ -29,6 +29,7 @@ import { hasCompletedWelcomeFlowSS } from "@/components/initialSetup/welcome/Wel
 import { fetchAssistantsSS } from "../assistants/fetchAssistantsSS";
 import { NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN } from "../constants";
 import { checkLLMSupportsImageInput } from "../llm/utils";
+import { redirect } from "next/navigation";
 
 interface FetchChatDataResult {
   user: User | null;
@@ -98,7 +99,15 @@ export async function fetchChatData(searchParams: {
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
-    return { redirect: "/auth/login" };
+    const headersList = headers();
+    const fullUrl = headersList.get("x-url") || "/chat";
+    const searchParamsString = new URLSearchParams(
+      searchParams as unknown as Record<string, string>
+    ).toString();
+    const redirectUrl = searchParamsString
+      ? `${fullUrl}?${searchParamsString}`
+      : fullUrl;
+    return redirect(`/auth/login?next=${encodeURIComponent(redirectUrl)}`);
   }
 
   if (user && !user.is_verified && authTypeMetadata?.requiresVerification) {
diff --git a/web/src/lib/userSS.ts b/web/src/lib/userSS.ts
index c1c5fc9d60e..81261cebea0 100644
--- a/web/src/lib/userSS.ts
+++ b/web/src/lib/userSS.ts
@@ -40,8 +40,12 @@ export const getAuthDisabledSS = async (): Promise<boolean> => {
   return (await getAuthTypeMetadataSS()).authType === "disabled";
 };
 
-const geOIDCAuthUrlSS = async (): Promise<string> => {
-  const res = await fetch(buildUrl("/auth/oidc/authorize"));
+const getOIDCAuthUrlSS = async (nextUrl: string | null): Promise<string> => {
+  const res = await fetch(
+    buildUrl(
+      `/auth/oidc/authorize${nextUrl ? `?next=${encodeURIComponent(nextUrl)}` : ""}`
+    )
+  );
   if (!res.ok) {
     throw new Error("Failed to fetch data");
   }
@@ -51,7 +55,7 @@ const geOIDCAuthUrlSS = async (): Promise<string> => {
 };
 
 const getGoogleOAuthUrlSS = async (): Promise<string> => {
-  const res = await fetch(buildUrl("/auth/oauth/authorize"));
+  const res = await fetch(buildUrl(`/auth/oauth/authorize`));
   if (!res.ok) {
     throw new Error("Failed to fetch data");
   }
@@ -70,7 +74,10 @@ const getSAMLAuthUrlSS = async (): Promise<string> => {
   return data.authorization_url;
 };
 
-export const getAuthUrlSS = async (authType: AuthType): Promise<string> => {
+export const getAuthUrlSS = async (
+  authType: AuthType,
+  nextUrl: string | null
+): Promise<string> => {
   // Returns the auth url for the given auth type
   switch (authType) {
     case "disabled":
@@ -84,7 +91,7 @@ export const getAuthUrlSS = async (authType: AuthType): Promise<string> => {
       return await getSAMLAuthUrlSS();
     }
     case "oidc": {
-      return await geOIDCAuthUrlSS();
+      return await getOIDCAuthUrlSS(nextUrl);
     }
   }
 };

From 0a0215ceee55fdbd1b06f442afd2cdb80a9a839d Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 16 Oct 2024 11:52:27 -0700
Subject: [PATCH 129/376] check last_pruned instead of is_pruning (#2748)

* check last_pruned instead of is_pruning

* try using the ThreadingHTTPServer class for stability and avoiding blocking single-threaded behavior

* add startup delay to web server in test

* just explicitly return None if we can't parse the datetime

* switch to uvicorn for test stability
---
 backend/danswer/server/documents/cc_pair.py   | 12 ++---
 .../common_utils/managers/cc_pair.py          | 23 +++++---
 .../connector_job_tests/slack/test_prune.py   |  3 +-
 .../integration/tests/pruning/test_pruning.py | 54 +++++++++++++++++--
 4 files changed, 74 insertions(+), 18 deletions(-)

diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index d835a25e26e..9cfe72275af 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -1,4 +1,5 @@
 import math
+from datetime import datetime
 from http import HTTPStatus
 
 from fastapi import APIRouter
@@ -204,12 +205,12 @@ def update_cc_pair_name(
         raise HTTPException(status_code=400, detail="Name must be unique")
 
 
-@router.get("/admin/cc-pair/{cc_pair_id}/prune")
-def get_cc_pair_latest_prune(
+@router.get("/admin/cc-pair/{cc_pair_id}/last_pruned")
+def get_cc_pair_last_pruned(
     cc_pair_id: int,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
-) -> bool:
+) -> datetime | None:
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id=cc_pair_id,
         db_session=db_session,
@@ -219,11 +220,10 @@ def get_cc_pair_latest_prune(
     if not cc_pair:
         raise HTTPException(
             status_code=400,
-            detail="Connection not found for current user's permissions",
+            detail="cc_pair not found for current user's permissions",
         )
 
-    rcp = RedisConnectorPruning(cc_pair.id)
-    return rcp.is_pruning(db_session, get_redis_client())
+    return cc_pair.last_pruned
 
 
 @router.post("/admin/cc-pair/{cc_pair_id}/prune")
diff --git a/backend/tests/integration/common_utils/managers/cc_pair.py b/backend/tests/integration/common_utils/managers/cc_pair.py
index 48ae805cbb9..64961b22e3c 100644
--- a/backend/tests/integration/common_utils/managers/cc_pair.py
+++ b/backend/tests/integration/common_utils/managers/cc_pair.py
@@ -274,31 +274,40 @@ def prune(
         result.raise_for_status()
 
     @staticmethod
-    def is_pruning(
+    def last_pruned(
         cc_pair: DATestCCPair,
         user_performing_action: DATestUser | None = None,
-    ) -> bool:
+    ) -> datetime | None:
         response = requests.get(
-            url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair.id}/prune",
+            url=f"{API_SERVER_URL}/manage/admin/cc-pair/{cc_pair.id}/last_pruned",
             headers=user_performing_action.headers
             if user_performing_action
             else GENERAL_HEADERS,
         )
         response.raise_for_status()
-        response_bool = response.json()
-        return response_bool
+        response_str = response.json()
+
+        # If the response itself is a datetime string, parse it
+        if not isinstance(response_str, str):
+            return None
+
+        try:
+            return datetime.fromisoformat(response_str)
+        except ValueError:
+            return None
 
     @staticmethod
     def wait_for_prune(
         cc_pair: DATestCCPair,
+        after: datetime,
         timeout: float = MAX_DELAY,
         user_performing_action: DATestUser | None = None,
     ) -> None:
         """after: The task register time must be after this time."""
         start = time.monotonic()
         while True:
-            result = CCPairManager.is_pruning(cc_pair, user_performing_action)
-            if not result:
+            last_pruned = CCPairManager.last_pruned(cc_pair, user_performing_action)
+            if last_pruned and last_pruned > after:
                 break
 
             elapsed = time.monotonic() - start
diff --git a/backend/tests/integration/connector_job_tests/slack/test_prune.py b/backend/tests/integration/connector_job_tests/slack/test_prune.py
index 96638417c4d..12cf7da7a97 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_prune.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_prune.py
@@ -195,8 +195,9 @@ def test_slack_prune(
     )
 
     # Prune the cc_pair
+    now = datetime.now(timezone.utc)
     CCPairManager.prune(cc_pair, user_performing_action=admin_user)
-    CCPairManager.wait_for_prune(cc_pair, user_performing_action=admin_user)
+    CCPairManager.wait_for_prune(cc_pair, now, user_performing_action=admin_user)
 
     # ----------------------------VERIFY THE CHANGES---------------------------
     # Ensure admin user can't see deleted messages
diff --git a/backend/tests/integration/tests/pruning/test_pruning.py b/backend/tests/integration/tests/pruning/test_pruning.py
index 4c4e82cdfb0..7498f760cef 100644
--- a/backend/tests/integration/tests/pruning/test_pruning.py
+++ b/backend/tests/integration/tests/pruning/test_pruning.py
@@ -10,6 +10,10 @@
 from time import sleep
 from typing import Any
 
+import uvicorn
+from fastapi import FastAPI
+from fastapi.staticfiles import StaticFiles
+
 from danswer.server.documents.models import DocumentSource
 from danswer.utils.logger import setup_logger
 from tests.integration.common_utils.managers.api_key import APIKeyManager
@@ -21,10 +25,50 @@
 logger = setup_logger()
 
 
+# FastAPI server for serving files
+def create_fastapi_app(directory: str) -> FastAPI:
+    app = FastAPI()
+
+    # Mount the directory to serve static files
+    app.mount("/", StaticFiles(directory=directory, html=True), name="static")
+
+    return app
+
+
+# as far as we know, this doesn't hang when crawled. This is good.
+@contextmanager
+def fastapi_server_context(
+    directory: str, port: int = 8000
+) -> Generator[None, None, None]:
+    app = create_fastapi_app(directory)
+
+    config = uvicorn.Config(app=app, host="0.0.0.0", port=port, log_level="info")
+    server = uvicorn.Server(config)
+
+    # Create a thread to run the FastAPI server
+    server_thread = threading.Thread(target=server.run)
+    server_thread.daemon = (
+        True  # Ensures the thread will exit when the main program exits
+    )
+
+    try:
+        # Start the server in the background
+        server_thread.start()
+        sleep(5)  # Give it a few seconds to start
+        yield  # Yield control back to the calling function (context manager in use)
+    finally:
+        # Shutdown the server
+        server.should_exit = True
+        server_thread.join()
+
+
+# Leaving this here for posterity and experimentation, but the reason we're
+# not using this is python's web servers hang frequently when crawled
+# this is obviously not good for a unit test
 @contextmanager
 def http_server_context(
     directory: str, port: int = 8000
-) -> Generator[http.server.HTTPServer, None, None]:
+) -> Generator[http.server.ThreadingHTTPServer, None, None]:
     # Create a handler that serves files from the specified directory
     def handler_class(
         *args: Any, **kwargs: Any
@@ -34,7 +78,7 @@ def handler_class(
         )
 
     # Create an HTTPServer instance
-    httpd = http.server.HTTPServer(("0.0.0.0", port), handler_class)
+    httpd = http.server.ThreadingHTTPServer(("0.0.0.0", port), handler_class)
 
     # Define a thread that runs the server in the background
     server_thread = threading.Thread(target=httpd.serve_forever)
@@ -45,6 +89,7 @@ def handler_class(
     try:
         # Start the server in the background
         server_thread.start()
+        sleep(5)  # give it a few seconds to start
         yield httpd
     finally:
         # Shutdown the server and wait for the thread to finish
@@ -70,7 +115,7 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
         website_src = os.path.join(test_directory, "website")
         website_tgt = os.path.join(temp_dir, "website")
         shutil.copytree(website_src, website_tgt)
-        with http_server_context(os.path.join(temp_dir, "website"), port):
+        with fastapi_server_context(os.path.join(temp_dir, "website"), port):
             sleep(1)  # sleep a tiny bit before starting everything
 
             hostname = os.getenv("TEST_WEB_HOSTNAME", "localhost")
@@ -105,9 +150,10 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
             logger.info("Removing courses.html.")
             os.remove(os.path.join(website_tgt, "courses.html"))
 
+            now = datetime.now(timezone.utc)
             CCPairManager.prune(cc_pair_1, user_performing_action=admin_user)
             CCPairManager.wait_for_prune(
-                cc_pair_1, timeout=60, user_performing_action=admin_user
+                cc_pair_1, now, timeout=60, user_performing_action=admin_user
             )
 
             selected_cc_pair = CCPairManager.get_one(

From f3fb7c572ed9b63a359587079a55f3a02da0ccf9 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 16 Oct 2024 13:21:04 -0700
Subject: [PATCH 130/376] ensure assistant response parsed correctly (#2823)

---
 web/src/app/chat/shared/[chatId]/page.tsx | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/web/src/app/chat/shared/[chatId]/page.tsx b/web/src/app/chat/shared/[chatId]/page.tsx
index d012b4a0d77..2e1b87a5614 100644
--- a/web/src/app/chat/shared/[chatId]/page.tsx
+++ b/web/src/app/chat/shared/[chatId]/page.tsx
@@ -9,7 +9,10 @@ import { redirect } from "next/navigation";
 import { BackendChatSession } from "../../interfaces";
 import { SharedChatDisplay } from "./SharedChatDisplay";
 import { Persona } from "@/app/admin/assistants/interfaces";
-import { fetchAssistantsSS } from "@/lib/assistants/fetchAssistantsSS";
+import {
+  FetchAssistantsResponse,
+  fetchAssistantsSS,
+} from "@/lib/assistants/fetchAssistantsSS";
 import FunctionalHeader from "@/components/chat_search/Header";
 import { defaultPersona } from "@/app/admin/assistants/lib";
 
@@ -44,7 +47,8 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   const authTypeMetadata = results[0] as AuthTypeMetadata | null;
   const user = results[1] as User | null;
   const chatSession = results[2] as BackendChatSession | null;
-  const availableAssistants = results[3] as Persona[] | null;
+  const assistantsResponse = results[3] as FetchAssistantsResponse | null;
+  const [availableAssistants, _] = assistantsResponse ?? [[], null];
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
@@ -54,11 +58,12 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   if (user && !user.is_verified && authTypeMetadata?.requiresVerification) {
     return redirect("/auth/waiting-on-verification");
   }
-  const persona = chatSession?.persona_id
-    ? (availableAssistants?.find((p) => p.id === chatSession.persona_id) ??
-      availableAssistants?.[0] ??
-      null)
-    : (availableAssistants?.[0] ?? defaultPersona);
+
+  const persona: Persona =
+    chatSession?.persona_id && availableAssistants?.length
+      ? (availableAssistants.find((p) => p.id === chatSession.persona_id) ??
+        defaultPersona)
+      : (availableAssistants?.[0] ?? defaultPersona);
 
   return (
     <div>

From db0779dd022e16d9cac0596f558efebcc313c5ed Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 16 Oct 2024 15:18:45 -0700
Subject: [PATCH 131/376] Session id: int -> UUID (#2814)

* session id: int -> UUID

* nit

* validated

* validated downgrade + upgrade + all functionality

* nit

* minor nit

* fix test case
---
 .../6756efa39ada_id_uuid_for_chat_session.py  | 153 ++++++++++++++++++
 backend/danswer/chat/chat_utils.py            |   3 +-
 backend/danswer/db/chat.py                    |  22 +--
 backend/danswer/db/models.py                  |  11 +-
 .../danswer/server/features/folder/models.py  |   4 +-
 .../server/query_and_chat/chat_backend.py     |   7 +-
 .../danswer/server/query_and_chat/models.py   |  22 ++-
 .../server/query_and_chat/query_backend.py    |   4 +-
 backend/danswer/tools/models.py               |   3 +-
 backend/ee/danswer/db/usage_export.py         |   2 +-
 .../danswer/server/query_and_chat/models.py   |   4 +-
 .../ee/danswer/server/query_history/api.py    |   9 +-
 .../server/reporting/usage_export_models.py   |   3 +-
 .../integration/common_utils/managers/chat.py |   3 +-
 .../integration/common_utils/test_models.py   |   4 +-
 .../danswer/tools/custom/test_custom_tools.py |   3 +-
 web/src/app/chat/ChatPage.tsx                 |  44 +++--
 web/src/app/chat/folders/FolderList.tsx       |   9 +-
 web/src/app/chat/folders/FolderManagement.tsx |   4 +-
 web/src/app/chat/input/ChatInputBar.tsx       |   2 +-
 web/src/app/chat/interfaces.ts                |   8 +-
 web/src/app/chat/lib.tsx                      |  15 +-
 .../app/chat/modal/ShareChatSessionModal.tsx  |   8 +-
 .../app/chat/modal/configuration/LlmTab.tsx   |   2 +-
 web/src/app/chat/sessionSidebar/PagesTab.tsx  |   7 +-
 web/src/app/chat/shared/[chatId]/page.tsx     |   5 +-
 web/src/components/search/SearchSection.tsx   |  13 +-
 web/src/lib/chat/fetchChatData.ts             |   6 +-
 web/src/lib/search/interfaces.ts              |   2 +-
 29 files changed, 276 insertions(+), 106 deletions(-)
 create mode 100644 backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py

diff --git a/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py b/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py
new file mode 100644
index 00000000000..057521d0f9c
--- /dev/null
+++ b/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py
@@ -0,0 +1,153 @@
+"""
+Revision ID: 6756efa39ada
+Revises: 5d12a446f5c0
+Create Date: 2024-10-15 17:47:44.108537
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "6756efa39ada"
+down_revision = "5d12a446f5c0"
+branch_labels = None
+depends_on = None
+
+"""
+Migrate chat_session and chat_message tables to use UUID primary keys.
+
+This script:
+1. Adds UUID columns to chat_session and chat_message
+2. Populates new columns with UUIDs
+3. Updates foreign key relationships
+4. Removes old integer ID columns
+
+Note: Downgrade will assign new integer IDs, not restore original ones.
+"""
+
+
+def upgrade() -> None:
+    op.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto;")
+
+    op.add_column(
+        "chat_session",
+        sa.Column(
+            "new_id",
+            sa.UUID(as_uuid=True),
+            server_default=sa.text("gen_random_uuid()"),
+            nullable=False,
+        ),
+    )
+
+    op.execute("UPDATE chat_session SET new_id = gen_random_uuid();")
+
+    op.add_column(
+        "chat_message",
+        sa.Column("new_chat_session_id", sa.UUID(as_uuid=True), nullable=True),
+    )
+
+    op.execute(
+        """
+        UPDATE chat_message
+        SET new_chat_session_id = cs.new_id
+        FROM chat_session cs
+        WHERE chat_message.chat_session_id = cs.id;
+        """
+    )
+
+    op.drop_constraint(
+        "chat_message_chat_session_id_fkey", "chat_message", type_="foreignkey"
+    )
+
+    op.drop_column("chat_message", "chat_session_id")
+    op.alter_column(
+        "chat_message", "new_chat_session_id", new_column_name="chat_session_id"
+    )
+
+    op.drop_constraint("chat_session_pkey", "chat_session", type_="primary")
+    op.drop_column("chat_session", "id")
+    op.alter_column("chat_session", "new_id", new_column_name="id")
+
+    op.create_primary_key("chat_session_pkey", "chat_session", ["id"])
+
+    op.create_foreign_key(
+        "chat_message_chat_session_id_fkey",
+        "chat_message",
+        "chat_session",
+        ["chat_session_id"],
+        ["id"],
+        ondelete="CASCADE",
+    )
+
+
+def downgrade() -> None:
+    op.drop_constraint(
+        "chat_message_chat_session_id_fkey", "chat_message", type_="foreignkey"
+    )
+
+    op.add_column(
+        "chat_session",
+        sa.Column("old_id", sa.Integer, autoincrement=True, nullable=True),
+    )
+
+    op.execute("CREATE SEQUENCE chat_session_old_id_seq OWNED BY chat_session.old_id;")
+    op.execute(
+        "ALTER TABLE chat_session ALTER COLUMN old_id SET DEFAULT nextval('chat_session_old_id_seq');"
+    )
+
+    op.execute(
+        "UPDATE chat_session SET old_id = nextval('chat_session_old_id_seq') WHERE old_id IS NULL;"
+    )
+
+    op.alter_column("chat_session", "old_id", nullable=False)
+
+    op.drop_constraint("chat_session_pkey", "chat_session", type_="primary")
+    op.create_primary_key("chat_session_pkey", "chat_session", ["old_id"])
+
+    op.add_column(
+        "chat_message",
+        sa.Column("old_chat_session_id", sa.Integer, nullable=True),
+    )
+
+    op.execute(
+        """
+        UPDATE chat_message
+        SET old_chat_session_id = cs.old_id
+        FROM chat_session cs
+        WHERE chat_message.chat_session_id = cs.id;
+        """
+    )
+
+    op.drop_column("chat_message", "chat_session_id")
+    op.alter_column(
+        "chat_message", "old_chat_session_id", new_column_name="chat_session_id"
+    )
+
+    op.create_foreign_key(
+        "chat_message_chat_session_id_fkey",
+        "chat_message",
+        "chat_session",
+        ["chat_session_id"],
+        ["old_id"],
+        ondelete="CASCADE",
+    )
+
+    op.drop_column("chat_session", "id")
+    op.alter_column("chat_session", "old_id", new_column_name="id")
+
+    op.alter_column(
+        "chat_session",
+        "id",
+        type_=sa.Integer(),
+        existing_type=sa.Integer(),
+        existing_nullable=False,
+        existing_server_default=False,
+    )
+
+    # Rename the sequence
+    op.execute("ALTER SEQUENCE chat_session_old_id_seq RENAME TO chat_session_id_seq;")
+
+    # Update the default value to use the renamed sequence
+    op.alter_column(
+        "chat_session",
+        "id",
+        server_default=sa.text("nextval('chat_session_id_seq'::regclass)"),
+    )
diff --git a/backend/danswer/chat/chat_utils.py b/backend/danswer/chat/chat_utils.py
index cf0fa28c145..f5961010ba5 100644
--- a/backend/danswer/chat/chat_utils.py
+++ b/backend/danswer/chat/chat_utils.py
@@ -1,5 +1,6 @@
 import re
 from typing import cast
+from uuid import UUID
 
 from fastapi.datastructures import Headers
 from sqlalchemy.orm import Session
@@ -34,7 +35,7 @@ def llm_doc_from_inference_section(inference_section: InferenceSection) -> LlmDo
 
 
 def create_chat_chain(
-    chat_session_id: int,
+    chat_session_id: UUID,
     db_session: Session,
     prefetch_tool_calls: bool = True,
     # Optional id at which we finish processing
diff --git a/backend/danswer/db/chat.py b/backend/danswer/db/chat.py
index feb2e2b4b51..d885e1efd63 100644
--- a/backend/danswer/db/chat.py
+++ b/backend/danswer/db/chat.py
@@ -43,7 +43,7 @@
 
 
 def get_chat_session_by_id(
-    chat_session_id: int,
+    chat_session_id: UUID,
     user_id: UUID | None,
     db_session: Session,
     include_deleted: bool = False,
@@ -87,9 +87,9 @@ def get_chat_sessions_by_slack_thread_id(
 
 
 def get_valid_messages_from_query_sessions(
-    chat_session_ids: list[int],
+    chat_session_ids: list[UUID],
     db_session: Session,
-) -> dict[int, str]:
+) -> dict[UUID, str]:
     user_message_subquery = (
         select(
             ChatMessage.chat_session_id, func.min(ChatMessage.id).label("user_msg_id")
@@ -196,7 +196,7 @@ def delete_orphaned_search_docs(db_session: Session) -> None:
 
 
 def delete_messages_and_files_from_chat_session(
-    chat_session_id: int, db_session: Session
+    chat_session_id: UUID, db_session: Session
 ) -> None:
     # Select messages older than cutoff_time with files
     messages_with_files = db_session.execute(
@@ -253,7 +253,7 @@ def create_chat_session(
 def update_chat_session(
     db_session: Session,
     user_id: UUID | None,
-    chat_session_id: int,
+    chat_session_id: UUID,
     description: str | None = None,
     sharing_status: ChatSessionSharedStatus | None = None,
 ) -> ChatSession:
@@ -276,7 +276,7 @@ def update_chat_session(
 
 def delete_chat_session(
     user_id: UUID | None,
-    chat_session_id: int,
+    chat_session_id: UUID,
     db_session: Session,
     hard_delete: bool = HARD_DELETE_CHATS,
 ) -> None:
@@ -337,7 +337,7 @@ def get_chat_message(
 
 
 def get_chat_messages_by_sessions(
-    chat_session_ids: list[int],
+    chat_session_ids: list[UUID],
     user_id: UUID | None,
     db_session: Session,
     skip_permission_check: bool = False,
@@ -370,7 +370,7 @@ def get_search_docs_for_chat_message(
 
 
 def get_chat_messages_by_session(
-    chat_session_id: int,
+    chat_session_id: UUID,
     user_id: UUID | None,
     db_session: Session,
     skip_permission_check: bool = False,
@@ -397,7 +397,7 @@ def get_chat_messages_by_session(
 
 
 def get_or_create_root_message(
-    chat_session_id: int,
+    chat_session_id: UUID,
     db_session: Session,
 ) -> ChatMessage:
     try:
@@ -433,7 +433,7 @@ def get_or_create_root_message(
 
 def reserve_message_id(
     db_session: Session,
-    chat_session_id: int,
+    chat_session_id: UUID,
     parent_message: int,
     message_type: MessageType,
 ) -> int:
@@ -460,7 +460,7 @@ def reserve_message_id(
 
 
 def create_new_chat_message(
-    chat_session_id: int,
+    chat_session_id: UUID,
     parent_message: ChatMessage,
     message: str,
     prompt_id: int | None,
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index c7cbbe9e8d0..9f8f1e2371a 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -5,9 +5,12 @@
 from typing import Literal
 from typing import NotRequired
 from typing import Optional
+from uuid import uuid4
 from typing_extensions import TypedDict  # noreorder
 from uuid import UUID
 
+from sqlalchemy.dialects.postgresql import UUID as PGUUID
+
 from fastapi_users_db_sqlalchemy import SQLAlchemyBaseOAuthAccountTableUUID
 from fastapi_users_db_sqlalchemy import SQLAlchemyBaseUserTableUUID
 from fastapi_users_db_sqlalchemy.access_token import SQLAlchemyBaseAccessTokenTableUUID
@@ -920,7 +923,9 @@ class ToolCall(Base):
 class ChatSession(Base):
     __tablename__ = "chat_session"
 
-    id: Mapped[int] = mapped_column(primary_key=True)
+    id: Mapped[UUID] = mapped_column(
+        PGUUID(as_uuid=True), primary_key=True, default=uuid4
+    )
     user_id: Mapped[UUID | None] = mapped_column(
         ForeignKey("user.id", ondelete="CASCADE"), nullable=True
     )
@@ -990,7 +995,9 @@ class ChatMessage(Base):
     __tablename__ = "chat_message"
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    chat_session_id: Mapped[int] = mapped_column(ForeignKey("chat_session.id"))
+    chat_session_id: Mapped[UUID] = mapped_column(
+        PGUUID(as_uuid=True), ForeignKey("chat_session.id")
+    )
 
     alternate_assistant_id = mapped_column(
         Integer, ForeignKey("persona.id"), nullable=True
diff --git a/backend/danswer/server/features/folder/models.py b/backend/danswer/server/features/folder/models.py
index d7b161414a3..3f7e1304cbc 100644
--- a/backend/danswer/server/features/folder/models.py
+++ b/backend/danswer/server/features/folder/models.py
@@ -1,3 +1,5 @@
+from uuid import UUID
+
 from pydantic import BaseModel
 
 from danswer.server.query_and_chat.models import ChatSessionDetails
@@ -23,7 +25,7 @@ class FolderUpdateRequest(BaseModel):
 
 
 class FolderChatSessionRequest(BaseModel):
-    chat_session_id: int
+    chat_session_id: UUID
 
 
 class DeleteFolderOptions(BaseModel):
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index c26bc9c5c8b..5b0cc30a1b4 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -4,6 +4,7 @@
 from collections.abc import Callable
 from collections.abc import Generator
 from typing import Tuple
+from uuid import UUID
 
 from fastapi import APIRouter
 from fastapi import Depends
@@ -131,7 +132,7 @@ def update_chat_session_model(
 
 @router.get("/get-chat-session/{session_id}")
 def get_chat_session(
-    session_id: int,
+    session_id: UUID,
     is_shared: bool = False,
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
@@ -254,7 +255,7 @@ def rename_chat_session(
 
 @router.patch("/chat-session/{session_id}")
 def patch_chat_session(
-    session_id: int,
+    session_id: UUID,
     chat_session_update_req: ChatSessionUpdateRequest,
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
@@ -271,7 +272,7 @@ def patch_chat_session(
 
 @router.delete("/delete-chat-session/{session_id}")
 def delete_chat_session_by_id(
-    session_id: int,
+    session_id: UUID,
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> None:
diff --git a/backend/danswer/server/query_and_chat/models.py b/backend/danswer/server/query_and_chat/models.py
index c9109b141c3..42f4100a24b 100644
--- a/backend/danswer/server/query_and_chat/models.py
+++ b/backend/danswer/server/query_and_chat/models.py
@@ -1,5 +1,6 @@
 from datetime import datetime
 from typing import Any
+from uuid import UUID
 
 from pydantic import BaseModel
 from pydantic import model_validator
@@ -34,7 +35,7 @@ class SimpleQueryRequest(BaseModel):
 
 class UpdateChatSessionThreadRequest(BaseModel):
     # If not specified, use Danswer default persona
-    chat_session_id: int
+    chat_session_id: UUID
     new_alternate_model: str
 
 
@@ -45,7 +46,7 @@ class ChatSessionCreationRequest(BaseModel):
 
 
 class CreateChatSessionID(BaseModel):
-    chat_session_id: int
+    chat_session_id: UUID
 
 
 class ChatFeedbackRequest(BaseModel):
@@ -75,7 +76,7 @@ def check_is_positive_or_feedback_text(self) -> "ChatFeedbackRequest":
 class CreateChatMessageRequest(ChunkContext):
     """Before creating messages, be sure to create a chat_session and get an id"""
 
-    chat_session_id: int
+    chat_session_id: UUID
     # This is the primary-key (unique identifier) for the previous message of the tree
     parent_message_id: int | None
     # New message contents
@@ -115,13 +116,18 @@ def check_search_doc_ids_or_retrieval_options(self) -> "CreateChatMessageRequest
             )
         return self
 
+    def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        data = super().model_dump(*args, **kwargs)
+        data["chat_session_id"] = str(data["chat_session_id"])
+        return data
+
 
 class ChatMessageIdentifier(BaseModel):
     message_id: int
 
 
 class ChatRenameRequest(BaseModel):
-    chat_session_id: int
+    chat_session_id: UUID
     name: str | None = None
 
 
@@ -134,7 +140,7 @@ class RenameChatSessionResponse(BaseModel):
 
 
 class ChatSessionDetails(BaseModel):
-    id: int
+    id: UUID
     name: str
     persona_id: int | None = None
     time_created: str
@@ -175,7 +181,7 @@ class ChatMessageDetail(BaseModel):
     overridden_model: str | None
     alternate_assistant_id: int | None = None
     # Dict mapping citation number to db_doc_id
-    chat_session_id: int | None = None
+    chat_session_id: UUID | None = None
     citations: dict[int, int] | None = None
     files: list[FileDescriptor]
     tool_calls: list[ToolCallFinalResult]
@@ -187,14 +193,14 @@ def model_dump(self, *args: list, **kwargs: dict[str, Any]) -> dict[str, Any]:
 
 
 class SearchSessionDetailResponse(BaseModel):
-    search_session_id: int
+    search_session_id: UUID
     description: str
     documents: list[SearchDoc]
     messages: list[ChatMessageDetail]
 
 
 class ChatSessionDetailResponse(BaseModel):
-    chat_session_id: int
+    chat_session_id: UUID
     description: str
     persona_id: int | None = None
     persona_name: str | None
diff --git a/backend/danswer/server/query_and_chat/query_backend.py b/backend/danswer/server/query_and_chat/query_backend.py
index 96f674276f4..703ef2c0475 100644
--- a/backend/danswer/server/query_and_chat/query_backend.py
+++ b/backend/danswer/server/query_and_chat/query_backend.py
@@ -1,3 +1,5 @@
+from uuid import UUID
+
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
@@ -186,7 +188,7 @@ def get_user_search_sessions(
 
 @basic_router.get("/search-session/{session_id}")
 def get_search_session(
-    session_id: int,
+    session_id: UUID,
     is_shared: bool = False,
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
diff --git a/backend/danswer/tools/models.py b/backend/danswer/tools/models.py
index 6317a95e2d3..4f56aecd372 100644
--- a/backend/danswer/tools/models.py
+++ b/backend/danswer/tools/models.py
@@ -1,4 +1,5 @@
 from typing import Any
+from uuid import UUID
 
 from pydantic import BaseModel
 from pydantic import model_validator
@@ -40,7 +41,7 @@ class ToolCallFinalResult(ToolCallKickoff):
 
 
 class DynamicSchemaInfo(BaseModel):
-    chat_session_id: int | None
+    chat_session_id: UUID | None
     message_id: int | None
 
 
diff --git a/backend/ee/danswer/db/usage_export.py b/backend/ee/danswer/db/usage_export.py
index bf53362e97e..ac9535bad4f 100644
--- a/backend/ee/danswer/db/usage_export.py
+++ b/backend/ee/danswer/db/usage_export.py
@@ -42,7 +42,7 @@ def get_empty_chat_messages_entries__paginated(
 
             message_skeletons.append(
                 ChatMessageSkeleton(
-                    message_id=chat_session.id,
+                    message_id=message.id,
                     chat_session_id=chat_session.id,
                     user_id=str(chat_session.user_id) if chat_session.user_id else None,
                     flow_type=flow_type,
diff --git a/backend/ee/danswer/server/query_and_chat/models.py b/backend/ee/danswer/server/query_and_chat/models.py
index ec9db73ecff..052be683e9e 100644
--- a/backend/ee/danswer/server/query_and_chat/models.py
+++ b/backend/ee/danswer/server/query_and_chat/models.py
@@ -1,3 +1,5 @@
+from uuid import UUID
+
 from pydantic import BaseModel
 from pydantic import Field
 
@@ -36,7 +38,7 @@ class BasicCreateChatMessageRequest(ChunkContext):
     Note, for simplicity this option only allows for a single linear chain of messages
     """
 
-    chat_session_id: int
+    chat_session_id: UUID
     # New message contents
     message: str
     # Defaults to using retrieval with no additional filters
diff --git a/backend/ee/danswer/server/query_history/api.py b/backend/ee/danswer/server/query_history/api.py
index 3fc0a98153a..1411f973e36 100644
--- a/backend/ee/danswer/server/query_history/api.py
+++ b/backend/ee/danswer/server/query_history/api.py
@@ -4,6 +4,7 @@
 from datetime import timedelta
 from datetime import timezone
 from typing import Literal
+from uuid import UUID
 
 from fastapi import APIRouter
 from fastapi import Depends
@@ -83,7 +84,7 @@ def build(cls, message: ChatMessage) -> "MessageSnapshot":
 
 
 class ChatSessionMinimal(BaseModel):
-    id: int
+    id: UUID
     user_email: str
     name: str | None
     first_user_message: str
@@ -95,7 +96,7 @@ class ChatSessionMinimal(BaseModel):
 
 
 class ChatSessionSnapshot(BaseModel):
-    id: int
+    id: UUID
     user_email: str
     name: str | None
     messages: list[MessageSnapshot]
@@ -105,7 +106,7 @@ class ChatSessionSnapshot(BaseModel):
 
 
 class QuestionAnswerPairSnapshot(BaseModel):
-    chat_session_id: int
+    chat_session_id: UUID
     # 1-indexed message number in the chat_session
     # e.g. the first message pair in the chat_session is 1, the second is 2, etc.
     message_pair_num: int
@@ -350,7 +351,7 @@ def get_chat_session_history(
 
 @router.get("/admin/chat-session-history/{chat_session_id}")
 def get_chat_session_admin(
-    chat_session_id: int,
+    chat_session_id: UUID,
     _: User | None = Depends(current_admin_user),
     db_session: Session = Depends(get_session),
 ) -> ChatSessionSnapshot:
diff --git a/backend/ee/danswer/server/reporting/usage_export_models.py b/backend/ee/danswer/server/reporting/usage_export_models.py
index 98d9021f816..21cd104e862 100644
--- a/backend/ee/danswer/server/reporting/usage_export_models.py
+++ b/backend/ee/danswer/server/reporting/usage_export_models.py
@@ -1,5 +1,6 @@
 from datetime import datetime
 from enum import Enum
+from uuid import UUID
 
 from pydantic import BaseModel
 
@@ -14,7 +15,7 @@ class FlowType(str, Enum):
 
 class ChatMessageSkeleton(BaseModel):
     message_id: int
-    chat_session_id: int
+    chat_session_id: UUID
     user_id: str | None
     flow_type: FlowType
     time_sent: datetime
diff --git a/backend/tests/integration/common_utils/managers/chat.py b/backend/tests/integration/common_utils/managers/chat.py
index 696baa2ad8b..a8643e9e83d 100644
--- a/backend/tests/integration/common_utils/managers/chat.py
+++ b/backend/tests/integration/common_utils/managers/chat.py
@@ -1,4 +1,5 @@
 import json
+from uuid import UUID
 
 import requests
 from requests.models import Response
@@ -44,7 +45,7 @@ def create(
 
     @staticmethod
     def send_message(
-        chat_session_id: int,
+        chat_session_id: UUID,
         message: str,
         parent_message_id: int | None = None,
         user_performing_action: DATestUser | None = None,
diff --git a/backend/tests/integration/common_utils/test_models.py b/backend/tests/integration/common_utils/test_models.py
index ca573663e72..af7cd882a68 100644
--- a/backend/tests/integration/common_utils/test_models.py
+++ b/backend/tests/integration/common_utils/test_models.py
@@ -123,14 +123,14 @@ class DATestPersona(BaseModel):
 
 #
 class DATestChatSession(BaseModel):
-    id: int
+    id: UUID
     persona_id: int
     description: str
 
 
 class DATestChatMessage(BaseModel):
     id: str | None = None
-    chat_session_id: int
+    chat_session_id: UUID
     parent_message_id: str | None
     message: str
     response: str
diff --git a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
index fcc48b98d21..5b07e21bb83 100644
--- a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
+++ b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
@@ -1,4 +1,5 @@
 import unittest
+import uuid
 from typing import Any
 from unittest.mock import patch
 
@@ -73,7 +74,7 @@ def setUp(self) -> None:
         }
         validate_openapi_schema(self.openapi_schema)
         self.dynamic_schema_info: DynamicSchemaInfo = DynamicSchemaInfo(
-            chat_session_id=10, message_id=20
+            chat_session_id=uuid.uuid4(), message_id=20
         )
 
     @patch("danswer.tools.custom.custom_tool.requests.request")
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index f218f7fab95..bed27f6a19d 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -145,19 +145,17 @@ export function ChatPage({
   const existingChatIdRaw = searchParams.get("chatId");
   const currentPersonaId = searchParams.get(SEARCH_PARAM_NAMES.PERSONA_ID);
 
-  const existingChatSessionId = existingChatIdRaw
-    ? parseInt(existingChatIdRaw)
-    : null;
+  const existingChatSessionId = existingChatIdRaw ? existingChatIdRaw : null;
 
   const selectedChatSession = chatSessions.find(
     (chatSession) => chatSession.id === existingChatSessionId
   );
 
-  const chatSessionIdRef = useRef<number | null>(existingChatSessionId);
+  const chatSessionIdRef = useRef<string | null>(existingChatSessionId);
 
   // Only updates on session load (ie. rename / switching chat session)
   // Useful for determining which session has been loaded (i.e. still on `new, empty session` or `previous session`)
-  const loadedIdSessionRef = useRef<number | null>(existingChatSessionId);
+  const loadedIdSessionRef = useRef<string | null>(existingChatSessionId);
 
   // Assistants in order
   const { finalAssistants } = useMemo(() => {
@@ -448,11 +446,11 @@ export function ChatPage({
   );
 
   const [completeMessageDetail, setCompleteMessageDetail] = useState<
-    Map<number | null, Map<number, Message>>
+    Map<string | null, Map<number, Message>>
   >(new Map());
 
   const updateCompleteMessageDetail = (
-    sessionId: number | null,
+    sessionId: string | null,
     messageMap: Map<number, Message>
   ) => {
     setCompleteMessageDetail((prevState) => {
@@ -463,13 +461,13 @@ export function ChatPage({
   };
 
   const currentMessageMap = (
-    messageDetail: Map<number | null, Map<number, Message>>
+    messageDetail: Map<string | null, Map<number, Message>>
   ) => {
     return (
       messageDetail.get(chatSessionIdRef.current) || new Map<number, Message>()
     );
   };
-  const currentSessionId = (): number => {
+  const currentSessionId = (): string => {
     return chatSessionIdRef.current!;
   };
 
@@ -484,7 +482,7 @@ export function ChatPage({
     // if calling this function repeatedly with short delay, stay may not update in time
     // and result in weird behavipr
     completeMessageMapOverride?: Map<number, Message> | null;
-    chatSessionId?: number;
+    chatSessionId?: string;
     replacementsMap?: Map<number, number> | null;
     makeLatestChildMessage?: boolean;
   }) => {
@@ -559,23 +557,23 @@ export function ChatPage({
 
   const [submittedMessage, setSubmittedMessage] = useState("");
 
-  const [chatState, setChatState] = useState<Map<number | null, ChatState>>(
+  const [chatState, setChatState] = useState<Map<string | null, ChatState>>(
     new Map([[chatSessionIdRef.current, "input"]])
   );
 
   const [regenerationState, setRegenerationState] = useState<
-    Map<number | null, RegenerationState | null>
+    Map<string | null, RegenerationState | null>
   >(new Map([[null, null]]));
 
   const [abortControllers, setAbortControllers] = useState<
-    Map<number | null, AbortController>
+    Map<string | null, AbortController>
   >(new Map());
 
   // Updates "null" session values to new session id for
   // regeneration, chat, and abort controller state, messagehistory
-  const updateStatesWithNewSessionId = (newSessionId: number) => {
+  const updateStatesWithNewSessionId = (newSessionId: string) => {
     const updateState = (
-      setState: Dispatch<SetStateAction<Map<number | null, any>>>,
+      setState: Dispatch<SetStateAction<Map<string | null, any>>>,
       defaultValue?: any
     ) => {
       setState((prevState) => {
@@ -610,7 +608,7 @@ export function ChatPage({
     chatSessionIdRef.current = newSessionId;
   };
 
-  const updateChatState = (newState: ChatState, sessionId?: number | null) => {
+  const updateChatState = (newState: ChatState, sessionId?: string | null) => {
     setChatState((prevState) => {
       const newChatState = new Map(prevState);
       newChatState.set(
@@ -635,7 +633,7 @@ export function ChatPage({
 
   const updateRegenerationState = (
     newState: RegenerationState | null,
-    sessionId?: number | null
+    sessionId?: string | null
   ) => {
     setRegenerationState((prevState) => {
       const newRegenerationState = new Map(prevState);
@@ -647,18 +645,18 @@ export function ChatPage({
     });
   };
 
-  const resetRegenerationState = (sessionId?: number | null) => {
+  const resetRegenerationState = (sessionId?: string | null) => {
     updateRegenerationState(null, sessionId);
   };
 
   const currentRegenerationState = (): RegenerationState | null => {
     return regenerationState.get(currentSessionId()) || null;
   };
-  const [canContinue, setCanContinue] = useState<Map<number | null, boolean>>(
+  const [canContinue, setCanContinue] = useState<Map<string | null, boolean>>(
     new Map([[null, false]])
   );
 
-  const updateCanContinue = (newState: boolean, sessionId?: number | null) => {
+  const updateCanContinue = (newState: boolean, sessionId?: string | null) => {
     setCanContinue((prevState) => {
       const newCanContinueState = new Map(prevState);
       newCanContinueState.set(
@@ -1003,7 +1001,7 @@ export function ChatPage({
 
     setAlternativeGeneratingAssistant(alternativeAssistantOverride);
     clientScrollToBottom();
-    let currChatSessionId: number;
+    let currChatSessionId: string;
     const isNewSession = chatSessionIdRef.current === null;
     const searchParamBasedChatSessionName =
       searchParams.get(SEARCH_PARAM_NAMES.TITLE) || null;
@@ -1014,7 +1012,7 @@ export function ChatPage({
         searchParamBasedChatSessionName
       );
     } else {
-      currChatSessionId = chatSessionIdRef.current as number;
+      currChatSessionId = chatSessionIdRef.current as string;
     }
     frozenSessionId = currChatSessionId;
 
@@ -1598,7 +1596,7 @@ export function ChatPage({
   }
 
   const [visibleRange, setVisibleRange] = useState<
-    Map<number | null, VisibleRange>
+    Map<string | null, VisibleRange>
   >(() => {
     const initialRange: VisibleRange = {
       start: 0,
diff --git a/web/src/app/chat/folders/FolderList.tsx b/web/src/app/chat/folders/FolderList.tsx
index 01e69b3a1a4..047cd1e9b32 100644
--- a/web/src/app/chat/folders/FolderList.tsx
+++ b/web/src/app/chat/folders/FolderList.tsx
@@ -30,7 +30,7 @@ const FolderItem = ({
   initiallySelected,
 }: {
   folder: Folder;
-  currentChatId?: number;
+  currentChatId?: string;
   isInitiallyExpanded: boolean;
   initiallySelected: boolean;
 }) => {
@@ -145,10 +145,7 @@ const FolderItem = ({
   const handleDrop = async (event: React.DragEvent<HTMLDivElement>) => {
     event.preventDefault();
     setIsDragOver(false);
-    const chatSessionId = parseInt(
-      event.dataTransfer.getData(CHAT_SESSION_ID_KEY),
-      10
-    );
+    const chatSessionId = event.dataTransfer.getData(CHAT_SESSION_ID_KEY);
     try {
       await addChatToFolder(folder.folder_id, chatSessionId);
       router.refresh(); // Refresh to show the updated folder contents
@@ -302,7 +299,7 @@ export const FolderList = ({
   newFolderId,
 }: {
   folders: Folder[];
-  currentChatId?: number;
+  currentChatId?: string;
   openedFolders?: { [key: number]: boolean };
   newFolderId: number | null;
 }) => {
diff --git a/web/src/app/chat/folders/FolderManagement.tsx b/web/src/app/chat/folders/FolderManagement.tsx
index 7e5abca0a42..417bb903f65 100644
--- a/web/src/app/chat/folders/FolderManagement.tsx
+++ b/web/src/app/chat/folders/FolderManagement.tsx
@@ -17,7 +17,7 @@ export async function createFolder(folderName: string): Promise<number> {
 // Function to add a chat session to a folder
 export async function addChatToFolder(
   folderId: number,
-  chatSessionId: number
+  chatSessionId: string
 ): Promise<void> {
   const response = await fetch(`/api/folder/${folderId}/add-chat-session`, {
     method: "POST",
@@ -34,7 +34,7 @@ export async function addChatToFolder(
 // Function to remove a chat session from a folder
 export async function removeChatFromFolder(
   folderId: number,
-  chatSessionId: number
+  chatSessionId: string
 ): Promise<void> {
   const response = await fetch(`/api/folder/${folderId}/remove-chat-session`, {
     method: "POST",
diff --git a/web/src/app/chat/input/ChatInputBar.tsx b/web/src/app/chat/input/ChatInputBar.tsx
index cce1bba53d9..e6e220f9ff1 100644
--- a/web/src/app/chat/input/ChatInputBar.tsx
+++ b/web/src/app/chat/input/ChatInputBar.tsx
@@ -86,7 +86,7 @@ export function ChatInputBar({
   setFiles: (files: FileDescriptor[]) => void;
   handleFileUpload: (files: File[]) => void;
   textAreaRef: React.RefObject<HTMLTextAreaElement>;
-  chatSessionId?: number;
+  chatSessionId?: string;
   refreshUser: () => void;
 }) {
   useEffect(() => {
diff --git a/web/src/app/chat/interfaces.ts b/web/src/app/chat/interfaces.ts
index dfc24aaa692..5e25566a7d5 100644
--- a/web/src/app/chat/interfaces.ts
+++ b/web/src/app/chat/interfaces.ts
@@ -60,7 +60,7 @@ export interface ToolCallFinalResult {
 }
 
 export interface ChatSession {
-  id: number;
+  id: string;
   name: string;
   persona_id: number;
   time_created: string;
@@ -70,7 +70,7 @@ export interface ChatSession {
 }
 
 export interface SearchSession {
-  search_session_id: number;
+  search_session_id: string;
   documents: SearchDanswerDocument[];
   messages: BackendMessage[];
   description: string;
@@ -97,7 +97,7 @@ export interface Message {
 }
 
 export interface BackendChatSession {
-  chat_session_id: number;
+  chat_session_id: string;
   description: string;
   persona_id: number;
   persona_name: string;
@@ -110,7 +110,7 @@ export interface BackendChatSession {
 export interface BackendMessage {
   message_id: number;
   comments: any;
-  chat_session_id: number;
+  chat_session_id: string;
   parent_message: number | null;
   latest_child_message: number | null;
   message: string;
diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx
index 7ce86ac2c61..3fbb9397abc 100644
--- a/web/src/app/chat/lib.tsx
+++ b/web/src/app/chat/lib.tsx
@@ -55,7 +55,7 @@ export function getChatRetentionInfo(
 }
 
 export async function updateModelOverrideForChatSession(
-  chatSessionId: number,
+  chatSessionId: string,
   newAlternateModel: string
 ) {
   const response = await fetch("/api/chat/update-chat-session-model", {
@@ -74,7 +74,7 @@ export async function updateModelOverrideForChatSession(
 export async function createChatSession(
   personaId: number,
   description: string | null
-): Promise<number> {
+): Promise<string> {
   const createChatSessionResponse = await fetch(
     "/api/chat/create-chat-session",
     {
@@ -131,7 +131,7 @@ export async function* sendMessage({
   message: string;
   fileDescriptors: FileDescriptor[];
   parentMessageId: number | null;
-  chatSessionId: number;
+  chatSessionId: string;
   promptId: number | null | undefined;
   filters: Filters | null;
   selectedDocumentIds: number[] | null;
@@ -203,7 +203,7 @@ export async function* sendMessage({
   yield* handleSSEStream<PacketType>(response);
 }
 
-export async function nameChatSession(chatSessionId: number, message: string) {
+export async function nameChatSession(chatSessionId: string, message: string) {
   const response = await fetch("/api/chat/rename-chat-session", {
     method: "PUT",
     headers: {
@@ -252,7 +252,7 @@ export async function handleChatFeedback(
   return response;
 }
 export async function renameChatSession(
-  chatSessionId: number,
+  chatSessionId: string,
   newName: string
 ) {
   const response = await fetch(`/api/chat/rename-chat-session`, {
@@ -269,7 +269,7 @@ export async function renameChatSession(
   return response;
 }
 
-export async function deleteChatSession(chatSessionId: number) {
+export async function deleteChatSession(chatSessionId: string) {
   const response = await fetch(
     `/api/chat/delete-chat-session/${chatSessionId}`,
     {
@@ -348,6 +348,7 @@ export function getCitedDocumentsFromMessage(message: Message) {
 }
 
 export function groupSessionsByDateRange(chatSessions: ChatSession[]) {
+  console.log(chatSessions);
   const today = new Date();
   today.setHours(0, 0, 0, 0); // Set to start of today for accurate comparison
 
@@ -584,7 +585,7 @@ const PARAMS_TO_SKIP = [
 
 export function buildChatUrl(
   existingSearchParams: ReadonlyURLSearchParams,
-  chatSessionId: number | null,
+  chatSessionId: string | null,
   personaId: number | null,
   search?: boolean
 ) {
diff --git a/web/src/app/chat/modal/ShareChatSessionModal.tsx b/web/src/app/chat/modal/ShareChatSessionModal.tsx
index e220fdb6d22..16a9147b52a 100644
--- a/web/src/app/chat/modal/ShareChatSessionModal.tsx
+++ b/web/src/app/chat/modal/ShareChatSessionModal.tsx
@@ -6,12 +6,12 @@ import { ChatSessionSharedStatus } from "../interfaces";
 import { FiCopy } from "react-icons/fi";
 import { CopyButton } from "@/components/CopyButton";
 
-function buildShareLink(chatSessionId: number) {
+function buildShareLink(chatSessionId: string) {
   const baseUrl = `${window.location.protocol}//${window.location.host}`;
   return `${baseUrl}/chat/shared/${chatSessionId}`;
 }
 
-async function generateShareLink(chatSessionId: number) {
+async function generateShareLink(chatSessionId: string) {
   const response = await fetch(`/api/chat/chat-session/${chatSessionId}`, {
     method: "PATCH",
     headers: {
@@ -26,7 +26,7 @@ async function generateShareLink(chatSessionId: number) {
   return null;
 }
 
-async function deleteShareLink(chatSessionId: number) {
+async function deleteShareLink(chatSessionId: string) {
   const response = await fetch(`/api/chat/chat-session/${chatSessionId}`, {
     method: "PATCH",
     headers: {
@@ -44,7 +44,7 @@ export function ShareChatSessionModal({
   onShare,
   onClose,
 }: {
-  chatSessionId: number;
+  chatSessionId: string;
   existingSharedStatus: ChatSessionSharedStatus;
   onShare?: (shared: boolean) => void;
   onClose: () => void;
diff --git a/web/src/app/chat/modal/configuration/LlmTab.tsx b/web/src/app/chat/modal/configuration/LlmTab.tsx
index 2cd40290db4..2d05e12b222 100644
--- a/web/src/app/chat/modal/configuration/LlmTab.tsx
+++ b/web/src/app/chat/modal/configuration/LlmTab.tsx
@@ -14,7 +14,7 @@ interface LlmTabProps {
   llmOverrideManager: LlmOverrideManager;
   currentLlm: string;
   openModelSettings: () => void;
-  chatSessionId?: number;
+  chatSessionId?: string;
   close: () => void;
   currentAssistant: Persona;
 }
diff --git a/web/src/app/chat/sessionSidebar/PagesTab.tsx b/web/src/app/chat/sessionSidebar/PagesTab.tsx
index 8477e197d10..be8de43ee23 100644
--- a/web/src/app/chat/sessionSidebar/PagesTab.tsx
+++ b/web/src/app/chat/sessionSidebar/PagesTab.tsx
@@ -23,7 +23,7 @@ export function PagesTab({
 }: {
   page: pageType;
   existingChats?: ChatSession[];
-  currentChatId?: number;
+  currentChatId?: string;
   folders?: Folder[];
   openedFolders?: { [key: number]: boolean };
   closeSidebar?: () => void;
@@ -44,10 +44,7 @@ export function PagesTab({
   ) => {
     event.preventDefault();
     setIsDragOver(false); // Reset drag over state on drop
-    const chatSessionId = parseInt(
-      event.dataTransfer.getData(CHAT_SESSION_ID_KEY),
-      10
-    );
+    const chatSessionId = event.dataTransfer.getData(CHAT_SESSION_ID_KEY);
     const folderId = event.dataTransfer.getData(FOLDER_ID_KEY);
 
     if (folderId) {
diff --git a/web/src/app/chat/shared/[chatId]/page.tsx b/web/src/app/chat/shared/[chatId]/page.tsx
index 2e1b87a5614..e620e85cded 100644
--- a/web/src/app/chat/shared/[chatId]/page.tsx
+++ b/web/src/app/chat/shared/[chatId]/page.tsx
@@ -48,7 +48,7 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   const user = results[1] as User | null;
   const chatSession = results[2] as BackendChatSession | null;
   const assistantsResponse = results[3] as FetchAssistantsResponse | null;
-  const [availableAssistants, _] = assistantsResponse ?? [[], null];
+  const [availableAssistants, error] = assistantsResponse ?? [[], null];
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
@@ -58,7 +58,6 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   if (user && !user.is_verified && authTypeMetadata?.requiresVerification) {
     return redirect("/auth/waiting-on-verification");
   }
-
   const persona: Persona =
     chatSession?.persona_id && availableAssistants?.length
       ? (availableAssistants.find((p) => p.id === chatSession.persona_id) ??
@@ -72,7 +71,7 @@ export default async function Page({ params }: { params: { chatId: string } }) {
       </div>
 
       <div className="flex relative bg-background text-default overflow-hidden pt-16 h-screen">
-        <SharedChatDisplay chatSession={chatSession} persona={persona!} />
+        <SharedChatDisplay chatSession={chatSession} persona={persona} />
       </div>
     </div>
   );
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 8ff4fdd3122..b1215a3708a 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -168,13 +168,10 @@ export const SearchSection = ({
     });
 
   const searchParams = useSearchParams();
-  const existingSearchIdRaw = searchParams.get("searchId");
-  const existingSearchessionId = existingSearchIdRaw
-    ? parseInt(existingSearchIdRaw)
-    : null;
+  const existingSearchessionId = searchParams.get("searchId");
 
   useEffect(() => {
-    if (existingSearchIdRaw == null) {
+    if (existingSearchessionId == null) {
       return;
     }
     function extractFirstMessageByType(
@@ -207,7 +204,7 @@ export const SearchSection = ({
           quotes: null,
           selectedDocIndices: null,
           error: null,
-          messageId: existingSearchIdRaw ? parseInt(existingSearchIdRaw) : null,
+          messageId: searchSession.messages[0].message_id,
           suggestedFlowType: null,
           additional_relevance: undefined,
         };
@@ -219,7 +216,7 @@ export const SearchSection = ({
       }
     }
     initialSessionFetch();
-  }, [existingSearchessionId, existingSearchIdRaw]);
+  }, [existingSearchessionId]);
 
   // Overrides for default behavior that only last a single query
   const [defaultOverrides, setDefaultOverrides] =
@@ -328,7 +325,7 @@ export const SearchSection = ({
   };
   const updateMessageAndThreadId = (
     messageId: number,
-    chat_session_id: number
+    chat_session_id: string
   ) => {
     setSearchResponse((prevState) => ({
       ...(prevState || initialSearchResponse),
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index c4188720689..82b10887dfa 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -136,8 +136,10 @@ export async function fetchChatData(searchParams: {
     );
   }
 
-  // Larger ID -> created later
-  chatSessions.sort((a, b) => (a.id > b.id ? -1 : 1));
+  chatSessions.sort(
+    (a, b) =>
+      new Date(b.time_created).getTime() - new Date(a.time_created).getTime()
+  );
 
   let documentSets: DocumentSet[] = [];
   if (documentSetsResponse?.ok) {
diff --git a/web/src/lib/search/interfaces.ts b/web/src/lib/search/interfaces.ts
index 6983bd3367f..1129bc67205 100644
--- a/web/src/lib/search/interfaces.ts
+++ b/web/src/lib/search/interfaces.ts
@@ -152,7 +152,7 @@ export interface SearchRequestArgs {
   updateError: (error: string) => void;
   updateMessageAndThreadId: (
     messageId: number,
-    chat_session_id: number
+    chat_session_id: string
   ) => void;
   finishedSearching: () => void;
   updateComments: (comments: any) => void;

From 33974fc12cd89b84a3cfede405724a47cc12bec7 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Wed, 16 Oct 2024 15:50:16 -0700
Subject: [PATCH 132/376] Add support for passthrough auth for custom tool
 calls (#2824)

* Add support for passthrough auth for custom tool calls

* Fix formatting
---
 backend/danswer/chat/process_message.py       | 14 +++-
 backend/danswer/configs/model_configs.py      | 16 ----
 backend/danswer/configs/tool_configs.py       | 22 ++++++
 backend/danswer/db/models.py                  |  3 +-
 backend/danswer/db/tools.py                   |  6 +-
 backend/danswer/llm/factory.py                |  2 +-
 backend/danswer/llm/headers.py                | 12 ---
 .../server/query_and_chat/chat_backend.py     |  6 +-
 backend/danswer/tools/custom/custom_tool.py   | 17 ++--
 .../tools/images/image_generation_tool.py     |  2 +-
 backend/danswer/utils/headers.py              | 79 +++++++++++++++++++
 .../danswer/tools/custom/test_custom_tools.py |  5 +-
 12 files changed, 134 insertions(+), 50 deletions(-)
 create mode 100644 backend/danswer/configs/tool_configs.py
 delete mode 100644 backend/danswer/llm/headers.py
 create mode 100644 backend/danswer/utils/headers.py

diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index 244aeb2b70f..ea4e7be93d4 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -105,6 +105,7 @@
 from danswer.tools.tool_runner import ToolCallFinalResult
 from danswer.tools.utils import compute_all_tool_tokens
 from danswer.tools.utils import explicit_tool_calling_supported
+from danswer.utils.headers import header_dict_to_header_list
 from danswer.utils.logger import setup_logger
 from danswer.utils.timing import log_generator_function_time
 
@@ -276,7 +277,7 @@ def stream_chat_message_objects(
     # on the `new_msg_req.message`. Currently, requires a state where the last message is a
     use_existing_user_message: bool = False,
     litellm_additional_headers: dict[str, str] | None = None,
-    tool_additional_headers: dict[str, str] | None = None,
+    custom_tool_additional_headers: dict[str, str] | None = None,
     is_connected: Callable[[], bool] | None = None,
     enforce_chat_session_id_for_search_docs: bool = True,
 ) -> ChatPacketStream:
@@ -640,7 +641,12 @@ def stream_chat_message_objects(
                             chat_session_id=chat_session_id,
                             message_id=user_message.id if user_message else None,
                         ),
-                        custom_headers=db_tool_model.custom_headers,
+                        custom_headers=(db_tool_model.custom_headers or [])
+                        + (
+                            header_dict_to_header_list(
+                                custom_tool_additional_headers or {}
+                            )
+                        ),
                     ),
                 )
 
@@ -863,7 +869,7 @@ def stream_chat_message(
     user: User | None,
     use_existing_user_message: bool = False,
     litellm_additional_headers: dict[str, str] | None = None,
-    tool_additional_headers: dict[str, str] | None = None,
+    custom_tool_additional_headers: dict[str, str] | None = None,
     is_connected: Callable[[], bool] | None = None,
 ) -> Iterator[str]:
     with get_session_context_manager() as db_session:
@@ -873,7 +879,7 @@ def stream_chat_message(
             db_session=db_session,
             use_existing_user_message=use_existing_user_message,
             litellm_additional_headers=litellm_additional_headers,
-            tool_additional_headers=tool_additional_headers,
+            custom_tool_additional_headers=custom_tool_additional_headers,
             is_connected=is_connected,
         )
         for obj in objects:
diff --git a/backend/danswer/configs/model_configs.py b/backend/danswer/configs/model_configs.py
index 4454eee159b..c9668cd8136 100644
--- a/backend/danswer/configs/model_configs.py
+++ b/backend/danswer/configs/model_configs.py
@@ -119,19 +119,3 @@
         logger.error(
             "Failed to parse LITELLM_PASS_THROUGH_HEADERS, must be a valid JSON object"
         )
-
-
-# List of headers to pass through to tool calls (e.g., API requests made by tools)
-# This allows for dynamic configuration of tool behavior based on incoming request headers
-TOOL_PASS_THROUGH_HEADERS: list[str] | None = None
-_TOOL_PASS_THROUGH_HEADERS_RAW = os.environ.get("TOOL_PASS_THROUGH_HEADERS")
-if _TOOL_PASS_THROUGH_HEADERS_RAW:
-    try:
-        TOOL_PASS_THROUGH_HEADERS = json.loads(_TOOL_PASS_THROUGH_HEADERS_RAW)
-    except Exception:
-        from danswer.utils.logger import setup_logger
-
-        logger = setup_logger()
-        logger.error(
-            "Failed to parse TOOL_PASS_THROUGH_HEADERS, must be a valid JSON object"
-        )
diff --git a/backend/danswer/configs/tool_configs.py b/backend/danswer/configs/tool_configs.py
new file mode 100644
index 00000000000..3170cb31ff9
--- /dev/null
+++ b/backend/danswer/configs/tool_configs.py
@@ -0,0 +1,22 @@
+import json
+import os
+
+
+# if specified, will pass through request headers to the call to API calls made by custom tools
+CUSTOM_TOOL_PASS_THROUGH_HEADERS: list[str] | None = None
+_CUSTOM_TOOL_PASS_THROUGH_HEADERS_RAW = os.environ.get(
+    "CUSTOM_TOOL_PASS_THROUGH_HEADERS"
+)
+if _CUSTOM_TOOL_PASS_THROUGH_HEADERS_RAW:
+    try:
+        CUSTOM_TOOL_PASS_THROUGH_HEADERS = json.loads(
+            _CUSTOM_TOOL_PASS_THROUGH_HEADERS_RAW
+        )
+    except Exception:
+        # need to import here to avoid circular imports
+        from danswer.utils.logger import setup_logger
+
+        logger = setup_logger()
+        logger.error(
+            "Failed to parse CUSTOM_TOOL_PASS_THROUGH_HEADERS, must be a valid JSON object"
+        )
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index 9f8f1e2371a..2101fc74e90 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -60,6 +60,7 @@
 from danswer.search.enums import RecencyBiasSetting
 from danswer.utils.encryption import decrypt_bytes_to_string
 from danswer.utils.encryption import encrypt_string_to_bytes
+from danswer.utils.headers import HeaderItemDict
 from shared_configs.enums import EmbeddingProvider
 from shared_configs.enums import RerankerProvider
 
@@ -1288,7 +1289,7 @@ class Tool(Base):
     openapi_schema: Mapped[dict[str, Any] | None] = mapped_column(
         postgresql.JSONB(), nullable=True
     )
-    custom_headers: Mapped[list[dict[str, str]] | None] = mapped_column(
+    custom_headers: Mapped[list[HeaderItemDict] | None] = mapped_column(
         postgresql.JSONB(), nullable=True
     )
     # user who created / owns the tool. Will be None for built-in tools.
diff --git a/backend/danswer/db/tools.py b/backend/danswer/db/tools.py
index 248744b5639..0fd126d0065 100644
--- a/backend/danswer/db/tools.py
+++ b/backend/danswer/db/tools.py
@@ -1,4 +1,5 @@
 from typing import Any
+from typing import cast
 from uuid import UUID
 
 from sqlalchemy import select
@@ -6,6 +7,7 @@
 
 from danswer.db.models import Tool
 from danswer.server.features.tool.models import Header
+from danswer.utils.headers import HeaderItemDict
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -67,7 +69,9 @@ def update_tool(
     if user_id is not None:
         tool.user_id = user_id
     if custom_headers is not None:
-        tool.custom_headers = [header.dict() for header in custom_headers]
+        tool.custom_headers = [
+            cast(HeaderItemDict, header.model_dump()) for header in custom_headers
+        ]
     db_session.commit()
 
     return tool
diff --git a/backend/danswer/llm/factory.py b/backend/danswer/llm/factory.py
index 904735d5ffe..f930c3d3358 100644
--- a/backend/danswer/llm/factory.py
+++ b/backend/danswer/llm/factory.py
@@ -7,9 +7,9 @@
 from danswer.db.models import Persona
 from danswer.llm.chat_llm import DefaultMultiLLM
 from danswer.llm.exceptions import GenAIDisabledException
-from danswer.llm.headers import build_llm_extra_headers
 from danswer.llm.interfaces import LLM
 from danswer.llm.override_models import LLMOverride
+from danswer.utils.headers import build_llm_extra_headers
 
 
 def get_main_llm_from_tuple(
diff --git a/backend/danswer/llm/headers.py b/backend/danswer/llm/headers.py
deleted file mode 100644
index 13622167d99..00000000000
--- a/backend/danswer/llm/headers.py
+++ /dev/null
@@ -1,12 +0,0 @@
-from danswer.configs.model_configs import LITELLM_EXTRA_HEADERS
-
-
-def build_llm_extra_headers(
-    additional_headers: dict[str, str] | None = None
-) -> dict[str, str]:
-    extra_headers: dict[str, str] = {}
-    if additional_headers:
-        extra_headers.update(additional_headers)
-    if LITELLM_EXTRA_HEADERS:
-        extra_headers.update(LITELLM_EXTRA_HEADERS)
-    return extra_headers
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index 5b0cc30a1b4..6e2d3c40988 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -25,7 +25,6 @@
 from danswer.configs.constants import FileOrigin
 from danswer.configs.constants import MessageType
 from danswer.configs.model_configs import LITELLM_PASS_THROUGH_HEADERS
-from danswer.configs.model_configs import TOOL_PASS_THROUGH_HEADERS
 from danswer.db.chat import create_chat_session
 from danswer.db.chat import create_new_chat_message
 from danswer.db.chat import delete_chat_session
@@ -74,6 +73,7 @@
 from danswer.server.query_and_chat.models import SearchFeedbackRequest
 from danswer.server.query_and_chat.models import UpdateChatSessionThreadRequest
 from danswer.server.query_and_chat.token_limit import check_token_rate_limits
+from danswer.utils.headers import get_custom_tool_additional_request_headers
 from danswer.utils.logger import setup_logger
 
 
@@ -338,8 +338,8 @@ def stream_generator() -> Generator[str, None, None]:
                 litellm_additional_headers=extract_headers(
                     request.headers, LITELLM_PASS_THROUGH_HEADERS
                 ),
-                tool_additional_headers=extract_headers(
-                    request.headers, TOOL_PASS_THROUGH_HEADERS
+                custom_tool_additional_headers=get_custom_tool_additional_request_headers(
+                    request.headers
                 ),
                 is_connected=is_disconnected_func,
             ):
diff --git a/backend/danswer/tools/custom/custom_tool.py b/backend/danswer/tools/custom/custom_tool.py
index 8f4a4b23fa8..ee431af70e1 100644
--- a/backend/danswer/tools/custom/custom_tool.py
+++ b/backend/danswer/tools/custom/custom_tool.py
@@ -29,6 +29,8 @@
 from danswer.tools.models import MESSAGE_ID_PLACEHOLDER
 from danswer.tools.tool import Tool
 from danswer.tools.tool import ToolResponse
+from danswer.utils.headers import header_list_to_header_dict
+from danswer.utils.headers import HeaderItemDict
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -46,8 +48,7 @@ def __init__(
         self,
         method_spec: MethodSpec,
         base_url: str,
-        custom_headers: list[dict[str, str]] | None = [],
-        tool_additional_headers: dict[str, str] | None = None,
+        custom_headers: list[HeaderItemDict] | None = None,
     ) -> None:
         self._base_url = base_url
         self._method_spec = method_spec
@@ -55,9 +56,9 @@ def __init__(
 
         self._name = self._method_spec.name
         self._description = self._method_spec.summary
-        self.headers = {
-            header["key"]: header["value"] for header in (custom_headers or [])
-        } | (tool_additional_headers or {})
+        self.headers = (
+            header_list_to_header_dict(custom_headers) if custom_headers else {}
+        )
 
     @property
     def name(self) -> str:
@@ -184,8 +185,7 @@ def final_result(self, *args: ToolResponse) -> JSON_ro:
 
 def build_custom_tools_from_openapi_schema_and_headers(
     openapi_schema: dict[str, Any],
-    tool_additional_headers: dict[str, str] | None = None,
-    custom_headers: list[dict[str, str]] | None = [],
+    custom_headers: list[HeaderItemDict] | None = None,
     dynamic_schema_info: DynamicSchemaInfo | None = None,
 ) -> list[CustomTool]:
     if dynamic_schema_info:
@@ -205,8 +205,7 @@ def build_custom_tools_from_openapi_schema_and_headers(
     url = openapi_to_url(openapi_schema)
     method_specs = openapi_to_method_specs(openapi_schema)
     return [
-        CustomTool(method_spec, url, custom_headers, tool_additional_headers)
-        for method_spec in method_specs
+        CustomTool(method_spec, url, custom_headers) for method_spec in method_specs
     ]
 
 
diff --git a/backend/danswer/tools/images/image_generation_tool.py b/backend/danswer/tools/images/image_generation_tool.py
index 3c1fa75c742..3584d50f77e 100644
--- a/backend/danswer/tools/images/image_generation_tool.py
+++ b/backend/danswer/tools/images/image_generation_tool.py
@@ -11,13 +11,13 @@
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
 from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
-from danswer.llm.headers import build_llm_extra_headers
 from danswer.llm.interfaces import LLM
 from danswer.llm.utils import build_content_with_imgs
 from danswer.llm.utils import message_to_string
 from danswer.prompts.constants import GENERAL_SEP_PAT
 from danswer.tools.tool import Tool
 from danswer.tools.tool import ToolResponse
+from danswer.utils.headers import build_llm_extra_headers
 from danswer.utils.logger import setup_logger
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
 
diff --git a/backend/danswer/utils/headers.py b/backend/danswer/utils/headers.py
new file mode 100644
index 00000000000..5ccf61a51e1
--- /dev/null
+++ b/backend/danswer/utils/headers.py
@@ -0,0 +1,79 @@
+from typing import TypedDict
+
+from fastapi.datastructures import Headers
+
+from danswer.configs.model_configs import LITELLM_EXTRA_HEADERS
+from danswer.configs.model_configs import LITELLM_PASS_THROUGH_HEADERS
+from danswer.configs.tool_configs import CUSTOM_TOOL_PASS_THROUGH_HEADERS
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class HeaderItemDict(TypedDict):
+    key: str
+    value: str
+
+
+def clean_header_list(headers_to_clean: list[HeaderItemDict]) -> dict[str, str]:
+    cleaned_headers: dict[str, str] = {}
+    for item in headers_to_clean:
+        key = item["key"]
+        value = item["value"]
+        if key in cleaned_headers:
+            logger.warning(
+                f"Duplicate header {key} found in custom headers, ignoring..."
+            )
+            continue
+        cleaned_headers[key] = value
+    return cleaned_headers
+
+
+def header_dict_to_header_list(header_dict: dict[str, str]) -> list[HeaderItemDict]:
+    return [{"key": key, "value": value} for key, value in header_dict.items()]
+
+
+def header_list_to_header_dict(header_list: list[HeaderItemDict]) -> dict[str, str]:
+    return {header["key"]: header["value"] for header in header_list}
+
+
+def get_relevant_headers(
+    headers: dict[str, str] | Headers, desired_headers: list[str] | None
+) -> dict[str, str]:
+    if not desired_headers:
+        return {}
+
+    pass_through_headers: dict[str, str] = {}
+    for key in desired_headers:
+        if key in headers:
+            pass_through_headers[key] = headers[key]
+        else:
+            # fastapi makes all header keys lowercase, handling that here
+            lowercase_key = key.lower()
+            if lowercase_key in headers:
+                pass_through_headers[lowercase_key] = headers[lowercase_key]
+
+    return pass_through_headers
+
+
+def get_litellm_additional_request_headers(
+    headers: dict[str, str] | Headers
+) -> dict[str, str]:
+    return get_relevant_headers(headers, LITELLM_PASS_THROUGH_HEADERS)
+
+
+def build_llm_extra_headers(
+    additional_headers: dict[str, str] | None = None
+) -> dict[str, str]:
+    extra_headers: dict[str, str] = {}
+    if additional_headers:
+        extra_headers.update(additional_headers)
+    if LITELLM_EXTRA_HEADERS:
+        extra_headers.update(LITELLM_EXTRA_HEADERS)
+    return extra_headers
+
+
+def get_custom_tool_additional_request_headers(
+    headers: dict[str, str] | Headers
+) -> dict[str, str]:
+    return get_relevant_headers(headers, CUSTOM_TOOL_PASS_THROUGH_HEADERS)
diff --git a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
index 5b07e21bb83..6139f41e62a 100644
--- a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
+++ b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
@@ -13,6 +13,7 @@
 from danswer.tools.custom.custom_tool import validate_openapi_schema
 from danswer.tools.models import DynamicSchemaInfo
 from danswer.tools.tool import ToolResponse
+from danswer.utils.headers import HeaderItemDict
 
 
 class TestCustomTool(unittest.TestCase):
@@ -143,7 +144,7 @@ def test_custom_tool_with_headers(
         Test the custom tool with custom headers.
         Verifies that the tool correctly includes the custom headers in the request.
         """
-        custom_headers: list[dict[str, str]] = [
+        custom_headers: list[HeaderItemDict] = [
             {"key": "Authorization", "value": "Bearer token123"},
             {"key": "Custom-Header", "value": "CustomValue"},
         ]
@@ -171,7 +172,7 @@ def test_custom_tool_with_empty_headers(
         Test the custom tool with an empty list of custom headers.
         Verifies that the tool correctly handles an empty list of headers.
         """
-        custom_headers: list[dict[str, str]] = []
+        custom_headers: list[HeaderItemDict] = []
         tools = build_custom_tools_from_openapi_schema_and_headers(
             self.openapi_schema,
             custom_headers=custom_headers,

From 5d390b65eb11c65e17c727781cd43059d65bb58f Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 17 Oct 2024 08:47:46 -0700
Subject: [PATCH 133/376] Added logging for when a member has no email or
 username

---
 .../ee/danswer/external_permissions/confluence/group_sync.py   | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/backend/ee/danswer/external_permissions/confluence/group_sync.py b/backend/ee/danswer/external_permissions/confluence/group_sync.py
index 9ef376ba641..a864ab1af4a 100644
--- a/backend/ee/danswer/external_permissions/confluence/group_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/group_sync.py
@@ -77,6 +77,9 @@ def _get_group_members_email_paginated(
                 email = get_user_email_from_username__server(
                     confluence_client, user_name
                 )
+            else:
+                logger.warning(f"Member {member} has no email or username")
+                email = None
 
             if email:
                 group_member_emails.add(email)

From 938a65628d48de4781b21c92327fd2aba3ef9b29 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 17 Oct 2024 09:01:51 -0700
Subject: [PATCH 134/376] rearrange logging

---
 .../ee/danswer/external_permissions/confluence/group_sync.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/ee/danswer/external_permissions/confluence/group_sync.py b/backend/ee/danswer/external_permissions/confluence/group_sync.py
index a864ab1af4a..5c729f51faf 100644
--- a/backend/ee/danswer/external_permissions/confluence/group_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/group_sync.py
@@ -78,7 +78,7 @@ def _get_group_members_email_paginated(
                     confluence_client, user_name
                 )
             else:
-                logger.warning(f"Member {member} has no email or username")
+                logger.warning(f"Member has no email or username: {member}")
                 email = None
 
             if email:

From a159779d39479c87d3ccd052fe71af2b854c508c Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 09:31:17 -0700
Subject: [PATCH 135/376] prevent alembic from configuring logger (#2826)

* k

* k
---
 backend/ee/danswer/server/tenants/provisioning.py | 3 +++
 web/src/app/auth/login/page.tsx                   | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/ee/danswer/server/tenants/provisioning.py b/backend/ee/danswer/server/tenants/provisioning.py
index 77d27e7a551..9ec7b8061aa 100644
--- a/backend/ee/danswer/server/tenants/provisioning.py
+++ b/backend/ee/danswer/server/tenants/provisioning.py
@@ -31,6 +31,9 @@ def run_alembic_migrations(schema_name: str) -> None:
             "script_location", os.path.join(root_dir, "alembic")
         )
 
+        # Ensure that logging isn't broken
+        alembic_cfg.attributes["configure_logger"] = False
+
         # Mimic command-line options by adding 'cmd_opts' to the config
         alembic_cfg.cmd_opts = SimpleNamespace()  # type: ignore
         alembic_cfg.cmd_opts.x = [f"schema={schema_name}"]  # type: ignore
diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index e47bdb420be..d7738144af6 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -11,7 +11,6 @@ import { SignInButton } from "./SignInButton";
 import { EmailPasswordForm } from "./EmailPasswordForm";
 import { Card, Title, Text } from "@tremor/react";
 import Link from "next/link";
-
 import { LoginText } from "./LoginText";
 import { getSecondsUntilExpiration } from "@/lib/time";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";

From 15afe4dc78ebda472639946001283e04bcbac8b2 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 11:05:35 -0700
Subject: [PATCH 136/376] bump litellm (#2827)

---
 backend/requirements/default.txt      | 4 ++--
 backend/requirements/model_server.txt | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 8ecfcdb1528..25e85b06f0b 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -29,7 +29,7 @@ trafilatura==1.12.2
 langchain==0.1.17
 langchain-core==0.1.50
 langchain-text-splitters==0.0.1
-litellm==1.48.7
+litellm==1.49.5
 lxml==5.3.0
 lxml_html_clean==0.2.2
 llama-index==0.9.45
@@ -38,7 +38,7 @@ msal==1.28.0
 nltk==3.8.1
 Office365-REST-Python-Client==2.5.9
 oauthlib==3.2.2
-openai==1.47.0
+openai==1.51.2
 openpyxl==3.1.2
 playwright==1.41.2
 psutil==5.9.5
diff --git a/backend/requirements/model_server.txt b/backend/requirements/model_server.txt
index 694a41a1ce6..1cda9a505fe 100644
--- a/backend/requirements/model_server.txt
+++ b/backend/requirements/model_server.txt
@@ -3,7 +3,7 @@ einops==0.8.0
 fastapi==0.109.2
 google-cloud-aiplatform==1.58.0
 numpy==1.26.4
-openai==1.47.0
+openai==1.51.2
 pydantic==2.8.2
 retry==0.9.2
 safetensors==0.4.2
@@ -12,4 +12,4 @@ torch==2.2.0
 transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
-litellm==1.48.7
+litellm==1.49.5
\ No newline at end of file

From 5063b944ec3bae191ec9f8536929424e9ea810e0 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 17 Oct 2024 11:36:59 -0700
Subject: [PATCH 137/376] Make flakey test still run but not fail CI

---
 .../connector_job_tests/slack/test_permission_sync.py          | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
index 9114c2ddecf..c1aa346a9ba 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
@@ -27,7 +27,7 @@
 from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
 
 
-@pytest.mark.skip(reason="flaky - see DAN-789 for example")
+@pytest.mark.xfail(reason="flaky - see DAN-789 for example", strict=False)
 def test_slack_permission_sync(
     reset: None,
     vespa_client: vespa_fixture,
@@ -251,3 +251,4 @@ def test_slack_permission_sync(
     # Ensure test_user_1 can only see messages from the public channel
     assert public_message in danswer_doc_message_strings
     assert private_message not in danswer_doc_message_strings
+    assert 1 == 2

From 0c102ebb5ce3d0d7ec3afe16e0f7860afdac5791 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 17 Oct 2024 12:13:42 -0700
Subject: [PATCH 138/376] simplified the document search function

---
 .../common_utils/managers/document_search.py  | 36 ++++++++
 .../slack/test_permission_sync.py             | 87 +++++--------------
 .../connector_job_tests/slack/test_prune.py   | 86 ++++--------------
 3 files changed, 75 insertions(+), 134 deletions(-)
 create mode 100644 backend/tests/integration/common_utils/managers/document_search.py

diff --git a/backend/tests/integration/common_utils/managers/document_search.py b/backend/tests/integration/common_utils/managers/document_search.py
new file mode 100644
index 00000000000..6ac012e4d0e
--- /dev/null
+++ b/backend/tests/integration/common_utils/managers/document_search.py
@@ -0,0 +1,36 @@
+import requests
+
+from danswer.search.enums import LLMEvaluationType
+from danswer.search.enums import SearchType
+from danswer.search.models import RetrievalDetails
+from danswer.search.models import SavedSearchDocWithContent
+from ee.danswer.server.query_and_chat.models import DocumentSearchRequest
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.test_models import DATestUser
+
+
+class DocumentSearchManager:
+    @staticmethod
+    def search_documents(
+        query: str,
+        search_type: SearchType = SearchType.KEYWORD,
+        user_performing_action: DATestUser | None = None,
+    ) -> list[str]:
+        search_request = DocumentSearchRequest(
+            message=query,
+            search_type=search_type,
+            retrieval_options=RetrievalDetails(),
+            evaluation_type=LLMEvaluationType.SKIP,
+        )
+        result = requests.post(
+            url=f"{API_SERVER_URL}/query/document-search",
+            json=search_request.model_dump(),
+            headers=user_performing_action.headers,
+        )
+        result.raise_for_status()
+        result_json = result.json()
+        top_documents: list[SavedSearchDocWithContent] = [
+            SavedSearchDocWithContent(**doc) for doc in result_json["top_documents"]
+        ]
+        document_content_list: list[str] = [doc.content for doc in top_documents]
+        return document_content_list
diff --git a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
index c1aa346a9ba..7fdb1a0428f 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
@@ -4,19 +4,16 @@
 from typing import Any
 
 import pytest
-import requests
 
 from danswer.connectors.models import InputType
 from danswer.db.enums import AccessType
-from danswer.search.enums import LLMEvaluationType
-from danswer.search.enums import SearchType
-from danswer.search.models import RetrievalDetails
 from danswer.server.documents.models import DocumentSource
-from ee.danswer.server.query_and_chat.models import DocumentSearchRequest
-from tests.integration.common_utils.constants import API_SERVER_URL
 from tests.integration.common_utils.managers.cc_pair import CCPairManager
 from tests.integration.common_utils.managers.connector import ConnectorManager
 from tests.integration.common_utils.managers.credential import CredentialManager
+from tests.integration.common_utils.managers.document_search import (
+    DocumentSearchManager,
+)
 from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
 from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.test_models import DATestCCPair
@@ -100,6 +97,7 @@ def test_slack_permission_sync(
     public_message = "Steve's favorite number is 809752"
     private_message = "Sara's favorite number is 346794"
 
+    # Add messages to channels
     SlackManager.add_message_to_channel(
         slack_client=slack_client,
         channel=public_channel,
@@ -133,42 +131,20 @@ def test_slack_permission_sync(
     )
 
     # Search as admin with access to both channels
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=admin_user.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=admin_user,
+    )
 
     # Ensure admin user can see messages from both channels
     assert public_message in danswer_doc_message_strings
     assert private_message in danswer_doc_message_strings
 
     # Search as test_user_2 with access to only the public channel
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=test_user_2.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=test_user_2,
+    )
     print(
         "\ntop_documents content before removing from private channel for test_user_2: ",
         danswer_doc_message_strings,
@@ -179,21 +155,10 @@ def test_slack_permission_sync(
     assert private_message not in danswer_doc_message_strings
 
     # Search as test_user_1 with access to both channels
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=test_user_1.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=test_user_1,
+    )
     print(
         "\ntop_documents content before removing from private channel for test_user_1: ",
         danswer_doc_message_strings,
@@ -228,21 +193,10 @@ def test_slack_permission_sync(
     # ----------------------------VERIFY THE CHANGES---------------------------
     # Ensure test_user_1 can no longer see messages from the private channel
     # Search as test_user_1 with access to only the public channel
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=test_user_1.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=test_user_1,
+    )
     print(
         "\ntop_documents content after removing from private channel for test_user_1: ",
         danswer_doc_message_strings,
@@ -251,4 +205,3 @@ def test_slack_permission_sync(
     # Ensure test_user_1 can only see messages from the public channel
     assert public_message in danswer_doc_message_strings
     assert private_message not in danswer_doc_message_strings
-    assert 1 == 2
diff --git a/backend/tests/integration/connector_job_tests/slack/test_prune.py b/backend/tests/integration/connector_job_tests/slack/test_prune.py
index 12cf7da7a97..3410a477cb4 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_prune.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_prune.py
@@ -3,19 +3,15 @@
 from datetime import timezone
 from typing import Any
 
-import requests
-
 from danswer.connectors.models import InputType
 from danswer.db.enums import AccessType
-from danswer.search.enums import LLMEvaluationType
-from danswer.search.enums import SearchType
-from danswer.search.models import RetrievalDetails
 from danswer.server.documents.models import DocumentSource
-from ee.danswer.server.query_and_chat.models import DocumentSearchRequest
-from tests.integration.common_utils.constants import API_SERVER_URL
 from tests.integration.common_utils.managers.cc_pair import CCPairManager
 from tests.integration.common_utils.managers.connector import ConnectorManager
 from tests.integration.common_utils.managers.credential import CredentialManager
+from tests.integration.common_utils.managers.document_search import (
+    DocumentSearchManager,
+)
 from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
 from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.test_models import DATestCCPair
@@ -134,21 +130,10 @@ def test_slack_prune(
 
     # ----------------------TEST THE SETUP--------------------------
     # Search as admin with access to both channels
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=admin_user.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=admin_user,
+    )
     print(
         "\ntop_documents content before deleting for admin: ",
         danswer_doc_message_strings,
@@ -160,21 +145,10 @@ def test_slack_prune(
     assert message_to_delete in danswer_doc_message_strings
 
     # Search as test_user_1 with access to both channels
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=test_user_1.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=test_user_1,
+    )
     print(
         "\ntop_documents content before deleting for test_user_1: ",
         danswer_doc_message_strings,
@@ -202,21 +176,10 @@ def test_slack_prune(
     # ----------------------------VERIFY THE CHANGES---------------------------
     # Ensure admin user can't see deleted messages
     # Search as admin user with access to only the public channel
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=admin_user.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=admin_user,
+    )
     print(
         "\ntop_documents content after deleting for admin: ",
         danswer_doc_message_strings,
@@ -229,21 +192,10 @@ def test_slack_prune(
 
     # Ensure test_user_1 can't see deleted messages
     # Search as test_user_1 with access to only the public channel
-    search_request = DocumentSearchRequest(
-        message="favorite number",
-        search_type=SearchType.KEYWORD,
-        retrieval_options=RetrievalDetails(),
-        evaluation_type=LLMEvaluationType.SKIP,
-    )
-    search_request_body = search_request.model_dump()
-    result = requests.post(
-        url=f"{API_SERVER_URL}/query/document-search",
-        json=search_request_body,
-        headers=test_user_1.headers,
-    )
-    result.raise_for_status()
-    found_docs = result.json()["top_documents"]
-    danswer_doc_message_strings = [doc["content"] for doc in found_docs]
+    danswer_doc_message_strings = DocumentSearchManager.search_documents(
+        query="favorite number",
+        user_performing_action=test_user_1,
+    )
     print(
         "\ntop_documents content after prune for test_user_1: ",
         danswer_doc_message_strings,

From 28ad01a51aa28e0d03f5f4dccddebe926426f514 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 17 Oct 2024 12:37:34 -0700
Subject: [PATCH 139/376] py

---
 .../integration/common_utils/managers/document_search.py     | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/backend/tests/integration/common_utils/managers/document_search.py b/backend/tests/integration/common_utils/managers/document_search.py
index 6ac012e4d0e..4fe2442b69a 100644
--- a/backend/tests/integration/common_utils/managers/document_search.py
+++ b/backend/tests/integration/common_utils/managers/document_search.py
@@ -6,6 +6,7 @@
 from danswer.search.models import SavedSearchDocWithContent
 from ee.danswer.server.query_and_chat.models import DocumentSearchRequest
 from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.constants import GENERAL_HEADERS
 from tests.integration.common_utils.test_models import DATestUser
 
 
@@ -25,7 +26,9 @@ def search_documents(
         result = requests.post(
             url=f"{API_SERVER_URL}/query/document-search",
             json=search_request.model_dump(),
-            headers=user_performing_action.headers,
+            headers=user_performing_action.headers
+            if user_performing_action
+            else GENERAL_HEADERS,
         )
         result.raise_for_status()
         result_json = result.json()

From 389c7b72db6e2889743be1fffc5c725e375e7e02 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 17 Oct 2024 12:43:19 -0700
Subject: [PATCH 140/376] Bugfix/monitor exceptions (#2830)

* do a rollback before more db work

* warn if not all doc_by_cc_pair entries were deleted

---------

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 .../background/celery/tasks/vespa/tasks.py    | 31 +++++++++++++------
 backend/danswer/db/document_set.py            |  5 +--
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index c43de3a85f8..e958d63ad30 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -31,6 +31,7 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.document import count_documents_by_needs_sync
 from danswer.db.document import get_document
+from danswer.db.document import get_document_ids_for_connector_credential_pair
 from danswer.db.document import mark_document_as_synced
 from danswer.db.document_set import delete_document_set
 from danswer.db.document_set import delete_document_set_cc_pair_relationship__no_commit
@@ -363,7 +364,7 @@ def monitor_connector_deletion_taskset(
 
     count = cast(int, r.scard(rcd.taskset_key))
     task_logger.info(
-        f"Connector deletion progress: cc_pair_id={cc_pair_id} remaining={count} initial={initial_count}"
+        f"Connector deletion progress: cc_pair={cc_pair_id} remaining={count} initial={initial_count}"
     )
     if count > 0:
         return
@@ -372,16 +373,27 @@ def monitor_connector_deletion_taskset(
         cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session)
         if not cc_pair:
             task_logger.warning(
-                f"monitor_connector_deletion_taskset - cc_pair_id not found: cc_pair_id={cc_pair_id}"
+                f"Connector deletion - cc_pair not found: cc_pair={cc_pair_id}"
             )
             return
 
         try:
+            doc_ids = get_document_ids_for_connector_credential_pair(
+                db_session, cc_pair.connector_id, cc_pair.credential_id
+            )
+            if len(doc_ids) > 0:
+                # if this happens, documents somehow got added while deletion was in progress. Likely a bug
+                # gating off pruning and indexing work before deletion starts
+                task_logger.warning(
+                    f"Connector deletion - documents still found after taskset completion: "
+                    f"cc_pair={cc_pair_id} num={len(doc_ids)}"
+                )
+
             # clean up the rest of the related Postgres entities
             # index attempts
             delete_index_attempts(
                 db_session=db_session,
-                cc_pair_id=cc_pair.id,
+                cc_pair_id=cc_pair_id,
             )
 
             # document sets
@@ -398,7 +410,7 @@ def monitor_connector_deletion_taskset(
                 noop_fallback,
             )
             cleanup_user_groups(
-                cc_pair_id=cc_pair.id,
+                cc_pair_id=cc_pair_id,
                 db_session=db_session,
             )
 
@@ -420,20 +432,21 @@ def monitor_connector_deletion_taskset(
                 db_session.delete(connector)
             db_session.commit()
         except Exception as e:
+            db_session.rollback()
             stack_trace = traceback.format_exc()
             error_message = f"Error: {str(e)}\n\nStack Trace:\n{stack_trace}"
-            add_deletion_failure_message(db_session, cc_pair.id, error_message)
+            add_deletion_failure_message(db_session, cc_pair_id, error_message)
             task_logger.exception(
                 f"Failed to run connector_deletion. "
-                f"cc_pair_id={cc_pair_id} connector_id={cc_pair.connector_id} credential_id={cc_pair.credential_id}"
+                f"cc_pair={cc_pair_id} connector={cc_pair.connector_id} credential={cc_pair.credential_id}"
             )
             raise e
 
     task_logger.info(
         f"Successfully deleted cc_pair: "
-        f"cc_pair_id={cc_pair_id} "
-        f"connector_id={cc_pair.connector_id} "
-        f"credential_id={cc_pair.credential_id} "
+        f"cc_pair={cc_pair_id} "
+        f"connector={cc_pair.connector_id} "
+        f"credential={cc_pair.credential_id} "
         f"docs_deleted={initial_count}"
     )
 
diff --git a/backend/danswer/db/document_set.py b/backend/danswer/db/document_set.py
index 0ba6c4e9ab3..b5af99b22d4 100644
--- a/backend/danswer/db/document_set.py
+++ b/backend/danswer/db/document_set.py
@@ -398,7 +398,7 @@ def mark_document_set_as_to_be_deleted(
 
 def delete_document_set_cc_pair_relationship__no_commit(
     connector_id: int, credential_id: int, db_session: Session
-) -> None:
+) -> int:
     """Deletes all rows from DocumentSet__ConnectorCredentialPair where the
     connector_credential_pair_id matches the given cc_pair_id."""
     delete_stmt = delete(DocumentSet__ConnectorCredentialPair).where(
@@ -409,7 +409,8 @@ def delete_document_set_cc_pair_relationship__no_commit(
             == ConnectorCredentialPair.id,
         )
     )
-    db_session.execute(delete_stmt)
+    result = db_session.execute(delete_stmt)
+    return result.rowcount  # type: ignore
 
 
 def fetch_document_sets(

From 114326d11a63925251ac4ca2dda632a7eeec2b75 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 17 Oct 2024 12:43:34 -0700
Subject: [PATCH 141/376] fix sync to use update_single (#2822)

---
 .../background/celery/tasks/shared/tasks.py        |  2 +-
 .../danswer/background/celery/tasks/vespa/tasks.py | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index b065122be84..05408d58f97 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -111,7 +111,7 @@ def document_by_cc_pair_cleanup_task(
                 pass
 
             task_logger.info(
-                f"document_id={document_id} refcount={count} action={action} chunks={chunks_affected}"
+                f"document_id={document_id} action={action} refcount={count} chunks={chunks_affected}"
             )
             db_session.commit()
     except SoftTimeLimitExceeded:
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index e958d63ad30..c50237b606a 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -45,7 +45,7 @@
 from danswer.db.models import UserGroup
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
-from danswer.document_index.interfaces import UpdateRequest
+from danswer.document_index.interfaces import VespaDocumentFields
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from danswer.utils.variable_functionality import (
@@ -609,20 +609,24 @@ def vespa_metadata_sync_task(
             doc_access = get_access_for_document(
                 document_id=document_id, db_session=db_session
             )
-            update_request = UpdateRequest(
-                document_ids=[document_id],
+
+            fields = VespaDocumentFields(
                 document_sets=update_doc_sets,
                 access=doc_access,
                 boost=doc.boost,
                 hidden=doc.hidden,
             )
 
-            # update Vespa
-            document_index.update(update_requests=[update_request])
+            # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
+            chunks_affected = document_index.update_single(document_id, fields=fields)
 
             # update db last. Worst case = we crash right before this and
             # the sync might repeat again later
             mark_document_as_synced(document_id, db_session)
+
+            task_logger.info(
+                f"document_id={document_id} action=sync chunks={chunks_affected}"
+            )
     except SoftTimeLimitExceeded:
         task_logger.info(f"SoftTimeLimitExceeded exception. doc_id={document_id}")
     except Exception as e:

From 0de487064af2b2965469a3d62ce529806847ae8f Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 17 Oct 2024 12:44:51 -0700
Subject: [PATCH 142/376] lock to avoid rare serializable errors (#2818)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 backend/danswer/db/index_attempt.py | 65 ++++++++++++++++++++++-------
 1 file changed, 51 insertions(+), 14 deletions(-)

diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index d9b1569e427..21a1bbd236f 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -105,27 +105,55 @@ def mark_attempt_in_progress(
     index_attempt: IndexAttempt,
     db_session: Session,
 ) -> None:
-    index_attempt.status = IndexingStatus.IN_PROGRESS
-    index_attempt.time_started = index_attempt.time_started or func.now()  # type: ignore
-    db_session.commit()
+    with db_session.begin_nested():
+        try:
+            attempt = db_session.execute(
+                select(IndexAttempt)
+                .where(IndexAttempt.id == index_attempt.id)
+                .with_for_update()
+            ).scalar_one()
+
+            attempt.status = IndexingStatus.IN_PROGRESS
+            attempt.time_started = index_attempt.time_started or func.now()  # type: ignore
+            db_session.commit()
+        except Exception:
+            db_session.rollback()
 
 
 def mark_attempt_succeeded(
     index_attempt: IndexAttempt,
     db_session: Session,
 ) -> None:
-    index_attempt.status = IndexingStatus.SUCCESS
-    db_session.add(index_attempt)
-    db_session.commit()
+    with db_session.begin_nested():
+        try:
+            attempt = db_session.execute(
+                select(IndexAttempt)
+                .where(IndexAttempt.id == index_attempt.id)
+                .with_for_update()
+            ).scalar_one()
+
+            attempt.status = IndexingStatus.SUCCESS
+            db_session.commit()
+        except Exception:
+            db_session.rollback()
 
 
 def mark_attempt_partially_succeeded(
     index_attempt: IndexAttempt,
     db_session: Session,
 ) -> None:
-    index_attempt.status = IndexingStatus.COMPLETED_WITH_ERRORS
-    db_session.add(index_attempt)
-    db_session.commit()
+    with db_session.begin_nested():
+        try:
+            attempt = db_session.execute(
+                select(IndexAttempt)
+                .where(IndexAttempt.id == index_attempt.id)
+                .with_for_update()
+            ).scalar_one()
+
+            attempt.status = IndexingStatus.COMPLETED_WITH_ERRORS
+            db_session.commit()
+        except Exception:
+            db_session.rollback()
 
 
 def mark_attempt_failed(
@@ -134,11 +162,20 @@ def mark_attempt_failed(
     failure_reason: str = "Unknown",
     full_exception_trace: str | None = None,
 ) -> None:
-    index_attempt.status = IndexingStatus.FAILED
-    index_attempt.error_msg = failure_reason
-    index_attempt.full_exception_trace = full_exception_trace
-    db_session.add(index_attempt)
-    db_session.commit()
+    with db_session.begin_nested():
+        try:
+            attempt = db_session.execute(
+                select(IndexAttempt)
+                .where(IndexAttempt.id == index_attempt.id)
+                .with_for_update()
+            ).scalar_one()
+
+            attempt.status = IndexingStatus.FAILED
+            attempt.error_msg = failure_reason
+            attempt.full_exception_trace = full_exception_trace
+            db_session.commit()
+        except Exception:
+            db_session.rollback()
 
     source = index_attempt.connector_credential_pair.connector.source
     optional_telemetry(record_type=RecordType.FAILURE, data={"connector": source})

From deb66a88aa80246fef8c0df15fd73973166c869b Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 17 Oct 2024 13:37:50 -0700
Subject: [PATCH 143/376] dont fail flaky tests

---
 .../tests/integration/connector_job_tests/slack/test_prune.py  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/backend/tests/integration/connector_job_tests/slack/test_prune.py b/backend/tests/integration/connector_job_tests/slack/test_prune.py
index 3410a477cb4..3abf7bd6fb0 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_prune.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_prune.py
@@ -3,6 +3,8 @@
 from datetime import timezone
 from typing import Any
 
+import pytest
+
 from danswer.connectors.models import InputType
 from danswer.db.enums import AccessType
 from danswer.server.documents.models import DocumentSource
@@ -22,6 +24,7 @@
 from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
 
 
+@pytest.mark.xfail(reason="flaky - see DAN-835 for example", strict=False)
 def test_slack_prune(
     reset: None,
     vespa_client: vespa_fixture,

From e48086b1c2ed98a76a50e0d91aa83809de9c857d Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 13:27:57 -0700
Subject: [PATCH 144/376] add slack markdown formatting (#2829)

* add slack markdown formatting

* nit

* k
---
 .../danswer/danswerbot/slack/formatting.py    | 66 +++++++++++++++++++
 .../slack/handlers/handle_regular_answer.py   |  4 +-
 backend/requirements/default.txt              |  1 +
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100644 backend/danswer/danswerbot/slack/formatting.py

diff --git a/backend/danswer/danswerbot/slack/formatting.py b/backend/danswer/danswerbot/slack/formatting.py
new file mode 100644
index 00000000000..604c879df27
--- /dev/null
+++ b/backend/danswer/danswerbot/slack/formatting.py
@@ -0,0 +1,66 @@
+from mistune import Markdown  # type: ignore
+from mistune import Renderer  # type: ignore
+
+
+def format_slack_message(message: str | None) -> str:
+    renderer = Markdown(renderer=SlackRenderer())
+    return renderer.render(message)
+
+
+class SlackRenderer(Renderer):
+    SPECIALS: dict[str, str] = {"&": "&amp;", "<": "&lt;", ">": "&gt;"}
+
+    def escape_special(self, text: str) -> str:
+        for special, replacement in self.SPECIALS.items():
+            text = text.replace(special, replacement)
+        return text
+
+    def header(self, text: str, level: int, raw: str | None = None) -> str:
+        return f"*{text}*\n"
+
+    def emphasis(self, text: str) -> str:
+        return f"_{text}_"
+
+    def double_emphasis(self, text: str) -> str:
+        return f"*{text}*"
+
+    def strikethrough(self, text: str) -> str:
+        return f"~{text}~"
+
+    def list(self, body: str, ordered: bool = True) -> str:
+        lines = body.split("\n")
+        count = 0
+        for i, line in enumerate(lines):
+            if line.startswith("li: "):
+                count += 1
+                prefix = f"{count}. " if ordered else "• "
+                lines[i] = f"{prefix}{line[4:]}"
+        return "\n".join(lines)
+
+    def list_item(self, text: str) -> str:
+        return f"li: {text}\n"
+
+    def link(self, link: str, title: str | None, content: str | None) -> str:
+        escaped_link = self.escape_special(link)
+        if content:
+            return f"<{escaped_link}|{content}>"
+        if title:
+            return f"<{escaped_link}|{title}>"
+        return f"<{escaped_link}>"
+
+    def image(self, src: str, title: str | None, text: str | None) -> str:
+        escaped_src = self.escape_special(src)
+        display_text = title or text
+        return f"<{escaped_src}|{display_text}>" if display_text else f"<{escaped_src}>"
+
+    def codespan(self, text: str) -> str:
+        return f"`{text}`"
+
+    def block_code(self, text: str, lang: str | None) -> str:
+        return f"```\n{text}\n```\n"
+
+    def paragraph(self, text: str) -> str:
+        return f"{text}\n"
+
+    def autolink(self, link: str, is_email: bool) -> str:
+        return link if is_email else self.link(link, None, None)
diff --git a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
index e864d92c702..c8aa5961ea7 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
@@ -27,6 +27,7 @@
 from danswer.danswerbot.slack.blocks import build_qa_response_blocks
 from danswer.danswerbot.slack.blocks import build_sources_blocks
 from danswer.danswerbot.slack.blocks import get_restate_blocks
+from danswer.danswerbot.slack.formatting import format_slack_message
 from danswer.danswerbot.slack.handlers.utils import send_team_member_message
 from danswer.danswerbot.slack.models import SlackMessageInfo
 from danswer.danswerbot.slack.utils import respond_in_thread
@@ -412,10 +413,11 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
 
     # If called with the DanswerBot slash command, the question is lost so we have to reshow it
     restate_question_block = get_restate_blocks(messages[-1].message, is_bot_msg)
+    formatted_answer = format_slack_message(answer.answer) if answer.answer else None
 
     answer_blocks = build_qa_response_blocks(
         message_id=answer.chat_message_id,
-        answer=answer.answer,
+        answer=formatted_answer,
         quotes=answer.quotes.quotes if answer.quotes else None,
         source_filters=retrieval_info.applied_source_filters,
         time_cutoff=retrieval_info.applied_time_cutoff,
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 25e85b06f0b..ef36de81257 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -81,3 +81,4 @@ dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
 ultimate_sitemap_parser==0.5
 stripe==10.12.0
+mistune==0.8.4
\ No newline at end of file

From b169f78699b34bd8442cf19f8a89177e59da2bf7 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 14:04:48 -0700
Subject: [PATCH 145/376] Push multi tenancy for slackbot (#2828)

* push multi tenancy for slackbot

* move to utils

* k

* k

---------

Co-authored-by: hagen-danswer <hagen@danswer.ai>
---
 .../slack/handlers/handle_buttons.py          |  26 ++--
 .../slack/handlers/handle_message.py          |  11 +-
 .../slack/handlers/handle_regular_answer.py   |  11 +-
 backend/danswer/danswerbot/slack/listener.py  | 142 ++++++++++--------
 backend/danswer/danswerbot/slack/utils.py     |  16 +-
 5 files changed, 116 insertions(+), 90 deletions(-)

diff --git a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py
index 9e1c171ee4f..f379e6af4ca 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_buttons.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_buttons.py
@@ -4,9 +4,7 @@
 from slack_sdk import WebClient
 from slack_sdk.models.blocks import SectionBlock
 from slack_sdk.models.views import View
-from slack_sdk.socket_mode import SocketModeClient
 from slack_sdk.socket_mode.request import SocketModeRequest
-from sqlalchemy.orm import Session
 
 from danswer.configs.constants import MessageType
 from danswer.configs.constants import SearchFeedbackType
@@ -35,20 +33,22 @@
 from danswer.danswerbot.slack.utils import get_feedback_visibility
 from danswer.danswerbot.slack.utils import read_slack_thread
 from danswer.danswerbot.slack.utils import respond_in_thread
+from danswer.danswerbot.slack.utils import TenantSocketModeClient
 from danswer.danswerbot.slack.utils import update_emote_react
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.feedback import create_chat_message_feedback
 from danswer.db.feedback import create_doc_retrieval_feedback
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 
 def handle_doc_feedback_button(
     req: SocketModeRequest,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
 ) -> None:
     if not (actions := req.payload.get("actions")):
         logger.error("Missing actions. Unable to build the source feedback view")
@@ -81,7 +81,7 @@ def handle_doc_feedback_button(
 
 def handle_generate_answer_button(
     req: SocketModeRequest,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
 ) -> None:
     channel_id = req.payload["channel"]["id"]
     channel_name = req.payload["channel"]["name"]
@@ -116,7 +116,7 @@ def handle_generate_answer_button(
         thread_ts=thread_ts,
     )
 
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(client.tenant_id) as db_session:
         slack_bot_config = get_slack_bot_config_for_channel(
             channel_name=channel_name, db_session=db_session
         )
@@ -136,6 +136,7 @@ def handle_generate_answer_button(
             slack_bot_config=slack_bot_config,
             receiver_ids=None,
             client=client.web_client,
+            tenant_id=client.tenant_id,
             channel=channel_id,
             logger=logger,
             feedback_reminder_id=None,
@@ -150,12 +151,11 @@ def handle_slack_feedback(
     user_id_to_post_confirmation: str,
     channel_id_to_post_confirmation: str,
     thread_ts_to_post_confirmation: str,
+    tenant_id: str | None,
 ) -> None:
-    engine = get_sqlalchemy_engine()
-
     message_id, doc_id, doc_rank = decompose_action_id(feedback_id)
 
-    with Session(engine) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         if feedback_type in [LIKE_BLOCK_ACTION_ID, DISLIKE_BLOCK_ACTION_ID]:
             create_chat_message_feedback(
                 is_positive=feedback_type == LIKE_BLOCK_ACTION_ID,
@@ -232,7 +232,7 @@ def handle_slack_feedback(
 
 def handle_followup_button(
     req: SocketModeRequest,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
 ) -> None:
     action_id = None
     if actions := req.payload.get("actions"):
@@ -252,7 +252,7 @@ def handle_followup_button(
 
     tag_ids: list[str] = []
     group_ids: list[str] = []
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(client.tenant_id) as db_session:
         channel_name, is_dm = get_channel_name_from_id(
             client=client.web_client, channel_id=channel_id
         )
@@ -295,7 +295,7 @@ def handle_followup_button(
 
 def get_clicker_name(
     req: SocketModeRequest,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
 ) -> str:
     clicker_name = req.payload.get("user", {}).get("name", "Someone")
     clicker_real_name = None
@@ -316,7 +316,7 @@ def get_clicker_name(
 
 def handle_followup_resolved_button(
     req: SocketModeRequest,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
     immediate: bool = False,
 ) -> None:
     channel_id = req.payload["container"]["channel_id"]
diff --git a/backend/danswer/danswerbot/slack/handlers/handle_message.py b/backend/danswer/danswerbot/slack/handlers/handle_message.py
index 0882796204d..ffbe902c5ec 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_message.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_message.py
@@ -2,7 +2,6 @@
 
 from slack_sdk import WebClient
 from slack_sdk.errors import SlackApiError
-from sqlalchemy.orm import Session
 
 from danswer.configs.danswerbot_configs import DANSWER_BOT_FEEDBACK_REMINDER
 from danswer.configs.danswerbot_configs import DANSWER_REACT_EMOJI
@@ -19,7 +18,7 @@
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.danswerbot.slack.utils import slack_usage_report
 from danswer.danswerbot.slack.utils import update_emote_react
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.models import SlackBotConfig
 from danswer.db.users import add_non_web_user_if_not_exists
 from danswer.utils.logger import setup_logger
@@ -110,6 +109,7 @@ def handle_message(
     slack_bot_config: SlackBotConfig | None,
     client: WebClient,
     feedback_reminder_id: str | None,
+    tenant_id: str | None,
 ) -> bool:
     """Potentially respond to the user message depending on filters and if an answer was generated
 
@@ -135,7 +135,9 @@ def handle_message(
         action = "slack_tag_message"
     elif is_bot_dm:
         action = "slack_dm_message"
-    slack_usage_report(action=action, sender_id=sender_id, client=client)
+    slack_usage_report(
+        action=action, sender_id=sender_id, client=client, tenant_id=tenant_id
+    )
 
     document_set_names: list[str] | None = None
     persona = slack_bot_config.persona if slack_bot_config else None
@@ -209,7 +211,7 @@ def handle_message(
     except SlackApiError as e:
         logger.error(f"Was not able to react to user message due to: {e}")
 
-    with Session(get_sqlalchemy_engine()) as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         if message_info.email:
             add_non_web_user_if_not_exists(db_session, message_info.email)
 
@@ -235,5 +237,6 @@ def handle_message(
             channel=channel,
             logger=logger,
             feedback_reminder_id=feedback_reminder_id,
+            tenant_id=tenant_id,
         )
         return issue_with_regular_answer
diff --git a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
index c8aa5961ea7..66f9ba54601 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
@@ -9,7 +9,6 @@
 from slack_sdk import WebClient
 from slack_sdk.models.blocks import DividerBlock
 from slack_sdk.models.blocks import SectionBlock
-from sqlalchemy.orm import Session
 
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.danswerbot_configs import DANSWER_BOT_ANSWER_GENERATION_TIMEOUT
@@ -33,7 +32,7 @@
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.danswerbot.slack.utils import SlackRateLimiter
 from danswer.danswerbot.slack.utils import update_emote_react
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.models import Persona
 from danswer.db.models import SlackBotConfig
 from danswer.db.models import SlackBotResponseType
@@ -88,6 +87,7 @@ def handle_regular_answer(
     channel: str,
     logger: DanswerLoggingAdapter,
     feedback_reminder_id: str | None,
+    tenant_id: str | None,
     num_retries: int = DANSWER_BOT_NUM_RETRIES,
     answer_generation_timeout: int = DANSWER_BOT_ANSWER_GENERATION_TIMEOUT,
     thread_context_percent: float = DANSWER_BOT_TARGET_CHUNK_PERCENTAGE,
@@ -104,8 +104,7 @@ def handle_regular_answer(
     user = None
     if message_info.is_bot_dm:
         if message_info.email:
-            engine = get_sqlalchemy_engine()
-            with Session(engine) as db_session:
+            with get_session_with_tenant(tenant_id) as db_session:
                 user = get_user_by_email(message_info.email, db_session)
 
     document_set_names: list[str] | None = None
@@ -152,7 +151,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
         max_document_tokens: int | None = None
         max_history_tokens: int | None = None
 
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             if len(new_message_request.messages) > 1:
                 if new_message_request.persona_config:
                     raise RuntimeError("Slack bot does not support persona config")
@@ -246,7 +245,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
         )
 
         # Always apply reranking settings if it exists, this is the non-streaming flow
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             saved_search_settings = get_current_search_settings(db_session)
 
         # This includes throwing out answer via reflexion
diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index dbf6eae24cd..86e59708820 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -4,11 +4,10 @@
 from typing import cast
 
 from slack_sdk import WebClient
-from slack_sdk.socket_mode import SocketModeClient
 from slack_sdk.socket_mode.request import SocketModeRequest
 from slack_sdk.socket_mode.response import SocketModeResponse
-from sqlalchemy.orm import Session
 
+from danswer.background.celery.celery_app import get_all_tenant_ids
 from danswer.configs.constants import MessageType
 from danswer.configs.danswerbot_configs import DANSWER_BOT_REPHRASE_MESSAGE
 from danswer.configs.danswerbot_configs import DANSWER_BOT_RESPOND_EVERY_CHANNEL
@@ -47,7 +46,8 @@
 from danswer.danswerbot.slack.utils import remove_danswer_bot_tag
 from danswer.danswerbot.slack.utils import rephrase_slack_message
 from danswer.danswerbot.slack.utils import respond_in_thread
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.danswerbot.slack.utils import TenantSocketModeClient
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.search_settings import get_current_search_settings
 from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
@@ -80,7 +80,7 @@
 _OFFICIAL_SLACKBOT_USER_ID = "USLACKBOT"
 
 
-def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool:
+def prefilter_requests(req: SocketModeRequest, client: TenantSocketModeClient) -> bool:
     """True to keep going, False to ignore this Slack request"""
     if req.type == "events_api":
         # Verify channel is valid
@@ -153,8 +153,7 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool
                 client=client.web_client, channel_id=channel
             )
 
-            engine = get_sqlalchemy_engine()
-            with Session(engine) as db_session:
+            with get_session_with_tenant(client.tenant_id) as db_session:
                 slack_bot_config = get_slack_bot_config_for_channel(
                     channel_name=channel_name, db_session=db_session
                 )
@@ -221,7 +220,7 @@ def prefilter_requests(req: SocketModeRequest, client: SocketModeClient) -> bool
     return True
 
 
-def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
+def process_feedback(req: SocketModeRequest, client: TenantSocketModeClient) -> None:
     if actions := req.payload.get("actions"):
         action = cast(dict[str, Any], actions[0])
         feedback_type = cast(str, action.get("action_id"))
@@ -243,6 +242,7 @@ def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
         user_id_to_post_confirmation=user_id,
         channel_id_to_post_confirmation=channel_id,
         thread_ts_to_post_confirmation=thread_ts,
+        tenant_id=client.tenant_id,
     )
 
     query_event_id, _, _ = decompose_action_id(feedback_id)
@@ -250,7 +250,7 @@ def process_feedback(req: SocketModeRequest, client: SocketModeClient) -> None:
 
 
 def build_request_details(
-    req: SocketModeRequest, client: SocketModeClient
+    req: SocketModeRequest, client: TenantSocketModeClient
 ) -> SlackMessageInfo:
     if req.type == "events_api":
         event = cast(dict[str, Any], req.payload["event"])
@@ -329,7 +329,7 @@ def build_request_details(
 
 def apologize_for_fail(
     details: SlackMessageInfo,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
 ) -> None:
     respond_in_thread(
         client=client.web_client,
@@ -341,7 +341,7 @@ def apologize_for_fail(
 
 def process_message(
     req: SocketModeRequest,
-    client: SocketModeClient,
+    client: TenantSocketModeClient,
     respond_every_channel: bool = DANSWER_BOT_RESPOND_EVERY_CHANNEL,
     notify_no_answer: bool = NOTIFY_SLACKBOT_NO_ANSWER,
 ) -> None:
@@ -357,8 +357,7 @@ def process_message(
         client=client.web_client, channel_id=channel
     )
 
-    engine = get_sqlalchemy_engine()
-    with Session(engine) as db_session:
+    with get_session_with_tenant(client.tenant_id) as db_session:
         slack_bot_config = get_slack_bot_config_for_channel(
             channel_name=channel_name, db_session=db_session
         )
@@ -390,6 +389,7 @@ def process_message(
             slack_bot_config=slack_bot_config,
             client=client.web_client,
             feedback_reminder_id=feedback_reminder_id,
+            tenant_id=client.tenant_id,
         )
 
         if failed:
@@ -404,12 +404,12 @@ def process_message(
                 apologize_for_fail(details, client)
 
 
-def acknowledge_message(req: SocketModeRequest, client: SocketModeClient) -> None:
+def acknowledge_message(req: SocketModeRequest, client: TenantSocketModeClient) -> None:
     response = SocketModeResponse(envelope_id=req.envelope_id)
     client.send_socket_mode_response(response)
 
 
-def action_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
+def action_routing(req: SocketModeRequest, client: TenantSocketModeClient) -> None:
     if actions := req.payload.get("actions"):
         action = cast(dict[str, Any], actions[0])
 
@@ -429,13 +429,13 @@ def action_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
             return handle_generate_answer_button(req, client)
 
 
-def view_routing(req: SocketModeRequest, client: SocketModeClient) -> None:
+def view_routing(req: SocketModeRequest, client: TenantSocketModeClient) -> None:
     if view := req.payload.get("view"):
         if view["callback_id"] == VIEW_DOC_FEEDBACK_ID:
             return process_feedback(req, client)
 
 
-def process_slack_event(client: SocketModeClient, req: SocketModeRequest) -> None:
+def process_slack_event(client: TenantSocketModeClient, req: SocketModeRequest) -> None:
     # Always respond right away, if Slack doesn't receive these frequently enough
     # it will assume the Bot is DEAD!!! :(
     acknowledge_message(req, client)
@@ -453,21 +453,24 @@ def process_slack_event(client: SocketModeClient, req: SocketModeRequest) -> Non
         logger.error(f"Slack request payload: {req.payload}")
 
 
-def _get_socket_client(slack_bot_tokens: SlackBotTokens) -> SocketModeClient:
+def _get_socket_client(
+    slack_bot_tokens: SlackBotTokens, tenant_id: str | None
+) -> TenantSocketModeClient:
     # For more info on how to set this up, checkout the docs:
     # https://docs.danswer.dev/slack_bot_setup
-    return SocketModeClient(
+    return TenantSocketModeClient(
         # This app-level token will be used only for establishing a connection
         app_token=slack_bot_tokens.app_token,
         web_client=WebClient(token=slack_bot_tokens.bot_token),
+        tenant_id=tenant_id,
     )
 
 
-def _initialize_socket_client(socket_client: SocketModeClient) -> None:
+def _initialize_socket_client(socket_client: TenantSocketModeClient) -> None:
     socket_client.socket_mode_request_listeners.append(process_slack_event)  # type: ignore
 
     # Establish a WebSocket connection to the Socket Mode servers
-    logger.notice("Listening for messages from Slack...")
+    logger.notice(f"Listening for messages from Slack {socket_client.tenant_id }...")
     socket_client.connect()
 
 
@@ -481,8 +484,8 @@ def _initialize_socket_client(socket_client: SocketModeClient) -> None:
 # NOTE: we are using Web Sockets so that you can run this from within a firewalled VPC
 # without issue.
 if __name__ == "__main__":
-    slack_bot_tokens: SlackBotTokens | None = None
-    socket_client: SocketModeClient | None = None
+    slack_bot_tokens: dict[str | None, SlackBotTokens] = {}
+    socket_clients: dict[str | None, TenantSocketModeClient] = {}
 
     set_is_ee_based_on_env_variable()
 
@@ -491,46 +494,59 @@ def _initialize_socket_client(socket_client: SocketModeClient) -> None:
 
     while True:
         try:
-            latest_slack_bot_tokens = fetch_tokens()
-
-            if latest_slack_bot_tokens != slack_bot_tokens:
-                if slack_bot_tokens is not None:
-                    logger.notice("Slack Bot tokens have changed - reconnecting")
-                else:
-                    # This happens on the very first time the listener process comes up
-                    # or the tokens have updated (set up for the first time)
-                    with Session(get_sqlalchemy_engine()) as db_session:
-                        search_settings = get_current_search_settings(db_session)
-                        embedding_model = EmbeddingModel.from_db_model(
-                            search_settings=search_settings,
-                            server_host=MODEL_SERVER_HOST,
-                            server_port=MODEL_SERVER_PORT,
-                        )
-
-                        warm_up_bi_encoder(
-                            embedding_model=embedding_model,
-                        )
-
-                slack_bot_tokens = latest_slack_bot_tokens
-                # potentially may cause a message to be dropped, but it is complicated
-                # to avoid + (1) if the user is changing tokens, they are likely okay with some
-                # "migration downtime" and (2) if a single message is lost it is okay
-                # as this should be a very rare occurrence
-                if socket_client:
-                    socket_client.close()
-
-                socket_client = _get_socket_client(slack_bot_tokens)
-                _initialize_socket_client(socket_client)
-
-            # Let the handlers run in the background + re-check for token updates every 60 seconds
+            tenant_ids = get_all_tenant_ids()  # Function to retrieve all tenant IDs
+
+            for tenant_id in tenant_ids:
+                with get_session_with_tenant(tenant_id) as db_session:
+                    try:
+                        latest_slack_bot_tokens = fetch_tokens()
+
+                        if (
+                            tenant_id not in slack_bot_tokens
+                            or latest_slack_bot_tokens != slack_bot_tokens[tenant_id]
+                        ):
+                            if tenant_id in slack_bot_tokens:
+                                logger.notice(
+                                    f"Slack Bot tokens have changed for tenant {tenant_id} - reconnecting"
+                                )
+                            else:
+                                # Initial setup for this tenant
+                                search_settings = get_current_search_settings(
+                                    db_session
+                                )
+                                embedding_model = EmbeddingModel.from_db_model(
+                                    search_settings=search_settings,
+                                    server_host=MODEL_SERVER_HOST,
+                                    server_port=MODEL_SERVER_PORT,
+                                )
+                                warm_up_bi_encoder(embedding_model=embedding_model)
+
+                            slack_bot_tokens[tenant_id] = latest_slack_bot_tokens
+
+                            # potentially may cause a message to be dropped, but it is complicated
+                            # to avoid + (1) if the user is changing tokens, they are likely okay with some
+                            # "migration downtime" and (2) if a single message is lost it is okay
+                            # as this should be a very rare occurrence
+                            if tenant_id in socket_clients:
+                                socket_clients[tenant_id].close()
+
+                            socket_client = _get_socket_client(
+                                latest_slack_bot_tokens, tenant_id
+                            )
+                            _initialize_socket_client(socket_client)
+
+                            socket_clients[tenant_id] = socket_client
+
+                    except KvKeyNotFoundError:
+                        logger.debug(f"Missing Slack Bot tokens for tenant {tenant_id}")
+                        if tenant_id in socket_clients:
+                            socket_clients[tenant_id].disconnect()
+                            del socket_clients[tenant_id]
+                            del slack_bot_tokens[tenant_id]
+
+            # Wait before checking for updates
             Event().wait(timeout=60)
-        except KvKeyNotFoundError:
-            # try again every 30 seconds. This is needed since the user may add tokens
-            # via the UI at any point in the programs lifecycle - if we just allow it to
-            # fail, then the user will need to restart the containers after adding tokens
-            logger.debug(
-                "Missing Slack Bot tokens - waiting 60 seconds and trying again"
-            )
-            if socket_client:
-                socket_client.disconnect()
+
+        except Exception:
+            logger.exception("An error occurred outside of main event loop")
             time.sleep(60)
diff --git a/backend/danswer/danswerbot/slack/utils.py b/backend/danswer/danswerbot/slack/utils.py
index 81209cf5c17..345f3605bd5 100644
--- a/backend/danswer/danswerbot/slack/utils.py
+++ b/backend/danswer/danswerbot/slack/utils.py
@@ -12,7 +12,7 @@
 from slack_sdk.errors import SlackApiError
 from slack_sdk.models.blocks import Block
 from slack_sdk.models.metadata import Metadata
-from sqlalchemy.orm import Session
+from slack_sdk.socket_mode import SocketModeClient
 
 from danswer.configs.app_configs import DISABLE_TELEMETRY
 from danswer.configs.constants import ID_SEPARATOR
@@ -31,7 +31,7 @@
 from danswer.connectors.slack.utils import SlackTextCleaner
 from danswer.danswerbot.slack.constants import FeedbackVisibility
 from danswer.danswerbot.slack.tokens import fetch_tokens
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.users import get_user_by_email
 from danswer.llm.exceptions import GenAIDisabledException
 from danswer.llm.factory import get_default_llms
@@ -489,7 +489,9 @@ def read_slack_thread(
     return thread_messages
 
 
-def slack_usage_report(action: str, sender_id: str | None, client: WebClient) -> None:
+def slack_usage_report(
+    action: str, sender_id: str | None, client: WebClient, tenant_id: str | None
+) -> None:
     if DISABLE_TELEMETRY:
         return
 
@@ -501,7 +503,7 @@ def slack_usage_report(action: str, sender_id: str | None, client: WebClient) ->
         logger.warning("Unable to find sender email")
 
     if sender_email is not None:
-        with Session(get_sqlalchemy_engine()) as db_session:
+        with get_session_with_tenant(tenant_id) as db_session:
             danswer_user = get_user_by_email(email=sender_email, db_session=db_session)
 
     optional_telemetry(
@@ -577,3 +579,9 @@ def get_feedback_visibility() -> FeedbackVisibility:
         return FeedbackVisibility(DANSWER_BOT_FEEDBACK_VISIBILITY.lower())
     except ValueError:
         return FeedbackVisibility.PRIVATE
+
+
+class TenantSocketModeClient(SocketModeClient):
+    def __init__(self, tenant_id: str | None, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.tenant_id = tenant_id

From 4c2cf8b13224dce731969cf4399d0a18f2a0cad6 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 17 Oct 2024 16:13:57 -0700
Subject: [PATCH 146/376] =?UTF-8?q?always=20finalize=20the=20serialized=20?=
 =?UTF-8?q?transaction=20so=20that=20it=20doesn't=20leak=20ou=E2=80=A6=20(?=
 =?UTF-8?q?#2843)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* always finalize the serialized transaction so that it doesn't leak outside the function

* re-raise the exception and log it
---
 .../background/indexing/run_indexing.py       | 51 +++++++++++--------
 1 file changed, 30 insertions(+), 21 deletions(-)

diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index c48d07ffd85..b4cfea97a2e 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -387,30 +387,39 @@ def _prepare_index_attempt(
     # after the next commit:
     # https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#setting-isolation-for-individual-transactions
     db_session.connection(execution_options={"isolation_level": "SERIALIZABLE"})  # type: ignore
-    if tenant_id is not None:
-        # Explicitly set the search path for the given tenant
-        db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
-        # Verify the search path was set correctly
-        result = db_session.execute(text("SHOW search_path"))
-        current_search_path = result.scalar()
-        logger.info(f"Current search path set to: {current_search_path}")
-
-    attempt = get_index_attempt(
-        db_session=db_session,
-        index_attempt_id=index_attempt_id,
-    )
+    try:
+        if tenant_id is not None:
+            # Explicitly set the search path for the given tenant
+            db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
+            # Verify the search path was set correctly
+            result = db_session.execute(text("SHOW search_path"))
+            current_search_path = result.scalar()
+            logger.info(f"Current search path set to: {current_search_path}")
+
+        attempt = get_index_attempt(
+            db_session=db_session,
+            index_attempt_id=index_attempt_id,
+        )
 
-    if attempt is None:
-        raise RuntimeError(f"Unable to find IndexAttempt for ID '{index_attempt_id}'")
+        if attempt is None:
+            raise RuntimeError(
+                f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
+            )
 
-    if attempt.status != IndexingStatus.NOT_STARTED:
-        raise RuntimeError(
-            f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
-            f"Current status is '{attempt.status}'."
-        )
+        if attempt.status != IndexingStatus.NOT_STARTED:
+            raise RuntimeError(
+                f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
+                f"Current status is '{attempt.status}'."
+            )
+
+        mark_attempt_in_progress(attempt, db_session)
 
-    # only commit once, to make sure this all happens in a single transaction
-    mark_attempt_in_progress(attempt, db_session)
+        # only commit once, to make sure this all happens in a single transaction
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        logger.exception("_prepare_index_attempt exceptioned.")
+        raise
 
     return attempt
 

From 61424de53133b8dab30fa7b18550f552d10b648a Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 16:20:37 -0700
Subject: [PATCH 147/376] add sentry (#2786)

* add sentry

* nit

* nit

* add requirement to ee

* try to ensure sentry is installed in integration tests
---
 .../danswer/background/celery/celery_app.py   |   14 +
 backend/danswer/main.py                       |   13 +
 backend/model_server/main.py                  |   13 +
 backend/requirements/default.txt              |    3 +-
 backend/requirements/ee.txt                   |    2 +-
 backend/requirements/model_server.txt         |    3 +-
 backend/shared_configs/configs.py             |    4 +
 .../docker_compose/docker-compose.dev.yml     |   12 +
 web/.gitignore                                |    2 +-
 web/instrumentation.ts                        |   12 +
 web/next.config.js                            |   12 +-
 web/package-lock.json                         | 2671 ++++++++++++++---
 web/package.json                              |    1 +
 web/sentry.client.config.ts                   |   23 +
 web/sentry.edge.config.ts                     |   16 +
 web/sentry.server.config.ts                   |   16 +
 web/src/app/global-error.tsx                  |   27 +
 17 files changed, 2460 insertions(+), 384 deletions(-)
 create mode 100644 web/instrumentation.ts
 create mode 100644 web/sentry.client.config.ts
 create mode 100644 web/sentry.edge.config.ts
 create mode 100644 web/sentry.server.config.ts
 create mode 100644 web/src/app/global-error.tsx

diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 0e9fb00b1fd..5477d416239 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -4,6 +4,7 @@
 from typing import Any
 
 import redis
+import sentry_sdk
 from celery import bootsteps  # type: ignore
 from celery import Celery
 from celery import current_task
@@ -16,6 +17,7 @@
 from celery.signals import worker_shutdown
 from celery.states import READY_STATES
 from celery.utils.log import get_task_logger
+from sentry_sdk.integrations.celery import CeleryIntegration
 
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
@@ -36,12 +38,24 @@
 from danswer.utils.logger import ColoredFormatter
 from danswer.utils.logger import PlainFormatter
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import SENTRY_DSN
 
 logger = setup_logger()
 
 # use this within celery tasks to get celery task specific logging
 task_logger = get_task_logger(__name__)
 
+if SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        integrations=[CeleryIntegration()],
+        traces_sample_rate=0.5,
+    )
+    logger.info("Sentry initialized")
+else:
+    logger.debug("Sentry DSN not provided, skipping Sentry initialization")
+
+
 celery_app = Celery(__name__)
 celery_app.config_from_object(
     "danswer.background.celery.celeryconfig"
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index cd0c5c195a6..ea338263279 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -5,6 +5,7 @@
 from typing import Any
 from typing import cast
 
+import sentry_sdk
 import uvicorn
 from fastapi import APIRouter
 from fastapi import FastAPI
@@ -15,6 +16,8 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from httpx_oauth.clients.google import GoogleOAuth2
+from sentry_sdk.integrations.fastapi import FastApiIntegration
+from sentry_sdk.integrations.starlette import StarletteIntegration
 from sqlalchemy.orm import Session
 
 from danswer import __version__
@@ -89,6 +92,7 @@
 from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 from shared_configs.configs import CORS_ALLOWED_ORIGIN
+from shared_configs.configs import SENTRY_DSN
 
 logger = setup_logger()
 
@@ -201,6 +205,15 @@ def get_application() -> FastAPI:
     application = FastAPI(
         title="Danswer Backend", version=__version__, lifespan=lifespan
     )
+    if SENTRY_DSN:
+        sentry_sdk.init(
+            dsn=SENTRY_DSN,
+            integrations=[StarletteIntegration(), FastApiIntegration()],
+            traces_sample_rate=0.5,
+        )
+        logger.info("Sentry initialized")
+    else:
+        logger.debug("Sentry DSN not provided, skipping Sentry initialization")
 
     # Add the custom exception handler
     application.add_exception_handler(status.HTTP_400_BAD_REQUEST, log_http_error)
diff --git a/backend/model_server/main.py b/backend/model_server/main.py
index 5c7979475c7..5505bbc8cdb 100644
--- a/backend/model_server/main.py
+++ b/backend/model_server/main.py
@@ -4,9 +4,12 @@
 from contextlib import asynccontextmanager
 from pathlib import Path
 
+import sentry_sdk
 import torch
 import uvicorn
 from fastapi import FastAPI
+from sentry_sdk.integrations.fastapi import FastApiIntegration
+from sentry_sdk.integrations.starlette import StarletteIntegration
 from transformers import logging as transformer_logging  # type:ignore
 
 from danswer import __version__
@@ -19,6 +22,7 @@
 from shared_configs.configs import MIN_THREADS_ML_MODELS
 from shared_configs.configs import MODEL_SERVER_ALLOWED_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
+from shared_configs.configs import SENTRY_DSN
 
 os.environ["TOKENIZERS_PARALLELISM"] = "false"
 os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
@@ -81,6 +85,15 @@ def get_model_app() -> FastAPI:
     application = FastAPI(
         title="Danswer Model Server", version=__version__, lifespan=lifespan
     )
+    if SENTRY_DSN:
+        sentry_sdk.init(
+            dsn=SENTRY_DSN,
+            integrations=[StarletteIntegration(), FastApiIntegration()],
+            traces_sample_rate=0.5,
+        )
+        logger.info("Sentry initialized")
+    else:
+        logger.debug("Sentry DSN not provided, skipping Sentry initialization")
 
     application.include_router(management_router)
     application.include_router(encoders_router)
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index ef36de81257..bc6c667cced 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -81,4 +81,5 @@ dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
 ultimate_sitemap_parser==0.5
 stripe==10.12.0
-mistune==0.8.4
\ No newline at end of file
+mistune==0.8.4
+sentry-sdk==2.14.0
diff --git a/backend/requirements/ee.txt b/backend/requirements/ee.txt
index 0717e3a67e7..1ca9c7eb924 100644
--- a/backend/requirements/ee.txt
+++ b/backend/requirements/ee.txt
@@ -1 +1 @@
-python3-saml==1.15.0
+python3-saml==1.15.0
\ No newline at end of file
diff --git a/backend/requirements/model_server.txt b/backend/requirements/model_server.txt
index 1cda9a505fe..2ea6df66d8b 100644
--- a/backend/requirements/model_server.txt
+++ b/backend/requirements/model_server.txt
@@ -12,4 +12,5 @@ torch==2.2.0
 transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
-litellm==1.49.5
\ No newline at end of file
+litellm==1.49.5
+sentry-sdk[fastapi,celery,starlette]==2.14.0
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index c5b79c4751a..ca452640071 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -71,6 +71,10 @@
     os.environ.get("STRICT_CHUNK_TOKEN_LIMIT", "").lower() == "true"
 )
 
+# Set up Sentry integration (for error logging)
+SENTRY_DSN = os.environ.get("SENTRY_DSN")
+
+
 # Fields which should only be set on new search setting
 PRESERVED_SEARCH_FIELDS = [
     "id",
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 5298859d13d..6f4619f83f7 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -90,6 +90,9 @@ services:
       - LOG_POSTGRES_CONN_COUNTS=${LOG_POSTGRES_CONN_COUNTS:-}
       - CELERY_BROKER_POOL_LIMIT=${CELERY_BROKER_POOL_LIMIT:-}
 
+      # Analytics Configs
+      - SENTRY_DSN=${SENTRY_DSN:-}
+
       # Chat Configs
       - HARD_DELETE_CHATS=${HARD_DELETE_CHATS:-}
       
@@ -197,6 +200,9 @@ services:
       - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
 
+      # Analytics Configs
+      - SENTRY_DSN=${SENTRY_DSN:-}
+
       # Enterprise Edition stuff
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${ENABLE_PAID_ENTERPRISE_EDITION_FEATURES:-false}
     extra_hosts:
@@ -254,6 +260,9 @@ services:
       - MIN_THREADS_ML_MODELS=${MIN_THREADS_ML_MODELS:-}
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
+
+      # Analytics Configs
+      - SENTRY_DSN=${SENTRY_DSN:-}
     volumes:
       # Not necessary, this is just to reduce download time during startup
       - model_cache_huggingface:/root/.cache/huggingface/
@@ -283,6 +292,9 @@ services:
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - CLIENT_EMBEDDING_TIMEOUT=${CLIENT_EMBEDDING_TIMEOUT:-}
+      
+      # Analytics Configs
+      - SENTRY_DSN=${SENTRY_DSN:-}
     volumes:
       # Not necessary, this is just to reduce download time during startup
       - indexing_huggingface_model_cache:/root/.cache/huggingface/
diff --git a/web/.gitignore b/web/.gitignore
index c87c9b392c0..e2a2a775c3b 100644
--- a/web/.gitignore
+++ b/web/.gitignore
@@ -1,5 +1,5 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
+.env.sentry-build-plugin
 # dependencies
 /node_modules
 /.pnp
diff --git a/web/instrumentation.ts b/web/instrumentation.ts
new file mode 100644
index 00000000000..dc0128848ea
--- /dev/null
+++ b/web/instrumentation.ts
@@ -0,0 +1,12 @@
+export async function register() {
+  if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
+    if (process.env.NEXT_RUNTIME === "nodejs") {
+      await import("./sentry.client.config");
+      await import("./sentry.server.config");
+    }
+
+    if (process.env.NEXT_RUNTIME === "edge") {
+      await import("./sentry.edge.config");
+    }
+  }
+}
diff --git a/web/next.config.js b/web/next.config.js
index 92812c513b7..684b43ffc16 100644
--- a/web/next.config.js
+++ b/web/next.config.js
@@ -13,4 +13,14 @@ const nextConfig = {
   },
 };
 
-module.exports = nextConfig;
+const { withSentryConfig } = require("@sentry/nextjs");
+
+module.exports = withSentryConfig(nextConfig, {
+  org: "danswer",
+  project: "javascript-nextjs",
+
+  // An auth token is required for uploading source maps.
+  authToken: process.env.SENTRY_AUTH_TOKEN,
+
+  silent: false, // Can be used to suppress logs
+});
diff --git a/web/package-lock.json b/web/package-lock.json
index 36a76cbe05c..4401222f737 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -15,6 +15,7 @@
         "@radix-ui/react-dialog": "^1.0.5",
         "@radix-ui/react-popover": "^1.0.7",
         "@radix-ui/react-tooltip": "^1.0.7",
+        "@sentry/nextjs": "^8.34.0",
         "@stripe/stripe-js": "^4.6.0",
         "@tremor/react": "^3.9.2",
         "@types/js-cookie": "^3.0.3",
@@ -72,7 +73,6 @@
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz",
       "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==",
-      "peer": true,
       "dependencies": {
         "@jridgewell/gen-mapping": "^0.3.5",
         "@jridgewell/trace-mapping": "^0.3.24"
@@ -97,7 +97,6 @@
       "version": "7.24.4",
       "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.24.4.tgz",
       "integrity": "sha512-vg8Gih2MLK+kOkHJp4gBEIkyaIi00jgWot2D9QOmmfLC8jINSOzmCLta6Bvz/JSBCqnegV0L80jhxkol5GWNfQ==",
-      "peer": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -106,7 +105,6 @@
       "version": "7.24.5",
       "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.24.5.tgz",
       "integrity": "sha512-tVQRucExLQ02Boi4vdPp49svNGcfL2GhdTCT9aldhXgCJVAI21EtRfBettiuLUwce/7r6bFdgs6JFkcdTiFttA==",
-      "peer": true,
       "dependencies": {
         "@ampproject/remapping": "^2.2.0",
         "@babel/code-frame": "^7.24.2",
@@ -136,7 +134,6 @@
       "version": "6.3.1",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
       "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "peer": true,
       "bin": {
         "semver": "bin/semver.js"
       }
@@ -170,7 +167,6 @@
       "version": "7.23.6",
       "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.23.6.tgz",
       "integrity": "sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==",
-      "peer": true,
       "dependencies": {
         "@babel/compat-data": "^7.23.5",
         "@babel/helper-validator-option": "^7.23.5",
@@ -186,7 +182,6 @@
       "version": "5.1.1",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
       "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
-      "peer": true,
       "dependencies": {
         "yallist": "^3.0.2"
       }
@@ -195,7 +190,6 @@
       "version": "6.3.1",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
       "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==",
-      "peer": true,
       "bin": {
         "semver": "bin/semver.js"
       }
@@ -246,7 +240,6 @@
       "version": "7.24.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.24.5.tgz",
       "integrity": "sha512-9GxeY8c2d2mdQUP1Dye0ks3VDyIMS98kt/llQ2nUId8IsWqTF0l1LkSX0/uP7l7MCDrzXS009Hyhe2gzTiGW8A==",
-      "peer": true,
       "dependencies": {
         "@babel/helper-environment-visitor": "^7.22.20",
         "@babel/helper-module-imports": "^7.24.3",
@@ -273,7 +266,6 @@
       "version": "7.24.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-simple-access/-/helper-simple-access-7.24.5.tgz",
       "integrity": "sha512-uH3Hmf5q5n7n8mz7arjUlDOCbttY/DW4DYhE6FUsjKJ/oYC1kQQUvwEQWxRwUpX9qQKRXeqLwWxrqilMrf32sQ==",
-      "peer": true,
       "dependencies": {
         "@babel/types": "^7.24.5"
       },
@@ -312,7 +304,6 @@
       "version": "7.23.5",
       "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.23.5.tgz",
       "integrity": "sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==",
-      "peer": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -321,7 +312,6 @@
       "version": "7.24.5",
       "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.24.5.tgz",
       "integrity": "sha512-CiQmBMMpMQHwM5m01YnrM6imUG1ebgYJ+fAIW4FZe6m4qHTPaRHti+R8cggAwkdz4oXhtO4/K9JWlh+8hIfR2Q==",
-      "peer": true,
       "dependencies": {
         "@babel/template": "^7.24.0",
         "@babel/traverse": "^7.24.5",
@@ -933,10 +923,20 @@
         "node": ">=6.0.0"
       }
     },
+    "node_modules/@jridgewell/source-map": {
+      "version": "0.3.6",
+      "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.6.tgz",
+      "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==",
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/gen-mapping": "^0.3.5",
+        "@jridgewell/trace-mapping": "^0.3.25"
+      }
+    },
     "node_modules/@jridgewell/sourcemap-codec": {
-      "version": "1.4.15",
-      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
-      "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz",
+      "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ=="
     },
     "node_modules/@jridgewell/trace-mapping": {
       "version": "0.3.25",
@@ -1128,315 +1128,565 @@
         "node": ">= 8"
       }
     },
-    "node_modules/@phosphor-icons/react": {
-      "version": "2.1.5",
-      "resolved": "https://registry.npmjs.org/@phosphor-icons/react/-/react-2.1.5.tgz",
-      "integrity": "sha512-B7vRm/w+P/+eavWZP5CB5Ul0ffK4Y7fpd/auWKuGvm+8pVgAJzbOK8O0s+DqzR+TwWkh5pHtJTuoAtaSvgCPzg==",
+    "node_modules/@opentelemetry/api": {
+      "version": "1.9.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz",
+      "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==",
       "engines": {
-        "node": ">=10"
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/@opentelemetry/api-logs": {
+      "version": "0.53.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.53.0.tgz",
+      "integrity": "sha512-8HArjKx+RaAI8uEIgcORbZIPklyh1YLjPSBus8hjRmvLi6DeFzgOcdZ7KwPabKj8mXF8dX0hyfAyGfycz0DbFw==",
+      "dependencies": {
+        "@opentelemetry/api": "^1.0.0"
       },
-      "peerDependencies": {
-        "react": ">= 16.8",
-        "react-dom": ">= 16.8"
+      "engines": {
+        "node": ">=14"
       }
     },
-    "node_modules/@pkgjs/parseargs": {
-      "version": "0.11.0",
-      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
-      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
-      "optional": true,
+    "node_modules/@opentelemetry/context-async-hooks": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-1.26.0.tgz",
+      "integrity": "sha512-HedpXXYzzbaoutw6DFLWLDket2FwLkLpil4hGCZ1xYEIMTcivdfwEOISgdbLEWyG3HW52gTq2V9mOVJrONgiwg==",
       "engines": {
         "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
       }
     },
-    "node_modules/@radix-ui/primitive": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.0.1.tgz",
-      "integrity": "sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==",
+    "node_modules/@opentelemetry/core": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.26.0.tgz",
+      "integrity": "sha512-1iKxXXE8415Cdv0yjG3G6hQnB5eVEsJce3QaawX8SjDn0mAS0ZM8fAbZZJD4ajvhC15cePvosSCut404KrIIvQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10"
+        "@opentelemetry/semantic-conventions": "1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
       }
     },
-    "node_modules/@radix-ui/react-arrow": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.0.3.tgz",
-      "integrity": "sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==",
+    "node_modules/@opentelemetry/instrumentation": {
+      "version": "0.53.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.53.0.tgz",
+      "integrity": "sha512-DMwg0hy4wzf7K73JJtl95m/e0boSoWhH07rfvHvYzQtBD3Bmv0Wc1x733vyZBqmFm8OjJD0/pfiUg1W3JjFX0A==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+        "@opentelemetry/api-logs": "0.53.0",
+        "@types/shimmer": "^1.2.0",
+        "import-in-the-middle": "^1.8.1",
+        "require-in-the-middle": "^7.1.1",
+        "semver": "^7.5.2",
+        "shimmer": "^1.2.1"
       },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      "engines": {
+        "node": ">=14"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
-      "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==",
+    "node_modules/@opentelemetry/instrumentation-amqplib": {
+      "version": "0.42.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-amqplib/-/instrumentation-amqplib-0.42.0.tgz",
+      "integrity": "sha512-fiuU6OKsqHJiydHWgTRQ7MnIrJ2lEqsdgFtNIH4LbAUJl/5XmrIeoDzDnox+hfkgWK65jsleFuQDtYb5hW1koQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10"
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+      "engines": {
+        "node": ">=14"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-context": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz",
-      "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==",
+    "node_modules/@opentelemetry/instrumentation-connect": {
+      "version": "0.39.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-connect/-/instrumentation-connect-0.39.0.tgz",
+      "integrity": "sha512-pGBiKevLq7NNglMgqzmeKczF4XQMTOUOTkK8afRHMZMnrK3fcETyTH7lVaSozwiOM3Ws+SuEmXZT7DYrrhxGlg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10"
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0",
+        "@types/connect": "3.4.36"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-dataloader": {
+      "version": "0.12.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-dataloader/-/instrumentation-dataloader-0.12.0.tgz",
+      "integrity": "sha512-pnPxatoFE0OXIZDQhL2okF//dmbiWFzcSc8pUg9TqofCLYZySSxDCgQc69CJBo5JnI3Gz1KP+mOjS4WAeRIH4g==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-dialog": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.0.5.tgz",
-      "integrity": "sha512-GjWJX/AUpB703eEBanuBnIWdIXg6NvJFCXcNlSZk4xdszCdhrJgBoUd1cGk67vFO+WdA2pfI/plOpqz/5GUP6Q==",
+    "node_modules/@opentelemetry/instrumentation-express": {
+      "version": "0.42.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-express/-/instrumentation-express-0.42.0.tgz",
+      "integrity": "sha512-YNcy7ZfGnLsVEqGXQPT+S0G1AE46N21ORY7i7yUQyfhGAL4RBjnZUqefMI0NwqIl6nGbr1IpF0rZGoN8Q7x12Q==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-dismissable-layer": "1.0.5",
-        "@radix-ui/react-focus-guards": "1.0.1",
-        "@radix-ui/react-focus-scope": "1.0.4",
-        "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-portal": "1.0.4",
-        "@radix-ui/react-presence": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-slot": "1.0.2",
-        "@radix-ui/react-use-controllable-state": "1.0.1",
-        "aria-hidden": "^1.1.1",
-        "react-remove-scroll": "2.5.5"
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-fastify": {
+      "version": "0.39.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fastify/-/instrumentation-fastify-0.39.0.tgz",
+      "integrity": "sha512-SS9uSlKcsWZabhBp2szErkeuuBDgxOUlllwkS92dVaWRnMmwysPhcEgHKB8rUe3BHg/GnZC1eo1hbTZv4YhfoA==",
+      "dependencies": {
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-dismissable-layer": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
-      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
+    "node_modules/@opentelemetry/instrumentation-fs": {
+      "version": "0.15.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.15.0.tgz",
+      "integrity": "sha512-JWVKdNLpu1skqZQA//jKOcKdJC66TWKqa2FUFq70rKohvaSq47pmXlnabNO+B/BvLfmidfiaN35XakT5RyMl2Q==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-escape-keydown": "1.0.3"
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-generic-pool": {
+      "version": "0.39.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-generic-pool/-/instrumentation-generic-pool-0.39.0.tgz",
+      "integrity": "sha512-y4v8Y+tSfRB3NNBvHjbjrn7rX/7sdARG7FuK6zR8PGb28CTa0kHpEGCJqvL9L8xkTNvTXo+lM36ajFGUaK1aNw==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-focus-guards": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
-      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
+    "node_modules/@opentelemetry/instrumentation-graphql": {
+      "version": "0.43.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-graphql/-/instrumentation-graphql-0.43.0.tgz",
+      "integrity": "sha512-aI3YMmC2McGd8KW5du1a2gBA0iOMOGLqg4s9YjzwbjFwjlmMNFSK1P3AIg374GWg823RPUGfVTIgZ/juk9CVOA==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10"
+        "@opentelemetry/instrumentation": "^0.53.0"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-hapi": {
+      "version": "0.41.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-hapi/-/instrumentation-hapi-0.41.0.tgz",
+      "integrity": "sha512-jKDrxPNXDByPlYcMdZjNPYCvw0SQJjN+B1A+QH+sx+sAHsKSAf9hwFiJSrI6C4XdOls43V/f/fkp9ITkHhKFbQ==",
+      "dependencies": {
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-focus-scope": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz",
-      "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==",
+    "node_modules/@opentelemetry/instrumentation-http": {
+      "version": "0.53.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-http/-/instrumentation-http-0.53.0.tgz",
+      "integrity": "sha512-H74ErMeDuZfj7KgYCTOFGWF5W9AfaPnqLQQxeFq85+D29wwV2yqHbz2IKLYpkOh7EI6QwDEl7rZCIxjJLyc/CQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
+        "@opentelemetry/core": "1.26.0",
+        "@opentelemetry/instrumentation": "0.53.0",
+        "@opentelemetry/semantic-conventions": "1.27.0",
+        "semver": "^7.5.2"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-ioredis": {
+      "version": "0.43.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.43.0.tgz",
+      "integrity": "sha512-i3Dke/LdhZbiUAEImmRG3i7Dimm/BD7t8pDDzwepSvIQ6s2X6FPia7561gw+64w+nx0+G9X14D7rEfaMEmmjig==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/redis-common": "^0.36.2",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-id": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
-      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
+    "node_modules/@opentelemetry/instrumentation-kafkajs": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-kafkajs/-/instrumentation-kafkajs-0.3.0.tgz",
+      "integrity": "sha512-UnkZueYK1ise8FXQeKlpBd7YYUtC7mM8J0wzUSccEfc/G8UqHQqAzIyYCUOUPUKp8GsjLnWOOK/3hJc4owb7Jg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-koa": {
+      "version": "0.43.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-koa/-/instrumentation-koa-0.43.0.tgz",
+      "integrity": "sha512-lDAhSnmoTIN6ELKmLJBplXzT/Jqs5jGZehuG22EdSMaTwgjMpxMDI1YtlKEhiWPWkrz5LUsd0aOO0ZRc9vn3AQ==",
+      "dependencies": {
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-popover": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.0.7.tgz",
-      "integrity": "sha512-shtvVnlsxT6faMnK/a7n0wptwBD23xc1Z5mdrtKLwVEfsEMXodS0r5s0/g5P0hX//EKYZS2sxUjqfzlg52ZSnQ==",
+    "node_modules/@opentelemetry/instrumentation-lru-memoizer": {
+      "version": "0.40.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-lru-memoizer/-/instrumentation-lru-memoizer-0.40.0.tgz",
+      "integrity": "sha512-21xRwZsEdMPnROu/QsaOIODmzw59IYpGFmuC4aFWvMj6stA8+Ei1tX67nkarJttlNjoM94um0N4X26AD7ff54A==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-dismissable-layer": "1.0.5",
-        "@radix-ui/react-focus-guards": "1.0.1",
-        "@radix-ui/react-focus-scope": "1.0.4",
-        "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-popper": "1.1.3",
-        "@radix-ui/react-portal": "1.0.4",
-        "@radix-ui/react-presence": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-slot": "1.0.2",
-        "@radix-ui/react-use-controllable-state": "1.0.1",
-        "aria-hidden": "^1.1.1",
-        "react-remove-scroll": "2.5.5"
+        "@opentelemetry/instrumentation": "^0.53.0"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-mongodb": {
+      "version": "0.47.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongodb/-/instrumentation-mongodb-0.47.0.tgz",
+      "integrity": "sha512-yqyXRx2SulEURjgOQyJzhCECSh5i1uM49NUaq9TqLd6fA7g26OahyJfsr9NE38HFqGRHpi4loyrnfYGdrsoVjQ==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/sdk-metrics": "^1.9.1",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-popper": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
-      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
+    "node_modules/@opentelemetry/instrumentation-mongoose": {
+      "version": "0.42.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mongoose/-/instrumentation-mongoose-0.42.0.tgz",
+      "integrity": "sha512-AnWv+RaR86uG3qNEMwt3plKX1ueRM7AspfszJYVkvkehiicC3bHQA6vWdb6Zvy5HAE14RyFbu9+2hUUjR2NSyg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@floating-ui/react-dom": "^2.0.0",
-        "@radix-ui/react-arrow": "1.0.3",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1",
-        "@radix-ui/react-use-rect": "1.0.1",
-        "@radix-ui/react-use-size": "1.0.1",
-        "@radix-ui/rect": "1.0.1"
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-mysql": {
+      "version": "0.41.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql/-/instrumentation-mysql-0.41.0.tgz",
+      "integrity": "sha512-jnvrV6BsQWyHS2qb2fkfbfSb1R/lmYwqEZITwufuRl37apTopswu9izc0b1CYRp/34tUG/4k/V39PND6eyiNvw==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0",
+        "@types/mysql": "2.15.26"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-portal": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
-      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+    "node_modules/@opentelemetry/instrumentation-mysql2": {
+      "version": "0.41.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-mysql2/-/instrumentation-mysql2-0.41.0.tgz",
+      "integrity": "sha512-REQB0x+IzVTpoNgVmy5b+UnH1/mDByrneimP6sbDHkp1j8QOl1HyWOrBH/6YWR0nrbU3l825Em5PlybjT3232g==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0",
+        "@opentelemetry/sql-common": "^0.40.1"
+      },
+      "engines": {
+        "node": ">=14"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-nestjs-core": {
+      "version": "0.40.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-nestjs-core/-/instrumentation-nestjs-core-0.40.0.tgz",
+      "integrity": "sha512-WF1hCUed07vKmf5BzEkL0wSPinqJgH7kGzOjjMAiTGacofNXjb/y4KQ8loj2sNsh5C/NN7s1zxQuCgbWbVTGKg==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
       }
     },
-    "node_modules/@radix-ui/react-presence": {
+    "node_modules/@opentelemetry/instrumentation-pg": {
+      "version": "0.44.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.44.0.tgz",
+      "integrity": "sha512-oTWVyzKqXud1BYEGX1loo2o4k4vaU1elr3vPO8NZolrBtFvQ34nx4HgUaexUDuEog00qQt+MLR5gws/p+JXMLQ==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0",
+        "@opentelemetry/sql-common": "^0.40.1",
+        "@types/pg": "8.6.1",
+        "@types/pg-pool": "2.0.6"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-redis-4": {
+      "version": "0.42.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-redis-4/-/instrumentation-redis-4-0.42.0.tgz",
+      "integrity": "sha512-NaD+t2JNcOzX/Qa7kMy68JbmoVIV37fT/fJYzLKu2Wwd+0NCxt+K2OOsOakA8GVg8lSpFdbx4V/suzZZ2Pvdjg==",
+      "dependencies": {
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/redis-common": "^0.36.2",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/instrumentation-undici": {
+      "version": "0.6.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation-undici/-/instrumentation-undici-0.6.0.tgz",
+      "integrity": "sha512-ABJBhm5OdhGmbh0S/fOTE4N69IZ00CsHC5ijMYfzbw3E5NwLgpQk5xsljaECrJ8wz1SfXbO03FiSuu5AyRAkvQ==",
+      "dependencies": {
+        "@opentelemetry/core": "^1.8.0",
+        "@opentelemetry/instrumentation": "^0.53.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.7.0"
+      }
+    },
+    "node_modules/@opentelemetry/redis-common": {
+      "version": "0.36.2",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/redis-common/-/redis-common-0.36.2.tgz",
+      "integrity": "sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@opentelemetry/resources": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.26.0.tgz",
+      "integrity": "sha512-CPNYchBE7MBecCSVy0HKpUISEeJOniWqcHaAHpmasZ3j9o6V3AyBzhRc90jdmemq0HOxDr6ylhUbDhBqqPpeNw==",
+      "dependencies": {
+        "@opentelemetry/core": "1.26.0",
+        "@opentelemetry/semantic-conventions": "1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/sdk-metrics": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-1.26.0.tgz",
+      "integrity": "sha512-0SvDXmou/JjzSDOjUmetAAvcKQW6ZrvosU0rkbDGpXvvZN+pQF6JbK/Kd4hNdK4q/22yeruqvukXEJyySTzyTQ==",
+      "dependencies": {
+        "@opentelemetry/core": "1.26.0",
+        "@opentelemetry/resources": "1.26.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.3.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/sdk-trace-base": {
+      "version": "1.26.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.26.0.tgz",
+      "integrity": "sha512-olWQldtvbK4v22ymrKLbIcBi9L2SpMO84sCPY54IVsJhP9fRsxJT194C/AVaAuJzLE30EdhhM1VmvVYR7az+cw==",
+      "dependencies": {
+        "@opentelemetry/core": "1.26.0",
+        "@opentelemetry/resources": "1.26.0",
+        "@opentelemetry/semantic-conventions": "1.27.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/semantic-conventions": {
+      "version": "1.27.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.27.0.tgz",
+      "integrity": "sha512-sAay1RrB+ONOem0OZanAR1ZI/k7yDpnOQSQmTMuGImUQb2y8EbSaCJ94FQluM74xoU03vlb2d2U90hZluL6nQg==",
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@opentelemetry/sql-common": {
+      "version": "0.40.1",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sql-common/-/sql-common-0.40.1.tgz",
+      "integrity": "sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==",
+      "dependencies": {
+        "@opentelemetry/core": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.1.0"
+      }
+    },
+    "node_modules/@phosphor-icons/react": {
+      "version": "2.1.5",
+      "resolved": "https://registry.npmjs.org/@phosphor-icons/react/-/react-2.1.5.tgz",
+      "integrity": "sha512-B7vRm/w+P/+eavWZP5CB5Ul0ffK4Y7fpd/auWKuGvm+8pVgAJzbOK8O0s+DqzR+TwWkh5pHtJTuoAtaSvgCPzg==",
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "react": ">= 16.8",
+        "react-dom": ">= 16.8"
+      }
+    },
+    "node_modules/@pkgjs/parseargs": {
+      "version": "0.11.0",
+      "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
+      "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
+      "optional": true,
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@prisma/instrumentation": {
+      "version": "5.19.1",
+      "resolved": "https://registry.npmjs.org/@prisma/instrumentation/-/instrumentation-5.19.1.tgz",
+      "integrity": "sha512-VLnzMQq7CWroL5AeaW0Py2huiNKeoMfCH3SUxstdzPrlWQi6UQ9UrfcbUkNHlVFqOMacqy8X/8YtE0kuKDpD9w==",
+      "dependencies": {
+        "@opentelemetry/api": "^1.8",
+        "@opentelemetry/instrumentation": "^0.49 || ^0.50 || ^0.51 || ^0.52.0",
+        "@opentelemetry/sdk-trace-base": "^1.22"
+      }
+    },
+    "node_modules/@prisma/instrumentation/node_modules/@opentelemetry/api-logs": {
+      "version": "0.52.1",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.52.1.tgz",
+      "integrity": "sha512-qnSqB2DQ9TPP96dl8cDubDvrUyWc0/sK81xHTK8eSUspzDM3bsewX903qclQFvVhgStjRWdC5bLb3kQqMkfV5A==",
+      "dependencies": {
+        "@opentelemetry/api": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@prisma/instrumentation/node_modules/@opentelemetry/instrumentation": {
+      "version": "0.52.1",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.52.1.tgz",
+      "integrity": "sha512-uXJbYU/5/MBHjMp1FqrILLRuiJCs3Ofk0MeRDk8g1S1gD47U8X3JnSwcMO1rtRo1x1a7zKaQHaoYu49p/4eSKw==",
+      "dependencies": {
+        "@opentelemetry/api-logs": "0.52.1",
+        "@types/shimmer": "^1.0.2",
+        "import-in-the-middle": "^1.8.1",
+        "require-in-the-middle": "^7.1.1",
+        "semver": "^7.5.2",
+        "shimmer": "^1.2.1"
+      },
+      "engines": {
+        "node": ">=14"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@radix-ui/primitive": {
       "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
-      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.0.1.tgz",
+      "integrity": "sha512-yQ8oGX2GVsEYMWGxcovu1uGWPCxV5BFfeeYxqPmuAzUyLT9qmaMXSAhXpb0WrspIeqYzdJpkh2vHModJPgRIaw==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      }
+    },
+    "node_modules/@radix-ui/react-arrow": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.0.3.tgz",
+      "integrity": "sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==",
       "dependencies": {
         "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "@radix-ui/react-primitive": "1.0.3"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -1453,36 +1703,29 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-primitive": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
-      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
+    "node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
+      "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-slot": "1.0.2"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-slot": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
-      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+    "node_modules/@radix-ui/react-context": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz",
+      "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -1494,24 +1737,26 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tooltip": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz",
-      "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==",
+    "node_modules/@radix-ui/react-dialog": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.0.5.tgz",
+      "integrity": "sha512-GjWJX/AUpB703eEBanuBnIWdIXg6NvJFCXcNlSZk4xdszCdhrJgBoUd1cGk67vFO+WdA2pfI/plOpqz/5GUP6Q==",
       "dependencies": {
         "@babel/runtime": "^7.13.10",
         "@radix-ui/primitive": "1.0.1",
         "@radix-ui/react-compose-refs": "1.0.1",
         "@radix-ui/react-context": "1.0.1",
         "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-focus-guards": "1.0.1",
+        "@radix-ui/react-focus-scope": "1.0.4",
         "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-popper": "1.1.3",
         "@radix-ui/react-portal": "1.0.4",
         "@radix-ui/react-presence": "1.0.1",
         "@radix-ui/react-primitive": "1.0.3",
         "@radix-ui/react-slot": "1.0.2",
         "@radix-ui/react-use-controllable-state": "1.0.1",
-        "@radix-ui/react-visually-hidden": "1.0.3"
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.5.5"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -1528,10 +1773,37 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-use-callback-ref": {
+    "node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
+      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-escape-keydown": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-focus-guards": {
       "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
-      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
+      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
       "dependencies": {
         "@babel/runtime": "^7.13.10"
       },
@@ -1545,31 +1817,38 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-use-controllable-state": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
-      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
+    "node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz",
+      "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==",
       "dependencies": {
         "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
         "@radix-ui/react-use-callback-ref": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-use-escape-keydown": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
-      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
+    "node_modules/@radix-ui/react-id": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
+      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
       "dependencies": {
         "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
+        "@radix-ui/react-use-layout-effect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -1581,95 +1860,928 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-use-layout-effect": {
+    "node_modules/@radix-ui/react-popover": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.0.7.tgz",
+      "integrity": "sha512-shtvVnlsxT6faMnK/a7n0wptwBD23xc1Z5mdrtKLwVEfsEMXodS0r5s0/g5P0hX//EKYZS2sxUjqfzlg52ZSnQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-focus-guards": "1.0.1",
+        "@radix-ui/react-focus-scope": "1.0.4",
+        "@radix-ui/react-id": "1.0.1",
+        "@radix-ui/react-popper": "1.1.3",
+        "@radix-ui/react-portal": "1.0.4",
+        "@radix-ui/react-presence": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-slot": "1.0.2",
+        "@radix-ui/react-use-controllable-state": "1.0.1",
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.5.5"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popper": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
+      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.0.3",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1",
+        "@radix-ui/react-use-rect": "1.0.1",
+        "@radix-ui/react-use-size": "1.0.1",
+        "@radix-ui/rect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-portal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
+      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-presence": {
       "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
-      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
+      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-primitive": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
+      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-slot": "1.0.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
         "react": "^16.8 || ^17.0 || ^18.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-tooltip": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz",
+      "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-id": "1.0.1",
+        "@radix-ui/react-popper": "1.1.3",
+        "@radix-ui/react-portal": "1.0.4",
+        "@radix-ui/react-presence": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-slot": "1.0.2",
+        "@radix-ui/react-use-controllable-state": "1.0.1",
+        "@radix-ui/react-visually-hidden": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
+      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
+      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-rect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz",
+      "integrity": "sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/rect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-use-size": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.0.1.tgz",
+      "integrity": "sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-visually-hidden": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.0.3.tgz",
+      "integrity": "sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/rect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.0.1.tgz",
+      "integrity": "sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      }
+    },
+    "node_modules/@rollup/plugin-commonjs": {
+      "version": "26.0.1",
+      "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-26.0.1.tgz",
+      "integrity": "sha512-UnsKoZK6/aGIH6AdkptXhNvhaqftcjq3zZdT+LY5Ftms6JR06nADcDsYp5hTU9E2lbJUEOhdlY5J4DNTneM+jQ==",
+      "dependencies": {
+        "@rollup/pluginutils": "^5.0.1",
+        "commondir": "^1.0.1",
+        "estree-walker": "^2.0.2",
+        "glob": "^10.4.1",
+        "is-reference": "1.2.1",
+        "magic-string": "^0.30.3"
+      },
+      "engines": {
+        "node": ">=16.0.0 || 14 >= 14.17"
+      },
+      "peerDependencies": {
+        "rollup": "^2.68.0||^3.0.0||^4.0.0"
+      },
+      "peerDependenciesMeta": {
+        "rollup": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@rollup/plugin-commonjs/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/@rollup/plugin-commonjs/node_modules/glob": {
+      "version": "10.4.5",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz",
+      "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==",
+      "dependencies": {
+        "foreground-child": "^3.1.0",
+        "jackspeak": "^3.1.2",
+        "minimatch": "^9.0.4",
+        "minipass": "^7.1.2",
+        "package-json-from-dist": "^1.0.0",
+        "path-scurry": "^1.11.1"
+      },
+      "bin": {
+        "glob": "dist/esm/bin.mjs"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@rollup/plugin-commonjs/node_modules/jackspeak": {
+      "version": "3.4.3",
+      "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz",
+      "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==",
+      "dependencies": {
+        "@isaacs/cliui": "^8.0.2"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      },
+      "optionalDependencies": {
+        "@pkgjs/parseargs": "^0.11.0"
+      }
+    },
+    "node_modules/@rollup/plugin-commonjs/node_modules/minimatch": {
+      "version": "9.0.5",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz",
+      "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@rollup/plugin-commonjs/node_modules/minipass": {
+      "version": "7.1.2",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz",
+      "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==",
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      }
+    },
+    "node_modules/@rollup/pluginutils": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/@rollup/pluginutils/-/pluginutils-5.1.2.tgz",
+      "integrity": "sha512-/FIdS3PyZ39bjZlwqFnWqCOVnW7o963LtKMwQOD0NhQqw22gSr2YY1afu3FxRip4ZCZNsD5jq6Aaz6QV3D/Njw==",
+      "dependencies": {
+        "@types/estree": "^1.0.0",
+        "estree-walker": "^2.0.2",
+        "picomatch": "^2.3.1"
+      },
+      "engines": {
+        "node": ">=14.0.0"
+      },
+      "peerDependencies": {
+        "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0"
+      },
+      "peerDependenciesMeta": {
+        "rollup": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@rushstack/eslint-patch": {
+      "version": "1.10.3",
+      "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.10.3.tgz",
+      "integrity": "sha512-qC/xYId4NMebE6w/V33Fh9gWxLgURiNYgVNObbJl2LZv0GUUItCcCqC5axQSwRaAgaxl2mELq1rMzlswaQ0Zxg==",
+      "dev": true
+    },
+    "node_modules/@sentry-internal/browser-utils": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/browser-utils/-/browser-utils-8.34.0.tgz",
+      "integrity": "sha512-4AcYOzPzD1tL5eSRQ/GpKv5enquZf4dMVUez99/Bh3va8qiJrNP55AcM7UzZ7WZLTqKygIYruJTU5Zu2SpEAPQ==",
+      "dependencies": {
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry-internal/feedback": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/feedback/-/feedback-8.34.0.tgz",
+      "integrity": "sha512-aYSM2KPUs0FLPxxbJCFSwCYG70VMzlT04xepD1Y/tTlPPOja/02tSv2tyOdZbv8Uw7xslZs3/8Lhj74oYcTBxw==",
+      "dependencies": {
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry-internal/replay": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/replay/-/replay-8.34.0.tgz",
+      "integrity": "sha512-EoMh9NYljNewZK1quY23YILgtNdGgrkzJ9TPsj6jXUG0LZ0Q7N7eFWd0xOEDBvFxrmI3cSXF1i4d1sBb+eyKRw==",
+      "dependencies": {
+        "@sentry-internal/browser-utils": "8.34.0",
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry-internal/replay-canvas": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry-internal/replay-canvas/-/replay-canvas-8.34.0.tgz",
+      "integrity": "sha512-x8KhZcCDpbKHqFOykYXiamX6x0LRxv6N1OJHoH+XCrMtiDBZr4Yo30d/MaS6rjmKGMtSRij30v+Uq+YWIgxUrg==",
+      "dependencies": {
+        "@sentry-internal/replay": "8.34.0",
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry/babel-plugin-component-annotate": {
+      "version": "2.22.3",
+      "resolved": "https://registry.npmjs.org/@sentry/babel-plugin-component-annotate/-/babel-plugin-component-annotate-2.22.3.tgz",
+      "integrity": "sha512-OlHA+i+vnQHRIdry4glpiS/xTOtgjmpXOt6IBOUqynx5Jd/iK1+fj+t8CckqOx9wRacO/hru2wfW/jFq0iViLg==",
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/@sentry/browser": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/browser/-/browser-8.34.0.tgz",
+      "integrity": "sha512-3HHG2NXxzHq1lVmDy2uRjYjGNf9NsJsTPlOC70vbQdOb+S49EdH/XMPy+J3ruIoyv6Cu0LwvA6bMOM6rHZOgNQ==",
+      "dependencies": {
+        "@sentry-internal/browser-utils": "8.34.0",
+        "@sentry-internal/feedback": "8.34.0",
+        "@sentry-internal/replay": "8.34.0",
+        "@sentry-internal/replay-canvas": "8.34.0",
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry/bundler-plugin-core": {
+      "version": "2.22.3",
+      "resolved": "https://registry.npmjs.org/@sentry/bundler-plugin-core/-/bundler-plugin-core-2.22.3.tgz",
+      "integrity": "sha512-DeoUl0WffcqZZRl5Wy9aHvX4WfZbbWt0QbJ7NJrcEViq+dRAI2FQTYECFLwdZi5Gtb3oyqZICO+P7k8wDnzsjQ==",
+      "dependencies": {
+        "@babel/core": "^7.18.5",
+        "@sentry/babel-plugin-component-annotate": "2.22.3",
+        "@sentry/cli": "^2.33.1",
+        "dotenv": "^16.3.1",
+        "find-up": "^5.0.0",
+        "glob": "^9.3.2",
+        "magic-string": "0.30.8",
+        "unplugin": "1.0.1"
+      },
+      "engines": {
+        "node": ">= 14"
+      }
+    },
+    "node_modules/@sentry/bundler-plugin-core/node_modules/brace-expansion": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
+      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+      "dependencies": {
+        "balanced-match": "^1.0.0"
+      }
+    },
+    "node_modules/@sentry/bundler-plugin-core/node_modules/glob": {
+      "version": "9.3.5",
+      "resolved": "https://registry.npmjs.org/glob/-/glob-9.3.5.tgz",
+      "integrity": "sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q==",
+      "dependencies": {
+        "fs.realpath": "^1.0.0",
+        "minimatch": "^8.0.2",
+        "minipass": "^4.2.4",
+        "path-scurry": "^1.6.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@sentry/bundler-plugin-core/node_modules/magic-string": {
+      "version": "0.30.8",
+      "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.8.tgz",
+      "integrity": "sha512-ISQTe55T2ao7XtlAStud6qwYPZjE4GK1S/BeVPus4jrq6JuOnQ00YKQC581RWhR122W7msZV263KzVeLoqidyQ==",
+      "dependencies": {
+        "@jridgewell/sourcemap-codec": "^1.4.15"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
+    "node_modules/@sentry/bundler-plugin-core/node_modules/minimatch": {
+      "version": "8.0.4",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-8.0.4.tgz",
+      "integrity": "sha512-W0Wvr9HyFXZRGIDgCicunpQ299OKXs9RgZfaukz4qAW/pJhcpUfupc9c+OObPOFueNy8VSrZgEmDtk6Kh4WzDA==",
+      "dependencies": {
+        "brace-expansion": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=16 || 14 >=14.17"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/isaacs"
+      }
+    },
+    "node_modules/@sentry/bundler-plugin-core/node_modules/minipass": {
+      "version": "4.2.8",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz",
+      "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@sentry/cli": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli/-/cli-2.37.0.tgz",
+      "integrity": "sha512-fM3V4gZRJR/s8lafc3O07hhOYRnvkySdPkvL/0e0XW0r+xRwqIAgQ5ECbsZO16A5weUiXVSf03ztDL1FcmbJCQ==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "https-proxy-agent": "^5.0.0",
+        "node-fetch": "^2.6.7",
+        "progress": "^2.0.3",
+        "proxy-from-env": "^1.1.0",
+        "which": "^2.0.2"
+      },
+      "bin": {
+        "sentry-cli": "bin/sentry-cli"
+      },
+      "engines": {
+        "node": ">= 10"
+      },
+      "optionalDependencies": {
+        "@sentry/cli-darwin": "2.37.0",
+        "@sentry/cli-linux-arm": "2.37.0",
+        "@sentry/cli-linux-arm64": "2.37.0",
+        "@sentry/cli-linux-i686": "2.37.0",
+        "@sentry/cli-linux-x64": "2.37.0",
+        "@sentry/cli-win32-i686": "2.37.0",
+        "@sentry/cli-win32-x64": "2.37.0"
+      }
+    },
+    "node_modules/@sentry/cli-darwin": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-darwin/-/cli-darwin-2.37.0.tgz",
+      "integrity": "sha512-CsusyMvO0eCPSN7H+sKHXS1pf637PWbS4rZak/7giz/z31/6qiXmeMlcL3f9lLZKtFPJmXVFO9uprn1wbBVF8A==",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/cli-linux-arm": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm/-/cli-linux-arm-2.37.0.tgz",
+      "integrity": "sha512-Dz0qH4Yt+gGUgoVsqVt72oDj4VQynRF1QB1/Sr8g76Vbi+WxWZmUh0iFwivYVwWxdQGu/OQrE0tx946HToCRyA==",
+      "cpu": [
+        "arm"
+      ],
+      "optional": true,
+      "os": [
+        "linux",
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/cli-linux-arm64": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm64/-/cli-linux-arm64-2.37.0.tgz",
+      "integrity": "sha512-2vzUWHLZ3Ct5gpcIlfd/2Qsha+y9M8LXvbZE26VxzYrIkRoLAWcnClBv8m4XsHLMURYvz3J9QSZHMZHSO7kAzw==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux",
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/cli-linux-i686": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-i686/-/cli-linux-i686-2.37.0.tgz",
+      "integrity": "sha512-MHRLGs4t/CQE1pG+mZBQixyWL6xDZfNalCjO8GMcTTbZFm44S3XRHfYJZNVCgdtnUP7b6OHGcu1v3SWE10LcwQ==",
+      "cpu": [
+        "x86",
+        "ia32"
+      ],
+      "optional": true,
+      "os": [
+        "linux",
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/cli-linux-x64": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-x64/-/cli-linux-x64-2.37.0.tgz",
+      "integrity": "sha512-k76ClefKZaDNJZU/H3mGeR8uAzAGPzDRG/A7grzKfBeyhP3JW09L7Nz9IQcSjCK+xr399qLhM2HFCaPWQ6dlMw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux",
+        "freebsd"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/cli-win32-i686": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-win32-i686/-/cli-win32-i686-2.37.0.tgz",
+      "integrity": "sha512-FFyi5RNYQQkEg4GkP2f3BJcgQn0F4fjFDMiWkjCkftNPXQG+HFUEtrGsWr6mnHPdFouwbYg3tEPUWNxAoypvTw==",
+      "cpu": [
+        "x86",
+        "ia32"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/cli-win32-x64": {
+      "version": "2.37.0",
+      "resolved": "https://registry.npmjs.org/@sentry/cli-win32-x64/-/cli-win32-x64-2.37.0.tgz",
+      "integrity": "sha512-nSMj4OcfQmyL+Tu/jWCJwhKCXFsCZW1MUk6wjjQlRt9SDLfgeapaMlK1ZvT1eZv5ZH6bj3qJfefwj4U8160uOA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/@sentry/core": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.34.0.tgz",
+      "integrity": "sha512-adrXCTK/zsg5pJ67lgtZqdqHvyx6etMjQW3P82NgWdj83c8fb+zH+K79Z47pD4zQjX0ou2Ws5nwwi4wJbz4bfA==",
+      "dependencies": {
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
       }
     },
-    "node_modules/@radix-ui/react-use-rect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz",
-      "integrity": "sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==",
+    "node_modules/@sentry/nextjs": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/nextjs/-/nextjs-8.34.0.tgz",
+      "integrity": "sha512-REHE3E21Mnm92B3BfJz3GTMsaZM8vaDJAe7RlAMDltESRECv+ELJ5qVRLgAp8Bd6w4mG8IRNINmK2UwHrAIi9g==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/rect": "1.0.1"
+        "@opentelemetry/instrumentation-http": "0.53.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0",
+        "@rollup/plugin-commonjs": "26.0.1",
+        "@sentry-internal/browser-utils": "8.34.0",
+        "@sentry/core": "8.34.0",
+        "@sentry/node": "8.34.0",
+        "@sentry/opentelemetry": "8.34.0",
+        "@sentry/react": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0",
+        "@sentry/vercel-edge": "8.34.0",
+        "@sentry/webpack-plugin": "2.22.3",
+        "chalk": "3.0.0",
+        "resolve": "1.22.8",
+        "rollup": "3.29.5",
+        "stacktrace-parser": "^0.1.10"
+      },
+      "engines": {
+        "node": ">=14.18"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "next": "^13.2.0 || ^14.0 || ^15.0.0-rc.0",
+        "webpack": ">=5.0.0"
       },
       "peerDependenciesMeta": {
-        "@types/react": {
+        "webpack": {
           "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-use-size": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.0.1.tgz",
-      "integrity": "sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==",
+    "node_modules/@sentry/nextjs/node_modules/chalk": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
+      "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "ansi-styles": "^4.1.0",
+        "supports-color": "^7.1.0"
       },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+      "engines": {
+        "node": ">=8"
+      }
+    },
+    "node_modules/@sentry/node": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/node/-/node-8.34.0.tgz",
+      "integrity": "sha512-Q7BPp7Y8yCcwD620xoziWSOuPi/PCIdttkczvB0BGzBRYh2s702h+qNusRijRpVNZmzmYOo9m1x7Y1O/b8/v2A==",
+      "dependencies": {
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/context-async-hooks": "^1.25.1",
+        "@opentelemetry/core": "^1.25.1",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/instrumentation-amqplib": "^0.42.0",
+        "@opentelemetry/instrumentation-connect": "0.39.0",
+        "@opentelemetry/instrumentation-dataloader": "0.12.0",
+        "@opentelemetry/instrumentation-express": "0.42.0",
+        "@opentelemetry/instrumentation-fastify": "0.39.0",
+        "@opentelemetry/instrumentation-fs": "0.15.0",
+        "@opentelemetry/instrumentation-generic-pool": "0.39.0",
+        "@opentelemetry/instrumentation-graphql": "0.43.0",
+        "@opentelemetry/instrumentation-hapi": "0.41.0",
+        "@opentelemetry/instrumentation-http": "0.53.0",
+        "@opentelemetry/instrumentation-ioredis": "0.43.0",
+        "@opentelemetry/instrumentation-kafkajs": "0.3.0",
+        "@opentelemetry/instrumentation-koa": "0.43.0",
+        "@opentelemetry/instrumentation-lru-memoizer": "0.40.0",
+        "@opentelemetry/instrumentation-mongodb": "0.47.0",
+        "@opentelemetry/instrumentation-mongoose": "0.42.0",
+        "@opentelemetry/instrumentation-mysql": "0.41.0",
+        "@opentelemetry/instrumentation-mysql2": "0.41.0",
+        "@opentelemetry/instrumentation-nestjs-core": "0.40.0",
+        "@opentelemetry/instrumentation-pg": "0.44.0",
+        "@opentelemetry/instrumentation-redis-4": "0.42.0",
+        "@opentelemetry/instrumentation-undici": "0.6.0",
+        "@opentelemetry/resources": "^1.26.0",
+        "@opentelemetry/sdk-trace-base": "^1.26.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0",
+        "@prisma/instrumentation": "5.19.1",
+        "@sentry/core": "8.34.0",
+        "@sentry/opentelemetry": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0",
+        "import-in-the-middle": "^1.11.0"
+      },
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry/opentelemetry": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/opentelemetry/-/opentelemetry-8.34.0.tgz",
+      "integrity": "sha512-WS91L+HVKGVIzOgt0szGp+24iKOs86BZsAHGt0HWnMR4kqWP6Ak+TLvqWDCxnuzniZMxdewDGA8p5hrBAPsmsA==",
+      "dependencies": {
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.9.0",
+        "@opentelemetry/core": "^1.25.1",
+        "@opentelemetry/instrumentation": "^0.53.0",
+        "@opentelemetry/sdk-trace-base": "^1.26.0",
+        "@opentelemetry/semantic-conventions": "^1.27.0"
       }
     },
-    "node_modules/@radix-ui/react-visually-hidden": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.0.3.tgz",
-      "integrity": "sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==",
+    "node_modules/@sentry/react": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/react/-/react-8.34.0.tgz",
+      "integrity": "sha512-gIgzhj7h67C+Sdq2ul4fOSK142Gf0uV99bqHRdtIiUlXw9yjzZQY5TKTtzbOaevn7qBJ0xrRKtIRUbOBMl0clw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+        "@sentry/browser": "8.34.0",
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0",
+        "hoist-non-react-statics": "^3.3.2"
+      },
+      "engines": {
+        "node": ">=14.18"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.14.0 || 17.x || 18.x || 19.x"
+      }
+    },
+    "node_modules/@sentry/types": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/types/-/types-8.34.0.tgz",
+      "integrity": "sha512-zLRc60CzohGCo6zNsNeQ9JF3SiEeRE4aDCP9fDDdIVCOKovS+mn1rtSip0qd0Vp2fidOu0+2yY0ALCz1A3PJSQ==",
+      "engines": {
+        "node": ">=14.18"
+      }
+    },
+    "node_modules/@sentry/utils": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/utils/-/utils-8.34.0.tgz",
+      "integrity": "sha512-W1KoRlFUjprlh3t86DZPFxLfM6mzjRzshVfMY7vRlJFymBelJsnJ3A1lPeBZM9nCraOSiw6GtOWu6k5BAkiGIg==",
+      "dependencies": {
+        "@sentry/types": "8.34.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
+      "engines": {
+        "node": ">=14.18"
       }
     },
-    "node_modules/@radix-ui/rect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.0.1.tgz",
-      "integrity": "sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==",
+    "node_modules/@sentry/vercel-edge": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/vercel-edge/-/vercel-edge-8.34.0.tgz",
+      "integrity": "sha512-yF6043FcVO9GqPawCJZp0psEL8iF9+5bOlAdQydCyaj2BtDgFvAeBVI19qlDeAHhqsXNfTD0JsIox2aJPNupwg==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10"
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0"
+      },
+      "engines": {
+        "node": ">=14.18"
       }
     },
-    "node_modules/@rushstack/eslint-patch": {
-      "version": "1.10.3",
-      "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.10.3.tgz",
-      "integrity": "sha512-qC/xYId4NMebE6w/V33Fh9gWxLgURiNYgVNObbJl2LZv0GUUItCcCqC5axQSwRaAgaxl2mELq1rMzlswaQ0Zxg==",
-      "dev": true
+    "node_modules/@sentry/webpack-plugin": {
+      "version": "2.22.3",
+      "resolved": "https://registry.npmjs.org/@sentry/webpack-plugin/-/webpack-plugin-2.22.3.tgz",
+      "integrity": "sha512-Sq1S6bL3nuoTP5typkj+HPjQ13dqftIE8kACAq4tKkXOpWO9bf6HtqcruEQCxMekbWDTdljsrknQ17ZBx2q66Q==",
+      "dependencies": {
+        "@sentry/bundler-plugin-core": "2.22.3",
+        "unplugin": "1.0.1",
+        "uuid": "^9.0.0"
+      },
+      "engines": {
+        "node": ">= 14"
+      },
+      "peerDependencies": {
+        "webpack": ">=4.40.0"
+      }
     },
     "node_modules/@stripe/stripe-js": {
       "version": "4.6.0",
@@ -1752,6 +2864,14 @@
         "react-dom": ">=16.6.0"
       }
     },
+    "node_modules/@types/connect": {
+      "version": "3.4.36",
+      "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz",
+      "integrity": "sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/d3-array": {
       "version": "3.2.1",
       "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.1.tgz",
@@ -1849,6 +2969,12 @@
       "resolved": "https://registry.npmjs.org/@types/js-cookie/-/js-cookie-3.0.6.tgz",
       "integrity": "sha512-wkw9yd1kEXOPnvEeEV1Go1MmxtBJL0RR79aOTAApecWFVu7w0NNXNqhcWgvw2YgZDYadliXkl14pa3WXw5jlCQ=="
     },
+    "node_modules/@types/json-schema": {
+      "version": "7.0.15",
+      "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
+      "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==",
+      "peer": true
+    },
     "node_modules/@types/json5": {
       "version": "0.0.29",
       "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz",
@@ -1873,6 +2999,14 @@
       "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz",
       "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g=="
     },
+    "node_modules/@types/mysql": {
+      "version": "2.15.26",
+      "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.26.tgz",
+      "integrity": "sha512-DSLCOXhkvfS5WNNPbfn2KdICAmk8lLc+/PNvnPnF7gOdMZCxopXduqv0OQ13y/yA/zXTSikZZqVgybUxOEg6YQ==",
+      "dependencies": {
+        "@types/node": "*"
+      }
+    },
     "node_modules/@types/node": {
       "version": "18.15.11",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.11.tgz",
@@ -1883,6 +3017,24 @@
       "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz",
       "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw=="
     },
+    "node_modules/@types/pg": {
+      "version": "8.6.1",
+      "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.6.1.tgz",
+      "integrity": "sha512-1Kc4oAGzAl7uqUStZCDvaLFqZrW9qWSjXOmBfdgyBP5La7Us6Mg4GBvRlSoaZMhQF/zSj1C8CtKMBkoiT8eL8w==",
+      "dependencies": {
+        "@types/node": "*",
+        "pg-protocol": "*",
+        "pg-types": "^2.2.0"
+      }
+    },
+    "node_modules/@types/pg-pool": {
+      "version": "2.0.6",
+      "resolved": "https://registry.npmjs.org/@types/pg-pool/-/pg-pool-2.0.6.tgz",
+      "integrity": "sha512-TaAUE5rq2VQYxab5Ts7WZhKNmuN78Q6PiFonTDdpbx8a1H0M1vhy3rhiMjl+e2iHmogyMw7jZF4FrE6eJUy5HQ==",
+      "dependencies": {
+        "@types/pg": "*"
+      }
+    },
     "node_modules/@types/prismjs": {
       "version": "1.26.4",
       "resolved": "https://registry.npmjs.org/@types/prismjs/-/prismjs-1.26.4.tgz",
@@ -1924,6 +3076,11 @@
       "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.23.0.tgz",
       "integrity": "sha512-YIoDCTH3Af6XM5VuwGG/QL/CJqga1Zm3NkU3HZ4ZHK2fRMPYP1VczsTUqtsf43PH/iJNVlPHAo2oWX7BSdB2Hw=="
     },
+    "node_modules/@types/shimmer": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@types/shimmer/-/shimmer-1.2.0.tgz",
+      "integrity": "sha512-UE7oxhQLLd9gub6JKIAhDq06T0F6FnztwMNRvYgjeQSBeMc1ZG/tA47EwfduvkuQS8apbkM/lpLpWsaCeYsXVg=="
+    },
     "node_modules/@types/unist": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.2.tgz",
@@ -2066,11 +3223,168 @@
       "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
       "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="
     },
+    "node_modules/@webassemblyjs/ast": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz",
+      "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/helper-numbers": "1.11.6",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6"
+      }
+    },
+    "node_modules/@webassemblyjs/floating-point-hex-parser": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz",
+      "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==",
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-api-error": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz",
+      "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==",
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-buffer": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz",
+      "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==",
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-numbers": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz",
+      "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/floating-point-hex-parser": "1.11.6",
+        "@webassemblyjs/helper-api-error": "1.11.6",
+        "@xtuc/long": "4.2.2"
+      }
+    },
+    "node_modules/@webassemblyjs/helper-wasm-bytecode": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz",
+      "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==",
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/helper-wasm-section": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz",
+      "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-buffer": "1.12.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/wasm-gen": "1.12.1"
+      }
+    },
+    "node_modules/@webassemblyjs/ieee754": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz",
+      "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==",
+      "peer": true,
+      "dependencies": {
+        "@xtuc/ieee754": "^1.2.0"
+      }
+    },
+    "node_modules/@webassemblyjs/leb128": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz",
+      "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==",
+      "peer": true,
+      "dependencies": {
+        "@xtuc/long": "4.2.2"
+      }
+    },
+    "node_modules/@webassemblyjs/utf8": {
+      "version": "1.11.6",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz",
+      "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==",
+      "peer": true
+    },
+    "node_modules/@webassemblyjs/wasm-edit": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz",
+      "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-buffer": "1.12.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/helper-wasm-section": "1.12.1",
+        "@webassemblyjs/wasm-gen": "1.12.1",
+        "@webassemblyjs/wasm-opt": "1.12.1",
+        "@webassemblyjs/wasm-parser": "1.12.1",
+        "@webassemblyjs/wast-printer": "1.12.1"
+      }
+    },
+    "node_modules/@webassemblyjs/wasm-gen": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz",
+      "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/ieee754": "1.11.6",
+        "@webassemblyjs/leb128": "1.11.6",
+        "@webassemblyjs/utf8": "1.11.6"
+      }
+    },
+    "node_modules/@webassemblyjs/wasm-opt": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz",
+      "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-buffer": "1.12.1",
+        "@webassemblyjs/wasm-gen": "1.12.1",
+        "@webassemblyjs/wasm-parser": "1.12.1"
+      }
+    },
+    "node_modules/@webassemblyjs/wasm-parser": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz",
+      "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/helper-api-error": "1.11.6",
+        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
+        "@webassemblyjs/ieee754": "1.11.6",
+        "@webassemblyjs/leb128": "1.11.6",
+        "@webassemblyjs/utf8": "1.11.6"
+      }
+    },
+    "node_modules/@webassemblyjs/wast-printer": {
+      "version": "1.12.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz",
+      "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==",
+      "peer": true,
+      "dependencies": {
+        "@webassemblyjs/ast": "1.12.1",
+        "@xtuc/long": "4.2.2"
+      }
+    },
+    "node_modules/@xtuc/ieee754": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
+      "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==",
+      "peer": true
+    },
+    "node_modules/@xtuc/long": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz",
+      "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==",
+      "peer": true
+    },
     "node_modules/acorn": {
       "version": "8.11.3",
       "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz",
       "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==",
-      "dev": true,
       "bin": {
         "acorn": "bin/acorn"
       },
@@ -2078,6 +3392,14 @@
         "node": ">=0.4.0"
       }
     },
+    "node_modules/acorn-import-attributes": {
+      "version": "1.9.5",
+      "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
+      "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
+      "peerDependencies": {
+        "acorn": "^8"
+      }
+    },
     "node_modules/acorn-jsx": {
       "version": "5.3.2",
       "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz",
@@ -2087,11 +3409,21 @@
         "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
       }
     },
+    "node_modules/agent-base": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+      "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+      "dependencies": {
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
     "node_modules/ajv": {
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
       "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==",
-      "dev": true,
       "dependencies": {
         "fast-deep-equal": "^3.1.1",
         "fast-json-stable-stringify": "^2.0.0",
@@ -2103,6 +3435,15 @@
         "url": "https://github.com/sponsors/epoberezkin"
       }
     },
+    "node_modules/ajv-keywords": {
+      "version": "3.5.2",
+      "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz",
+      "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==",
+      "peer": true,
+      "peerDependencies": {
+        "ajv": "^6.9.1"
+      }
+    },
     "node_modules/ansi-regex": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz",
@@ -2530,6 +3871,12 @@
         "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7"
       }
     },
+    "node_modules/buffer-from": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
+      "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==",
+      "peer": true
+    },
     "node_modules/busboy": {
       "version": "1.6.0",
       "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz",
@@ -2697,6 +4044,20 @@
         "node": ">= 6"
       }
     },
+    "node_modules/chrome-trace-event": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz",
+      "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==",
+      "peer": true,
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
+    "node_modules/cjs-module-lexer": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz",
+      "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA=="
+    },
     "node_modules/client-only": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
@@ -2743,6 +4104,11 @@
         "node": ">= 6"
       }
     },
+    "node_modules/commondir": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz",
+      "integrity": "sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg=="
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -2752,8 +4118,7 @@
     "node_modules/convert-source-map": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
-      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==",
-      "peer": true
+      "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="
     },
     "node_modules/cosmiconfig": {
       "version": "7.1.0",
@@ -3002,11 +4367,11 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.3.4",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
-      "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==",
+      "version": "4.3.7",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.7.tgz",
+      "integrity": "sha512-Er2nc/H7RrMXZBFCEim6TCmMk02Z8vLC2Rbi1KEBggpo0fS6l0S1nnapwmIi3yW/+GOJap1Krg4w0Hg80oCqgQ==",
       "dependencies": {
-        "ms": "2.1.2"
+        "ms": "^2.1.3"
       },
       "engines": {
         "node": ">=6.0"
@@ -3149,6 +4514,17 @@
         "csstype": "^3.0.2"
       }
     },
+    "node_modules/dotenv": {
+      "version": "16.4.5",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz",
+      "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "url": "https://dotenvx.com"
+      }
+    },
     "node_modules/eastasianwidth": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
@@ -3165,10 +4541,9 @@
       "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
     },
     "node_modules/enhanced-resolve": {
-      "version": "5.16.1",
-      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.16.1.tgz",
-      "integrity": "sha512-4U5pNsuDl0EhuZpq46M5xPslstkviJuhrdobaRDBk2Jy2KO37FDAJl4lb2KlNabxT0m4MTK2UHNrsAcphE8nyw==",
-      "dev": true,
+      "version": "5.17.1",
+      "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz",
+      "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==",
       "dependencies": {
         "graceful-fs": "^4.2.4",
         "tapable": "^2.2.0"
@@ -3305,6 +4680,12 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/es-module-lexer": {
+      "version": "1.5.4",
+      "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.5.4.tgz",
+      "integrity": "sha512-MVNK56NiMrOwitFB7cqDwq0CQutbw+0BvLshJSse0MUNU+y1FC3bUS/AQg7oUng+/wKrrki7JfmwtVHkVfPLlw==",
+      "peer": true
+    },
     "node_modules/es-object-atoms": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.0.0.tgz",
@@ -3762,7 +5143,6 @@
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
       "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
-      "dev": true,
       "dependencies": {
         "estraverse": "^5.2.0"
       },
@@ -3774,7 +5154,6 @@
       "version": "5.3.0",
       "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz",
       "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==",
-      "dev": true,
       "engines": {
         "node": ">=4.0"
       }
@@ -3788,6 +5167,11 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/estree-walker": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz",
+      "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="
+    },
     "node_modules/esutils": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz",
@@ -3802,6 +5186,15 @@
       "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz",
       "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="
     },
+    "node_modules/events": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
+      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
+      "peer": true,
+      "engines": {
+        "node": ">=0.8.x"
+      }
+    },
     "node_modules/extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
@@ -3810,8 +5203,7 @@
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
-      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==",
-      "dev": true
+      "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="
     },
     "node_modules/fast-equals": {
       "version": "5.0.1",
@@ -3850,8 +5242,7 @@
     "node_modules/fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
-      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==",
-      "dev": true
+      "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="
     },
     "node_modules/fast-levenshtein": {
       "version": "2.0.6",
@@ -3910,7 +5301,6 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
       "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==",
-      "dev": true,
       "dependencies": {
         "locate-path": "^6.0.0",
         "path-exists": "^4.0.0"
@@ -4005,8 +5395,7 @@
     "node_modules/fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-      "dev": true
+      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="
     },
     "node_modules/fsevents": {
       "version": "2.3.3",
@@ -4060,7 +5449,6 @@
       "version": "1.0.0-beta.2",
       "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz",
       "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==",
-      "peer": true,
       "engines": {
         "node": ">=6.9.0"
       }
@@ -4152,6 +5540,12 @@
         "node": ">=10.13.0"
       }
     },
+    "node_modules/glob-to-regexp": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
+      "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==",
+      "peer": true
+    },
     "node_modules/glob/node_modules/brace-expansion": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
@@ -4260,7 +5654,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
       "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
-      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -4509,6 +5902,18 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/https-proxy-agent": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+      "dependencies": {
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/ignore": {
       "version": "5.3.1",
       "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz",
@@ -4533,6 +5938,17 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/import-in-the-middle": {
+      "version": "1.11.2",
+      "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-1.11.2.tgz",
+      "integrity": "sha512-gK6Rr6EykBcc6cVWRSBR5TWf8nn6hZMYSRYqCcHa0l0d1fPK7JSYo6+Mlmck76jIX9aL/IZ71c06U2VpFwl1zA==",
+      "dependencies": {
+        "acorn": "^8.8.2",
+        "acorn-import-attributes": "^1.9.5",
+        "cjs-module-lexer": "^1.2.2",
+        "module-details-from-path": "^1.0.3"
+      }
+    },
     "node_modules/imurmurhash": {
       "version": "0.1.4",
       "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz",
@@ -4877,6 +6293,14 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/is-reference": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/is-reference/-/is-reference-1.2.1.tgz",
+      "integrity": "sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==",
+      "dependencies": {
+        "@types/estree": "*"
+      }
+    },
     "node_modules/is-regex": {
       "version": "1.1.4",
       "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.4.tgz",
@@ -5046,6 +6470,35 @@
         "@pkgjs/parseargs": "^0.11.0"
       }
     },
+    "node_modules/jest-worker": {
+      "version": "27.5.1",
+      "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-27.5.1.tgz",
+      "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==",
+      "peer": true,
+      "dependencies": {
+        "@types/node": "*",
+        "merge-stream": "^2.0.0",
+        "supports-color": "^8.0.0"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      }
+    },
+    "node_modules/jest-worker/node_modules/supports-color": {
+      "version": "8.1.1",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz",
+      "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==",
+      "peer": true,
+      "dependencies": {
+        "has-flag": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/chalk/supports-color?sponsor=1"
+      }
+    },
     "node_modules/jiti": {
       "version": "1.21.0",
       "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.0.tgz",
@@ -5104,8 +6557,7 @@
     "node_modules/json-schema-traverse": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz",
-      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==",
-      "dev": true
+      "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg=="
     },
     "node_modules/json-stable-stringify-without-jsonify": {
       "version": "1.0.1",
@@ -5117,7 +6569,6 @@
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
       "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==",
-      "peer": true,
       "bin": {
         "json5": "lib/cli.js"
       },
@@ -5193,11 +6644,19 @@
       "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz",
       "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="
     },
+    "node_modules/loader-runner": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
+      "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==",
+      "peer": true,
+      "engines": {
+        "node": ">=6.11.5"
+      }
+    },
     "node_modules/locate-path": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
       "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==",
-      "dev": true,
       "dependencies": {
         "p-locate": "^5.0.0"
       },
@@ -5264,6 +6723,14 @@
         "node": "14 || >=16.14"
       }
     },
+    "node_modules/magic-string": {
+      "version": "0.30.12",
+      "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz",
+      "integrity": "sha512-Ea8I3sQMVXr8JhN4z+H/d8zwo+tYDgHE9+5G4Wnrwhs0gaK9fXTKx0Tw5Xwsd/bCPTTZNRAdpyzvoeORe9LYpw==",
+      "dependencies": {
+        "@jridgewell/sourcemap-codec": "^1.5.0"
+      }
+    },
     "node_modules/markdown-table": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz",
@@ -5544,6 +7011,12 @@
       "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-6.0.0.tgz",
       "integrity": "sha512-rkpe71W0N0c0Xz6QD0eJETuWAJGnJ9afsl1srmwPrI+yBCkge5EycXXbYRyvL29zZVUWQCY7InPRCv3GDXuZNw=="
     },
+    "node_modules/merge-stream": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
+      "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==",
+      "peer": true
+    },
     "node_modules/merge2": {
       "version": "1.4.1",
       "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
@@ -6099,6 +7572,27 @@
         "node": ">=8.6"
       }
     },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "peer": true,
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "peer": true,
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/minimatch": {
       "version": "3.1.2",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@@ -6128,10 +7622,15 @@
         "node": ">=16 || 14 >=14.17"
       }
     },
+    "node_modules/module-details-from-path": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/module-details-from-path/-/module-details-from-path-1.0.3.tgz",
+      "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A=="
+    },
     "node_modules/ms": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
-      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
     "node_modules/mz": {
       "version": "2.7.0",
@@ -6166,6 +7665,12 @@
       "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==",
       "dev": true
     },
+    "node_modules/neo-async": {
+      "version": "2.6.2",
+      "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz",
+      "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==",
+      "peer": true
+    },
     "node_modules/next": {
       "version": "14.2.3",
       "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz",
@@ -6242,6 +7747,25 @@
         "node": "^10 || ^12 || >=14"
       }
     },
+    "node_modules/node-fetch": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/node-releases": {
       "version": "2.0.14",
       "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",
@@ -8818,7 +10342,6 @@
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz",
       "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==",
-      "dev": true,
       "dependencies": {
         "yocto-queue": "^0.1.0"
       },
@@ -8833,7 +10356,6 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz",
       "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==",
-      "dev": true,
       "dependencies": {
         "p-limit": "^3.0.2"
       },
@@ -8844,6 +10366,11 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/package-json-from-dist": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz",
+      "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="
+    },
     "node_modules/parent-module": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz",
@@ -8916,7 +10443,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
       "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
-      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -8966,6 +10492,34 @@
         "node": ">=8"
       }
     },
+    "node_modules/pg-int8": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+      "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/pg-protocol": {
+      "version": "1.7.0",
+      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.7.0.tgz",
+      "integrity": "sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ=="
+    },
+    "node_modules/pg-types": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+      "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+      "dependencies": {
+        "pg-int8": "1.0.1",
+        "postgres-array": "~2.0.0",
+        "postgres-bytea": "~1.0.0",
+        "postgres-date": "~1.0.4",
+        "postgres-interval": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/picocolors": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
@@ -9161,6 +10715,41 @@
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
     },
+    "node_modules/postgres-array": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+      "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/postgres-bytea": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+      "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-date": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+      "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-interval": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+      "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+      "dependencies": {
+        "xtend": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -9193,6 +10782,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/progress": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
+      "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/prop-types": {
       "version": "15.8.1",
       "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
@@ -9217,11 +10814,15 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/proxy-from-env": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+    },
     "node_modules/punycode": {
       "version": "2.3.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
       "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==",
-      "dev": true,
       "engines": {
         "node": ">=6"
       }
@@ -9259,6 +10860,15 @@
         }
       ]
     },
+    "node_modules/randombytes": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz",
+      "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==",
+      "peer": true,
+      "dependencies": {
+        "safe-buffer": "^5.1.0"
+      }
+    },
     "node_modules/react": {
       "version": "18.3.1",
       "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz",
@@ -9708,6 +11318,19 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/require-in-the-middle": {
+      "version": "7.4.0",
+      "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.4.0.tgz",
+      "integrity": "sha512-X34iHADNbNDfr6OTStIAHWSAvvKQRYgLO6duASaVf7J2VA3lvmNYboAHOuLC2huav1IwgZJtyEcJCKVzFxOSMQ==",
+      "dependencies": {
+        "debug": "^4.3.5",
+        "module-details-from-path": "^1.0.3",
+        "resolve": "^1.22.8"
+      },
+      "engines": {
+        "node": ">=8.6.0"
+      }
+    },
     "node_modules/resolve": {
       "version": "1.22.8",
       "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz",
@@ -9785,6 +11408,21 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/rollup": {
+      "version": "3.29.5",
+      "resolved": "https://registry.npmjs.org/rollup/-/rollup-3.29.5.tgz",
+      "integrity": "sha512-GVsDdsbJzzy4S/v3dqWPJ7EfvZJfCHiDqe80IyrF59LYuP+e6U1LJoUqeuqRbwAWoMNoXivMNeNAOf5E22VA1w==",
+      "bin": {
+        "rollup": "dist/bin/rollup"
+      },
+      "engines": {
+        "node": ">=14.18.0",
+        "npm": ">=8.0.0"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
+      }
+    },
     "node_modules/run-parallel": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -9825,6 +11463,26 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/safe-buffer": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
+      "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "peer": true
+    },
     "node_modules/safe-regex-test": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.3.tgz",
@@ -9850,6 +11508,24 @@
         "loose-envify": "^1.1.0"
       }
     },
+    "node_modules/schema-utils": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz",
+      "integrity": "sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==",
+      "peer": true,
+      "dependencies": {
+        "@types/json-schema": "^7.0.8",
+        "ajv": "^6.12.5",
+        "ajv-keywords": "^3.5.2"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      }
+    },
     "node_modules/semver": {
       "version": "7.6.2",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
@@ -9861,6 +11537,15 @@
         "node": ">=10"
       }
     },
+    "node_modules/serialize-javascript": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz",
+      "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==",
+      "peer": true,
+      "dependencies": {
+        "randombytes": "^2.1.0"
+      }
+    },
     "node_modules/set-function-length": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
@@ -9916,6 +11601,11 @@
         "node": ">=8"
       }
     },
+    "node_modules/shimmer": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/shimmer/-/shimmer-1.2.1.tgz",
+      "integrity": "sha512-sQTKC1Re/rM6XyFM6fIAGHRPVGvyXfgzIDvzoq608vM+jeyVD0Tu1E6Np0Kc2zAIFWIj963V2800iF/9LPieQw=="
+    },
     "node_modules/side-channel": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.6.tgz",
@@ -9969,6 +11659,25 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/source-map-support": {
+      "version": "0.5.21",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz",
+      "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==",
+      "peer": true,
+      "dependencies": {
+        "buffer-from": "^1.0.0",
+        "source-map": "^0.6.0"
+      }
+    },
+    "node_modules/source-map-support/node_modules/source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==",
+      "peer": true,
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/space-separated-tokens": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz",
@@ -9978,6 +11687,25 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/stacktrace-parser": {
+      "version": "0.1.10",
+      "resolved": "https://registry.npmjs.org/stacktrace-parser/-/stacktrace-parser-0.1.10.tgz",
+      "integrity": "sha512-KJP1OCML99+8fhOHxwwzyWrlUuVX5GQ0ZpJTd1DFXhdkrvg1szxfHhawXUZ3g9TkXORQd4/WG68jMlQZ2p8wlg==",
+      "dependencies": {
+        "type-fest": "^0.7.1"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
+    "node_modules/stacktrace-parser/node_modules/type-fest": {
+      "version": "0.7.1",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.7.1.tgz",
+      "integrity": "sha512-Ne2YiiGN8bmrmJJEuTWTLJR32nh/JdL1+PSicowtNb0WFpn59GK8/lfD61bVtzguz7b3PBt74nxpv/Pw5po5Rg==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/streamsearch": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz",
@@ -10303,7 +12031,6 @@
       "version": "7.2.0",
       "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
       "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
-      "dev": true,
       "dependencies": {
         "has-flag": "^4.0.0"
       },
@@ -10400,11 +12127,68 @@
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz",
       "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==",
-      "dev": true,
       "engines": {
         "node": ">=6"
       }
     },
+    "node_modules/terser": {
+      "version": "5.34.1",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.34.1.tgz",
+      "integrity": "sha512-FsJZ7iZLd/BXkz+4xrRTGJ26o/6VTjQytUk8b8OxkwcD2I+79VPJlz7qss1+zE7h8GNIScFqXcDyJ/KqBYZFVA==",
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/source-map": "^0.3.3",
+        "acorn": "^8.8.2",
+        "commander": "^2.20.0",
+        "source-map-support": "~0.5.20"
+      },
+      "bin": {
+        "terser": "bin/terser"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
+    "node_modules/terser-webpack-plugin": {
+      "version": "5.3.10",
+      "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-5.3.10.tgz",
+      "integrity": "sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==",
+      "peer": true,
+      "dependencies": {
+        "@jridgewell/trace-mapping": "^0.3.20",
+        "jest-worker": "^27.4.5",
+        "schema-utils": "^3.1.1",
+        "serialize-javascript": "^6.0.1",
+        "terser": "^5.26.0"
+      },
+      "engines": {
+        "node": ">= 10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependencies": {
+        "webpack": "^5.1.0"
+      },
+      "peerDependenciesMeta": {
+        "@swc/core": {
+          "optional": true
+        },
+        "esbuild": {
+          "optional": true
+        },
+        "uglify-js": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/terser/node_modules/commander": {
+      "version": "2.20.3",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz",
+      "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==",
+      "peer": true
+    },
     "node_modules/text-table": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@@ -10469,6 +12253,11 @@
       "resolved": "https://registry.npmjs.org/toposort/-/toposort-2.0.2.tgz",
       "integrity": "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg=="
     },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
+    },
     "node_modules/trim-lines": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz",
@@ -10761,6 +12550,17 @@
         "url": "https://opencollective.com/unified"
       }
     },
+    "node_modules/unplugin": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-1.0.1.tgz",
+      "integrity": "sha512-aqrHaVBWW1JVKBHmGo33T5TxeL0qWzfvjWokObHA9bYmN7eNDkwOxmLjhioHl9878qDFMAaT51XNroRyuz7WxA==",
+      "dependencies": {
+        "acorn": "^8.8.1",
+        "chokidar": "^3.5.3",
+        "webpack-sources": "^3.2.3",
+        "webpack-virtual-modules": "^0.5.0"
+      }
+    },
     "node_modules/update-browserslist-db": {
       "version": "1.0.16",
       "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz",
@@ -10794,7 +12594,6 @@
       "version": "4.4.1",
       "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz",
       "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==",
-      "dev": true,
       "dependencies": {
         "punycode": "^2.1.0"
       }
@@ -10939,6 +12738,19 @@
         "d3-timer": "^3.0.1"
       }
     },
+    "node_modules/watchpack": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.2.tgz",
+      "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==",
+      "peer": true,
+      "dependencies": {
+        "glob-to-regexp": "^0.4.1",
+        "graceful-fs": "^4.1.2"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
     "node_modules/web-namespaces": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz",
@@ -10948,6 +12760,101 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
+    },
+    "node_modules/webpack": {
+      "version": "5.95.0",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.95.0.tgz",
+      "integrity": "sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q==",
+      "peer": true,
+      "dependencies": {
+        "@types/estree": "^1.0.5",
+        "@webassemblyjs/ast": "^1.12.1",
+        "@webassemblyjs/wasm-edit": "^1.12.1",
+        "@webassemblyjs/wasm-parser": "^1.12.1",
+        "acorn": "^8.7.1",
+        "acorn-import-attributes": "^1.9.5",
+        "browserslist": "^4.21.10",
+        "chrome-trace-event": "^1.0.2",
+        "enhanced-resolve": "^5.17.1",
+        "es-module-lexer": "^1.2.1",
+        "eslint-scope": "5.1.1",
+        "events": "^3.2.0",
+        "glob-to-regexp": "^0.4.1",
+        "graceful-fs": "^4.2.11",
+        "json-parse-even-better-errors": "^2.3.1",
+        "loader-runner": "^4.2.0",
+        "mime-types": "^2.1.27",
+        "neo-async": "^2.6.2",
+        "schema-utils": "^3.2.0",
+        "tapable": "^2.1.1",
+        "terser-webpack-plugin": "^5.3.10",
+        "watchpack": "^2.4.1",
+        "webpack-sources": "^3.2.3"
+      },
+      "bin": {
+        "webpack": "bin/webpack.js"
+      },
+      "engines": {
+        "node": ">=10.13.0"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/webpack"
+      },
+      "peerDependenciesMeta": {
+        "webpack-cli": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/webpack-sources": {
+      "version": "3.2.3",
+      "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz",
+      "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==",
+      "engines": {
+        "node": ">=10.13.0"
+      }
+    },
+    "node_modules/webpack-virtual-modules": {
+      "version": "0.5.0",
+      "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.5.0.tgz",
+      "integrity": "sha512-kyDivFZ7ZM0BVOUteVbDFhlRt7Ah/CSPwJdi8hBpkK7QLumUqdLtVfm/PX/hkcnrvr0i77fO5+TjZ94Pe+C9iw=="
+    },
+    "node_modules/webpack/node_modules/eslint-scope": {
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+      "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+      "peer": true,
+      "dependencies": {
+        "esrecurse": "^4.3.0",
+        "estraverse": "^4.1.1"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/webpack/node_modules/estraverse": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz",
+      "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==",
+      "peer": true,
+      "engines": {
+        "node": ">=4.0"
+      }
+    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
     "node_modules/which": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@@ -11143,11 +13050,18 @@
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
       "dev": true
     },
+    "node_modules/xtend": {
+      "version": "4.0.2",
+      "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
+      "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
+      "engines": {
+        "node": ">=0.4"
+      }
+    },
     "node_modules/yallist": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
-      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
-      "peer": true
+      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
     },
     "node_modules/yaml": {
       "version": "2.4.2",
@@ -11164,7 +13078,6 @@
       "version": "0.1.0",
       "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz",
       "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==",
-      "dev": true,
       "engines": {
         "node": ">=10"
       },
diff --git a/web/package.json b/web/package.json
index 39d4b316e1e..c3efcd78d3f 100644
--- a/web/package.json
+++ b/web/package.json
@@ -16,6 +16,7 @@
     "@radix-ui/react-dialog": "^1.0.5",
     "@radix-ui/react-popover": "^1.0.7",
     "@radix-ui/react-tooltip": "^1.0.7",
+    "@sentry/nextjs": "^8.34.0",
     "@stripe/stripe-js": "^4.6.0",
     "@tremor/react": "^3.9.2",
     "@types/js-cookie": "^3.0.3",
diff --git a/web/sentry.client.config.ts b/web/sentry.client.config.ts
new file mode 100644
index 00000000000..f6e0e7d1ad7
--- /dev/null
+++ b/web/sentry.client.config.ts
@@ -0,0 +1,23 @@
+import * as Sentry from "@sentry/nextjs";
+
+if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
+    // Replay may only be enabled for the client-side
+    integrations: [Sentry.replayIntegration()],
+
+    // Set tracesSampleRate to 1.0 to capture 100%
+    // of transactions for tracing.
+    // We recommend adjusting this value in production
+    tracesSampleRate: 1.0,
+
+    // Capture Replay for 10% of all sessions,
+    // plus for 100% of sessions with an error
+    replaysSessionSampleRate: 0.1,
+    replaysOnErrorSampleRate: 1.0,
+
+    // Note: if you want to override the automatic release value, do not set a
+    // `release` value here - use the environment variable `SENTRY_RELEASE`, so
+    // that it will also get attached to your source maps
+  });
+}
diff --git a/web/sentry.edge.config.ts b/web/sentry.edge.config.ts
new file mode 100644
index 00000000000..4c0eac9e310
--- /dev/null
+++ b/web/sentry.edge.config.ts
@@ -0,0 +1,16 @@
+import * as Sentry from "@sentry/nextjs";
+
+if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
+
+    // Set tracesSampleRate to 1.0 to capture 100%
+    // of transactions for tracing.
+    // We recommend adjusting this value in production
+    tracesSampleRate: 1.0,
+
+    // Note: if you want to override the automatic release value, do not set a
+    // `release` value here - use the environment variable `SENTRY_RELEASE`, so
+    // that it will also get attached to your source maps
+  });
+}
diff --git a/web/sentry.server.config.ts b/web/sentry.server.config.ts
new file mode 100644
index 00000000000..4c0eac9e310
--- /dev/null
+++ b/web/sentry.server.config.ts
@@ -0,0 +1,16 @@
+import * as Sentry from "@sentry/nextjs";
+
+if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
+  Sentry.init({
+    dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
+
+    // Set tracesSampleRate to 1.0 to capture 100%
+    // of transactions for tracing.
+    // We recommend adjusting this value in production
+    tracesSampleRate: 1.0,
+
+    // Note: if you want to override the automatic release value, do not set a
+    // `release` value here - use the environment variable `SENTRY_RELEASE`, so
+    // that it will also get attached to your source maps
+  });
+}
diff --git a/web/src/app/global-error.tsx b/web/src/app/global-error.tsx
new file mode 100644
index 00000000000..1ef2941b270
--- /dev/null
+++ b/web/src/app/global-error.tsx
@@ -0,0 +1,27 @@
+"use client";
+
+import * as Sentry from "@sentry/nextjs";
+import NextError from "next/error";
+import { useEffect } from "react";
+
+// This global error page is necessary to capture errors that occur in the app.
+export default function GlobalError({
+  error,
+}: {
+  error: Error & { digest?: string };
+}) {
+  useEffect(() => {
+    Sentry.captureException(error);
+  }, [error]);
+
+  return (
+    <html>
+      <body>
+        {/* NextError require  a `statusCode` prop. However, since the App Router
+        does not expose status codes for errors, we simply pass 0 to render a
+        generic error message. */}
+        <NextError statusCode={0} />
+      </body>
+    </html>
+  );
+}

From 6e54c9732617e4f66508ec145b6e4e5bc91e0b47 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 17:54:02 -0700
Subject: [PATCH 148/376] multitenant setup (#2845)

---
 backend/danswer/main.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index ea338263279..9db1b9cd35c 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -84,6 +84,7 @@
     router as token_rate_limit_settings_router,
 )
 from danswer.setup import setup_danswer
+from danswer.setup import setup_multitenant_danswer
 from danswer.utils.logger import setup_logger
 from danswer.utils.telemetry import get_or_generate_uuid
 from danswer.utils.telemetry import optional_telemetry
@@ -182,6 +183,8 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
         # If we are multi-tenant, we need to only set up initial public tables
         with Session(engine) as db_session:
             setup_danswer(db_session)
+    else:
+        setup_multitenant_danswer()
 
     optional_telemetry(record_type=RecordType.VERSION, data={"version": __version__})
     yield
@@ -312,6 +315,7 @@ def get_application() -> FastAPI:
             prefix="/auth/oauth",
             tags=["auth"],
         )
+
         # Need basic auth router for `logout` endpoint
         include_router_with_global_prefix_prepended(
             application,

From 7906d9edc837767834fc51d15d2a7313a8d888d2 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 17 Oct 2024 19:55:05 -0700
Subject: [PATCH 149/376] Add all-tenants migration for K8 job (#2846)

* add migration

* update migration logic for tenants

* k

* k

* k

* k
---
 backend/alembic/env.py | 171 +++++++++++++++++++++++++++++------------
 1 file changed, 120 insertions(+), 51 deletions(-)

diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index afa5a9669c1..c89d3455227 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -1,10 +1,11 @@
+from sqlalchemy.engine.base import Connection
 from typing import Any
 import asyncio
 from logging.config import fileConfig
+import logging
 
 from alembic import context
 from sqlalchemy import pool
-from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.sql import text
 
@@ -12,6 +13,7 @@
 from danswer.db.engine import build_connection_string
 from danswer.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore
+from danswer.background.celery.celery_app import get_all_tenant_ids
 
 # Alembic Config object
 config = context.config
@@ -22,66 +24,42 @@
 ):
     fileConfig(config.config_file_name)
 
-# Add your model's MetaData object here
-# for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
+# Add your model's MetaData object here for 'autogenerate' support
 target_metadata = [Base.metadata, ResultModelBase.metadata]
 
-
-def get_schema_options() -> tuple[str, bool]:
-    x_args_raw = context.get_x_argument()
-    x_args = {}
-    for arg in x_args_raw:
-        for pair in arg.split(","):
-            if "=" in pair:
-                key, value = pair.split("=", 1)
-                x_args[key.strip()] = value.strip()
-    schema_name = x_args.get("schema", "public")
-    create_schema = x_args.get("create_schema", "true").lower() == "true"
-    return schema_name, create_schema
-
-
 EXCLUDE_TABLES = {"kombu_queue", "kombu_message"}
 
+# Set up logging
+logger = logging.getLogger(__name__)
+
 
 def include_object(
     object: Any, name: str, type_: str, reflected: bool, compare_to: Any
 ) -> bool:
+    """
+    Determines whether a database object should be included in migrations.
+    Excludes specified tables from migrations.
+    """
     if type_ == "table" and name in EXCLUDE_TABLES:
         return False
     return True
 
 
-def run_migrations_offline() -> None:
-    """Run migrations in 'offline' mode.
-    This configures the context with just a URL
-    and not an Engine, though an Engine is acceptable
-    here as well.  By skipping the Engine creation
-    we don't even need a DBAPI to be available.
-    Calls to context.execute() here emit the given string to the
-    script output.
+def get_schema_options() -> tuple[str, bool, bool]:
     """
-    schema_name, _ = get_schema_options()
-    url = build_connection_string()
-
-    context.configure(
-        url=url,
-        target_metadata=target_metadata,  # type: ignore
-        literal_binds=True,
-        include_object=include_object,
-        version_table_schema=schema_name,
-        include_schemas=True,
-        script_location=config.get_main_option("script_location"),
-        dialect_opts={"paramstyle": "named"},
-    )
-
-    with context.begin_transaction():
-        context.run_migrations()
-
-
-def do_run_migrations(connection: Connection) -> None:
-    schema_name, create_schema = get_schema_options()
+    Parses command-line options passed via '-x' in Alembic commands.
+    Recognizes 'schema', 'create_schema', and 'upgrade_all_tenants' options.
+    """
+    x_args_raw = context.get_x_argument()
+    x_args = {}
+    for arg in x_args_raw:
+        for pair in arg.split(","):
+            if "=" in pair:
+                key, value = pair.split("=", 1)
+                x_args[key.strip()] = value.strip()
+    schema_name = x_args.get("schema", "public")
+    create_schema = x_args.get("create_schema", "true").lower() == "true"
+    upgrade_all_tenants = x_args.get("upgrade_all_tenants", "false").lower() == "true"
 
     if MULTI_TENANT and schema_name == "public":
         raise ValueError(
@@ -89,6 +67,17 @@ def do_run_migrations(connection: Connection) -> None:
             "Please specify a tenant-specific schema."
         )
 
+    return schema_name, create_schema, upgrade_all_tenants
+
+
+def do_run_migrations(
+    connection: Connection, schema_name: str, create_schema: bool
+) -> None:
+    """
+    Executes migrations in the specified schema.
+    """
+    logger.info(f"About to migrate schema: {schema_name}")
+
     if create_schema:
         connection.execute(text(f'CREATE SCHEMA IF NOT EXISTS "{schema_name}"'))
         connection.execute(text("COMMIT"))
@@ -112,18 +101,98 @@ def do_run_migrations(connection: Connection) -> None:
 
 
 async def run_async_migrations() -> None:
-    connectable = create_async_engine(
+    """
+    Determines whether to run migrations for a single schema or all schemas,
+    and executes migrations accordingly.
+    """
+    schema_name, create_schema, upgrade_all_tenants = get_schema_options()
+
+    engine = create_async_engine(
         build_connection_string(),
         poolclass=pool.NullPool,
     )
 
-    async with connectable.connect() as connection:
-        await connection.run_sync(do_run_migrations)
+    if upgrade_all_tenants:
+        # Run migrations for all tenant schemas sequentially
+        tenant_schemas = get_all_tenant_ids()
+
+        for schema in tenant_schemas:
+            try:
+                logger.info(f"Migrating schema: {schema}")
+                async with engine.connect() as connection:
+                    await connection.run_sync(
+                        do_run_migrations,
+                        schema_name=schema,
+                        create_schema=create_schema,
+                    )
+            except Exception as e:
+                logger.error(f"Error migrating schema {schema}: {e}")
+                raise
+    else:
+        try:
+            logger.info(f"Migrating schema: {schema_name}")
+            async with engine.connect() as connection:
+                await connection.run_sync(
+                    do_run_migrations,
+                    schema_name=schema_name,
+                    create_schema=create_schema,
+                )
+        except Exception as e:
+            logger.error(f"Error migrating schema {schema_name}: {e}")
+            raise
+
+    await engine.dispose()
+
+
+def run_migrations_offline() -> None:
+    """
+    Run migrations in 'offline' mode.
+    """
+    schema_name, _, upgrade_all_tenants = get_schema_options()
+    url = build_connection_string()
+
+    if upgrade_all_tenants:
+        # Run offline migrations for all tenant schemas
+        engine = create_async_engine(url)
+        tenant_schemas = get_all_tenant_ids()
+        engine.sync_engine.dispose()
+
+        for schema in tenant_schemas:
+            logger.info(f"Migrating schema: {schema}")
+            context.configure(
+                url=url,
+                target_metadata=target_metadata,  # type: ignore
+                literal_binds=True,
+                include_object=include_object,
+                version_table_schema=schema,
+                include_schemas=True,
+                script_location=config.get_main_option("script_location"),
+                dialect_opts={"paramstyle": "named"},
+            )
+
+            with context.begin_transaction():
+                context.run_migrations()
+    else:
+        logger.info(f"Migrating schema: {schema_name}")
+        context.configure(
+            url=url,
+            target_metadata=target_metadata,  # type: ignore
+            literal_binds=True,
+            include_object=include_object,
+            version_table_schema=schema_name,
+            include_schemas=True,
+            script_location=config.get_main_option("script_location"),
+            dialect_opts={"paramstyle": "named"},
+        )
 
-    await connectable.dispose()
+        with context.begin_transaction():
+            context.run_migrations()
 
 
 def run_migrations_online() -> None:
+    """
+    Runs migrations in 'online' mode using an asynchronous engine.
+    """
     asyncio.run(run_async_migrations())
 
 

From e12785d277296cd9749c8ee7d8959b4df62f957e Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 18 Oct 2024 11:07:54 -0700
Subject: [PATCH 150/376] no serializable, use with_for_update to lock the row.

---
 .../background/indexing/run_indexing.py       | 52 ++++---------------
 backend/danswer/db/index_attempt.py           | 38 ++++++++++++++
 2 files changed, 48 insertions(+), 42 deletions(-)

diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index b4cfea97a2e..ca749dd40b1 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -20,11 +20,10 @@
 from danswer.db.connector_credential_pair import update_connector_credential_pair
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
-from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import mark_attempt_failed
-from danswer.db.index_attempt import mark_attempt_in_progress
 from danswer.db.index_attempt import mark_attempt_partially_succeeded
 from danswer.db.index_attempt import mark_attempt_succeeded
+from danswer.db.index_attempt import transition_attempt_to_in_progress
 from danswer.db.index_attempt import update_docs_indexed
 from danswer.db.models import IndexAttempt
 from danswer.db.models import IndexingStatus
@@ -382,46 +381,15 @@ def _run_indexing(
 def _prepare_index_attempt(
     db_session: Session, index_attempt_id: int, tenant_id: str | None
 ) -> IndexAttempt:
-    # make sure that the index attempt can't change in between checking the
-    # status and marking it as in_progress. This setting will be discarded
-    # after the next commit:
-    # https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#setting-isolation-for-individual-transactions
-    db_session.connection(execution_options={"isolation_level": "SERIALIZABLE"})  # type: ignore
-    try:
-        if tenant_id is not None:
-            # Explicitly set the search path for the given tenant
-            db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
-            # Verify the search path was set correctly
-            result = db_session.execute(text("SHOW search_path"))
-            current_search_path = result.scalar()
-            logger.info(f"Current search path set to: {current_search_path}")
-
-        attempt = get_index_attempt(
-            db_session=db_session,
-            index_attempt_id=index_attempt_id,
-        )
-
-        if attempt is None:
-            raise RuntimeError(
-                f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
-            )
-
-        if attempt.status != IndexingStatus.NOT_STARTED:
-            raise RuntimeError(
-                f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
-                f"Current status is '{attempt.status}'."
-            )
-
-        mark_attempt_in_progress(attempt, db_session)
-
-        # only commit once, to make sure this all happens in a single transaction
-        db_session.commit()
-    except Exception:
-        db_session.rollback()
-        logger.exception("_prepare_index_attempt exceptioned.")
-        raise
-
-    return attempt
+    if tenant_id is not None:
+        # Explicitly set the search path for the given tenant
+        db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
+        # Verify the search path was set correctly
+        result = db_session.execute(text("SHOW search_path"))
+        current_search_path = result.scalar()
+        logger.info(f"Current search path set to: {current_search_path}")
+
+    return transition_attempt_to_in_progress(index_attempt_id, db_session)
 
 
 def run_indexing_entrypoint(
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index 21a1bbd236f..f1b58878dc2 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -101,6 +101,40 @@ def get_not_started_index_attempts(db_session: Session) -> list[IndexAttempt]:
     return list(new_attempts.all())
 
 
+def transition_attempt_to_in_progress(
+    index_attempt_id: int,
+    db_session: Session,
+) -> IndexAttempt:
+    """Locks the row when we try to update"""
+    with db_session.begin_nested():
+        try:
+            attempt = db_session.execute(
+                select(IndexAttempt)
+                .where(IndexAttempt.id == index_attempt_id)
+                .with_for_update()
+            ).scalar_one()
+
+            if attempt is None:
+                raise RuntimeError(
+                    f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
+                )
+
+            if attempt.status != IndexingStatus.NOT_STARTED:
+                raise RuntimeError(
+                    f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
+                    f"Current status is '{attempt.status}'."
+                )
+
+            attempt.status = IndexingStatus.IN_PROGRESS
+            attempt.time_started = attempt.time_started or func.now()  # type: ignore
+            db_session.commit()
+            return attempt
+        except Exception:
+            db_session.rollback()
+            logger.exception("transition_attempt_to_in_progress exceptioned.")
+            raise
+
+
 def mark_attempt_in_progress(
     index_attempt: IndexAttempt,
     db_session: Session,
@@ -118,6 +152,7 @@ def mark_attempt_in_progress(
             db_session.commit()
         except Exception:
             db_session.rollback()
+            raise
 
 
 def mark_attempt_succeeded(
@@ -136,6 +171,7 @@ def mark_attempt_succeeded(
             db_session.commit()
         except Exception:
             db_session.rollback()
+            raise
 
 
 def mark_attempt_partially_succeeded(
@@ -154,6 +190,7 @@ def mark_attempt_partially_succeeded(
             db_session.commit()
         except Exception:
             db_session.rollback()
+            raise
 
 
 def mark_attempt_failed(
@@ -176,6 +213,7 @@ def mark_attempt_failed(
             db_session.commit()
         except Exception:
             db_session.rollback()
+            raise
 
     source = index_attempt.connector_credential_pair.connector.source
     optional_telemetry(record_type=RecordType.FAILURE, data={"connector": source})

From 59364aadd7cf728b5bbc21ad35a031f076b5f551 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 18 Oct 2024 11:10:09 -0700
Subject: [PATCH 151/376] Revert "no serializable, use with_for_update to lock
 the row."

This reverts commit e12785d277296cd9749c8ee7d8959b4df62f957e.
---
 .../background/indexing/run_indexing.py       | 52 +++++++++++++++----
 backend/danswer/db/index_attempt.py           | 38 --------------
 2 files changed, 42 insertions(+), 48 deletions(-)

diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index ca749dd40b1..b4cfea97a2e 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -20,10 +20,11 @@
 from danswer.db.connector_credential_pair import update_connector_credential_pair
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import mark_attempt_failed
+from danswer.db.index_attempt import mark_attempt_in_progress
 from danswer.db.index_attempt import mark_attempt_partially_succeeded
 from danswer.db.index_attempt import mark_attempt_succeeded
-from danswer.db.index_attempt import transition_attempt_to_in_progress
 from danswer.db.index_attempt import update_docs_indexed
 from danswer.db.models import IndexAttempt
 from danswer.db.models import IndexingStatus
@@ -381,15 +382,46 @@ def _run_indexing(
 def _prepare_index_attempt(
     db_session: Session, index_attempt_id: int, tenant_id: str | None
 ) -> IndexAttempt:
-    if tenant_id is not None:
-        # Explicitly set the search path for the given tenant
-        db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
-        # Verify the search path was set correctly
-        result = db_session.execute(text("SHOW search_path"))
-        current_search_path = result.scalar()
-        logger.info(f"Current search path set to: {current_search_path}")
-
-    return transition_attempt_to_in_progress(index_attempt_id, db_session)
+    # make sure that the index attempt can't change in between checking the
+    # status and marking it as in_progress. This setting will be discarded
+    # after the next commit:
+    # https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#setting-isolation-for-individual-transactions
+    db_session.connection(execution_options={"isolation_level": "SERIALIZABLE"})  # type: ignore
+    try:
+        if tenant_id is not None:
+            # Explicitly set the search path for the given tenant
+            db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
+            # Verify the search path was set correctly
+            result = db_session.execute(text("SHOW search_path"))
+            current_search_path = result.scalar()
+            logger.info(f"Current search path set to: {current_search_path}")
+
+        attempt = get_index_attempt(
+            db_session=db_session,
+            index_attempt_id=index_attempt_id,
+        )
+
+        if attempt is None:
+            raise RuntimeError(
+                f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
+            )
+
+        if attempt.status != IndexingStatus.NOT_STARTED:
+            raise RuntimeError(
+                f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
+                f"Current status is '{attempt.status}'."
+            )
+
+        mark_attempt_in_progress(attempt, db_session)
+
+        # only commit once, to make sure this all happens in a single transaction
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        logger.exception("_prepare_index_attempt exceptioned.")
+        raise
+
+    return attempt
 
 
 def run_indexing_entrypoint(
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index f1b58878dc2..21a1bbd236f 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -101,40 +101,6 @@ def get_not_started_index_attempts(db_session: Session) -> list[IndexAttempt]:
     return list(new_attempts.all())
 
 
-def transition_attempt_to_in_progress(
-    index_attempt_id: int,
-    db_session: Session,
-) -> IndexAttempt:
-    """Locks the row when we try to update"""
-    with db_session.begin_nested():
-        try:
-            attempt = db_session.execute(
-                select(IndexAttempt)
-                .where(IndexAttempt.id == index_attempt_id)
-                .with_for_update()
-            ).scalar_one()
-
-            if attempt is None:
-                raise RuntimeError(
-                    f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
-                )
-
-            if attempt.status != IndexingStatus.NOT_STARTED:
-                raise RuntimeError(
-                    f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
-                    f"Current status is '{attempt.status}'."
-                )
-
-            attempt.status = IndexingStatus.IN_PROGRESS
-            attempt.time_started = attempt.time_started or func.now()  # type: ignore
-            db_session.commit()
-            return attempt
-        except Exception:
-            db_session.rollback()
-            logger.exception("transition_attempt_to_in_progress exceptioned.")
-            raise
-
-
 def mark_attempt_in_progress(
     index_attempt: IndexAttempt,
     db_session: Session,
@@ -152,7 +118,6 @@ def mark_attempt_in_progress(
             db_session.commit()
         except Exception:
             db_session.rollback()
-            raise
 
 
 def mark_attempt_succeeded(
@@ -171,7 +136,6 @@ def mark_attempt_succeeded(
             db_session.commit()
         except Exception:
             db_session.rollback()
-            raise
 
 
 def mark_attempt_partially_succeeded(
@@ -190,7 +154,6 @@ def mark_attempt_partially_succeeded(
             db_session.commit()
         except Exception:
             db_session.rollback()
-            raise
 
 
 def mark_attempt_failed(
@@ -213,7 +176,6 @@ def mark_attempt_failed(
             db_session.commit()
         except Exception:
             db_session.rollback()
-            raise
 
     source = index_attempt.connector_credential_pair.connector.source
     optional_telemetry(record_type=RecordType.FAILURE, data={"connector": source})

From 5b782998802bbd550ea2501a5ef464025019de00 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Fri, 18 Oct 2024 11:15:43 -0700
Subject: [PATCH 152/376] use native rate limiting in the confluence client
 (#2837)

* use native rate limiting in the confluence client

* upgrade urllib3 to v2.2.3 to support retries in confluence client

* improve logging so that progress is visible.
---
 .../connectors/confluence/connector.py        |  57 +++-
 .../confluence/rate_limit_handler.py          | 275 ++++++++++++------
 backend/danswer/db/connector.py               |   2 +-
 .../confluence/sync_utils.py                  |   3 +
 backend/requirements/default.txt              |   3 +-
 5 files changed, 230 insertions(+), 110 deletions(-)

diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index 82174c8b931..03b91fa29d4 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -105,6 +105,7 @@ def _get_user(user_id: str, confluence_client: DanswerConfluence) -> str:
         confluence_client.get_user_details_by_accountid
     )
     try:
+        logger.info(f"_get_user - get_user_details_by_accountid: id={user_id}")
         return get_user_details_by_accountid(user_id).get("displayName", user_not_found)
     except Exception as e:
         logger.warning(
@@ -156,6 +157,9 @@ def _comment_dfs(
             comment_html, confluence_client
         )
         try:
+            logger.info(
+                f"_comment_dfs - get_page_by_child_type: id={comment_page['id']}"
+            )
             child_comment_pages = get_page_child_by_type(
                 comment_page["id"],
                 type="comment",
@@ -212,13 +216,16 @@ def _fetch_origin_page(self) -> dict[str, Any]:
             self.confluence_client.get_page_by_id
         )
         try:
+            logger.info(
+                f"_fetch_origin_page - get_page_by_id: id={self.origin_page_id}"
+            )
             origin_page = get_page_by_id(
                 self.origin_page_id, expand="body.storage.value,version,space"
             )
             return origin_page
-        except Exception as e:
-            logger.warning(
-                f"Appending origin page with id {self.origin_page_id} failed: {e}"
+        except Exception:
+            logger.exception(
+                f"Appending origin page with id {self.origin_page_id} failed."
             )
             return {}
 
@@ -230,6 +237,10 @@ def recurse_children_pages(
         queue: list[str] = [page_id]
         visited_pages: set[str] = set()
 
+        get_page_by_id = make_confluence_call_handle_rate_limit(
+            self.confluence_client.get_page_by_id
+        )
+
         get_page_child_by_type = make_confluence_call_handle_rate_limit(
             self.confluence_client.get_page_child_by_type
         )
@@ -242,12 +253,15 @@ def recurse_children_pages(
 
             try:
                 # Fetch the page itself
-                page = self.confluence_client.get_page_by_id(
+                logger.info(
+                    f"recurse_children_pages - get_page_by_id: id={current_page_id}"
+                )
+                page = get_page_by_id(
                     current_page_id, expand="body.storage.value,version,space"
                 )
                 pages.append(page)
-            except Exception as e:
-                logger.warning(f"Failed to fetch page {current_page_id}: {e}")
+            except Exception:
+                logger.exception(f"Failed to fetch page {current_page_id}.")
                 continue
 
             if not self.index_recursively:
@@ -256,6 +270,9 @@ def recurse_children_pages(
             # Fetch child pages
             start = 0
             while True:
+                logger.info(
+                    f"recurse_children_pages - get_page_by_child_type: id={current_page_id}"
+                )
                 child_pages_response = get_page_child_by_type(
                     current_page_id,
                     type="page",
@@ -323,11 +340,17 @@ def __init__(
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         username = credentials["confluence_username"]
         access_token = credentials["confluence_access_token"]
+
+        # see https://github.com/atlassian-api/atlassian-python-api/blob/master/atlassian/rest_client.py
+        # for a list of other hidden constructor args
         self.confluence_client = DanswerConfluence(
             url=self.wiki_base,
             username=username if self.is_cloud else None,
             password=access_token if self.is_cloud else None,
             token=access_token if not self.is_cloud else None,
+            backoff_and_retry=True,
+            max_backoff_retries=60,
+            max_backoff_seconds=60,
         )
         return None
 
@@ -354,6 +377,9 @@ def _fetch_space(
             )
 
             try:
+                logger.info(
+                    f"_fetch_space - get_all_pages: cursor={cursor} limit={batch_size}"
+                )
                 response = get_all_pages(
                     cql=self.cql_query,
                     cursor=cursor,
@@ -380,6 +406,9 @@ def _fetch_space(
                 view_pages: list[dict[str, Any]] = []
                 for _ in range(self.batch_size):
                     try:
+                        logger.info(
+                            f"_fetch_space - get_all_pages: cursor={cursor} limit=1"
+                        )
                         response = get_all_pages(
                             cql=self.cql_query,
                             cursor=cursor,
@@ -406,6 +435,9 @@ def _fetch_space(
                             f"Page failed with cql {self.cql_query} with cursor {cursor}, "
                             f"trying alternative expand option: {e}"
                         )
+                        logger.info(
+                            f"_fetch_space - get_all_pages - trying alternative expand: cursor={cursor} limit=1"
+                        )
                         response = get_all_pages(
                             cql=self.cql_query,
                             cursor=cursor,
@@ -464,6 +496,7 @@ def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
         )
 
         try:
+            logger.info(f"_fetch_comments - get_page_child_by_type: id={page_id}")
             comment_pages = list(
                 get_page_child_by_type(
                     page_id,
@@ -478,9 +511,7 @@ def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
             if not self.continue_on_failure:
                 raise e
 
-            logger.exception(
-                "Ran into exception when fetching comments from Confluence"
-            )
+            logger.exception("Fetching comments from Confluence exceptioned")
             return ""
 
     def _fetch_labels(self, confluence_client: Confluence, page_id: str) -> list[str]:
@@ -488,13 +519,14 @@ def _fetch_labels(self, confluence_client: Confluence, page_id: str) -> list[str
             confluence_client.get_page_labels
         )
         try:
+            logger.info(f"_fetch_labels - get_page_labels: id={page_id}")
             labels_response = get_page_labels(page_id)
             return [label["name"] for label in labels_response["results"]]
         except Exception as e:
             if not self.continue_on_failure:
                 raise e
 
-            logger.exception("Ran into exception when fetching labels from Confluence")
+            logger.exception("Fetching labels from Confluence exceptioned")
             return []
 
     @classmethod
@@ -531,6 +563,7 @@ def _attachment_to_content(
             )
             return None
 
+        logger.info(f"_attachment_to_content - _session.get: link={download_link}")
         response = confluence_client._session.get(download_link)
         if response.status_code != 200:
             logger.warning(
@@ -589,9 +622,7 @@ def _fetch_attachments(
                 return "", []
             if not self.continue_on_failure:
                 raise e
-            logger.exception(
-                f"Ran into exception when fetching attachments from Confluence: {e}"
-            )
+            logger.exception("Fetching attachments from Confluence exceptioned.")
 
         return "\n".join(files_attachment_content), unused_attachments
 
diff --git a/backend/danswer/connectors/confluence/rate_limit_handler.py b/backend/danswer/connectors/confluence/rate_limit_handler.py
index c05754bb105..8dbdeba1ab6 100644
--- a/backend/danswer/connectors/confluence/rate_limit_handler.py
+++ b/backend/danswer/connectors/confluence/rate_limit_handler.py
@@ -5,11 +5,8 @@
 from typing import cast
 from typing import TypeVar
 
-from redis.exceptions import ConnectionError
 from requests import HTTPError
 
-from danswer.connectors.interfaces import BaseConnector
-from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -25,110 +22,198 @@ class ConfluenceRateLimitError(Exception):
     pass
 
 
+# commenting out while we try using confluence's rate limiter instead
+# # https://developer.atlassian.com/cloud/confluence/rate-limiting/
+# def make_confluence_call_handle_rate_limit(confluence_call: F) -> F:
+#     def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
+#         max_retries = 5
+#         starting_delay = 5
+#         backoff = 2
+
+#         # max_delay is used when the server doesn't hand back "Retry-After"
+#         # and we have to decide the retry delay ourselves
+#         max_delay = 30  # Atlassian uses max_delay = 30 in their examples
+
+#         # max_retry_after is used when we do get a "Retry-After" header
+#         max_retry_after = 300  # should we really cap the maximum retry delay?
+
+#         NEXT_RETRY_KEY = BaseConnector.REDIS_KEY_PREFIX + "confluence_next_retry"
+
+#         # for testing purposes, rate limiting is written to fall back to a simpler
+#         # rate limiting approach when redis is not available
+#         r = get_redis_client()
+
+#         for attempt in range(max_retries):
+#             try:
+#                 # if multiple connectors are waiting for the next attempt, there could be an issue
+#                 # where many connectors are "released" onto the server at the same time.
+#                 # That's not ideal ... but coming up with a mechanism for queueing
+#                 # all of these connectors is a bigger problem that we want to take on
+#                 # right now
+#                 try:
+#                     next_attempt = r.get(NEXT_RETRY_KEY)
+#                     if next_attempt is None:
+#                         next_attempt = 0
+#                     else:
+#                         next_attempt = int(cast(int, next_attempt))
+
+#                     # TODO: all connectors need to be interruptible moving forward
+#                     while time.monotonic() < next_attempt:
+#                         time.sleep(1)
+#                 except ConnectionError:
+#                     pass
+
+#                 return confluence_call(*args, **kwargs)
+#             except HTTPError as e:
+#                 # Check if the response or headers are None to avoid potential AttributeError
+#                 if e.response is None or e.response.headers is None:
+#                     logger.warning("HTTPError with `None` as response or as headers")
+#                     raise e
+
+#                 retry_after_header = e.response.headers.get("Retry-After")
+#                 if (
+#                     e.response.status_code == 429
+#                     or RATE_LIMIT_MESSAGE_LOWERCASE in e.response.text.lower()
+#                 ):
+#                     retry_after = None
+#                     if retry_after_header is not None:
+#                         try:
+#                             retry_after = int(retry_after_header)
+#                         except ValueError:
+#                             pass
+
+#                     if retry_after is not None:
+#                         if retry_after > max_retry_after:
+#                             logger.warning(
+#                                 f"Clamping retry_after from {retry_after} to {max_delay} seconds..."
+#                             )
+#                             retry_after = max_delay
+
+#                         logger.warning(
+#                             f"Rate limit hit. Retrying after {retry_after} seconds..."
+#                         )
+#                         try:
+#                             r.set(
+#                                 NEXT_RETRY_KEY,
+#                                 math.ceil(time.monotonic() + retry_after),
+#                             )
+#                         except ConnectionError:
+#                             pass
+#                     else:
+#                         logger.warning(
+#                             "Rate limit hit. Retrying with exponential backoff..."
+#                         )
+#                         delay = min(starting_delay * (backoff**attempt), max_delay)
+#                         delay_until = math.ceil(time.monotonic() + delay)
+
+#                         try:
+#                             r.set(NEXT_RETRY_KEY, delay_until)
+#                         except ConnectionError:
+#                             while time.monotonic() < delay_until:
+#                                 time.sleep(1)
+#                 else:
+#                     # re-raise, let caller handle
+#                     raise
+#             except AttributeError as e:
+#                 # Some error within the Confluence library, unclear why it fails.
+#                 # Users reported it to be intermittent, so just retry
+#                 logger.warning(f"Confluence Internal Error, retrying... {e}")
+#                 delay = min(starting_delay * (backoff**attempt), max_delay)
+#                 delay_until = math.ceil(time.monotonic() + delay)
+#                 try:
+#                     r.set(NEXT_RETRY_KEY, delay_until)
+#                 except ConnectionError:
+#                     while time.monotonic() < delay_until:
+#                         time.sleep(1)
+
+#                 if attempt == max_retries - 1:
+#                     raise e
+
+#     return cast(F, wrapped_call)
+
+
+def _handle_http_error(e: HTTPError, attempt: int) -> int:
+    MIN_DELAY = 2
+    MAX_DELAY = 60
+    STARTING_DELAY = 5
+    BACKOFF = 2
+
+    # Check if the response or headers are None to avoid potential AttributeError
+    if e.response is None or e.response.headers is None:
+        logger.warning("HTTPError with `None` as response or as headers")
+        raise e
+
+    if (
+        e.response.status_code != 429
+        and RATE_LIMIT_MESSAGE_LOWERCASE not in e.response.text.lower()
+    ):
+        raise e
+
+    retry_after = None
+
+    retry_after_header = e.response.headers.get("Retry-After")
+    if retry_after_header is not None:
+        try:
+            retry_after = int(retry_after_header)
+            if retry_after > MAX_DELAY:
+                logger.warning(
+                    f"Clamping retry_after from {retry_after} to {MAX_DELAY} seconds..."
+                )
+                retry_after = MAX_DELAY
+            if retry_after < MIN_DELAY:
+                retry_after = MIN_DELAY
+        except ValueError:
+            pass
+
+    if retry_after is not None:
+        logger.warning(
+            f"Rate limiting with retry header. Retrying after {retry_after} seconds..."
+        )
+        delay = retry_after
+    else:
+        logger.warning(
+            "Rate limiting without retry header. Retrying with exponential backoff..."
+        )
+        delay = min(STARTING_DELAY * (BACKOFF**attempt), MAX_DELAY)
+
+    delay_until = math.ceil(time.monotonic() + delay)
+    return delay_until
+
+
 # https://developer.atlassian.com/cloud/confluence/rate-limiting/
+# this uses the native rate limiting option provided by the
+# confluence client and otherwise applies a simpler set of error handling
 def make_confluence_call_handle_rate_limit(confluence_call: F) -> F:
     def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
-        max_retries = 5
-        starting_delay = 5
-        backoff = 2
-
-        # max_delay is used when the server doesn't hand back "Retry-After"
-        # and we have to decide the retry delay ourselves
-        max_delay = 30  # Atlassian uses max_delay = 30 in their examples
-
-        # max_retry_after is used when we do get a "Retry-After" header
-        max_retry_after = 300  # should we really cap the maximum retry delay?
+        MAX_RETRIES = 5
 
-        NEXT_RETRY_KEY = BaseConnector.REDIS_KEY_PREFIX + "confluence_next_retry"
+        TIMEOUT = 3600
+        timeout_at = time.monotonic() + TIMEOUT
 
-        # for testing purposes, rate limiting is written to fall back to a simpler
-        # rate limiting approach when redis is not available
-        r = get_redis_client()
+        for attempt in range(MAX_RETRIES):
+            if time.monotonic() > timeout_at:
+                raise TimeoutError(
+                    f"Confluence call attempts took longer than {TIMEOUT} seconds."
+                )
 
-        for attempt in range(max_retries):
             try:
-                # if multiple connectors are waiting for the next attempt, there could be an issue
-                # where many connectors are "released" onto the server at the same time.
-                # That's not ideal ... but coming up with a mechanism for queueing
-                # all of these connectors is a bigger problem that we want to take on
-                # right now
-                try:
-                    next_attempt = r.get(NEXT_RETRY_KEY)
-                    if next_attempt is None:
-                        next_attempt = 0
-                    else:
-                        next_attempt = int(cast(int, next_attempt))
-
-                    # TODO: all connectors need to be interruptible moving forward
-                    while time.monotonic() < next_attempt:
-                        time.sleep(1)
-                except ConnectionError:
-                    pass
-
+                # we're relying more on the client to rate limit itself
+                # and applying our own retries in a more specific set of circumstances
                 return confluence_call(*args, **kwargs)
             except HTTPError as e:
-                # Check if the response or headers are None to avoid potential AttributeError
-                if e.response is None or e.response.headers is None:
-                    logger.warning("HTTPError with `None` as response or as headers")
-                    raise e
-
-                retry_after_header = e.response.headers.get("Retry-After")
-                if (
-                    e.response.status_code == 429
-                    or RATE_LIMIT_MESSAGE_LOWERCASE in e.response.text.lower()
-                ):
-                    retry_after = None
-                    if retry_after_header is not None:
-                        try:
-                            retry_after = int(retry_after_header)
-                        except ValueError:
-                            pass
-
-                    if retry_after is not None:
-                        if retry_after > max_retry_after:
-                            logger.warning(
-                                f"Clamping retry_after from {retry_after} to {max_delay} seconds..."
-                            )
-                            retry_after = max_delay
-
-                        logger.warning(
-                            f"Rate limit hit. Retrying after {retry_after} seconds..."
-                        )
-                        try:
-                            r.set(
-                                NEXT_RETRY_KEY,
-                                math.ceil(time.monotonic() + retry_after),
-                            )
-                        except ConnectionError:
-                            pass
-                    else:
-                        logger.warning(
-                            "Rate limit hit. Retrying with exponential backoff..."
-                        )
-                        delay = min(starting_delay * (backoff**attempt), max_delay)
-                        delay_until = math.ceil(time.monotonic() + delay)
-
-                        try:
-                            r.set(NEXT_RETRY_KEY, delay_until)
-                        except ConnectionError:
-                            while time.monotonic() < delay_until:
-                                time.sleep(1)
-                else:
-                    # re-raise, let caller handle
-                    raise
+                delay_until = _handle_http_error(e, attempt)
+                while time.monotonic() < delay_until:
+                    # in the future, check a signal here to exit
+                    time.sleep(1)
             except AttributeError as e:
                 # Some error within the Confluence library, unclear why it fails.
                 # Users reported it to be intermittent, so just retry
-                logger.warning(f"Confluence Internal Error, retrying... {e}")
-                delay = min(starting_delay * (backoff**attempt), max_delay)
-                delay_until = math.ceil(time.monotonic() + delay)
-                try:
-                    r.set(NEXT_RETRY_KEY, delay_until)
-                except ConnectionError:
-                    while time.monotonic() < delay_until:
-                        time.sleep(1)
-
-                if attempt == max_retries - 1:
+                if attempt == MAX_RETRIES - 1:
                     raise e
 
+                logger.exception(
+                    "Confluence Client raised an AttributeError. Retrying..."
+                )
+                time.sleep(5)
+
     return cast(F, wrapped_call)
diff --git a/backend/danswer/db/connector.py b/backend/danswer/db/connector.py
index 0f777d30ec9..835f74d437c 100644
--- a/backend/danswer/db/connector.py
+++ b/backend/danswer/db/connector.py
@@ -248,7 +248,7 @@ def create_initial_default_connector(db_session: Session) -> None:
             logger.warning(
                 "Default connector does not have expected values. Updating to proper state."
             )
-            # Ensure default connector has correct valuesg
+            # Ensure default connector has correct values
             default_connector.source = DocumentSource.INGESTION_API
             default_connector.input_type = InputType.LOAD_STATE
             default_connector.refresh_freq = None
diff --git a/backend/ee/danswer/external_permissions/confluence/sync_utils.py b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
index 183e390595e..d6eb225007a 100644
--- a/backend/ee/danswer/external_permissions/confluence/sync_utils.py
+++ b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
@@ -20,6 +20,9 @@ def build_confluence_client(
         username=credentials_json["confluence_username"] if is_cloud else None,
         password=credentials_json["confluence_access_token"] if is_cloud else None,
         token=credentials_json["confluence_access_token"] if not is_cloud else None,
+        backoff_and_retry=True,
+        max_backoff_retries=60,
+        max_backoff_seconds=60,
     )
 
 
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index bc6c667cced..354103f200a 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -1,7 +1,7 @@
 aiohttp==3.10.2
 alembic==1.10.4
 asyncpg==0.27.0
-atlassian-python-api==3.37.0
+atlassian-python-api==3.41.16
 beautifulsoup4==4.12.3
 boto3==1.34.84
 celery==5.5.0b4
@@ -81,5 +81,6 @@ dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
 ultimate_sitemap_parser==0.5
 stripe==10.12.0
+urllib3==2.2.3
 mistune==0.8.4
 sentry-sdk==2.14.0

From 36134021c547ca7c071f7fb79efd66ebe8877598 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Fri, 18 Oct 2024 11:25:27 -0700
Subject: [PATCH 153/376] Refactor + add global timeout env variable (#2844)

* Refactor + add global timeout env variable

* remove model

* mypy

* Remove unused
---
 backend/model_server/encoders.py              | 147 ++++++++----------
 backend/shared_configs/configs.py             |   9 +-
 .../tests/daily/embedding/test_embeddings.py  |  20 +++
 3 files changed, 90 insertions(+), 86 deletions(-)

diff --git a/backend/model_server/encoders.py b/backend/model_server/encoders.py
index f252b29d172..003953cb29a 100644
--- a/backend/model_server/encoders.py
+++ b/backend/model_server/encoders.py
@@ -1,5 +1,5 @@
 import json
-from typing import Any
+from typing import cast
 from typing import Optional
 
 import httpx
@@ -25,6 +25,7 @@
 from model_server.constants import EmbeddingModelTextType
 from model_server.constants import EmbeddingProvider
 from model_server.utils import simple_log_function_time
+from shared_configs.configs import API_BASED_EMBEDDING_TIMEOUT
 from shared_configs.configs import INDEXING_ONLY
 from shared_configs.configs import OPENAI_EMBEDDING_TIMEOUT
 from shared_configs.enums import EmbedTextType
@@ -54,32 +55,6 @@
 _COHERE_MAX_INPUT_LEN = 96
 
 
-def _initialize_client(
-    api_key: str,
-    provider: EmbeddingProvider,
-    model: str | None = None,
-    api_url: str | None = None,
-    api_version: str | None = None,
-) -> Any:
-    if provider == EmbeddingProvider.OPENAI:
-        return openai.OpenAI(api_key=api_key, timeout=OPENAI_EMBEDDING_TIMEOUT)
-    elif provider == EmbeddingProvider.COHERE:
-        return CohereClient(api_key=api_key)
-    elif provider == EmbeddingProvider.VOYAGE:
-        return voyageai.Client(api_key=api_key)
-    elif provider == EmbeddingProvider.GOOGLE:
-        credentials = service_account.Credentials.from_service_account_info(
-            json.loads(api_key)
-        )
-        project_id = json.loads(api_key)["project_id"]
-        vertexai.init(project=project_id, credentials=credentials)
-        return TextEmbeddingModel.from_pretrained(model or DEFAULT_VERTEX_MODEL)
-    elif provider == EmbeddingProvider.AZURE:
-        return {"api_key": api_key, "api_url": api_url, "api_version": api_version}
-    else:
-        raise ValueError(f"Unsupported provider: {provider}")
-
-
 class CloudEmbedding:
     def __init__(
         self,
@@ -87,25 +62,22 @@ def __init__(
         provider: EmbeddingProvider,
         api_url: str | None = None,
         api_version: str | None = None,
-        # Only for Google as is needed on client setup
-        model: str | None = None,
     ) -> None:
         self.provider = provider
-        self.client = _initialize_client(
-            api_key, self.provider, model, api_url, api_version
-        )
+        self.api_key = api_key
+        self.api_url = api_url
+        self.api_version = api_version
 
     def _embed_openai(self, texts: list[str], model: str | None) -> list[Embedding]:
         if not model:
             model = DEFAULT_OPENAI_MODEL
 
-        # OpenAI does not seem to provide truncation option, however
-        # the context lengths used by Danswer currently are smaller than the max token length
-        # for OpenAI embeddings so it's not a big deal
+        client = openai.OpenAI(api_key=self.api_key, timeout=OPENAI_EMBEDDING_TIMEOUT)
+
         final_embeddings: list[Embedding] = []
         try:
             for text_batch in batch_list(texts, _OPENAI_MAX_INPUT_LEN):
-                response = self.client.embeddings.create(input=text_batch, model=model)
+                response = client.embeddings.create(input=text_batch, model=model)
                 final_embeddings.extend(
                     [embedding.embedding for embedding in response.data]
                 )
@@ -126,17 +98,19 @@ def _embed_cohere(
         if not model:
             model = DEFAULT_COHERE_MODEL
 
+        client = CohereClient(api_key=self.api_key, timeout=API_BASED_EMBEDDING_TIMEOUT)
+
         final_embeddings: list[Embedding] = []
         for text_batch in batch_list(texts, _COHERE_MAX_INPUT_LEN):
             # Does not use the same tokenizer as the Danswer API server but it's approximately the same
             # empirically it's only off by a very few tokens so it's not a big deal
-            response = self.client.embed(
+            response = client.embed(
                 texts=text_batch,
                 model=model,
                 input_type=embedding_type,
                 truncate="END",
             )
-            final_embeddings.extend(response.embeddings)
+            final_embeddings.extend(cast(list[Embedding], response.embeddings))
         return final_embeddings
 
     def _embed_voyage(
@@ -145,13 +119,15 @@ def _embed_voyage(
         if not model:
             model = DEFAULT_VOYAGE_MODEL
 
-        # Similar to Cohere, the API server will do approximate size chunking
-        # it's acceptable to miss by a few tokens
-        response = self.client.embed(
+        client = voyageai.Client(
+            api_key=self.api_key, timeout=API_BASED_EMBEDDING_TIMEOUT
+        )
+
+        response = client.embed(
             texts,
             model=model,
             input_type=embedding_type,
-            truncation=True,  # Also this is default
+            truncation=True,
         )
         return response.embeddings
 
@@ -159,9 +135,10 @@ def _embed_azure(self, texts: list[str], model: str | None) -> list[Embedding]:
         response = embedding(
             model=model,
             input=texts,
-            api_key=self.client["api_key"],
-            api_base=self.client["api_url"],
-            api_version=self.client["api_version"],
+            timeout=API_BASED_EMBEDDING_TIMEOUT,
+            api_key=self.api_key,
+            api_base=self.api_url,
+            api_version=self.api_version,
         )
         embeddings = [embedding["embedding"] for embedding in response.data]
 
@@ -173,7 +150,14 @@ def _embed_vertex(
         if not model:
             model = DEFAULT_VERTEX_MODEL
 
-        embeddings = self.client.get_embeddings(
+        credentials = service_account.Credentials.from_service_account_info(
+            json.loads(self.api_key)
+        )
+        project_id = json.loads(self.api_key)["project_id"]
+        vertexai.init(project=project_id, credentials=credentials)
+        client = TextEmbeddingModel.from_pretrained(model)
+
+        embeddings = client.get_embeddings(
             [
                 TextEmbeddingInput(
                     text,
@@ -185,6 +169,33 @@ def _embed_vertex(
         )
         return [embedding.values for embedding in embeddings]
 
+    def _embed_litellm_proxy(
+        self, texts: list[str], model_name: str | None
+    ) -> list[Embedding]:
+        if not model_name:
+            raise ValueError("Model name is required for LiteLLM proxy embedding.")
+
+        if not self.api_url:
+            raise ValueError("API URL is required for LiteLLM proxy embedding.")
+
+        headers = (
+            {} if not self.api_key else {"Authorization": f"Bearer {self.api_key}"}
+        )
+
+        with httpx.Client() as client:
+            response = client.post(
+                self.api_url,
+                json={
+                    "model": model_name,
+                    "input": texts,
+                },
+                headers=headers,
+                timeout=API_BASED_EMBEDDING_TIMEOUT,
+            )
+            response.raise_for_status()
+            result = response.json()
+            return [embedding["embedding"] for embedding in result["data"]]
+
     @retry(tries=_RETRY_TRIES, delay=_RETRY_DELAY)
     def embed(
         self,
@@ -199,6 +210,9 @@ def embed(
                 return self._embed_openai(texts, model_name)
             elif self.provider == EmbeddingProvider.AZURE:
                 return self._embed_azure(texts, f"azure/{deployment_name}")
+            elif self.provider == EmbeddingProvider.LITELLM:
+                return self._embed_litellm_proxy(texts, model_name)
+
             embedding_type = EmbeddingModelTextType.get_type(self.provider, text_type)
             if self.provider == EmbeddingProvider.COHERE:
                 return self._embed_cohere(texts, model_name, embedding_type)
@@ -218,12 +232,11 @@ def embed(
     def create(
         api_key: str,
         provider: EmbeddingProvider,
-        model: str | None = None,
         api_url: str | None = None,
         api_version: str | None = None,
     ) -> "CloudEmbedding":
         logger.debug(f"Creating Embedding instance for provider: {provider}")
-        return CloudEmbedding(api_key, provider, model, api_url, api_version)
+        return CloudEmbedding(api_key, provider, api_url, api_version)
 
 
 def get_embedding_model(
@@ -266,25 +279,6 @@ def get_local_reranking_model(
     return _RERANK_MODEL
 
 
-def embed_with_litellm_proxy(
-    texts: list[str], api_url: str, model_name: str, api_key: str | None
-) -> list[Embedding]:
-    headers = {} if not api_key else {"Authorization": f"Bearer {api_key}"}
-
-    with httpx.Client() as client:
-        response = client.post(
-            api_url,
-            json={
-                "model": model_name,
-                "input": texts,
-            },
-            headers=headers,
-        )
-        response.raise_for_status()
-        result = response.json()
-        return [embedding["embedding"] for embedding in result["data"]]
-
-
 @simple_log_function_time()
 def embed_text(
     texts: list[str],
@@ -309,23 +303,7 @@ def embed_text(
         logger.error("No texts provided for embedding")
         raise ValueError("No texts provided for embedding.")
 
-    if provider_type == EmbeddingProvider.LITELLM:
-        logger.debug(f"Using LiteLLM proxy for embedding with URL: {api_url}")
-        if not api_url:
-            logger.error("API URL not provided for LiteLLM proxy")
-            raise ValueError("API URL is required for LiteLLM proxy embedding.")
-        try:
-            return embed_with_litellm_proxy(
-                texts=texts,
-                api_url=api_url,
-                model_name=model_name or "",
-                api_key=api_key,
-            )
-        except Exception as e:
-            logger.exception(f"Error during LiteLLM proxy embedding: {str(e)}")
-            raise
-
-    elif provider_type is not None:
+    if provider_type is not None:
         logger.debug(f"Using cloud provider {provider_type} for embedding")
         if api_key is None:
             logger.error("API key not provided for cloud model")
@@ -341,7 +319,6 @@ def embed_text(
         cloud_model = CloudEmbedding(
             api_key=api_key,
             provider=provider_type,
-            model=model_name,
             api_url=api_url,
             api_version=api_version,
         )
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index ca452640071..f10855f103f 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -63,8 +63,15 @@
 # notset, debug, info, notice, warning, error, or critical
 LOG_LEVEL = os.environ.get("LOG_LEVEL", "notice")
 
+# Timeout for API-based embedding models
+# NOTE: does not apply for Google VertexAI, since the python client doesn't
+# allow us to specify a custom timeout
+API_BASED_EMBEDDING_TIMEOUT = int(os.environ.get("API_BASED_EMBEDDING_TIMEOUT", "600"))
+
 # Only used for OpenAI
-OPENAI_EMBEDDING_TIMEOUT = int(os.environ.get("OPENAI_EMBEDDING_TIMEOUT", "600"))
+OPENAI_EMBEDDING_TIMEOUT = int(
+    os.environ.get("OPENAI_EMBEDDING_TIMEOUT", API_BASED_EMBEDDING_TIMEOUT)
+)
 
 # Whether or not to strictly enforce token limit for chunking.
 STRICT_CHUNK_TOKEN_LIMIT = (
diff --git a/backend/tests/daily/embedding/test_embeddings.py b/backend/tests/daily/embedding/test_embeddings.py
index b736f374741..10a1dd850f6 100644
--- a/backend/tests/daily/embedding/test_embeddings.py
+++ b/backend/tests/daily/embedding/test_embeddings.py
@@ -61,6 +61,26 @@ def test_cohere_embedding(cohere_embedding_model: EmbeddingModel) -> None:
     _run_embeddings(TOO_LONG_SAMPLE, cohere_embedding_model, 384)
 
 
+@pytest.fixture
+def litellm_embedding_model() -> EmbeddingModel:
+    return EmbeddingModel(
+        server_host="localhost",
+        server_port=9000,
+        model_name="text-embedding-3-small",
+        normalize=True,
+        query_prefix=None,
+        passage_prefix=None,
+        api_key=os.getenv("LITE_LLM_API_KEY"),
+        provider_type=EmbeddingProvider.LITELLM,
+        api_url=os.getenv("LITE_LLM_API_URL"),
+    )
+
+
+def test_litellm_embedding(litellm_embedding_model: EmbeddingModel) -> None:
+    _run_embeddings(VALID_SAMPLE, litellm_embedding_model, 1536)
+    _run_embeddings(TOO_LONG_SAMPLE, litellm_embedding_model, 1536)
+
+
 @pytest.fixture
 def local_nomic_embedding_model() -> EmbeddingModel:
     return EmbeddingModel(

From 55de519364c1f342ddb9c5e00e37488b1ac87b67 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Fri, 18 Oct 2024 11:44:24 -0700
Subject: [PATCH 154/376] Cleanup connector form (#2849)

* move "advanced options" to the bottom of the form and cleanup curator frontend

* troll
---
 .../[connector]/AddConnectorPage.tsx          |  4 +-
 .../pages/DynamicConnectorCreationForm.tsx    |  8 +++
 .../admin/connectors/AccessTypeForm.tsx       | 53 ++++++++-----------
 3 files changed, 31 insertions(+), 34 deletions(-)

diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index fee2042b3b4..ce846851820 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -431,10 +431,8 @@ export default function AddConnector({
                   config={configuration}
                   setSelectedFiles={setSelectedFiles}
                   selectedFiles={selectedFiles}
+                  connector={connector}
                 />
-
-                <AccessTypeForm connector={connector} />
-                <AccessTypeGroupSelector />
               </Card>
             )}
 
diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
index bc064b727ec..f02111d262c 100644
--- a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
@@ -10,12 +10,16 @@ import { TextFormField } from "@/components/admin/connectors/Field";
 import ListInput from "./ConnectorInput/ListInput";
 import FileInput from "./ConnectorInput/FileInput";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
+import { AccessTypeForm } from "@/components/admin/connectors/AccessTypeForm";
+import { AccessTypeGroupSelector } from "@/components/admin/connectors/AccessTypeGroupSelector";
+import { ConfigurableSources } from "@/lib/types";
 
 export interface DynamicConnectionFormProps {
   config: ConnectionConfiguration;
   selectedFiles: File[];
   setSelectedFiles: Dispatch<SetStateAction<File[]>>;
   values: any;
+  connector: ConfigurableSources;
 }
 
 const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
@@ -23,6 +27,7 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
   selectedFiles,
   setSelectedFiles,
   values,
+  connector,
 }) => {
   const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
 
@@ -96,6 +101,9 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
 
       {config.values.map((field) => !field.hidden && renderField(field))}
 
+      <AccessTypeForm connector={connector} />
+      <AccessTypeGroupSelector />
+
       {config.advanced_values.length > 0 && (
         <>
           <AdvancedOptionsToggle
diff --git a/web/src/components/admin/connectors/AccessTypeForm.tsx b/web/src/components/admin/connectors/AccessTypeForm.tsx
index dab628ecd44..108b657b568 100644
--- a/web/src/components/admin/connectors/AccessTypeForm.tsx
+++ b/web/src/components/admin/connectors/AccessTypeForm.tsx
@@ -57,39 +57,30 @@ export function AccessTypeForm({
 
   return (
     <>
-      {isPaidEnterpriseEnabled && (
+      {isPaidEnterpriseEnabled && isAdmin && (
         <>
-          <div>
-            <div className="flex gap-x-2 items-center">
-              <label className="text-text-950 font-medium">
-                Document Access
-              </label>
-            </div>
-            <p className="text-sm text-text-500 mb-2">
-              Control who has access to the documents indexed by this connector.
-            </p>
-
-            {isAdmin && (
-              <>
-                <DefaultDropdown
-                  options={options}
-                  selected={access_type.value}
-                  onSelect={(selected) =>
-                    access_type_helpers.setValue(selected as AccessType)
-                  }
-                  includeDefault={false}
-                />
-
-                {access_type.value === "sync" && isAutoSyncSupported && (
-                  <div>
-                    <AutoSyncOptions
-                      connectorType={connector as ValidAutoSyncSources}
-                    />
-                  </div>
-                )}
-              </>
-            )}
+          <div className="flex gap-x-2 items-center">
+            <label className="text-text-950 font-medium">Document Access</label>
           </div>
+          <p className="text-sm text-text-500 mb-2">
+            Control who has access to the documents indexed by this connector.
+          </p>
+          <DefaultDropdown
+            options={options}
+            selected={access_type.value}
+            onSelect={(selected) =>
+              access_type_helpers.setValue(selected as AccessType)
+            }
+            includeDefault={false}
+          />
+
+          {access_type.value === "sync" && isAutoSyncSupported && (
+            <div>
+              <AutoSyncOptions
+                connectorType={connector as ValidAutoSyncSources}
+              />
+            </div>
+          )}
         </>
       )}
     </>

From 12cbbe6ceeca812cb0ebf523b5aff29c6d264db2 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Fri, 18 Oct 2024 13:35:23 -0700
Subject: [PATCH 155/376] use with for update instead of serializable (#2848)

* use with for update instead of serializable

* remove tenant logic handled now by get_session_with_tenant

* remove usage of begin_nested ... it's not necessary
---
 .../background/indexing/run_indexing.py       |  51 +------
 backend/danswer/db/index_attempt.py           | 129 +++++++++++-------
 2 files changed, 83 insertions(+), 97 deletions(-)

diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index b4cfea97a2e..32878bfa4ec 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -4,7 +4,6 @@
 from datetime import timedelta
 from datetime import timezone
 
-from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from danswer.background.indexing.checkpointing import get_time_windows_for_index_attempt
@@ -20,11 +19,10 @@
 from danswer.db.connector_credential_pair import update_connector_credential_pair
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
-from danswer.db.index_attempt import get_index_attempt
 from danswer.db.index_attempt import mark_attempt_failed
-from danswer.db.index_attempt import mark_attempt_in_progress
 from danswer.db.index_attempt import mark_attempt_partially_succeeded
 from danswer.db.index_attempt import mark_attempt_succeeded
+from danswer.db.index_attempt import transition_attempt_to_in_progress
 from danswer.db.index_attempt import update_docs_indexed
 from danswer.db.models import IndexAttempt
 from danswer.db.models import IndexingStatus
@@ -379,51 +377,6 @@ def _run_indexing(
         )
 
 
-def _prepare_index_attempt(
-    db_session: Session, index_attempt_id: int, tenant_id: str | None
-) -> IndexAttempt:
-    # make sure that the index attempt can't change in between checking the
-    # status and marking it as in_progress. This setting will be discarded
-    # after the next commit:
-    # https://docs.sqlalchemy.org/en/20/orm/session_transaction.html#setting-isolation-for-individual-transactions
-    db_session.connection(execution_options={"isolation_level": "SERIALIZABLE"})  # type: ignore
-    try:
-        if tenant_id is not None:
-            # Explicitly set the search path for the given tenant
-            db_session.execute(text(f'SET search_path TO "{tenant_id}"'))
-            # Verify the search path was set correctly
-            result = db_session.execute(text("SHOW search_path"))
-            current_search_path = result.scalar()
-            logger.info(f"Current search path set to: {current_search_path}")
-
-        attempt = get_index_attempt(
-            db_session=db_session,
-            index_attempt_id=index_attempt_id,
-        )
-
-        if attempt is None:
-            raise RuntimeError(
-                f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
-            )
-
-        if attempt.status != IndexingStatus.NOT_STARTED:
-            raise RuntimeError(
-                f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
-                f"Current status is '{attempt.status}'."
-            )
-
-        mark_attempt_in_progress(attempt, db_session)
-
-        # only commit once, to make sure this all happens in a single transaction
-        db_session.commit()
-    except Exception:
-        db_session.rollback()
-        logger.exception("_prepare_index_attempt exceptioned.")
-        raise
-
-    return attempt
-
-
 def run_indexing_entrypoint(
     index_attempt_id: int,
     tenant_id: str | None,
@@ -440,7 +393,7 @@ def run_indexing_entrypoint(
             index_attempt_id, connector_credential_pair_id
         )
         with get_session_with_tenant(tenant_id) as db_session:
-            attempt = _prepare_index_attempt(db_session, index_attempt_id, tenant_id)
+            attempt = transition_attempt_to_in_progress(index_attempt_id, db_session)
 
             logger.info(
                 f"Indexing starting for tenant {tenant_id}: "
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index 21a1bbd236f..5d214d77836 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -101,59 +101,92 @@ def get_not_started_index_attempts(db_session: Session) -> list[IndexAttempt]:
     return list(new_attempts.all())
 
 
+def transition_attempt_to_in_progress(
+    index_attempt_id: int,
+    db_session: Session,
+) -> IndexAttempt:
+    """Locks the row when we try to update"""
+    try:
+        attempt = db_session.execute(
+            select(IndexAttempt)
+            .where(IndexAttempt.id == index_attempt_id)
+            .with_for_update()
+        ).scalar_one()
+
+        if attempt is None:
+            raise RuntimeError(
+                f"Unable to find IndexAttempt for ID '{index_attempt_id}'"
+            )
+
+        if attempt.status != IndexingStatus.NOT_STARTED:
+            raise RuntimeError(
+                f"Indexing attempt with ID '{index_attempt_id}' is not in NOT_STARTED status. "
+                f"Current status is '{attempt.status}'."
+            )
+
+        attempt.status = IndexingStatus.IN_PROGRESS
+        attempt.time_started = attempt.time_started or func.now()  # type: ignore
+        db_session.commit()
+        return attempt
+    except Exception:
+        db_session.rollback()
+        logger.exception("transition_attempt_to_in_progress exceptioned.")
+        raise
+
+
 def mark_attempt_in_progress(
     index_attempt: IndexAttempt,
     db_session: Session,
 ) -> None:
-    with db_session.begin_nested():
-        try:
-            attempt = db_session.execute(
-                select(IndexAttempt)
-                .where(IndexAttempt.id == index_attempt.id)
-                .with_for_update()
-            ).scalar_one()
+    try:
+        attempt = db_session.execute(
+            select(IndexAttempt)
+            .where(IndexAttempt.id == index_attempt.id)
+            .with_for_update()
+        ).scalar_one()
 
-            attempt.status = IndexingStatus.IN_PROGRESS
-            attempt.time_started = index_attempt.time_started or func.now()  # type: ignore
-            db_session.commit()
-        except Exception:
-            db_session.rollback()
+        attempt.status = IndexingStatus.IN_PROGRESS
+        attempt.time_started = index_attempt.time_started or func.now()  # type: ignore
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
 
 
 def mark_attempt_succeeded(
     index_attempt: IndexAttempt,
     db_session: Session,
 ) -> None:
-    with db_session.begin_nested():
-        try:
-            attempt = db_session.execute(
-                select(IndexAttempt)
-                .where(IndexAttempt.id == index_attempt.id)
-                .with_for_update()
-            ).scalar_one()
+    try:
+        attempt = db_session.execute(
+            select(IndexAttempt)
+            .where(IndexAttempt.id == index_attempt.id)
+            .with_for_update()
+        ).scalar_one()
 
-            attempt.status = IndexingStatus.SUCCESS
-            db_session.commit()
-        except Exception:
-            db_session.rollback()
+        attempt.status = IndexingStatus.SUCCESS
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
 
 
 def mark_attempt_partially_succeeded(
     index_attempt: IndexAttempt,
     db_session: Session,
 ) -> None:
-    with db_session.begin_nested():
-        try:
-            attempt = db_session.execute(
-                select(IndexAttempt)
-                .where(IndexAttempt.id == index_attempt.id)
-                .with_for_update()
-            ).scalar_one()
+    try:
+        attempt = db_session.execute(
+            select(IndexAttempt)
+            .where(IndexAttempt.id == index_attempt.id)
+            .with_for_update()
+        ).scalar_one()
 
-            attempt.status = IndexingStatus.COMPLETED_WITH_ERRORS
-            db_session.commit()
-        except Exception:
-            db_session.rollback()
+        attempt.status = IndexingStatus.COMPLETED_WITH_ERRORS
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
 
 
 def mark_attempt_failed(
@@ -162,20 +195,20 @@ def mark_attempt_failed(
     failure_reason: str = "Unknown",
     full_exception_trace: str | None = None,
 ) -> None:
-    with db_session.begin_nested():
-        try:
-            attempt = db_session.execute(
-                select(IndexAttempt)
-                .where(IndexAttempt.id == index_attempt.id)
-                .with_for_update()
-            ).scalar_one()
-
-            attempt.status = IndexingStatus.FAILED
-            attempt.error_msg = failure_reason
-            attempt.full_exception_trace = full_exception_trace
-            db_session.commit()
-        except Exception:
-            db_session.rollback()
+    try:
+        attempt = db_session.execute(
+            select(IndexAttempt)
+            .where(IndexAttempt.id == index_attempt.id)
+            .with_for_update()
+        ).scalar_one()
+
+        attempt.status = IndexingStatus.FAILED
+        attempt.error_msg = failure_reason
+        attempt.full_exception_trace = full_exception_trace
+        db_session.commit()
+    except Exception:
+        db_session.rollback()
+        raise
 
     source = index_attempt.connector_credential_pair.connector.source
     optional_telemetry(record_type=RecordType.FAILURE, data={"connector": source})

From 6913efef908de199e431d35f114b7af3584dab96 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Fri, 18 Oct 2024 15:40:05 -0700
Subject: [PATCH 156/376] fresh indexing feature branch (#2790)

* fresh indexing feature branch

* cherry pick test

* Revert "cherry pick test"

This reverts commit 2a624220687affdda3de347e30f2011136f64bda.

* set multitenant so that vespa fields match when indexing

* cleanup pass

* mypy

* pass through env var to control celery indexing concurrency

* comments on task kickoff and some logging improvements

* use get_session_with_tenant

* comment out all of update.py

* rename to RedisConnectorIndexingFenceData

* first check num_indexing_workers

* refactor RedisConnectorIndexingFenceData

* comment out on_worker_process_init

* fix where num_indexing_workers falls back

* remove extra brace
---
 .../danswer/background/celery/celery_app.py   |  109 +-
 .../danswer/background/celery/celery_redis.py |  113 +-
 .../danswer/background/celery/celery_utils.py |   34 +-
 .../danswer/background/celery/celeryconfig.py |    5 +
 .../background/celery/tasks/indexing/tasks.py |  452 +++++++
 .../background/celery/tasks/pruning/tasks.py  |  170 ++-
 .../background/celery/tasks/shared/tasks.py   |   22 +-
 .../background/celery/tasks/vespa/tasks.py    |  188 ++-
 .../background/indexing/run_indexing.py       |   17 +-
 backend/danswer/background/update.py          | 1068 ++++++++---------
 backend/danswer/configs/constants.py          |   19 +-
 backend/danswer/db/index_attempt.py           |   17 +-
 backend/danswer/document_index/factory.py     |    5 +-
 backend/danswer/document_index/vespa/index.py |    9 +-
 .../document_index/vespa/indexing_utils.py    |   11 +-
 backend/danswer/server/documents/connector.py |   56 +-
 backend/danswer/server/documents/models.py    |    4 +
 backend/danswer/utils/logger.py               |   21 +
 .../danswer/background/celery/celery_app.py   |    2 +-
 .../background/celery/tasks/vespa/tasks.py    |   10 +-
 backend/scripts/dev_run_background_jobs.py    |   54 +-
 backend/supervisord.conf                      |   28 +-
 .../common_utils/managers/cc_pair.py          |   12 +-
 .../common_utils/managers/document_set.py     |    1 +
 .../common_utils/managers/user_group.py       |    1 +
 .../docker_compose/docker-compose.dev.yml     |    5 +-
 .../docker_compose/docker-compose.gpu-dev.yml |    5 +-
 .../status/CCPairIndexingStatusTable.tsx      |    1 +
 web/src/lib/types.ts                          |    1 +
 29 files changed, 1677 insertions(+), 763 deletions(-)
 create mode 100644 backend/danswer/background/celery/tasks/indexing/tasks.py

diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 5477d416239..702ab520589 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -1,4 +1,5 @@
 import logging
+import multiprocessing
 import time
 from datetime import timedelta
 from typing import Any
@@ -12,6 +13,7 @@
 from celery import Task
 from celery.exceptions import WorkerShutdown
 from celery.signals import beat_init
+from celery.signals import celeryd_init
 from celery.signals import worker_init
 from celery.signals import worker_ready
 from celery.signals import worker_shutdown
@@ -21,23 +23,32 @@
 
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
-from danswer.background.update import get_all_tenant_ids
+from danswer.background.celery.celery_utils import get_all_tenant_ids
 from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.configs.constants import POSTGRES_CELERY_BEAT_APP_NAME
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_HEAVY_APP_NAME
+from danswer.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_APP_NAME
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_LIGHT_APP_NAME
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import SqlEngine
+from danswer.db.search_settings import get_current_search_settings
+from danswer.db.swap_index import check_index_swap
+from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
+from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import ColoredFormatter
 from danswer.utils.logger import PlainFormatter
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
+from shared_configs.configs import MODEL_SERVER_PORT
 from shared_configs.configs import SENTRY_DSN
 
 logger = setup_logger()
@@ -62,8 +73,20 @@
 )  # Load configuration from 'celeryconfig.py'
 
 
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    pass
+
+
 @signals.task_postrun.connect
-def celery_task_postrun(
+def on_task_postrun(
     sender: Any | None = None,
     task_id: str | None = None,
     task: Task | None = None,
@@ -80,6 +103,9 @@ def celery_task_postrun(
     This function runs after any task completes (both success and failure)
     Note that this signal does not fire on a task that failed to complete and is going
     to be retried.
+
+    This also does not fire if a worker with acks_late=False crashes (which all of our
+    long running workers are)
     """
     if not task:
         return
@@ -101,32 +127,38 @@ def celery_task_postrun(
     if task_id.startswith(RedisDocumentSet.PREFIX):
         document_set_id = RedisDocumentSet.get_id_from_task_id(task_id)
         if document_set_id is not None:
-            rds = RedisDocumentSet(document_set_id)
+            rds = RedisDocumentSet(int(document_set_id))
             r.srem(rds.taskset_key, task_id)
         return
 
     if task_id.startswith(RedisUserGroup.PREFIX):
         usergroup_id = RedisUserGroup.get_id_from_task_id(task_id)
         if usergroup_id is not None:
-            rug = RedisUserGroup(usergroup_id)
+            rug = RedisUserGroup(int(usergroup_id))
             r.srem(rug.taskset_key, task_id)
         return
 
     if task_id.startswith(RedisConnectorDeletion.PREFIX):
         cc_pair_id = RedisConnectorDeletion.get_id_from_task_id(task_id)
         if cc_pair_id is not None:
-            rcd = RedisConnectorDeletion(cc_pair_id)
+            rcd = RedisConnectorDeletion(int(cc_pair_id))
             r.srem(rcd.taskset_key, task_id)
         return
 
     if task_id.startswith(RedisConnectorPruning.SUBTASK_PREFIX):
         cc_pair_id = RedisConnectorPruning.get_id_from_task_id(task_id)
         if cc_pair_id is not None:
-            rcp = RedisConnectorPruning(cc_pair_id)
+            rcp = RedisConnectorPruning(int(cc_pair_id))
             r.srem(rcp.taskset_key, task_id)
         return
 
 
+@celeryd_init.connect
+def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
+    """The first signal sent on celery worker startup"""
+    multiprocessing.set_start_method("spawn")  # fork is unsafe, set to spawn
+
+
 @beat_init.connect
 def on_beat_init(sender: Any, **kwargs: Any) -> None:
     SqlEngine.set_app_name(POSTGRES_CELERY_BEAT_APP_NAME)
@@ -135,6 +167,9 @@ def on_beat_init(sender: Any, **kwargs: Any) -> None:
 
 @worker_init.connect
 def on_worker_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("worker_init signal received.")
+    logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
+
     # decide some initial startup settings based on the celery worker's hostname
     # (set at the command line)
     hostname = sender.hostname
@@ -144,6 +179,30 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     elif hostname.startswith("heavy"):
         SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_HEAVY_APP_NAME)
         SqlEngine.init_engine(pool_size=8, max_overflow=0)
+    elif hostname.startswith("indexing"):
+        SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_APP_NAME)
+        SqlEngine.init_engine(pool_size=8, max_overflow=0)
+
+        # TODO: why is this necessary for the indexer to do?
+        with get_session_with_tenant(tenant_id) as db_session:
+            check_index_swap(db_session=db_session)
+            search_settings = get_current_search_settings(db_session)
+
+            # So that the first time users aren't surprised by really slow speed of first
+            # batch of documents indexed
+
+            if search_settings.provider_type is None:
+                logger.notice("Running a first inference to warm up embedding model")
+                embedding_model = EmbeddingModel.from_db_model(
+                    search_settings=search_settings,
+                    server_host=INDEXING_MODEL_SERVER_HOST,
+                    server_port=MODEL_SERVER_PORT,
+                )
+
+                warm_up_bi_encoder(
+                    embedding_model=embedding_model,
+                )
+                logger.notice("First inference complete.")
     else:
         SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
         SqlEngine.init_engine(pool_size=8, max_overflow=0)
@@ -234,6 +293,8 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
 
     sender.primary_worker_lock = lock
 
+    # As currently designed, when this worker starts as "primary", we reinitialize redis
+    # to a clean state (for our purposes, anyway)
     r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
     r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
@@ -270,6 +331,31 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
         r.delete(key)
 
+    for key in r.scan_iter(RedisConnectorIndexing.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_COMPLETE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_PROGRESS_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
+        r.delete(key)
+
+
+# @worker_process_init.connect
+# def on_worker_process_init(sender: Any, **kwargs: Any) -> None:
+#     """This only runs inside child processes when the worker is in pool=prefork mode.
+#     This may be technically unnecessary since we're finding prefork pools to be
+#     unstable and currently aren't planning on using them."""
+#     logger.info("worker_process_init signal received.")
+#     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME)
+#     SqlEngine.init_engine(pool_size=5, max_overflow=0)
+
+#     # https://stackoverflow.com/questions/43944787/sqlalchemy-celery-with-scoped-session-error
+#     SqlEngine.get_engine().dispose(close=False)
+
 
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:
@@ -318,7 +404,7 @@ def on_setup_logging(
     # TODO: could unhardcode format and colorize and accept these as options from
     # celery's config
 
-    # reformats celery's worker logger
+    # reformats the root logger
     root_logger = logging.getLogger()
 
     root_handler = logging.StreamHandler()  # Set up a handler for the root logger
@@ -441,6 +527,7 @@ def stop(self, worker: Any) -> None:
 celery_app.autodiscover_tasks(
     [
         "danswer.background.celery.tasks.connector_deletion",
+        "danswer.background.celery.tasks.indexing",
         "danswer.background.celery.tasks.periodic",
         "danswer.background.celery.tasks.pruning",
         "danswer.background.celery.tasks.shared",
@@ -467,9 +554,15 @@ def stop(self, worker: Any) -> None:
         "schedule": timedelta(seconds=60),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
+    {
+        "name": "check-for-indexing",
+        "task": "check_for_indexing",
+        "schedule": timedelta(seconds=10),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
     {
         "name": "check-for-prune",
-        "task": "check_for_prune_task_2",
+        "task": "check_for_pruning",
         "schedule": timedelta(seconds=10),
         "options": {"priority": DanswerCeleryPriority.HIGH},
     },
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index 1506a4b9be1..53f20946077 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -29,8 +29,8 @@ class RedisObjectHelper(ABC):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
-    def __init__(self, id: int):
-        self._id: int = id
+    def __init__(self, id: str):
+        self._id: str = id
 
     @property
     def task_id_prefix(self) -> str:
@@ -47,7 +47,7 @@ def taskset_key(self) -> str:
         return f"{self.TASKSET_PREFIX}_{self._id}"
 
     @staticmethod
-    def get_id_from_fence_key(key: str) -> int | None:
+    def get_id_from_fence_key(key: str) -> str | None:
         """
         Extracts the object ID from a fence key in the format `PREFIX_fence_X`.
 
@@ -61,15 +61,11 @@ def get_id_from_fence_key(key: str) -> int | None:
         if len(parts) != 3:
             return None
 
-        try:
-            object_id = int(parts[2])
-        except ValueError:
-            return None
-
+        object_id = parts[2]
         return object_id
 
     @staticmethod
-    def get_id_from_task_id(task_id: str) -> int | None:
+    def get_id_from_task_id(task_id: str) -> str | None:
         """
         Extracts the object ID from a task ID string.
 
@@ -93,11 +89,7 @@ def get_id_from_task_id(task_id: str) -> int | None:
         if len(parts) != 3:
             return None
 
-        try:
-            object_id = int(parts[1])
-        except ValueError:
-            return None
-
+        object_id = parts[1]
         return object_id
 
     @abstractmethod
@@ -117,6 +109,9 @@ class RedisDocumentSet(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
+    def __init__(self, id: int) -> None:
+        super().__init__(str(id))
+
     def generate_tasks(
         self,
         celery_app: Celery,
@@ -128,7 +123,7 @@ def generate_tasks(
         last_lock_time = time.monotonic()
 
         async_results = []
-        stmt = construct_document_select_by_docset(self._id, current_only=False)
+        stmt = construct_document_select_by_docset(int(self._id), current_only=False)
         for doc in db_session.scalars(stmt).yield_per(1):
             current_time = time.monotonic()
             if current_time - last_lock_time >= (
@@ -164,6 +159,9 @@ class RedisUserGroup(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
+    def __init__(self, id: int) -> None:
+        super().__init__(str(id))
+
     def generate_tasks(
         self,
         celery_app: Celery,
@@ -187,7 +185,7 @@ def generate_tasks(
         except ModuleNotFoundError:
             return 0
 
-        stmt = construct_document_select_by_usergroup(self._id)
+        stmt = construct_document_select_by_usergroup(int(self._id))
         for doc in db_session.scalars(stmt).yield_per(1):
             current_time = time.monotonic()
             if current_time - last_lock_time >= (
@@ -219,13 +217,19 @@ def generate_tasks(
 
 
 class RedisConnectorCredentialPair(RedisObjectHelper):
-    """This class differs from the default in that the taskset used spans
+    """This class is used to scan documents by cc_pair in the db and collect them into
+    a unified set for syncing.
+
+    It differs from the other redis helpers in that the taskset used spans
     all connectors and is not per connector."""
 
     PREFIX = "connectorsync"
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
+    def __init__(self, id: int) -> None:
+        super().__init__(str(id))
+
     @classmethod
     def get_fence_key(cls) -> str:
         return RedisConnectorCredentialPair.FENCE_PREFIX
@@ -252,7 +256,7 @@ def generate_tasks(
         last_lock_time = time.monotonic()
 
         async_results = []
-        cc_pair = get_connector_credential_pair_from_id(self._id, db_session)
+        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
         if not cc_pair:
             return None
 
@@ -298,6 +302,9 @@ class RedisConnectorDeletion(RedisObjectHelper):
     FENCE_PREFIX = PREFIX + "_fence"
     TASKSET_PREFIX = PREFIX + "_taskset"
 
+    def __init__(self, id: int) -> None:
+        super().__init__(str(id))
+
     def generate_tasks(
         self,
         celery_app: Celery,
@@ -309,7 +316,7 @@ def generate_tasks(
         last_lock_time = time.monotonic()
 
         async_results = []
-        cc_pair = get_connector_credential_pair_from_id(self._id, db_session)
+        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
         if not cc_pair:
             return None
 
@@ -386,9 +393,7 @@ class RedisConnectorPruning(RedisObjectHelper):
     )  # a signal that the generator has finished
 
     def __init__(self, id: int) -> None:
-        """id: the cc_pair_id of the connector credential pair"""
-
-        super().__init__(id)
+        super().__init__(str(id))
         self.documents_to_prune: set[str] = set()
 
     @property
@@ -420,7 +425,7 @@ def generate_tasks(
         last_lock_time = time.monotonic()
 
         async_results = []
-        cc_pair = get_connector_credential_pair_from_id(self._id, db_session)
+        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
         if not cc_pair:
             return None
 
@@ -463,7 +468,7 @@ def generate_tasks(
     def is_pruning(self, db_session: Session, redis_client: Redis) -> bool:
         """A single example of a helper method being refactored into the redis helper"""
         cc_pair = get_connector_credential_pair_from_id(
-            cc_pair_id=self._id, db_session=db_session
+            cc_pair_id=int(self._id), db_session=db_session
         )
         if not cc_pair:
             raise ValueError(f"cc_pair_id {self._id} does not exist.")
@@ -474,6 +479,66 @@ def is_pruning(self, db_session: Session, redis_client: Redis) -> bool:
         return False
 
 
+class RedisConnectorIndexing(RedisObjectHelper):
+    """Celery will kick off a long running indexing task to crawl the connector and
+    find any new or updated docs docs, which will each then get a new sync task or be
+    indexed inline.
+
+    ID should be a concatenation of cc_pair_id and search_setting_id, delimited by "/".
+    e.g. "2/5"
+    """
+
+    PREFIX = "connectorindexing"
+    FENCE_PREFIX = PREFIX + "_fence"  # a fence for the entire indexing process
+    GENERATOR_TASK_PREFIX = PREFIX + "+generator"
+
+    TASKSET_PREFIX = PREFIX + "_taskset"  # stores a list of prune tasks id's
+    SUBTASK_PREFIX = PREFIX + "+sub"
+
+    GENERATOR_LOCK_PREFIX = "da_lock:indexing"
+    GENERATOR_PROGRESS_PREFIX = (
+        PREFIX + "_generator_progress"
+    )  # a signal that contains generator progress
+    GENERATOR_COMPLETE_PREFIX = (
+        PREFIX + "_generator_complete"
+    )  # a signal that the generator has finished
+
+    def __init__(self, cc_pair_id: int, search_settings_id: int) -> None:
+        super().__init__(f"{cc_pair_id}/{search_settings_id}")
+
+    @property
+    def generator_lock_key(self) -> str:
+        return f"{self.GENERATOR_LOCK_PREFIX}_{self._id}"
+
+    @property
+    def generator_task_id_prefix(self) -> str:
+        return f"{self.GENERATOR_TASK_PREFIX}_{self._id}"
+
+    @property
+    def generator_progress_key(self) -> str:
+        # example: connectorpruning_generator_progress_1
+        return f"{self.GENERATOR_PROGRESS_PREFIX}_{self._id}"
+
+    @property
+    def generator_complete_key(self) -> str:
+        # example: connectorpruning_generator_complete_1
+        return f"{self.GENERATOR_COMPLETE_PREFIX}_{self._id}"
+
+    @property
+    def subtask_id_prefix(self) -> str:
+        return f"{self.SUBTASK_PREFIX}_{self._id}"
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock | None,
+        tenant_id: str | None,
+    ) -> int | None:
+        return None
+
+
 def celery_get_queue_length(queue: str, r: Redis) -> int:
     """This is a redis specific way to get the length of a celery queue.
     It is priority aware and knows how to count across the multiple redis lists
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 03ca82d500d..b76e148e237 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -3,10 +3,13 @@
 from datetime import timezone
 from typing import Any
 
+from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
+from danswer.configs.app_configs import MULTI_TENANT
+from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
 )
@@ -16,6 +19,7 @@
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.models import Document
 from danswer.db.connector_credential_pair import get_connector_credential_pair
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import TaskStatus
 from danswer.db.models import TaskQueueState
 from danswer.redis.redis_pool import get_redis_client
@@ -124,10 +128,30 @@ def celery_is_worker_primary(worker: Any) -> bool:
     for the celery worker, which can be done either in celeryconfig.py or on the
     command line with '--hostname'."""
     hostname = worker.hostname
-    if hostname.startswith("light"):
-        return False
+    if hostname.startswith("primary"):
+        return True
+
+    return False
 
-    if hostname.startswith("heavy"):
-        return False
 
-    return True
+def get_all_tenant_ids() -> list[str] | list[None]:
+    if not MULTI_TENANT:
+        return [None]
+    with get_session_with_tenant(tenant_id="public") as session:
+        result = session.execute(
+            text(
+                """
+            SELECT schema_name
+            FROM information_schema.schemata
+            WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
+            )
+        )
+        tenant_ids = [row[0] for row in result]
+
+    valid_tenants = [
+        tenant
+        for tenant in tenant_ids
+        if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
+    ]
+
+    return valid_tenants
diff --git a/backend/danswer/background/celery/celeryconfig.py b/backend/danswer/background/celery/celeryconfig.py
index 31d36d99533..3f96364de1e 100644
--- a/backend/danswer/background/celery/celeryconfig.py
+++ b/backend/danswer/background/celery/celeryconfig.py
@@ -41,6 +41,11 @@
 # can stall other tasks.
 worker_prefetch_multiplier = 4
 
+# Leaving this to the default of True may cause double logging since both our own app
+# and celery think they are controlling the logger.
+# TODO: Configure celery's logger entirely manually and set this to False
+# worker_hijack_root_logger = False
+
 broker_connection_retry_on_startup = True
 broker_pool_limit = CELERY_BROKER_POOL_LIMIT
 
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
new file mode 100644
index 00000000000..fefbae03220
--- /dev/null
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -0,0 +1,452 @@
+from datetime import datetime
+from datetime import timezone
+from http import HTTPStatus
+from time import sleep
+from typing import cast
+from uuid import uuid4
+
+from celery import shared_task
+from celery.exceptions import SoftTimeLimitExceeded
+from redis import Redis
+from sqlalchemy.orm import Session
+
+from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
+from danswer.background.celery.tasks.shared.tasks import RedisConnectorIndexingFenceData
+from danswer.background.indexing.job_client import SimpleJobClient
+from danswer.background.indexing.run_indexing import run_indexing_entrypoint
+from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+from danswer.configs.constants import CELERY_INDEXING_LOCK_TIMEOUT
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.configs.constants import DanswerRedisLocks
+from danswer.configs.constants import DocumentSource
+from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
+from danswer.db.engine import get_db_current_time
+from danswer.db.engine import get_session_with_tenant
+from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.enums import IndexingStatus
+from danswer.db.enums import IndexModelStatus
+from danswer.db.index_attempt import create_index_attempt
+from danswer.db.index_attempt import get_index_attempt
+from danswer.db.index_attempt import get_last_attempt_for_cc_pair
+from danswer.db.index_attempt import mark_attempt_failed
+from danswer.db.models import ConnectorCredentialPair
+from danswer.db.models import IndexAttempt
+from danswer.db.models import SearchSettings
+from danswer.db.search_settings import get_current_search_settings
+from danswer.db.search_settings import get_secondary_search_settings
+from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import setup_logger
+from danswer.utils.variable_functionality import global_version
+
+logger = setup_logger()
+
+
+@shared_task(
+    name="check_for_indexing",
+    soft_time_limit=300,
+)
+def check_for_indexing(tenant_id: str | None) -> int | None:
+    tasks_created = 0
+
+    r = get_redis_client()
+
+    lock_beat = r.lock(
+        DanswerRedisLocks.CHECK_INDEXING_BEAT_LOCK,
+        timeout=CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT,
+    )
+
+    try:
+        # these tasks should never overlap
+        if not lock_beat.acquire(blocking=False):
+            return None
+
+        with get_session_with_tenant(tenant_id) as db_session:
+            # Get the primary search settings
+            primary_search_settings = get_current_search_settings(db_session)
+            search_settings = [primary_search_settings]
+
+            # Check for secondary search settings
+            secondary_search_settings = get_secondary_search_settings(db_session)
+            if secondary_search_settings is not None:
+                # If secondary settings exist, add them to the list
+                search_settings.append(secondary_search_settings)
+
+            cc_pairs = fetch_connector_credential_pairs(db_session)
+            for cc_pair in cc_pairs:
+                for search_settings_instance in search_settings:
+                    rci = RedisConnectorIndexing(
+                        cc_pair.id, search_settings_instance.id
+                    )
+                    if r.exists(rci.fence_key):
+                        continue
+
+                    last_attempt = get_last_attempt_for_cc_pair(
+                        cc_pair.id, search_settings_instance.id, db_session
+                    )
+                    if not _should_index(
+                        cc_pair=cc_pair,
+                        last_index=last_attempt,
+                        search_settings_instance=search_settings_instance,
+                        secondary_index_building=len(search_settings) > 1,
+                        db_session=db_session,
+                    ):
+                        continue
+
+                    # using a task queue and only allowing one task per cc_pair/search_setting
+                    # prevents us from starving out certain attempts
+                    attempt_id = try_creating_indexing_task(
+                        cc_pair,
+                        search_settings_instance,
+                        False,
+                        db_session,
+                        r,
+                        tenant_id,
+                    )
+                    if attempt_id:
+                        task_logger.info(
+                            f"Indexing queued: cc_pair_id={cc_pair.id} index_attempt_id={attempt_id}"
+                        )
+                        tasks_created += 1
+    except SoftTimeLimitExceeded:
+        task_logger.info(
+            "Soft time limit exceeded, task is being terminated gracefully."
+        )
+    except Exception:
+        task_logger.exception("Unexpected exception")
+    finally:
+        if lock_beat.owned():
+            lock_beat.release()
+
+    return tasks_created
+
+
+def _should_index(
+    cc_pair: ConnectorCredentialPair,
+    last_index: IndexAttempt | None,
+    search_settings_instance: SearchSettings,
+    secondary_index_building: bool,
+    db_session: Session,
+) -> bool:
+    """Checks various global settings and past indexing attempts to determine if
+    we should try to start indexing the cc pair / search setting combination.
+
+    Note that tactical checks such as preventing overlap with a currently running task
+    are not handled here.
+
+    Return True if we should try to index, False if not.
+    """
+    connector = cc_pair.connector
+
+    # uncomment for debugging
+    # task_logger.info(f"_should_index: "
+    #                  f"cc_pair={cc_pair.id} "
+    #                  f"connector={cc_pair.connector_id} "
+    #                  f"refresh_freq={connector.refresh_freq}")
+
+    # don't kick off indexing for `NOT_APPLICABLE` sources
+    if connector.source == DocumentSource.NOT_APPLICABLE:
+        return False
+
+    # User can still manually create single indexing attempts via the UI for the
+    # currently in use index
+    if DISABLE_INDEX_UPDATE_ON_SWAP:
+        if (
+            search_settings_instance.status == IndexModelStatus.PRESENT
+            and secondary_index_building
+        ):
+            return False
+
+    # When switching over models, always index at least once
+    if search_settings_instance.status == IndexModelStatus.FUTURE:
+        if last_index:
+            # No new index if the last index attempt succeeded
+            # Once is enough. The model will never be able to swap otherwise.
+            if last_index.status == IndexingStatus.SUCCESS:
+                return False
+
+            # No new index if the last index attempt is waiting to start
+            if last_index.status == IndexingStatus.NOT_STARTED:
+                return False
+
+            # No new index if the last index attempt is running
+            if last_index.status == IndexingStatus.IN_PROGRESS:
+                return False
+        else:
+            if (
+                connector.id == 0 or connector.source == DocumentSource.INGESTION_API
+            ):  # Ingestion API
+                return False
+        return True
+
+    # If the connector is paused or is the ingestion API, don't index
+    # NOTE: during an embedding model switch over, the following logic
+    # is bypassed by the above check for a future model
+    if (
+        not cc_pair.status.is_active()
+        or connector.id == 0
+        or connector.source == DocumentSource.INGESTION_API
+    ):
+        return False
+
+    # if no attempt has ever occurred, we should index regardless of refresh_freq
+    if not last_index:
+        return True
+
+    if connector.refresh_freq is None:
+        return False
+
+    current_db_time = get_db_current_time(db_session)
+    time_since_index = current_db_time - last_index.time_updated
+    if time_since_index.total_seconds() < connector.refresh_freq:
+        return False
+
+    return True
+
+
+def try_creating_indexing_task(
+    cc_pair: ConnectorCredentialPair,
+    search_settings: SearchSettings,
+    reindex: bool,
+    db_session: Session,
+    r: Redis,
+    tenant_id: str | None,
+) -> int | None:
+    """Checks for any conditions that should block the indexing task from being
+    created, then creates the task.
+
+    Does not check for scheduling related conditions as this function
+    is used to trigger indexing immediately.
+    """
+
+    LOCK_TIMEOUT = 30
+
+    # we need to serialize any attempt to trigger indexing since it can be triggered
+    # either via celery beat or manually (API call)
+    lock = r.lock(
+        DANSWER_REDIS_FUNCTION_LOCK_PREFIX + "try_creating_indexing_task",
+        timeout=LOCK_TIMEOUT,
+    )
+
+    acquired = lock.acquire(blocking_timeout=LOCK_TIMEOUT / 2)
+    if not acquired:
+        return None
+
+    try:
+        rci = RedisConnectorIndexing(cc_pair.id, search_settings.id)
+
+        # skip if already indexing
+        if r.exists(rci.fence_key):
+            return None
+
+        # skip indexing if the cc_pair is deleting
+        db_session.refresh(cc_pair)
+        if cc_pair.status == ConnectorCredentialPairStatus.DELETING:
+            return None
+
+        # add a long running generator task to the queue
+        r.delete(rci.generator_complete_key)
+        r.delete(rci.taskset_key)
+
+        custom_task_id = f"{rci.generator_task_id_prefix}_{uuid4()}"
+
+        # create the index attempt ... just for tracking purposes
+        index_attempt_id = create_index_attempt(
+            cc_pair.id,
+            search_settings.id,
+            from_beginning=reindex,
+            db_session=db_session,
+        )
+
+        result = celery_app.send_task(
+            "connector_indexing_proxy_task",
+            kwargs=dict(
+                index_attempt_id=index_attempt_id,
+                cc_pair_id=cc_pair.id,
+                search_settings_id=search_settings.id,
+                tenant_id=tenant_id,
+            ),
+            queue=DanswerCeleryQueues.CONNECTOR_INDEXING,
+            task_id=custom_task_id,
+            priority=DanswerCeleryPriority.MEDIUM,
+        )
+        if not result:
+            return None
+
+        # set this only after all tasks have been added
+        fence_value = RedisConnectorIndexingFenceData(
+            index_attempt_id=index_attempt_id,
+            started=None,
+            submitted=datetime.now(timezone.utc),
+            celery_task_id=result.id,
+        )
+        r.set(rci.fence_key, fence_value.model_dump_json())
+    except Exception:
+        task_logger.exception("Unexpected exception")
+        return None
+    finally:
+        if lock.owned():
+            lock.release()
+
+    return index_attempt_id
+
+
+@shared_task(name="connector_indexing_proxy_task", acks_late=False, track_started=True)
+def connector_indexing_proxy_task(
+    index_attempt_id: int,
+    cc_pair_id: int,
+    search_settings_id: int,
+    tenant_id: str | None,
+) -> None:
+    """celery tasks are forked, but forking is unstable.  This proxies work to a spawned task."""
+
+    client = SimpleJobClient()
+
+    job = client.submit(
+        connector_indexing_task,
+        index_attempt_id,
+        cc_pair_id,
+        search_settings_id,
+        tenant_id,
+        global_version.is_ee_version(),
+        pure=False,
+    )
+
+    if not job:
+        return
+
+    while True:
+        sleep(10)
+        with get_session_with_tenant(tenant_id) as db_session:
+            index_attempt = get_index_attempt(
+                db_session=db_session, index_attempt_id=index_attempt_id
+            )
+
+            # do nothing for ongoing jobs that haven't been stopped
+            if not job.done():
+                if not index_attempt:
+                    continue
+
+                if not index_attempt.is_finished():
+                    continue
+
+            if job.status == "error":
+                logger.error(job.exception())
+
+            job.release()
+            break
+
+    return
+
+
+def connector_indexing_task(
+    index_attempt_id: int,
+    cc_pair_id: int,
+    search_settings_id: int,
+    tenant_id: str | None,
+    is_ee: bool,
+) -> int | None:
+    """Indexing task. For a cc pair, this task pulls all document IDs from the source
+    and compares those IDs to locally stored documents and deletes all locally stored IDs missing
+    from the most recently pulled document ID list
+
+    acks_late must be set to False. Otherwise, celery's visibility timeout will
+    cause any task that runs longer than the timeout to be redispatched by the broker.
+    There appears to be no good workaround for this, so we need to handle redispatching
+    manually.
+
+    Returns None if the task did not run (possibly due to a conflict).
+    Otherwise, returns an int >= 0 representing the number of indexed docs.
+    """
+
+    attempt = None
+    n_final_progress = 0
+
+    r = get_redis_client()
+
+    rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
+
+    lock = r.lock(
+        rci.generator_lock_key,
+        timeout=CELERY_INDEXING_LOCK_TIMEOUT,
+    )
+
+    acquired = lock.acquire(blocking=False)
+    if not acquired:
+        task_logger.warning(
+            f"Indexing task already running, exiting...: "
+            f"cc_pair_id={cc_pair_id} search_settings_id={search_settings_id}"
+        )
+        # r.set(rci.generator_complete_key, HTTPStatus.CONFLICT.value)
+        return None
+
+    try:
+        with get_session_with_tenant(tenant_id) as db_session:
+            attempt = get_index_attempt(db_session, index_attempt_id)
+            if not attempt:
+                raise ValueError(
+                    f"Index attempt not found: index_attempt_id={index_attempt_id}"
+                )
+
+            cc_pair = get_connector_credential_pair_from_id(
+                cc_pair_id=cc_pair_id,
+                db_session=db_session,
+            )
+
+            if not cc_pair:
+                raise ValueError(f"cc_pair not found: cc_pair_id={cc_pair_id}")
+
+            if not cc_pair.connector:
+                raise ValueError(
+                    f"Connector not found: connector_id={cc_pair.connector_id}"
+                )
+
+            if not cc_pair.credential:
+                raise ValueError(
+                    f"Credential not found: credential_id={cc_pair.credential_id}"
+                )
+
+            rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
+
+            # Define the callback function
+            def redis_increment_callback(amount: int) -> None:
+                lock.reacquire()
+                r.incrby(rci.generator_progress_key, amount)
+
+            run_indexing_entrypoint(
+                index_attempt_id,
+                tenant_id,
+                cc_pair_id,
+                is_ee,
+                progress_callback=redis_increment_callback,
+            )
+
+            # get back the total number of indexed docs and return it
+            generator_progress_value = r.get(rci.generator_progress_key)
+            if generator_progress_value is not None:
+                try:
+                    n_final_progress = int(cast(int, generator_progress_value))
+                except ValueError:
+                    pass
+
+            r.set(rci.generator_complete_key, HTTPStatus.OK.value)
+    except Exception as e:
+        task_logger.exception(f"Failed to run indexing for cc_pair_id={cc_pair_id}.")
+        if attempt:
+            mark_attempt_failed(attempt, db_session, failure_reason=str(e))
+
+        r.delete(rci.generator_lock_key)
+        r.delete(rci.generator_progress_key)
+        r.delete(rci.taskset_key)
+        r.delete(rci.fence_key)
+        raise e
+    finally:
+        if lock.owned():
+            lock.release()
+
+    return n_final_progress
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index 28149bb82a3..ee5adfd10c0 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -3,7 +3,6 @@
 from datetime import timezone
 from uuid import uuid4
 
-import redis
 from celery import shared_task
 from celery.exceptions import SoftTimeLimitExceeded
 from redis import Redis
@@ -15,7 +14,9 @@
 from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector
 from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
 from danswer.configs.app_configs import JOB_TIMEOUT
+from danswer.configs.constants import CELERY_PRUNING_LOCK_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
 from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerCeleryQueues
 from danswer.configs.constants import DanswerRedisLocks
@@ -30,15 +31,14 @@
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 
-
 logger = setup_logger()
 
 
 @shared_task(
-    name="check_for_prune_task_2",
+    name="check_for_pruning",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_for_prune_task_2(tenant_id: str | None) -> None:
+def check_for_pruning(tenant_id: str | None) -> None:
     r = get_redis_client()
 
     lock_beat = r.lock(
@@ -54,13 +54,17 @@ def check_for_prune_task_2(tenant_id: str | None) -> None:
         with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = get_connector_credential_pairs(db_session)
             for cc_pair in cc_pairs:
-                tasks_created = ccpair_pruning_generator_task_creation_helper(
-                    cc_pair, db_session, tenant_id, r, lock_beat
+                lock_beat.reacquire()
+                if not is_pruning_due(cc_pair, db_session, r):
+                    continue
+
+                tasks_created = try_creating_prune_generator_task(
+                    cc_pair, db_session, r, tenant_id
                 )
                 if not tasks_created:
                     continue
 
-                task_logger.info(f"Pruning started: cc_pair_id={cc_pair.id}")
+                task_logger.info(f"Pruning queued: cc_pair_id={cc_pair.id}")
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -72,13 +76,11 @@ def check_for_prune_task_2(tenant_id: str | None) -> None:
             lock_beat.release()
 
 
-def ccpair_pruning_generator_task_creation_helper(
+def is_pruning_due(
     cc_pair: ConnectorCredentialPair,
     db_session: Session,
-    tenant_id: str | None,
     r: Redis,
-    lock_beat: redis.lock.Lock,
-) -> int | None:
+) -> bool:
     """Returns an int if pruning is triggered.
     The int represents the number of prune tasks generated (in this case, only one
     because the task is a long running generator task.)
@@ -89,24 +91,30 @@ def ccpair_pruning_generator_task_creation_helper(
     try_creating_prune_generator_task.
     """
 
-    lock_beat.reacquire()
-
     # skip pruning if no prune frequency is set
     # pruning can still be forced via the API which will run a pruning task directly
     if not cc_pair.connector.prune_freq:
-        return None
+        return False
+
+    # skip pruning if not active
+    if cc_pair.status != ConnectorCredentialPairStatus.ACTIVE:
+        return False
 
     # skip pruning if the next scheduled prune time hasn't been reached yet
     last_pruned = cc_pair.last_pruned
     if not last_pruned:
-        # if never pruned, use the connector time created as the last_pruned time
-        last_pruned = cc_pair.connector.time_created
+        if not cc_pair.last_successful_index_time:
+            # if we've never indexed, we can't prune
+            return False
+
+        # if never pruned, use the last time the connector indexed successfully
+        last_pruned = cc_pair.last_successful_index_time
 
     next_prune = last_pruned + timedelta(seconds=cc_pair.connector.prune_freq)
     if datetime.now(timezone.utc) < next_prune:
-        return None
+        return False
 
-    return try_creating_prune_generator_task(cc_pair, db_session, r, tenant_id)
+    return True
 
 
 def try_creating_prune_generator_task(
@@ -119,50 +127,78 @@ def try_creating_prune_generator_task(
     created, then creates the task.
 
     Does not check for scheduling related conditions as this function
-    is used to trigger prunes immediately.
+    is used to trigger prunes immediately, e.g. via the web ui.
     """
 
     if not ALLOW_SIMULTANEOUS_PRUNING:
         for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
             return None
 
-    rcp = RedisConnectorPruning(cc_pair.id)
+    LOCK_TIMEOUT = 30
 
-    # skip pruning if already pruning
-    if r.exists(rcp.fence_key):
-        return None
+    # we need to serialize starting pruning since it can be triggered either via
+    # celery beat or manually (API call)
+    lock = r.lock(
+        DANSWER_REDIS_FUNCTION_LOCK_PREFIX + "try_creating_prune_generator_task",
+        timeout=LOCK_TIMEOUT,
+    )
 
-    # skip pruning if the cc_pair is deleting
-    db_session.refresh(cc_pair)
-    if cc_pair.status == ConnectorCredentialPairStatus.DELETING:
+    acquired = lock.acquire(blocking_timeout=LOCK_TIMEOUT / 2)
+    if not acquired:
         return None
 
-    # add a long running generator task to the queue
-    r.delete(rcp.generator_complete_key)
-    r.delete(rcp.taskset_key)
-
-    custom_task_id = f"{rcp.generator_task_id_prefix}_{uuid4()}"
-
-    celery_app.send_task(
-        "connector_pruning_generator_task",
-        kwargs=dict(
-            connector_id=cc_pair.connector_id,
-            credential_id=cc_pair.credential_id,
-            tenant_id=tenant_id,
-        ),
-        queue=DanswerCeleryQueues.CONNECTOR_PRUNING,
-        task_id=custom_task_id,
-        priority=DanswerCeleryPriority.LOW,
-    )
+    try:
+        rcp = RedisConnectorPruning(cc_pair.id)
+
+        # skip pruning if already pruning
+        if r.exists(rcp.fence_key):
+            return None
+
+        # skip pruning if the cc_pair is deleting
+        db_session.refresh(cc_pair)
+        if cc_pair.status == ConnectorCredentialPairStatus.DELETING:
+            return None
+
+        # add a long running generator task to the queue
+        r.delete(rcp.generator_complete_key)
+        r.delete(rcp.taskset_key)
+
+        custom_task_id = f"{rcp.generator_task_id_prefix}_{uuid4()}"
+
+        celery_app.send_task(
+            "connector_pruning_generator_task",
+            kwargs=dict(
+                cc_pair_id=cc_pair.id,
+                connector_id=cc_pair.connector_id,
+                credential_id=cc_pair.credential_id,
+                tenant_id=tenant_id,
+            ),
+            queue=DanswerCeleryQueues.CONNECTOR_PRUNING,
+            task_id=custom_task_id,
+            priority=DanswerCeleryPriority.LOW,
+        )
+
+        # set this only after all tasks have been added
+        r.set(rcp.fence_key, 1)
+    except Exception:
+        task_logger.exception("Unexpected exception")
+        return None
+    finally:
+        if lock.owned():
+            lock.release()
 
-    # set this only after all tasks have been added
-    r.set(rcp.fence_key, 1)
     return 1
 
 
-@shared_task(name="connector_pruning_generator_task", soft_time_limit=JOB_TIMEOUT)
+@shared_task(
+    name="connector_pruning_generator_task",
+    acks_late=False,
+    soft_time_limit=JOB_TIMEOUT,
+    track_started=True,
+    trail=False,
+)
 def connector_pruning_generator_task(
-    connector_id: int, credential_id: int, tenant_id: str | None
+    cc_pair_id: int, connector_id: int, credential_id: int, tenant_id: str | None
 ) -> None:
     """connector pruning task. For a cc pair, this task pulls all document IDs from the source
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
@@ -170,8 +206,22 @@ def connector_pruning_generator_task(
 
     r = get_redis_client()
 
-    with get_session_with_tenant(tenant_id) as db_session:
-        try:
+    rcp = RedisConnectorPruning(cc_pair_id)
+
+    lock = r.lock(
+        DanswerRedisLocks.PRUNING_LOCK_PREFIX + f"_{rcp._id}",
+        timeout=CELERY_PRUNING_LOCK_TIMEOUT,
+    )
+
+    acquired = lock.acquire(blocking=False)
+    if not acquired:
+        task_logger.warning(
+            f"Pruning task already running, exiting...: cc_pair_id={cc_pair_id}"
+        )
+        return None
+
+    try:
+        with get_session_with_tenant(tenant_id) as db_session:
             cc_pair = get_connector_credential_pair(
                 db_session=db_session,
                 connector_id=connector_id,
@@ -180,14 +230,13 @@ def connector_pruning_generator_task(
 
             if not cc_pair:
                 task_logger.warning(
-                    f"ccpair not found for {connector_id} {credential_id}"
+                    f"cc_pair not found for {connector_id} {credential_id}"
                 )
                 return
 
-            rcp = RedisConnectorPruning(cc_pair.id)
-
             # Define the callback function
             def redis_increment_callback(amount: int) -> None:
+                lock.reacquire()
                 r.incrby(rcp.generator_progress_key, amount)
 
             runnable_connector = instantiate_connector(
@@ -240,12 +289,13 @@ def redis_increment_callback(amount: int) -> None:
             )
 
             r.set(rcp.generator_complete_key, tasks_generated)
-        except Exception as e:
-            task_logger.exception(
-                f"Failed to run pruning for connector id {connector_id}."
-            )
+    except Exception as e:
+        task_logger.exception(f"Failed to run pruning for connector id {connector_id}.")
 
-            r.delete(rcp.generator_progress_key)
-            r.delete(rcp.taskset_key)
-            r.delete(rcp.fence_key)
-            raise e
+        r.delete(rcp.generator_progress_key)
+        r.delete(rcp.taskset_key)
+        r.delete(rcp.fence_key)
+        raise e
+    finally:
+        if lock.owned():
+            lock.release()
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 05408d58f97..6fc0b5f1f67 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -1,6 +1,9 @@
+from datetime import datetime
+
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
+from pydantic import BaseModel
 
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.celery_app import task_logger
@@ -17,6 +20,13 @@
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 
 
+class RedisConnectorIndexingFenceData(BaseModel):
+    index_attempt_id: int
+    started: datetime | None
+    submitted: datetime
+    celery_task_id: str
+
+
 @shared_task(
     name="document_by_cc_pair_cleanup_task",
     bind=True,
@@ -46,6 +56,8 @@ def document_by_cc_pair_cleanup_task(
     connector / credential pair from the access list
     (6) delete all relevant entries from postgres
     """
+    task_logger.info(f"document_id={document_id}")
+
     try:
         with get_session_with_tenant(tenant_id) as db_session:
             action = "skip"
@@ -111,11 +123,17 @@ def document_by_cc_pair_cleanup_task(
                 pass
 
             task_logger.info(
-                f"document_id={document_id} action={action} refcount={count} chunks={chunks_affected}"
+                f"tenant_id={tenant_id} "
+                f"document_id={document_id} "
+                f"action={action} "
+                f"refcount={count} "
+                f"chunks={chunks_affected}"
             )
             db_session.commit()
     except SoftTimeLimitExceeded:
-        task_logger.info(f"SoftTimeLimitExceeded exception. doc_id={document_id}")
+        task_logger.info(
+            f"SoftTimeLimitExceeded exception. tenant_id={tenant_id} doc_id={document_id}"
+        )
     except Exception as e:
         task_logger.exception("Unexpected exception")
 
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index c50237b606a..9830d71f778 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -1,10 +1,15 @@
 import traceback
+from datetime import datetime
+from datetime import timezone
+from http import HTTPStatus
 from typing import cast
 
 import redis
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
+from celery.result import AsyncResult
+from celery.states import READY_STATES
 from redis import Redis
 from sqlalchemy.orm import Session
 
@@ -14,9 +19,11 @@
 from danswer.background.celery.celery_redis import celery_get_queue_length
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
+from danswer.background.celery.tasks.shared.tasks import RedisConnectorIndexingFenceData
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryQueues
@@ -40,8 +47,13 @@
 from danswer.db.document_set import get_document_set_by_id
 from danswer.db.document_set import mark_document_set_as_synced
 from danswer.db.engine import get_session_with_tenant
+from danswer.db.enums import IndexingStatus
 from danswer.db.index_attempt import delete_index_attempts
+from danswer.db.index_attempt import get_all_index_attempts_by_status
+from danswer.db.index_attempt import get_index_attempt
+from danswer.db.index_attempt import mark_attempt_failed
 from danswer.db.models import DocumentSet
+from danswer.db.models import IndexAttempt
 from danswer.db.models import UserGroup
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
@@ -296,11 +308,13 @@ def monitor_document_set_taskset(
     key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
-    document_set_id = RedisDocumentSet.get_id_from_fence_key(fence_key)
-    if document_set_id is None:
+    document_set_id_str = RedisDocumentSet.get_id_from_fence_key(fence_key)
+    if document_set_id_str is None:
         task_logger.warning(f"could not parse document set id from {fence_key}")
         return
 
+    document_set_id = int(document_set_id_str)
+
     rds = RedisDocumentSet(document_set_id)
 
     fence_value = r.get(rds.fence_key)
@@ -315,7 +329,8 @@ def monitor_document_set_taskset(
 
     count = cast(int, r.scard(rds.taskset_key))
     task_logger.info(
-        f"Document set sync progress: document_set_id={document_set_id} remaining={count} initial={initial_count}"
+        f"Document set sync progress: document_set_id={document_set_id} "
+        f"remaining={count} initial={initial_count}"
     )
     if count > 0:
         return
@@ -345,11 +360,13 @@ def monitor_connector_deletion_taskset(
     key_bytes: bytes, r: Redis, tenant_id: str | None
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
-    cc_pair_id = RedisConnectorDeletion.get_id_from_fence_key(fence_key)
-    if cc_pair_id is None:
+    cc_pair_id_str = RedisConnectorDeletion.get_id_from_fence_key(fence_key)
+    if cc_pair_id_str is None:
         task_logger.warning(f"could not parse cc_pair_id from {fence_key}")
         return
 
+    cc_pair_id = int(cc_pair_id_str)
+
     rcd = RedisConnectorDeletion(cc_pair_id)
 
     fence_value = r.get(rcd.fence_key)
@@ -458,13 +475,15 @@ def monitor_ccpair_pruning_taskset(
     key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
-    cc_pair_id = RedisConnectorPruning.get_id_from_fence_key(fence_key)
-    if cc_pair_id is None:
+    cc_pair_id_str = RedisConnectorPruning.get_id_from_fence_key(fence_key)
+    if cc_pair_id_str is None:
         task_logger.warning(
-            f"monitor_connector_pruning_taskset: could not parse cc_pair_id from {fence_key}"
+            f"monitor_ccpair_pruning_taskset: could not parse cc_pair_id from {fence_key}"
         )
         return
 
+    cc_pair_id = int(cc_pair_id_str)
+
     rcp = RedisConnectorPruning(cc_pair_id)
 
     fence_value = r.get(rcp.fence_key)
@@ -488,7 +507,7 @@ def monitor_ccpair_pruning_taskset(
     if count > 0:
         return
 
-    mark_ccpair_as_pruned(cc_pair_id, db_session)
+    mark_ccpair_as_pruned(int(cc_pair_id), db_session)
     task_logger.info(
         f"Successfully pruned connector credential pair. cc_pair_id={cc_pair_id}"
     )
@@ -499,14 +518,127 @@ def monitor_ccpair_pruning_taskset(
     r.delete(rcp.fence_key)
 
 
+def monitor_ccpair_indexing_taskset(
+    key_bytes: bytes, r: Redis, db_session: Session
+) -> None:
+    # if the fence doesn't exist, there's nothing to do
+    fence_key = key_bytes.decode("utf-8")
+    composite_id = RedisConnectorIndexing.get_id_from_fence_key(fence_key)
+    if composite_id is None:
+        task_logger.warning(
+            f"monitor_ccpair_indexing_taskset: could not parse composite_id from {fence_key}"
+        )
+        return
+
+    # parse out metadata and initialize the helper class with it
+    parts = composite_id.split("/")
+    if len(parts) != 2:
+        return
+
+    cc_pair_id = int(parts[0])
+    search_settings_id = int(parts[1])
+
+    rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
+
+    # read related data and evaluate/print task progress
+    fence_value = cast(bytes, r.get(rci.fence_key))
+    if fence_value is None:
+        return
+
+    try:
+        fence_json = fence_value.decode("utf-8")
+        fence_data = RedisConnectorIndexingFenceData.model_validate_json(
+            cast(str, fence_json)
+        )
+    except ValueError:
+        task_logger.exception(
+            "monitor_ccpair_indexing_taskset: fence_data not decodeable."
+        )
+        raise
+
+    elapsed_submitted = datetime.now(timezone.utc) - fence_data.submitted
+
+    generator_progress_value = r.get(rci.generator_progress_key)
+    if generator_progress_value is not None:
+        try:
+            progress_count = int(cast(int, generator_progress_value))
+
+            task_logger.info(
+                f"Connector indexing progress: cc_pair_id={cc_pair_id} "
+                f"search_settings_id={search_settings_id} "
+                f"progress={progress_count} "
+                f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
+            )
+        except ValueError:
+            task_logger.error(
+                "monitor_ccpair_indexing_taskset: generator_progress_value is not an integer."
+            )
+
+    # Read result state BEFORE generator_complete_key to avoid a race condition
+    result: AsyncResult = AsyncResult(fence_data.celery_task_id)
+    result_state = result.state
+
+    generator_complete_value = r.get(rci.generator_complete_key)
+    if generator_complete_value is None:
+        if result_state in READY_STATES:
+            # IF the task state is READY, THEN generator_complete should be set
+            # if it isn't, then the worker crashed
+            task_logger.info(
+                f"Connector indexing aborted: "
+                f"cc_pair_id={cc_pair_id} "
+                f"search_settings_id={search_settings_id} "
+                f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
+            )
+
+            index_attempt = get_index_attempt(db_session, fence_data.index_attempt_id)
+            if index_attempt:
+                mark_attempt_failed(
+                    index_attempt=index_attempt,
+                    db_session=db_session,
+                    failure_reason="Connector indexing aborted or exceptioned.",
+                )
+
+            r.delete(rci.generator_lock_key)
+            r.delete(rci.taskset_key)
+            r.delete(rci.generator_progress_key)
+            r.delete(rci.generator_complete_key)
+            r.delete(rci.fence_key)
+        return
+
+    status_enum = HTTPStatus.INTERNAL_SERVER_ERROR
+    try:
+        status_value = int(cast(int, generator_complete_value))
+        status_enum = HTTPStatus(status_value)
+    except ValueError:
+        task_logger.error(
+            f"monitor_ccpair_indexing_taskset: "
+            f"generator_complete_value=f{generator_complete_value} could not be parsed."
+        )
+
+    task_logger.info(
+        f"Connector indexing finished: cc_pair_id={cc_pair_id} "
+        f"search_settings_id={search_settings_id} "
+        f"status={status_enum.name} "
+        f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
+    )
+
+    r.delete(rci.generator_lock_key)
+    r.delete(rci.taskset_key)
+    r.delete(rci.generator_progress_key)
+    r.delete(rci.generator_complete_key)
+    r.delete(rci.fence_key)
+
+
 @shared_task(name="monitor_vespa_sync", soft_time_limit=300, bind=True)
-def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
+def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
     """This is a celery beat task that monitors and finalizes metadata sync tasksets.
     It scans for fence values and then gets the counts of any associated tasksets.
     If the count is 0, that means all tasks finished and we should clean up.
 
     This task lock timeout is CELERY_METADATA_SYNC_BEAT_LOCK_TIMEOUT seconds, so don't
     do anything too expensive in this function!
+
+    Returns True if the task actually did work, False
     """
     r = get_redis_client()
 
@@ -518,11 +650,14 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
     try:
         # prevent overlapping tasks
         if not lock_beat.acquire(blocking=False):
-            return
+            return False
 
         # print current queue lengths
         r_celery = self.app.broker_connection().channel().client  # type: ignore
         n_celery = celery_get_queue_length("celery", r)
+        n_indexing = celery_get_queue_length(
+            DanswerCeleryQueues.CONNECTOR_INDEXING, r_celery
+        )
         n_sync = celery_get_queue_length(
             DanswerCeleryQueues.VESPA_METADATA_SYNC, r_celery
         )
@@ -534,7 +669,11 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
         )
 
         task_logger.info(
-            f"Queue lengths: celery={n_celery} sync={n_sync} deletion={n_deletion} pruning={n_pruning}"
+            f"Queue lengths: celery={n_celery} "
+            f"indexing={n_indexing} "
+            f"sync={n_sync} "
+            f"deletion={n_deletion} "
+            f"pruning={n_pruning}"
         )
 
         lock_beat.reacquire()
@@ -565,6 +704,29 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
             for key_bytes in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
                 monitor_ccpair_pruning_taskset(key_bytes, r, db_session)
 
+            # do some cleanup before clearing fences
+            # check the db for any outstanding index attempts
+            attempts: list[IndexAttempt] = []
+            attempts.extend(
+                get_all_index_attempts_by_status(IndexingStatus.NOT_STARTED, db_session)
+            )
+            attempts.extend(
+                get_all_index_attempts_by_status(IndexingStatus.IN_PROGRESS, db_session)
+            )
+
+            for a in attempts:
+                # if attempts exist in the db but we don't detect them in redis, mark them as failed
+                rci = RedisConnectorIndexing(
+                    a.connector_credential_pair_id, a.search_settings_id
+                )
+                failure_reason = f"Unknown index attempt {a.id}. Might be left over from a process restart."
+                if not r.exists(rci.fence_key):
+                    mark_attempt_failed(a, db_session, failure_reason=failure_reason)
+
+            lock_beat.reacquire()
+            for key_bytes in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
+                monitor_ccpair_indexing_taskset(key_bytes, r, db_session)
+
         # uncomment for debugging if needed
         # r_celery = celery_app.broker_connection().channel().client
         # length = celery_get_queue_length(DanswerCeleryQueues.VESPA_METADATA_SYNC, r_celery)
@@ -577,6 +739,8 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> None:
         if lock_beat.owned():
             lock_beat.release()
 
+    return True
+
 
 @shared_task(
     name="vespa_metadata_sync_task",
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index 32878bfa4ec..cb507390452 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -1,5 +1,6 @@
 import time
 import traceback
+from collections.abc import Callable
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -88,12 +89,18 @@ def _get_connector_runner(
 
 
 def _run_indexing(
-    db_session: Session, index_attempt: IndexAttempt, tenant_id: str | None
+    db_session: Session,
+    index_attempt: IndexAttempt,
+    tenant_id: str | None,
+    progress_callback: Callable[[int], None] | None = None,
 ) -> None:
     """
     1. Get documents which are either new or updated from specified application
     2. Embed and index these documents into the chosen datastore (vespa)
     3. Updates Postgres to record the indexed documents + the outcome of this run
+
+    TODO: do not change index attempt statuses here ... instead, set signals in redis
+    and allow the monitor function to clean them up
     """
     start_time = time.time()
 
@@ -236,6 +243,8 @@ def _run_indexing(
                 logger.debug(f"Indexing batch of documents: {batch_description}")
 
                 index_attempt_md.batch_num = batch_num + 1  # use 1-index for this
+
+                # real work happens here!
                 new_docs, total_batch_chunks = indexing_pipeline(
                     document_batch=doc_batch,
                     index_attempt_metadata=index_attempt_md,
@@ -254,6 +263,9 @@ def _run_indexing(
                 # be inaccurate
                 db_session.commit()
 
+                if progress_callback:
+                    progress_callback(len(doc_batch))
+
                 # This new value is updated every batch, so UI can refresh per batch update
                 update_docs_indexed(
                     db_session=db_session,
@@ -382,6 +394,7 @@ def run_indexing_entrypoint(
     tenant_id: str | None,
     connector_credential_pair_id: int,
     is_ee: bool = False,
+    progress_callback: Callable[[int], None] | None = None,
 ) -> None:
     try:
         if is_ee:
@@ -404,7 +417,7 @@ def run_indexing_entrypoint(
                 f"credentials='{attempt.connector_credential_pair.connector_id}'"
             )
 
-            _run_indexing(db_session, attempt, tenant_id)
+            _run_indexing(db_session, attempt, tenant_id, progress_callback)
 
             logger.info(
                 f"Indexing finished for tenant {tenant_id}: "
diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
index b981a90e315..b408f289724 100755
--- a/backend/danswer/background/update.py
+++ b/backend/danswer/background/update.py
@@ -1,574 +1,494 @@
-import logging
-import time
-from datetime import datetime
-
-import dask
-from dask.distributed import Client
-from dask.distributed import Future
-from distributed import LocalCluster
-from sqlalchemy import text
-from sqlalchemy.exc import ProgrammingError
-from sqlalchemy.orm import Session
-
-from danswer.background.indexing.dask_utils import ResourceLogger
-from danswer.background.indexing.job_client import SimpleJob
-from danswer.background.indexing.job_client import SimpleJobClient
-from danswer.background.indexing.run_indexing import run_indexing_entrypoint
-from danswer.configs.app_configs import CLEANUP_INDEXING_JOBS_TIMEOUT
-from danswer.configs.app_configs import DASK_JOB_CLIENT_ENABLED
-from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
-from danswer.configs.app_configs import MULTI_TENANT
-from danswer.configs.app_configs import NUM_INDEXING_WORKERS
-from danswer.configs.app_configs import NUM_SECONDARY_INDEXING_WORKERS
-from danswer.configs.constants import DocumentSource
-from danswer.configs.constants import POSTGRES_INDEXER_APP_NAME
-from danswer.configs.constants import TENANT_ID_PREFIX
-from danswer.db.connector import fetch_connectors
-from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
-from danswer.db.engine import get_db_current_time
-from danswer.db.engine import get_session_with_tenant
-from danswer.db.engine import get_sqlalchemy_engine
-from danswer.db.engine import SqlEngine
-from danswer.db.index_attempt import create_index_attempt
-from danswer.db.index_attempt import get_index_attempt
-from danswer.db.index_attempt import get_inprogress_index_attempts
-from danswer.db.index_attempt import get_last_attempt_for_cc_pair
-from danswer.db.index_attempt import get_not_started_index_attempts
-from danswer.db.index_attempt import mark_attempt_failed
-from danswer.db.models import ConnectorCredentialPair
-from danswer.db.models import IndexAttempt
-from danswer.db.models import IndexingStatus
-from danswer.db.models import IndexModelStatus
-from danswer.db.models import SearchSettings
-from danswer.db.search_settings import get_current_search_settings
-from danswer.db.search_settings import get_secondary_search_settings
-from danswer.db.swap_index import check_index_swap
-from danswer.document_index.vespa.index import VespaIndex
-from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
-from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
-from danswer.utils.logger import setup_logger
-from danswer.utils.variable_functionality import global_version
-from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
-from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
-from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
-from shared_configs.configs import LOG_LEVEL
-
-logger = setup_logger()
-
-# If the indexing dies, it's most likely due to resource constraints,
-# restarting just delays the eventual failure, not useful to the user
-dask.config.set({"distributed.scheduler.allowed-failures": 0})
-
-_UNEXPECTED_STATE_FAILURE_REASON = (
-    "Stopped mid run, likely due to the background process being killed"
-)
-
-
-def _should_create_new_indexing(
-    cc_pair: ConnectorCredentialPair,
-    last_index: IndexAttempt | None,
-    search_settings_instance: SearchSettings,
-    secondary_index_building: bool,
-    db_session: Session,
-) -> bool:
-    connector = cc_pair.connector
-
-    # don't kick off indexing for `NOT_APPLICABLE` sources
-    if connector.source == DocumentSource.NOT_APPLICABLE:
-        return False
-
-    # User can still manually create single indexing attempts via the UI for the
-    # currently in use index
-    if DISABLE_INDEX_UPDATE_ON_SWAP:
-        if (
-            search_settings_instance.status == IndexModelStatus.PRESENT
-            and secondary_index_building
-        ):
-            return False
-
-    # When switching over models, always index at least once
-    if search_settings_instance.status == IndexModelStatus.FUTURE:
-        if last_index:
-            # No new index if the last index attempt succeeded
-            # Once is enough. The model will never be able to swap otherwise.
-            if last_index.status == IndexingStatus.SUCCESS:
-                return False
-
-            # No new index if the last index attempt is waiting to start
-            if last_index.status == IndexingStatus.NOT_STARTED:
-                return False
-
-            # No new index if the last index attempt is running
-            if last_index.status == IndexingStatus.IN_PROGRESS:
-                return False
-        else:
-            if (
-                connector.id == 0 or connector.source == DocumentSource.INGESTION_API
-            ):  # Ingestion API
-                return False
-        return True
-
-    # If the connector is paused or is the ingestion API, don't index
-    # NOTE: during an embedding model switch over, the following logic
-    # is bypassed by the above check for a future model
-    if (
-        not cc_pair.status.is_active()
-        or connector.id == 0
-        or connector.source == DocumentSource.INGESTION_API
-    ):
-        return False
-
-    if not last_index:
-        return True
-
-    if connector.refresh_freq is None:
-        return False
-
-    # Only one scheduled/ongoing job per connector at a time
-    # this prevents cases where
-    # (1) the "latest" index_attempt is scheduled so we show
-    #     that in the UI despite another index_attempt being in-progress
-    # (2) multiple scheduled index_attempts at a time
-    if (
-        last_index.status == IndexingStatus.NOT_STARTED
-        or last_index.status == IndexingStatus.IN_PROGRESS
-    ):
-        return False
-
-    current_db_time = get_db_current_time(db_session)
-    time_since_index = current_db_time - last_index.time_updated
-    return time_since_index.total_seconds() >= connector.refresh_freq
-
-
-def _mark_run_failed(
-    db_session: Session, index_attempt: IndexAttempt, failure_reason: str
-) -> None:
-    """Marks the `index_attempt` row as failed + updates the `
-    connector_credential_pair` to reflect that the run failed"""
-    logger.warning(
-        f"Marking in-progress attempt 'connector: {index_attempt.connector_credential_pair.connector_id}, "
-        f"credential: {index_attempt.connector_credential_pair.credential_id}' as failed due to {failure_reason}"
-    )
-    mark_attempt_failed(
-        index_attempt=index_attempt,
-        db_session=db_session,
-        failure_reason=failure_reason,
-    )
-
-
-"""Main funcs"""
-
-
-def create_indexing_jobs(
-    existing_jobs: dict[int, Future | SimpleJob], tenant_id: str | None
-) -> None:
-    """Creates new indexing jobs for each connector / credential pair which is:
-    1. Enabled
-    2. `refresh_frequency` time has passed since the last indexing run for this pair
-    3. There is not already an ongoing indexing attempt for this pair
-    """
-    with get_session_with_tenant(tenant_id) as db_session:
-        ongoing: set[tuple[int | None, int]] = set()
-        for attempt_id in existing_jobs:
-            attempt = get_index_attempt(
-                db_session=db_session, index_attempt_id=attempt_id
-            )
-            if attempt is None:
-                logger.error(
-                    f"Unable to find IndexAttempt for ID '{attempt_id}' when creating "
-                    "indexing jobs"
-                )
-                continue
-            ongoing.add(
-                (
-                    attempt.connector_credential_pair_id,
-                    attempt.search_settings_id,
-                )
-            )
-
-        # Get the primary search settings
-        primary_search_settings = get_current_search_settings(db_session)
-        search_settings = [primary_search_settings]
-
-        # Check for secondary search settings
-        secondary_search_settings = get_secondary_search_settings(db_session)
-        if secondary_search_settings is not None:
-            # If secondary settings exist, add them to the list
-            search_settings.append(secondary_search_settings)
-
-        all_connector_credential_pairs = fetch_connector_credential_pairs(db_session)
-        for cc_pair in all_connector_credential_pairs:
-            for search_settings_instance in search_settings:
-                # Check if there is an ongoing indexing attempt for this connector credential pair
-                if (cc_pair.id, search_settings_instance.id) in ongoing:
-                    continue
-
-                last_attempt = get_last_attempt_for_cc_pair(
-                    cc_pair.id, search_settings_instance.id, db_session
-                )
-                if not _should_create_new_indexing(
-                    cc_pair=cc_pair,
-                    last_index=last_attempt,
-                    search_settings_instance=search_settings_instance,
-                    secondary_index_building=len(search_settings) > 1,
-                    db_session=db_session,
-                ):
-                    continue
-
-                create_index_attempt(
-                    cc_pair.id, search_settings_instance.id, db_session
-                )
-
-
-def cleanup_indexing_jobs(
-    existing_jobs: dict[int, Future | SimpleJob],
-    tenant_id: str | None,
-    timeout_hours: int = CLEANUP_INDEXING_JOBS_TIMEOUT,
-) -> dict[int, Future | SimpleJob]:
-    existing_jobs_copy = existing_jobs.copy()
-    # clean up completed jobs
-    with get_session_with_tenant(tenant_id) as db_session:
-        for attempt_id, job in existing_jobs.items():
-            index_attempt = get_index_attempt(
-                db_session=db_session, index_attempt_id=attempt_id
-            )
-
-            # do nothing for ongoing jobs that haven't been stopped
-            if not job.done():
-                if not index_attempt:
-                    continue
-
-                if not index_attempt.is_finished():
-                    continue
-
-            if job.status == "error":
-                logger.error(job.exception())
-
-            job.release()
-            del existing_jobs_copy[attempt_id]
-
-            if not index_attempt:
-                logger.error(
-                    f"Unable to find IndexAttempt for ID '{attempt_id}' when cleaning "
-                    "up indexing jobs"
-                )
-                continue
-
-            if (
-                index_attempt.status == IndexingStatus.IN_PROGRESS
-                or job.status == "error"
-            ):
-                _mark_run_failed(
-                    db_session=db_session,
-                    index_attempt=index_attempt,
-                    failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
-                )
-
-        # clean up in-progress jobs that were never completed
-        try:
-            connectors = fetch_connectors(db_session)
-            for connector in connectors:
-                in_progress_indexing_attempts = get_inprogress_index_attempts(
-                    connector.id, db_session
-                )
-
-                for index_attempt in in_progress_indexing_attempts:
-                    if index_attempt.id in existing_jobs:
-                        # If index attempt is canceled, stop the run
-                        if index_attempt.status == IndexingStatus.FAILED:
-                            existing_jobs[index_attempt.id].cancel()
-                        # check to see if the job has been updated in last `timeout_hours` hours, if not
-                        # assume it to frozen in some bad state and just mark it as failed. Note: this relies
-                        # on the fact that the `time_updated` field is constantly updated every
-                        # batch of documents indexed
-                        current_db_time = get_db_current_time(db_session=db_session)
-                        time_since_update = current_db_time - index_attempt.time_updated
-                        if time_since_update.total_seconds() > 60 * 60 * timeout_hours:
-                            existing_jobs[index_attempt.id].cancel()
-                            _mark_run_failed(
-                                db_session=db_session,
-                                index_attempt=index_attempt,
-                                failure_reason="Indexing run frozen - no updates in the last three hours. "
-                                "The run will be re-attempted at next scheduled indexing time.",
-                            )
-                    else:
-                        # If job isn't known, simply mark it as failed
-                        _mark_run_failed(
-                            db_session=db_session,
-                            index_attempt=index_attempt,
-                            failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
-                        )
-        except ProgrammingError:
-            logger.debug(f"No Connector Table exists for: {tenant_id}")
-    return existing_jobs_copy
-
-
-def kickoff_indexing_jobs(
-    existing_jobs: dict[int, Future | SimpleJob],
-    client: Client | SimpleJobClient,
-    secondary_client: Client | SimpleJobClient,
-    tenant_id: str | None,
-) -> dict[int, Future | SimpleJob]:
-    existing_jobs_copy = existing_jobs.copy()
-
-    current_session = get_session_with_tenant(tenant_id)
-
-    # Don't include jobs waiting in the Dask queue that just haven't started running
-    # Also (rarely) don't include for jobs that started but haven't updated the indexing tables yet
-    with current_session as db_session:
-        # get_not_started_index_attempts orders its returned results from oldest to newest
-        # we must process attempts in a FIFO manner to prevent connector starvation
-        new_indexing_attempts = [
-            (attempt, attempt.search_settings)
-            for attempt in get_not_started_index_attempts(db_session)
-            if attempt.id not in existing_jobs
-        ]
-
-    logger.debug(f"Found {len(new_indexing_attempts)} new indexing task(s).")
-
-    if not new_indexing_attempts:
-        return existing_jobs
-
-    indexing_attempt_count = 0
-
-    primary_client_full = False
-    secondary_client_full = False
-    for attempt, search_settings in new_indexing_attempts:
-        if primary_client_full and secondary_client_full:
-            break
-
-        use_secondary_index = (
-            search_settings.status == IndexModelStatus.FUTURE
-            if search_settings is not None
-            else False
-        )
-        if attempt.connector_credential_pair.connector is None:
-            logger.warning(
-                f"Skipping index attempt as Connector has been deleted: {attempt}"
-            )
-            with current_session as db_session:
-                mark_attempt_failed(
-                    attempt, db_session, failure_reason="Connector is null"
-                )
-            continue
-        if attempt.connector_credential_pair.credential is None:
-            logger.warning(
-                f"Skipping index attempt as Credential has been deleted: {attempt}"
-            )
-            with current_session as db_session:
-                mark_attempt_failed(
-                    attempt, db_session, failure_reason="Credential is null"
-                )
-            continue
-
-        if not use_secondary_index:
-            if not primary_client_full:
-                run = client.submit(
-                    run_indexing_entrypoint,
-                    attempt.id,
-                    tenant_id,
-                    attempt.connector_credential_pair_id,
-                    global_version.is_ee_version(),
-                    pure=False,
-                )
-                if not run:
-                    primary_client_full = True
-        else:
-            if not secondary_client_full:
-                run = secondary_client.submit(
-                    run_indexing_entrypoint,
-                    attempt.id,
-                    tenant_id,
-                    attempt.connector_credential_pair_id,
-                    global_version.is_ee_version(),
-                    pure=False,
-                )
-                if not run:
-                    secondary_client_full = True
-
-        if run:
-            if indexing_attempt_count == 0:
-                logger.info(
-                    f"Indexing dispatch starts: pending={len(new_indexing_attempts)}"
-                )
-
-            indexing_attempt_count += 1
-            secondary_str = " (secondary index)" if use_secondary_index else ""
-            logger.info(
-                f"Indexing dispatched{secondary_str}: "
-                f"attempt_id={attempt.id} "
-                f"connector='{attempt.connector_credential_pair.connector.name}' "
-                f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
-                f"credentials='{attempt.connector_credential_pair.credential_id}'"
-            )
-            existing_jobs_copy[attempt.id] = run
-
-    if indexing_attempt_count > 0:
-        logger.info(
-            f"Indexing dispatch results: "
-            f"initial_pending={len(new_indexing_attempts)} "
-            f"started={indexing_attempt_count} "
-            f"remaining={len(new_indexing_attempts) - indexing_attempt_count}"
-        )
-
-    return existing_jobs_copy
-
-
-def get_all_tenant_ids() -> list[str] | list[None]:
-    if not MULTI_TENANT:
-        return [None]
-    with get_session_with_tenant(tenant_id="public") as session:
-        result = session.execute(
-            text(
-                """
-            SELECT schema_name
-            FROM information_schema.schemata
-            WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
-            )
-        )
-        tenant_ids = [row[0] for row in result]
-
-    valid_tenants = [
-        tenant
-        for tenant in tenant_ids
-        if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
-    ]
-
-    return valid_tenants
-
-
-def update_loop(
-    delay: int = 10,
-    num_workers: int = NUM_INDEXING_WORKERS,
-    num_secondary_workers: int = NUM_SECONDARY_INDEXING_WORKERS,
-) -> None:
-    if not MULTI_TENANT:
-        # We can use this function as we are certain only the public schema exists
-        # (explicitly for the non-`MULTI_TENANT` case)
-        engine = get_sqlalchemy_engine()
-        with Session(engine) as db_session:
-            check_index_swap(db_session=db_session)
-
-            search_settings = get_current_search_settings(db_session)
-
-            # So that the first time users aren't surprised by really slow speed of first
-            # batch of documents indexed
-            if search_settings.provider_type is None:
-                logger.notice("Running a first inference to warm up embedding model")
-                embedding_model = EmbeddingModel.from_db_model(
-                    search_settings=search_settings,
-                    server_host=INDEXING_MODEL_SERVER_HOST,
-                    server_port=INDEXING_MODEL_SERVER_PORT,
-                )
-
-                warm_up_bi_encoder(
-                    embedding_model=embedding_model,
-                )
-                logger.notice("First inference complete.")
-
-    client_primary: Client | SimpleJobClient
-    client_secondary: Client | SimpleJobClient
-    if DASK_JOB_CLIENT_ENABLED:
-        cluster_primary = LocalCluster(
-            n_workers=num_workers,
-            threads_per_worker=1,
-            silence_logs=logging.ERROR,
-        )
-        cluster_secondary = LocalCluster(
-            n_workers=num_secondary_workers,
-            threads_per_worker=1,
-            silence_logs=logging.ERROR,
-        )
-        client_primary = Client(cluster_primary)
-        client_secondary = Client(cluster_secondary)
-        if LOG_LEVEL.lower() == "debug":
-            client_primary.register_worker_plugin(ResourceLogger())
-    else:
-        client_primary = SimpleJobClient(n_workers=num_workers)
-        client_secondary = SimpleJobClient(n_workers=num_secondary_workers)
-
-    existing_jobs: dict[str | None, dict[int, Future | SimpleJob]] = {}
-
-    logger.notice("Startup complete. Waiting for indexing jobs...")
-    while True:
-        start = time.time()
-        start_time_utc = datetime.utcfromtimestamp(start).strftime("%Y-%m-%d %H:%M:%S")
-        logger.debug(f"Running update, current UTC time: {start_time_utc}")
-
-        if existing_jobs:
-            logger.debug(
-                "Found existing indexing jobs: "
-                f"{[(tenant_id, list(jobs.keys())) for tenant_id, jobs in existing_jobs.items()]}"
-            )
-
-        try:
-            tenants = get_all_tenant_ids()
-
-            for tenant_id in tenants:
-                try:
-                    logger.debug(
-                        f"Processing {'index attempts' if tenant_id is None else f'tenant {tenant_id}'}"
-                    )
-                    with get_session_with_tenant(tenant_id) as db_session:
-                        index_to_expire = check_index_swap(db_session=db_session)
-
-                        if index_to_expire and tenant_id and MULTI_TENANT:
-                            VespaIndex.delete_entries_by_tenant_id(
-                                tenant_id=tenant_id,
-                                index_name=index_to_expire.index_name,
-                            )
-
-                        if not MULTI_TENANT:
-                            search_settings = get_current_search_settings(db_session)
-                            if search_settings.provider_type is None:
-                                logger.notice(
-                                    "Running a first inference to warm up embedding model"
-                                )
-                                embedding_model = EmbeddingModel.from_db_model(
-                                    search_settings=search_settings,
-                                    server_host=INDEXING_MODEL_SERVER_HOST,
-                                    server_port=INDEXING_MODEL_SERVER_PORT,
-                                )
-                                warm_up_bi_encoder(embedding_model=embedding_model)
-                                logger.notice("First inference complete.")
-
-                    tenant_jobs = existing_jobs.get(tenant_id, {})
-
-                    tenant_jobs = cleanup_indexing_jobs(
-                        existing_jobs=tenant_jobs, tenant_id=tenant_id
-                    )
-                    create_indexing_jobs(existing_jobs=tenant_jobs, tenant_id=tenant_id)
-                    tenant_jobs = kickoff_indexing_jobs(
-                        existing_jobs=tenant_jobs,
-                        client=client_primary,
-                        secondary_client=client_secondary,
-                        tenant_id=tenant_id,
-                    )
-
-                    existing_jobs[tenant_id] = tenant_jobs
-
-                except Exception as e:
-                    logger.exception(
-                        f"Failed to process tenant {tenant_id or 'default'}: {e}"
-                    )
-
-        except Exception as e:
-            logger.exception(f"Failed to run update due to {e}")
-
-        sleep_time = delay - (time.time() - start)
-        if sleep_time > 0:
-            time.sleep(sleep_time)
-
-
-def update__main() -> None:
-    set_is_ee_based_on_env_variable()
-
-    # initialize the Postgres connection pool
-    SqlEngine.set_app_name(POSTGRES_INDEXER_APP_NAME)
-
-    logger.notice("Starting indexing service")
-    update_loop()
-
-
-if __name__ == "__main__":
-    update__main()
+# TODO(rkuo): delete after background indexing via celery is fully vetted
+# import logging
+# import time
+# from datetime import datetime
+# import dask
+# from dask.distributed import Client
+# from dask.distributed import Future
+# from distributed import LocalCluster
+# from sqlalchemy import text
+# from sqlalchemy.exc import ProgrammingError
+# from sqlalchemy.orm import Session
+# from danswer.background.indexing.dask_utils import ResourceLogger
+# from danswer.background.indexing.job_client import SimpleJob
+# from danswer.background.indexing.job_client import SimpleJobClient
+# from danswer.background.indexing.run_indexing import run_indexing_entrypoint
+# from danswer.configs.app_configs import CLEANUP_INDEXING_JOBS_TIMEOUT
+# from danswer.configs.app_configs import DASK_JOB_CLIENT_ENABLED
+# from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+# from danswer.configs.app_configs import MULTI_TENANT
+# from danswer.configs.app_configs import NUM_INDEXING_WORKERS
+# from danswer.configs.app_configs import NUM_SECONDARY_INDEXING_WORKERS
+# from danswer.configs.constants import DocumentSource
+# from danswer.configs.constants import POSTGRES_INDEXER_APP_NAME
+# from danswer.configs.constants import TENANT_ID_PREFIX
+# from danswer.db.connector import fetch_connectors
+# from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
+# from danswer.db.engine import get_db_current_time
+# from danswer.db.engine import get_session_with_tenant
+# from danswer.db.engine import get_sqlalchemy_engine
+# from danswer.db.engine import SqlEngine
+# from danswer.db.index_attempt import create_index_attempt
+# from danswer.db.index_attempt import get_index_attempt
+# from danswer.db.index_attempt import get_inprogress_index_attempts
+# from danswer.db.index_attempt import get_last_attempt_for_cc_pair
+# from danswer.db.index_attempt import get_not_started_index_attempts
+# from danswer.db.index_attempt import mark_attempt_failed
+# from danswer.db.models import ConnectorCredentialPair
+# from danswer.db.models import IndexAttempt
+# from danswer.db.models import IndexingStatus
+# from danswer.db.models import IndexModelStatus
+# from danswer.db.models import SearchSettings
+# from danswer.db.search_settings import get_current_search_settings
+# from danswer.db.search_settings import get_secondary_search_settings
+# from danswer.db.swap_index import check_index_swap
+# from danswer.document_index.vespa.index import VespaIndex
+# from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
+# from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
+# from danswer.utils.logger import setup_logger
+# from danswer.utils.variable_functionality import global_version
+# from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
+# from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
+# from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
+# from shared_configs.configs import LOG_LEVEL
+# logger = setup_logger()
+# # If the indexing dies, it's most likely due to resource constraints,
+# # restarting just delays the eventual failure, not useful to the user
+# dask.config.set({"distributed.scheduler.allowed-failures": 0})
+# _UNEXPECTED_STATE_FAILURE_REASON = (
+#     "Stopped mid run, likely due to the background process being killed"
+# )
+# def _should_create_new_indexing(
+#     cc_pair: ConnectorCredentialPair,
+#     last_index: IndexAttempt | None,
+#     search_settings_instance: SearchSettings,
+#     secondary_index_building: bool,
+#     db_session: Session,
+# ) -> bool:
+#     connector = cc_pair.connector
+#     # don't kick off indexing for `NOT_APPLICABLE` sources
+#     if connector.source == DocumentSource.NOT_APPLICABLE:
+#         return False
+#     # User can still manually create single indexing attempts via the UI for the
+#     # currently in use index
+#     if DISABLE_INDEX_UPDATE_ON_SWAP:
+#         if (
+#             search_settings_instance.status == IndexModelStatus.PRESENT
+#             and secondary_index_building
+#         ):
+#             return False
+#     # When switching over models, always index at least once
+#     if search_settings_instance.status == IndexModelStatus.FUTURE:
+#         if last_index:
+#             # No new index if the last index attempt succeeded
+#             # Once is enough. The model will never be able to swap otherwise.
+#             if last_index.status == IndexingStatus.SUCCESS:
+#                 return False
+#             # No new index if the last index attempt is waiting to start
+#             if last_index.status == IndexingStatus.NOT_STARTED:
+#                 return False
+#             # No new index if the last index attempt is running
+#             if last_index.status == IndexingStatus.IN_PROGRESS:
+#                 return False
+#         else:
+#             if (
+#                 connector.id == 0 or connector.source == DocumentSource.INGESTION_API
+#             ):  # Ingestion API
+#                 return False
+#         return True
+#     # If the connector is paused or is the ingestion API, don't index
+#     # NOTE: during an embedding model switch over, the following logic
+#     # is bypassed by the above check for a future model
+#     if (
+#         not cc_pair.status.is_active()
+#         or connector.id == 0
+#         or connector.source == DocumentSource.INGESTION_API
+#     ):
+#         return False
+#     if not last_index:
+#         return True
+#     if connector.refresh_freq is None:
+#         return False
+#     # Only one scheduled/ongoing job per connector at a time
+#     # this prevents cases where
+#     # (1) the "latest" index_attempt is scheduled so we show
+#     #     that in the UI despite another index_attempt being in-progress
+#     # (2) multiple scheduled index_attempts at a time
+#     if (
+#         last_index.status == IndexingStatus.NOT_STARTED
+#         or last_index.status == IndexingStatus.IN_PROGRESS
+#     ):
+#         return False
+#     current_db_time = get_db_current_time(db_session)
+#     time_since_index = current_db_time - last_index.time_updated
+#     return time_since_index.total_seconds() >= connector.refresh_freq
+# def _mark_run_failed(
+#     db_session: Session, index_attempt: IndexAttempt, failure_reason: str
+# ) -> None:
+#     """Marks the `index_attempt` row as failed + updates the `
+#     connector_credential_pair` to reflect that the run failed"""
+#     logger.warning(
+#         f"Marking in-progress attempt 'connector: {index_attempt.connector_credential_pair.connector_id}, "
+#         f"credential: {index_attempt.connector_credential_pair.credential_id}' as failed due to {failure_reason}"
+#     )
+#     mark_attempt_failed(
+#         index_attempt=index_attempt,
+#         db_session=db_session,
+#         failure_reason=failure_reason,
+#     )
+# """Main funcs"""
+# def create_indexing_jobs(
+#     existing_jobs: dict[int, Future | SimpleJob], tenant_id: str | None
+# ) -> None:
+#     """Creates new indexing jobs for each connector / credential pair which is:
+#     1. Enabled
+#     2. `refresh_frequency` time has passed since the last indexing run for this pair
+#     3. There is not already an ongoing indexing attempt for this pair
+#     """
+#     with get_session_with_tenant(tenant_id) as db_session:
+#         ongoing: set[tuple[int | None, int]] = set()
+#         for attempt_id in existing_jobs:
+#             attempt = get_index_attempt(
+#                 db_session=db_session, index_attempt_id=attempt_id
+#             )
+#             if attempt is None:
+#                 logger.error(
+#                     f"Unable to find IndexAttempt for ID '{attempt_id}' when creating "
+#                     "indexing jobs"
+#                 )
+#                 continue
+#             ongoing.add(
+#                 (
+#                     attempt.connector_credential_pair_id,
+#                     attempt.search_settings_id,
+#                 )
+#             )
+#         # Get the primary search settings
+#         primary_search_settings = get_current_search_settings(db_session)
+#         search_settings = [primary_search_settings]
+#         # Check for secondary search settings
+#         secondary_search_settings = get_secondary_search_settings(db_session)
+#         if secondary_search_settings is not None:
+#             # If secondary settings exist, add them to the list
+#             search_settings.append(secondary_search_settings)
+#         all_connector_credential_pairs = fetch_connector_credential_pairs(db_session)
+#         for cc_pair in all_connector_credential_pairs:
+#             for search_settings_instance in search_settings:
+#                 # Check if there is an ongoing indexing attempt for this connector credential pair
+#                 if (cc_pair.id, search_settings_instance.id) in ongoing:
+#                     continue
+#                 last_attempt = get_last_attempt_for_cc_pair(
+#                     cc_pair.id, search_settings_instance.id, db_session
+#                 )
+#                 if not _should_create_new_indexing(
+#                     cc_pair=cc_pair,
+#                     last_index=last_attempt,
+#                     search_settings_instance=search_settings_instance,
+#                     secondary_index_building=len(search_settings) > 1,
+#                     db_session=db_session,
+#                 ):
+#                     continue
+#                 create_index_attempt(
+#                     cc_pair.id, search_settings_instance.id, db_session
+#                 )
+# def cleanup_indexing_jobs(
+#     existing_jobs: dict[int, Future | SimpleJob],
+#     tenant_id: str | None,
+#     timeout_hours: int = CLEANUP_INDEXING_JOBS_TIMEOUT,
+# ) -> dict[int, Future | SimpleJob]:
+#     existing_jobs_copy = existing_jobs.copy()
+#     # clean up completed jobs
+#     with get_session_with_tenant(tenant_id) as db_session:
+#         for attempt_id, job in existing_jobs.items():
+#             index_attempt = get_index_attempt(
+#                 db_session=db_session, index_attempt_id=attempt_id
+#             )
+#             # do nothing for ongoing jobs that haven't been stopped
+#             if not job.done():
+#                 if not index_attempt:
+#                     continue
+#                 if not index_attempt.is_finished():
+#                     continue
+#             if job.status == "error":
+#                 logger.error(job.exception())
+#             job.release()
+#             del existing_jobs_copy[attempt_id]
+#             if not index_attempt:
+#                 logger.error(
+#                     f"Unable to find IndexAttempt for ID '{attempt_id}' when cleaning "
+#                     "up indexing jobs"
+#                 )
+#                 continue
+#             if (
+#                 index_attempt.status == IndexingStatus.IN_PROGRESS
+#                 or job.status == "error"
+#             ):
+#                 _mark_run_failed(
+#                     db_session=db_session,
+#                     index_attempt=index_attempt,
+#                     failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
+#                 )
+#         # clean up in-progress jobs that were never completed
+#         try:
+#             connectors = fetch_connectors(db_session)
+#             for connector in connectors:
+#                 in_progress_indexing_attempts = get_inprogress_index_attempts(
+#                     connector.id, db_session
+#                 )
+#                 for index_attempt in in_progress_indexing_attempts:
+#                     if index_attempt.id in existing_jobs:
+#                         # If index attempt is canceled, stop the run
+#                         if index_attempt.status == IndexingStatus.FAILED:
+#                             existing_jobs[index_attempt.id].cancel()
+#                         # check to see if the job has been updated in last `timeout_hours` hours, if not
+#                         # assume it to frozen in some bad state and just mark it as failed. Note: this relies
+#                         # on the fact that the `time_updated` field is constantly updated every
+#                         # batch of documents indexed
+#                         current_db_time = get_db_current_time(db_session=db_session)
+#                         time_since_update = current_db_time - index_attempt.time_updated
+#                         if time_since_update.total_seconds() > 60 * 60 * timeout_hours:
+#                             existing_jobs[index_attempt.id].cancel()
+#                             _mark_run_failed(
+#                                 db_session=db_session,
+#                                 index_attempt=index_attempt,
+#                                 failure_reason="Indexing run frozen - no updates in the last three hours. "
+#                                 "The run will be re-attempted at next scheduled indexing time.",
+#                             )
+#                     else:
+#                         # If job isn't known, simply mark it as failed
+#                         _mark_run_failed(
+#                             db_session=db_session,
+#                             index_attempt=index_attempt,
+#                             failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
+#                         )
+#         except ProgrammingError:
+#             logger.debug(f"No Connector Table exists for: {tenant_id}")
+#     return existing_jobs_copy
+# def kickoff_indexing_jobs(
+#     existing_jobs: dict[int, Future | SimpleJob],
+#     client: Client | SimpleJobClient,
+#     secondary_client: Client | SimpleJobClient,
+#     tenant_id: str | None,
+# ) -> dict[int, Future | SimpleJob]:
+#     existing_jobs_copy = existing_jobs.copy()
+#     current_session = get_session_with_tenant(tenant_id)
+#     # Don't include jobs waiting in the Dask queue that just haven't started running
+#     # Also (rarely) don't include for jobs that started but haven't updated the indexing tables yet
+#     with current_session as db_session:
+#         # get_not_started_index_attempts orders its returned results from oldest to newest
+#         # we must process attempts in a FIFO manner to prevent connector starvation
+#         new_indexing_attempts = [
+#             (attempt, attempt.search_settings)
+#             for attempt in get_not_started_index_attempts(db_session)
+#             if attempt.id not in existing_jobs
+#         ]
+#     logger.debug(f"Found {len(new_indexing_attempts)} new indexing task(s).")
+#     if not new_indexing_attempts:
+#         return existing_jobs
+#     indexing_attempt_count = 0
+#     primary_client_full = False
+#     secondary_client_full = False
+#     for attempt, search_settings in new_indexing_attempts:
+#         if primary_client_full and secondary_client_full:
+#             break
+#         use_secondary_index = (
+#             search_settings.status == IndexModelStatus.FUTURE
+#             if search_settings is not None
+#             else False
+#         )
+#         if attempt.connector_credential_pair.connector is None:
+#             logger.warning(
+#                 f"Skipping index attempt as Connector has been deleted: {attempt}"
+#             )
+#             with current_session as db_session:
+#                 mark_attempt_failed(
+#                     attempt, db_session, failure_reason="Connector is null"
+#                 )
+#             continue
+#         if attempt.connector_credential_pair.credential is None:
+#             logger.warning(
+#                 f"Skipping index attempt as Credential has been deleted: {attempt}"
+#             )
+#             with current_session as db_session:
+#                 mark_attempt_failed(
+#                     attempt, db_session, failure_reason="Credential is null"
+#                 )
+#             continue
+#         if not use_secondary_index:
+#             if not primary_client_full:
+#                 run = client.submit(
+#                     run_indexing_entrypoint,
+#                     attempt.id,
+#                     tenant_id,
+#                     attempt.connector_credential_pair_id,
+#                     global_version.is_ee_version(),
+#                     pure=False,
+#                 )
+#                 if not run:
+#                     primary_client_full = True
+#         else:
+#             if not secondary_client_full:
+#                 run = secondary_client.submit(
+#                     run_indexing_entrypoint,
+#                     attempt.id,
+#                     tenant_id,
+#                     attempt.connector_credential_pair_id,
+#                     global_version.is_ee_version(),
+#                     pure=False,
+#                 )
+#                 if not run:
+#                     secondary_client_full = True
+#         if run:
+#             if indexing_attempt_count == 0:
+#                 logger.info(
+#                     f"Indexing dispatch starts: pending={len(new_indexing_attempts)}"
+#                 )
+#             indexing_attempt_count += 1
+#             secondary_str = " (secondary index)" if use_secondary_index else ""
+#             logger.info(
+#                 f"Indexing dispatched{secondary_str}: "
+#                 f"attempt_id={attempt.id} "
+#                 f"connector='{attempt.connector_credential_pair.connector.name}' "
+#                 f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
+#                 f"credentials='{attempt.connector_credential_pair.credential_id}'"
+#             )
+#             existing_jobs_copy[attempt.id] = run
+#     if indexing_attempt_count > 0:
+#         logger.info(
+#             f"Indexing dispatch results: "
+#             f"initial_pending={len(new_indexing_attempts)} "
+#             f"started={indexing_attempt_count} "
+#             f"remaining={len(new_indexing_attempts) - indexing_attempt_count}"
+#         )
+#     return existing_jobs_copy
+# def get_all_tenant_ids() -> list[str] | list[None]:
+#     if not MULTI_TENANT:
+#         return [None]
+#     with get_session_with_tenant(tenant_id="public") as session:
+#         result = session.execute(
+#             text(
+#                 """
+#             SELECT schema_name
+#             FROM information_schema.schemata
+#             WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
+#             )
+#         )
+#         tenant_ids = [row[0] for row in result]
+#     valid_tenants = [
+#         tenant
+#         for tenant in tenant_ids
+#         if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
+#     ]
+#     return valid_tenants
+# def update_loop(
+#     delay: int = 10,
+#     num_workers: int = NUM_INDEXING_WORKERS,
+#     num_secondary_workers: int = NUM_SECONDARY_INDEXING_WORKERS,
+# ) -> None:
+#     if not MULTI_TENANT:
+#         # We can use this function as we are certain only the public schema exists
+#         # (explicitly for the non-`MULTI_TENANT` case)
+#         engine = get_sqlalchemy_engine()
+#         with Session(engine) as db_session:
+#             check_index_swap(db_session=db_session)
+#             search_settings = get_current_search_settings(db_session)
+#             # So that the first time users aren't surprised by really slow speed of first
+#             # batch of documents indexed
+#             if search_settings.provider_type is None:
+#                 logger.notice("Running a first inference to warm up embedding model")
+#                 embedding_model = EmbeddingModel.from_db_model(
+#                     search_settings=search_settings,
+#                     server_host=INDEXING_MODEL_SERVER_HOST,
+#                     server_port=INDEXING_MODEL_SERVER_PORT,
+#                 )
+#                 warm_up_bi_encoder(
+#                     embedding_model=embedding_model,
+#                 )
+#                 logger.notice("First inference complete.")
+#     client_primary: Client | SimpleJobClient
+#     client_secondary: Client | SimpleJobClient
+#     if DASK_JOB_CLIENT_ENABLED:
+#         cluster_primary = LocalCluster(
+#             n_workers=num_workers,
+#             threads_per_worker=1,
+#             silence_logs=logging.ERROR,
+#         )
+#         cluster_secondary = LocalCluster(
+#             n_workers=num_secondary_workers,
+#             threads_per_worker=1,
+#             silence_logs=logging.ERROR,
+#         )
+#         client_primary = Client(cluster_primary)
+#         client_secondary = Client(cluster_secondary)
+#         if LOG_LEVEL.lower() == "debug":
+#             client_primary.register_worker_plugin(ResourceLogger())
+#     else:
+#         client_primary = SimpleJobClient(n_workers=num_workers)
+#         client_secondary = SimpleJobClient(n_workers=num_secondary_workers)
+#     existing_jobs: dict[str | None, dict[int, Future | SimpleJob]] = {}
+#     logger.notice("Startup complete. Waiting for indexing jobs...")
+#     while True:
+#         start = time.time()
+#         start_time_utc = datetime.utcfromtimestamp(start).strftime("%Y-%m-%d %H:%M:%S")
+#         logger.debug(f"Running update, current UTC time: {start_time_utc}")
+#         if existing_jobs:
+#             logger.debug(
+#                 "Found existing indexing jobs: "
+#                 f"{[(tenant_id, list(jobs.keys())) for tenant_id, jobs in existing_jobs.items()]}"
+#             )
+#         try:
+#             tenants = get_all_tenant_ids()
+#             for tenant_id in tenants:
+#                 try:
+#                     logger.debug(
+#                         f"Processing {'index attempts' if tenant_id is None else f'tenant {tenant_id}'}"
+#                     )
+#                     with get_session_with_tenant(tenant_id) as db_session:
+#                         index_to_expire = check_index_swap(db_session=db_session)
+#                         if index_to_expire and tenant_id and MULTI_TENANT:
+#                             VespaIndex.delete_entries_by_tenant_id(
+#                                 tenant_id=tenant_id,
+#                                 index_name=index_to_expire.index_name,
+#                             )
+#                         if not MULTI_TENANT:
+#                             search_settings = get_current_search_settings(db_session)
+#                             if search_settings.provider_type is None:
+#                                 logger.notice(
+#                                     "Running a first inference to warm up embedding model"
+#                                 )
+#                                 embedding_model = EmbeddingModel.from_db_model(
+#                                     search_settings=search_settings,
+#                                     server_host=INDEXING_MODEL_SERVER_HOST,
+#                                     server_port=INDEXING_MODEL_SERVER_PORT,
+#                                 )
+#                                 warm_up_bi_encoder(embedding_model=embedding_model)
+#                                 logger.notice("First inference complete.")
+#                     tenant_jobs = existing_jobs.get(tenant_id, {})
+#                     tenant_jobs = cleanup_indexing_jobs(
+#                         existing_jobs=tenant_jobs, tenant_id=tenant_id
+#                     )
+#                     create_indexing_jobs(existing_jobs=tenant_jobs, tenant_id=tenant_id)
+#                     tenant_jobs = kickoff_indexing_jobs(
+#                         existing_jobs=tenant_jobs,
+#                         client=client_primary,
+#                         secondary_client=client_secondary,
+#                         tenant_id=tenant_id,
+#                     )
+#                     existing_jobs[tenant_id] = tenant_jobs
+#                 except Exception as e:
+#                     logger.exception(
+#                         f"Failed to process tenant {tenant_id or 'default'}: {e}"
+#                     )
+#         except Exception as e:
+#             logger.exception(f"Failed to run update due to {e}")
+#         sleep_time = delay - (time.time() - start)
+#         if sleep_time > 0:
+#             time.sleep(sleep_time)
+# def update__main() -> None:
+#     set_is_ee_based_on_env_variable()
+#     # initialize the Postgres connection pool
+#     SqlEngine.set_app_name(POSTGRES_INDEXER_APP_NAME)
+#     logger.notice("Starting indexing service")
+#     update_loop()
+# if __name__ == "__main__":
+#     update__main()
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 6b167246a66..fb0a146c8b0 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -42,6 +42,8 @@
 POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME = "celery_worker_primary"
 POSTGRES_CELERY_WORKER_LIGHT_APP_NAME = "celery_worker_light"
 POSTGRES_CELERY_WORKER_HEAVY_APP_NAME = "celery_worker_heavy"
+POSTGRES_CELERY_WORKER_INDEXING_APP_NAME = "celery_worker_indexing"
+POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME = "celery_worker_indexing_child"
 POSTGRES_PERMISSIONS_APP_NAME = "permissions"
 POSTGRES_UNKNOWN_APP_NAME = "unknown"
 POSTGRES_DEFAULT_SCHEMA = "public"
@@ -73,6 +75,16 @@
 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT = 60
 CELERY_PRIMARY_WORKER_LOCK_TIMEOUT = 120
 
+# needs to be long enough to cover the maximum time it takes to download an object
+# if we can get callbacks as object bytes download, we could lower this a lot.
+CELERY_INDEXING_LOCK_TIMEOUT = 60 * 60  # 60 min
+
+# needs to be long enough to cover the maximum time it takes to download an object
+# if we can get callbacks as object bytes download, we could lower this a lot.
+CELERY_PRUNING_LOCK_TIMEOUT = 300  # 5 min
+
+DANSWER_REDIS_FUNCTION_LOCK_PREFIX = "da_function_lock:"
+
 
 class DocumentSource(str, Enum):
     # Special case, document passed in via Danswer APIs without specifying a source type
@@ -196,14 +208,19 @@ class DanswerCeleryQueues:
     VESPA_METADATA_SYNC = "vespa_metadata_sync"
     CONNECTOR_DELETION = "connector_deletion"
     CONNECTOR_PRUNING = "connector_pruning"
+    CONNECTOR_INDEXING = "connector_indexing"
 
 
 class DanswerRedisLocks:
     PRIMARY_WORKER = "da_lock:primary_worker"
     CHECK_VESPA_SYNC_BEAT_LOCK = "da_lock:check_vespa_sync_beat"
-    MONITOR_VESPA_SYNC_BEAT_LOCK = "da_lock:monitor_vespa_sync_beat"
     CHECK_CONNECTOR_DELETION_BEAT_LOCK = "da_lock:check_connector_deletion_beat"
     CHECK_PRUNE_BEAT_LOCK = "da_lock:check_prune_beat"
+    CHECK_INDEXING_BEAT_LOCK = "da_lock:check_indexing_beat"
+    MONITOR_VESPA_SYNC_BEAT_LOCK = "da_lock:monitor_vespa_sync_beat"
+
+    PRUNING_LOCK_PREFIX = "da_lock:pruning"
+    INDEXING_METADATA_PREFIX = "da_metadata:indexing"
 
 
 class DanswerCeleryPriority(int, Enum):
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index 5d214d77836..fbbbef1bbfc 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -1,4 +1,6 @@
 from collections.abc import Sequence
+from datetime import datetime
+from datetime import timezone
 
 from sqlalchemy import and_
 from sqlalchemy import delete
@@ -19,8 +21,6 @@
 from danswer.server.documents.models import ConnectorCredentialPair
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 from danswer.utils.logger import setup_logger
-from danswer.utils.telemetry import optional_telemetry
-from danswer.utils.telemetry import RecordType
 
 logger = setup_logger()
 
@@ -66,7 +66,7 @@ def create_index_attempt(
     return new_attempt.id
 
 
-def get_inprogress_index_attempts(
+def get_in_progress_index_attempts(
     connector_id: int | None,
     db_session: Session,
 ) -> list[IndexAttempt]:
@@ -81,13 +81,15 @@ def get_inprogress_index_attempts(
     return list(incomplete_attempts.all())
 
 
-def get_not_started_index_attempts(db_session: Session) -> list[IndexAttempt]:
+def get_all_index_attempts_by_status(
+    status: IndexingStatus, db_session: Session
+) -> list[IndexAttempt]:
     """This eagerly loads the connector and credential so that the db_session can be expired
     before running long-living indexing jobs, which causes increasing memory usage.
 
     Results are ordered by time_created (oldest to newest)."""
     stmt = select(IndexAttempt)
-    stmt = stmt.where(IndexAttempt.status == IndexingStatus.NOT_STARTED)
+    stmt = stmt.where(IndexAttempt.status == status)
     stmt = stmt.order_by(IndexAttempt.time_created)
     stmt = stmt.options(
         joinedload(IndexAttempt.connector_credential_pair).joinedload(
@@ -202,6 +204,8 @@ def mark_attempt_failed(
             .with_for_update()
         ).scalar_one()
 
+        if not attempt.time_started:
+            attempt.time_started = datetime.now(timezone.utc)
         attempt.status = IndexingStatus.FAILED
         attempt.error_msg = failure_reason
         attempt.full_exception_trace = full_exception_trace
@@ -210,9 +214,6 @@ def mark_attempt_failed(
         db_session.rollback()
         raise
 
-    source = index_attempt.connector_credential_pair.connector.source
-    optional_telemetry(record_type=RecordType.FAILURE, data={"connector": source})
-
 
 def update_docs_indexed(
     db_session: Session,
diff --git a/backend/danswer/document_index/factory.py b/backend/danswer/document_index/factory.py
index aedaec147d0..4f188ebc0ba 100644
--- a/backend/danswer/document_index/factory.py
+++ b/backend/danswer/document_index/factory.py
@@ -1,5 +1,6 @@
 from sqlalchemy.orm import Session
 
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.search_settings import get_current_search_settings
 from danswer.document_index.interfaces import DocumentIndex
 from danswer.document_index.vespa.index import VespaIndex
@@ -14,7 +15,9 @@ def get_default_document_index(
     index both need to be updated, updates are applied to both indices"""
     # Currently only supporting Vespa
     return VespaIndex(
-        index_name=primary_index_name, secondary_index_name=secondary_index_name
+        index_name=primary_index_name,
+        secondary_index_name=secondary_index_name,
+        multitenant=MULTI_TENANT,
     )
 
 
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index fd546307674..d71d198aea7 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -124,9 +124,15 @@ def add_ngrams_to_schema(schema_content: str) -> str:
 
 
 class VespaIndex(DocumentIndex):
-    def __init__(self, index_name: str, secondary_index_name: str | None) -> None:
+    def __init__(
+        self,
+        index_name: str,
+        secondary_index_name: str | None,
+        multitenant: bool = False,
+    ) -> None:
         self.index_name = index_name
         self.secondary_index_name = secondary_index_name
+        self.multitenant = multitenant
 
     def ensure_indices_exist(
         self,
@@ -341,6 +347,7 @@ def index(
                     chunks=chunk_batch,
                     index_name=self.index_name,
                     http_client=http_client,
+                    multitenant=self.multitenant,
                     executor=executor,
                 )
 
diff --git a/backend/danswer/document_index/vespa/indexing_utils.py b/backend/danswer/document_index/vespa/indexing_utils.py
index 35ebd52430f..28ff31c8071 100644
--- a/backend/danswer/document_index/vespa/indexing_utils.py
+++ b/backend/danswer/document_index/vespa/indexing_utils.py
@@ -123,6 +123,7 @@ def _index_vespa_chunk(
     chunk: DocMetadataAwareIndexChunk,
     index_name: str,
     http_client: httpx.Client,
+    multitenant: bool,
 ) -> None:
     json_header = {
         "Content-Type": "application/json",
@@ -179,8 +180,9 @@ def _index_vespa_chunk(
         BOOST: chunk.boost,
     }
 
-    if chunk.tenant_id:
-        vespa_document_fields[TENANT_ID] = chunk.tenant_id
+    if multitenant:
+        if chunk.tenant_id:
+            vespa_document_fields[TENANT_ID] = chunk.tenant_id
 
     vespa_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{vespa_chunk_id}"
     logger.debug(f'Indexing to URL "{vespa_url}"')
@@ -200,6 +202,7 @@ def batch_index_vespa_chunks(
     chunks: list[DocMetadataAwareIndexChunk],
     index_name: str,
     http_client: httpx.Client,
+    multitenant: bool,
     executor: concurrent.futures.ThreadPoolExecutor | None = None,
 ) -> None:
     external_executor = True
@@ -210,7 +213,9 @@ def batch_index_vespa_chunks(
 
     try:
         chunk_index_future = {
-            executor.submit(_index_vespa_chunk, chunk, index_name, http_client): chunk
+            executor.submit(
+                _index_vespa_chunk, chunk, index_name, http_client, multitenant
+            ): chunk
             for chunk in chunks
         }
         for future in concurrent.futures.as_completed(chunk_index_future):
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index 732d2511976..8de42db3863 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -15,7 +15,9 @@
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
+from danswer.background.celery.tasks.indexing.tasks import try_creating_indexing_task
 from danswer.configs.app_configs import ENABLED_CONNECTOR_TYPES
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import FileOrigin
@@ -63,19 +65,22 @@
 from danswer.db.credentials import fetch_credential_by_id
 from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed
 from danswer.db.document import get_document_counts_for_cc_pairs
+from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import AccessType
-from danswer.db.index_attempt import create_index_attempt
 from danswer.db.index_attempt import get_index_attempts_for_cc_pair
 from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
 from danswer.db.index_attempt import get_latest_index_attempts
 from danswer.db.index_attempt import get_latest_index_attempts_by_status
 from danswer.db.models import IndexingStatus
+from danswer.db.models import SearchSettings
 from danswer.db.models import User
 from danswer.db.models import UserRole
 from danswer.db.search_settings import get_current_search_settings
+from danswer.db.search_settings import get_secondary_search_settings
 from danswer.file_store.file_store import get_default_file_store
 from danswer.key_value_store.interface import KvKeyNotFoundError
+from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import AuthStatus
 from danswer.server.documents.models import AuthUrl
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
@@ -480,6 +485,8 @@ def get_connector_indexing_status(
 ) -> list[ConnectorIndexingStatus]:
     indexing_statuses: list[ConnectorIndexingStatus] = []
 
+    r = get_redis_client()
+
     # NOTE: If the connector is deleting behind the scenes,
     # accessing cc_pairs can be inconsistent and members like
     # connector or credential may be None.
@@ -531,6 +538,12 @@ def get_connector_indexing_status(
             relationship.user_group_id
         )
 
+    search_settings: SearchSettings | None = None
+    if not secondary_index:
+        search_settings = get_current_search_settings(db_session)
+    else:
+        search_settings = get_secondary_search_settings(db_session)
+
     for cc_pair in cc_pairs:
         # TODO remove this to enable ingestion API
         if cc_pair.name == "DefaultCCPair":
@@ -542,6 +555,12 @@ def get_connector_indexing_status(
             # This may happen if background deletion is happening
             continue
 
+        in_progress = False
+        if search_settings:
+            rci = RedisConnectorIndexing(cc_pair.id, search_settings.id)
+            if r.exists(rci.fence_key):
+                in_progress = True
+
         latest_index_attempt = cc_pair_to_latest_index_attempt.get(
             (connector.id, credential.id)
         )
@@ -595,6 +614,7 @@ def get_connector_indexing_status(
                     allow_scheduled=True,
                 )
                 is None,
+                in_progress=in_progress,
             )
         )
 
@@ -750,7 +770,13 @@ def connector_run_once(
     run_info: RunConnectorRequest,
     _: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
+    tenant_id: str = Depends(get_current_tenant_id),
 ) -> StatusResponse[list[int]]:
+    """Used to trigger indexing on a set of cc_pairs associated with a
+    single connector."""
+
+    r = get_redis_client()
+
     connector_id = run_info.connector_id
     specified_credential_ids = run_info.credential_ids
 
@@ -804,16 +830,24 @@ def connector_run_once(
         if credential_id not in skipped_credentials
     ]
 
-    index_attempt_ids = [
-        create_index_attempt(
-            connector_credential_pair_id=connector_credential_pair.id,
-            search_settings_id=search_settings.id,
-            from_beginning=run_info.from_beginning,
-            db_session=db_session,
-        )
-        for connector_credential_pair in connector_credential_pairs
-        if connector_credential_pair is not None
-    ]
+    index_attempt_ids = []
+    for cc_pair in connector_credential_pairs:
+        if cc_pair is not None:
+            attempt_id = try_creating_indexing_task(
+                cc_pair,
+                search_settings,
+                run_info.from_beginning,
+                db_session,
+                r,
+                tenant_id,
+            )
+            if attempt_id:
+                logger.info(
+                    f"try_creating_indexing_task succeeded: cc_pair={cc_pair.id} attempt_id={attempt_id}"
+                )
+                index_attempt_ids.append(attempt_id)
+            else:
+                logger.info(f"try_creating_indexing_task failed: cc_pair={cc_pair.id}")
 
     if not index_attempt_ids:
         raise HTTPException(
diff --git a/backend/danswer/server/documents/models.py b/backend/danswer/server/documents/models.py
index 15354a8f381..780d8a3f28e 100644
--- a/backend/danswer/server/documents/models.py
+++ b/backend/danswer/server/documents/models.py
@@ -307,6 +307,10 @@ class ConnectorIndexingStatus(BaseModel):
     deletion_attempt: DeletionAttemptSnapshot | None
     is_deletable: bool
 
+    # index attempt in db can be marked successful while celery/redis
+    # is stil running/cleaning up
+    in_progress: bool
+
 
 class ConnectorCredentialPairIdentifier(BaseModel):
     connector_id: int
diff --git a/backend/danswer/utils/logger.py b/backend/danswer/utils/logger.py
index 96d4ae2a25e..065d6282cbf 100644
--- a/backend/danswer/utils/logger.py
+++ b/backend/danswer/utils/logger.py
@@ -182,3 +182,24 @@ def setup_logger(
     logger.notice = lambda msg, *args, **kwargs: logger.log(logging.getLevelName("NOTICE"), msg, *args, **kwargs)  # type: ignore
 
     return DanswerLoggingAdapter(logger, extra=extra)
+
+
+def print_loggers() -> None:
+    root_logger = logging.getLogger()
+    loggers: list[logging.Logger | logging.PlaceHolder] = [root_logger]
+    loggers.extend(logging.Logger.manager.loggerDict.values())
+
+    for logger in loggers:
+        if isinstance(logger, logging.PlaceHolder):
+            # Skip placeholders that aren't actual loggers
+            continue
+
+        print(f"Logger: '{logger.name}' (Level: {logging.getLevelName(logger.level)})")
+        if logger.handlers:
+            for handler in logger.handlers:
+                print(f"  Handler: {handler}")
+        else:
+            print("  No handlers")
+
+        print(f"  Propagate: {logger.propagate}")
+        print()
diff --git a/backend/ee/danswer/background/celery/celery_app.py b/backend/ee/danswer/background/celery/celery_app.py
index de57794ee5a..afc77c1466d 100644
--- a/backend/ee/danswer/background/celery/celery_app.py
+++ b/backend/ee/danswer/background/celery/celery_app.py
@@ -1,8 +1,8 @@
 from datetime import timedelta
 
 from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.celery_utils import get_all_tenant_ids
 from danswer.background.task_utils import build_celery_task_wrapper
-from danswer.background.update import get_all_tenant_ids
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.chat import delete_chat_sessions_older_than
diff --git a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
index d305f723daf..259f2474928 100644
--- a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
@@ -16,11 +16,17 @@
 def monitor_usergroup_taskset(key_bytes: bytes, r: Redis, db_session: Session) -> None:
     """This function is likely to move in the worker refactor happening next."""
     fence_key = key_bytes.decode("utf-8")
-    usergroup_id = RedisUserGroup.get_id_from_fence_key(fence_key)
-    if not usergroup_id:
+    usergroup_id_str = RedisUserGroup.get_id_from_fence_key(fence_key)
+    if not usergroup_id_str:
         task_logger.warning(f"Could not parse usergroup id from {fence_key}")
         return
 
+    try:
+        usergroup_id = int(usergroup_id_str)
+    except ValueError:
+        task_logger.exception(f"usergroup_id ({usergroup_id_str}) is not an integer!")
+        raise
+
     rug = RedisUserGroup(usergroup_id)
     fence_value = r.get(rug.fence_key)
     if fence_value is None:
diff --git a/backend/scripts/dev_run_background_jobs.py b/backend/scripts/dev_run_background_jobs.py
index 84a92f4826b..4c87a3d3703 100644
--- a/backend/scripts/dev_run_background_jobs.py
+++ b/backend/scripts/dev_run_background_jobs.py
@@ -1,5 +1,4 @@
 import argparse
-import os
 import subprocess
 import threading
 
@@ -17,7 +16,7 @@ def monitor_process(process_name: str, process: subprocess.Popen) -> None:
             break
 
 
-def run_jobs(exclude_indexing: bool) -> None:
+def run_jobs() -> None:
     # command setup
     cmd_worker_primary = [
         "celery",
@@ -26,6 +25,7 @@ def run_jobs(exclude_indexing: bool) -> None:
         "worker",
         "--pool=threads",
         "--concurrency=6",
+        "--prefetch-multiplier=1",
         "--loglevel=INFO",
         "-n",
         "primary@%n",
@@ -40,6 +40,7 @@ def run_jobs(exclude_indexing: bool) -> None:
         "worker",
         "--pool=threads",
         "--concurrency=16",
+        "--prefetch-multiplier=8",
         "--loglevel=INFO",
         "-n",
         "light@%n",
@@ -54,6 +55,7 @@ def run_jobs(exclude_indexing: bool) -> None:
         "worker",
         "--pool=threads",
         "--concurrency=6",
+        "--prefetch-multiplier=1",
         "--loglevel=INFO",
         "-n",
         "heavy@%n",
@@ -61,6 +63,20 @@ def run_jobs(exclude_indexing: bool) -> None:
         "connector_pruning",
     ]
 
+    cmd_worker_indexing = [
+        "celery",
+        "-A",
+        "ee.danswer.background.celery.celery_app",
+        "worker",
+        "--pool=threads",
+        "--concurrency=1",
+        "--prefetch-multiplier=1",
+        "--loglevel=INFO",
+        "-n",
+        "indexing@%n",
+        "--queues=connector_indexing",
+    ]
+
     cmd_beat = [
         "celery",
         "-A",
@@ -82,6 +98,10 @@ def run_jobs(exclude_indexing: bool) -> None:
         cmd_worker_heavy, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
     )
 
+    worker_indexing_process = subprocess.Popen(
+        cmd_worker_indexing, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
+    )
+
     beat_process = subprocess.Popen(
         cmd_beat, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
     )
@@ -96,44 +116,26 @@ def run_jobs(exclude_indexing: bool) -> None:
     worker_heavy_thread = threading.Thread(
         target=monitor_process, args=("HEAVY", worker_heavy_process)
     )
+    worker_indexing_thread = threading.Thread(
+        target=monitor_process, args=("INDEX", worker_indexing_process)
+    )
     beat_thread = threading.Thread(target=monitor_process, args=("BEAT", beat_process))
 
     worker_primary_thread.start()
     worker_light_thread.start()
     worker_heavy_thread.start()
+    worker_indexing_thread.start()
     beat_thread.start()
 
-    if not exclude_indexing:
-        update_env = os.environ.copy()
-        update_env["PYTHONPATH"] = "."
-        cmd_indexing = ["python", "danswer/background/update.py"]
-
-        indexing_process = subprocess.Popen(
-            cmd_indexing,
-            env=update_env,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            text=True,
-        )
-
-        indexing_thread = threading.Thread(
-            target=monitor_process, args=("INDEXING", indexing_process)
-        )
-
-        indexing_thread.start()
-        indexing_thread.join()
-
     worker_primary_thread.join()
     worker_light_thread.join()
     worker_heavy_thread.join()
+    worker_indexing_thread.join()
     beat_thread.join()
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Run background jobs.")
-    parser.add_argument(
-        "--no-indexing", action="store_true", help="Do not run indexing process"
-    )
     args = parser.parse_args()
 
-    run_jobs(args.no_indexing)
+    run_jobs()
diff --git a/backend/supervisord.conf b/backend/supervisord.conf
index ebe56761381..76026bc5667 100644
--- a/backend/supervisord.conf
+++ b/backend/supervisord.conf
@@ -3,17 +3,6 @@ nodaemon=true
 user=root
 logfile=/var/log/supervisord.log
 
-# Indexing is the heaviest job, also requires some CPU intensive steps
-# Cannot place this in Celery for now because Celery must run as a single process (see note below)
-# Indexing uses multi-processing to speed things up
-[program:document_indexing]
-environment=CURRENT_PROCESS_IS_AN_INDEXING_JOB=true
-command=python danswer/background/update.py
-stdout_logfile=/var/log/document_indexing.log
-stdout_logfile_maxbytes=16MB
-redirect_stderr=true
-autorestart=true
-
 # Background jobs that must be run async due to long time to completion
 # NOTE: due to an issue with Celery + SQLAlchemy 
 # (https://github.com/celery/celery/issues/7007#issuecomment-1740139367)
@@ -73,6 +62,21 @@ autorestart=true
 startsecs=10
 stopasgroup=true
 
+[program:celery_worker_indexing]
+command=bash -c "celery -A danswer.background.celery.celery_run:celery_app worker \
+    --pool=threads \
+    --concurrency=${CELERY_WORKER_INDEXING_CONCURRENCY:-${NUM_INDEXING_WORKERS:-1}} \
+    --prefetch-multiplier=1 \
+    --loglevel=INFO \
+    --hostname=indexing@%%n \
+    -Q connector_indexing"
+stdout_logfile=/var/log/celery_worker_indexing.log
+stdout_logfile_maxbytes=16MB
+redirect_stderr=true
+autorestart=true
+startsecs=10
+stopasgroup=true
+
 # Job scheduler for periodic tasks
 [program:celery_beat]
 command=celery -A danswer.background.celery.celery_run:celery_app beat
@@ -103,7 +107,7 @@ command=tail -qF
     /var/log/celery_worker_primary.log
     /var/log/celery_worker_light.log
     /var/log/celery_worker_heavy.log
-    /var/log/document_indexing.log
+    /var/log/celery_worker_indexing.log
     /var/log/slack_bot.log
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes = 0  # must be set to 0 when stdout_logfile=/dev/stdout
diff --git a/backend/tests/integration/common_utils/managers/cc_pair.py b/backend/tests/integration/common_utils/managers/cc_pair.py
index 64961b22e3c..99d8a82a2b3 100644
--- a/backend/tests/integration/common_utils/managers/cc_pair.py
+++ b/backend/tests/integration/common_utils/managers/cc_pair.py
@@ -239,24 +239,24 @@ def wait_for_indexing(
                 if fetched_cc_pair.cc_pair_id != cc_pair.id:
                     continue
 
+                if fetched_cc_pair.in_progress:
+                    continue
+
                 if (
                     fetched_cc_pair.last_success
                     and fetched_cc_pair.last_success > after
                 ):
-                    print(f"cc_pair {cc_pair.id} indexing complete.")
+                    print(f"CC pair {cc_pair.id} indexing complete.")
                     return
-                else:
-                    print("cc_pair found but not finished:")
-                    # print(fetched_cc_pair.__dict__)
 
             elapsed = time.monotonic() - start
             if elapsed > timeout:
                 raise TimeoutError(
-                    f"CC pair indexing was not completed within {timeout} seconds"
+                    f"CC pair {cc_pair.id} indexing was not completed within {timeout} seconds"
                 )
 
             print(
-                f"Waiting for CC indexing to complete. elapsed={elapsed:.2f} timeout={timeout}"
+                f"CC pair {cc_pair.id} indexing to complete. elapsed={elapsed:.2f} timeout={timeout}"
             )
             time.sleep(5)
 
diff --git a/backend/tests/integration/common_utils/managers/document_set.py b/backend/tests/integration/common_utils/managers/document_set.py
index cd6936602ea..7670f42fa3c 100644
--- a/backend/tests/integration/common_utils/managers/document_set.py
+++ b/backend/tests/integration/common_utils/managers/document_set.py
@@ -135,6 +135,7 @@ def wait_for_sync(
             all_up_to_date = all(doc_set.is_up_to_date for doc_set in doc_sets)
 
             if all_up_to_date:
+                print("Document sets synced successfully.")
                 break
 
             if time.time() - start > MAX_DELAY:
diff --git a/backend/tests/integration/common_utils/managers/user_group.py b/backend/tests/integration/common_utils/managers/user_group.py
index baf2008b965..e8a26fa34a7 100644
--- a/backend/tests/integration/common_utils/managers/user_group.py
+++ b/backend/tests/integration/common_utils/managers/user_group.py
@@ -146,6 +146,7 @@ def wait_for_sync(
                     if user_group.id in check_ids
                 ]
             if all(ug.is_up_to_date for ug in user_groups):
+                print("User groups synced successfully.")
                 return
 
             if time.time() - start > MAX_DELAY:
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 6f4619f83f7..d22bde5b465 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -174,8 +174,9 @@ services:
       - GONG_CONNECTOR_START_TIME=${GONG_CONNECTOR_START_TIME:-}
       - NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP=${NOTION_CONNECTOR_ENABLE_RECURSIVE_PAGE_LOOKUP:-}
       - GITHUB_CONNECTOR_BASE_URL=${GITHUB_CONNECTOR_BASE_URL:-}
-      # Celery Configs (defaults are set in the supervisord.conf file, prefer doing that to have on source
-      # of defaults)
+      # Celery Configs (defaults are set in the supervisord.conf file.
+      # prefer doing that to have one source of defaults)
+      - CELERY_WORKER_INDEXING_CONCURRENCY=${CELERY_WORKER_INDEXING_CONCURRENCY:-}
       - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
       - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
 
diff --git a/deployment/docker_compose/docker-compose.gpu-dev.yml b/deployment/docker_compose/docker-compose.gpu-dev.yml
index 6397f657c19..a7e0a2afe97 100644
--- a/deployment/docker_compose/docker-compose.gpu-dev.yml
+++ b/deployment/docker_compose/docker-compose.gpu-dev.yml
@@ -186,8 +186,9 @@ services:
       # Log all of Danswer prompts and interactions with the LLM
       - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
-      # Celery Configs (defaults are set in the supervisord.conf file, prefer doing that to have on source
-      # of defaults)
+      # Celery Configs (defaults are set in the supervisord.conf file.
+      # prefer doing that to have one source of defaults)
+      - CELERY_WORKER_INDEXING_CONCURRENCY=${CELERY_WORKER_INDEXING_CONCURRENCY:-}
       - CELERY_WORKER_LIGHT_CONCURRENCY=${CELERY_WORKER_LIGHT_CONCURRENCY:-}
       - CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-}
 
diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
index 39c238870e5..7d60d786986 100644
--- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
+++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
@@ -439,6 +439,7 @@ export function CCPairIndexingStatusTable({
             error_msg: "",
             deletion_attempt: null,
             is_deletable: true,
+            in_progress: false,
             groups: [], // Add this line
           }}
           isEditable={false}
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index c760074f403..d4f43a024d6 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -111,6 +111,7 @@ export interface ConnectorIndexingStatus<
   latest_index_attempt: IndexAttemptSnapshot | null;
   deletion_attempt: DeletionAttemptSnapshot | null;
   is_deletable: boolean;
+  in_progress: boolean;
 }
 
 export interface CCPairBasicInfo {

From 8b220d2dbae5c2e2c88cd2a762fcb096097a1fc9 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 18 Oct 2024 18:21:11 -0700
Subject: [PATCH 157/376] Add assistant notifications + update assistant
 context (#2816)

* add assistant notifications

* nit

* update context

* validated

* ensure context passed properly

* validated + cleaned

* nit: naming

* k

* k

* final validation + new ui

* nit + video

* nit

* nit

* nit

* k

* fix typos
---
 ...30_add_additional_data_to_notifications.py |  26 ++
 backend/danswer/configs/constants.py          |   1 +
 backend/danswer/db/models.py                  |   3 +
 backend/danswer/db/notification.py            |  27 +-
 backend/danswer/main.py                       |   2 +
 .../server/features/notifications/api.py      |  47 ++++
 .../danswer/server/features/persona/api.py    |  48 +++-
 .../danswer/server/features/persona/models.py |   4 +
 backend/danswer/server/settings/api.py        |  26 +-
 backend/danswer/server/settings/models.py     |   2 +
 web/src/app/admin/settings/interfaces.ts      |  12 +-
 web/src/app/assistants/SidebarWrapper.tsx     |  14 +-
 .../assistants/gallery/AssistantsGallery.tsx  |  17 +-
 .../gallery/WrappedAssistantsGallery.tsx      |  19 +-
 web/src/app/assistants/gallery/page.tsx       |  27 +-
 .../assistants/mine/AssistantSharingModal.tsx |   9 +-
 .../app/assistants/mine/AssistantsList.tsx    |  80 +++---
 .../assistants/mine/WrappedAssistantsMine.tsx |  21 +-
 .../assistants/mine/WrappedInputPrompts.tsx   |  37 +--
 web/src/app/assistants/mine/page.tsx          |  16 +-
 web/src/app/chat/ChatPage.tsx                 |  26 +-
 web/src/app/chat/input/ChatInputBar.tsx       |   8 +-
 .../app/chat/modal/SetDefaultModelModal.tsx   |   4 +-
 .../modal/configuration/AssistantsTab.tsx     |  19 +-
 web/src/app/chat/page.tsx                     |  43 ++--
 web/src/app/chat/shared/[chatId]/page.tsx     |   2 +-
 web/src/app/prompts/page.tsx                  |  11 +-
 web/src/app/search/page.tsx                   |  48 ++--
 web/src/components/UserDropdown.tsx           | 171 +++++++------
 web/src/components/admin/ClientLayout.tsx     |   2 +-
 web/src/components/chat_search/Header.tsx     |   5 +-
 .../components/chat_search/Notifications.tsx  | 231 ++++++++++++++++++
 .../components/context/AssistantsContext.tsx  | 119 +++++++++
 web/src/components/context/ChatContext.tsx    |  13 +-
 web/src/components/icons/icons.tsx            |  23 ++
 web/src/components/search/SearchSection.tsx   |   1 -
 web/src/lib/chat/fetchChatData.ts             |   4 +
 37 files changed, 822 insertions(+), 346 deletions(-)
 create mode 100644 backend/alembic/versions/1b10e1fda030_add_additional_data_to_notifications.py
 create mode 100644 backend/danswer/server/features/notifications/api.py
 create mode 100644 web/src/components/chat_search/Notifications.tsx
 create mode 100644 web/src/components/context/AssistantsContext.tsx

diff --git a/backend/alembic/versions/1b10e1fda030_add_additional_data_to_notifications.py b/backend/alembic/versions/1b10e1fda030_add_additional_data_to_notifications.py
new file mode 100644
index 00000000000..71c31e2c862
--- /dev/null
+++ b/backend/alembic/versions/1b10e1fda030_add_additional_data_to_notifications.py
@@ -0,0 +1,26 @@
+"""add additional data to notifications
+
+Revision ID: 1b10e1fda030
+Revises: 6756efa39ada
+Create Date: 2024-10-15 19:26:44.071259
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = "1b10e1fda030"
+down_revision = "6756efa39ada"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "notification", sa.Column("additional_data", postgresql.JSONB(), nullable=True)
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("notification", "additional_data")
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index fb0a146c8b0..9858f2354b9 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -135,6 +135,7 @@ class DocumentSource(str, Enum):
 
 class NotificationType(str, Enum):
     REINDEX = "reindex"
+    PERSONA_SHARED = "persona_shared"
 
 
 class BlobType(str, Enum):
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index 2101fc74e90..bee69353437 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -235,6 +235,9 @@ class Notification(Base):
     first_shown: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True))
 
     user: Mapped[User] = relationship("User", back_populates="notifications")
+    additional_data: Mapped[dict | None] = mapped_column(
+        postgresql.JSONB(), nullable=True
+    )
 
 
 """
diff --git a/backend/danswer/db/notification.py b/backend/danswer/db/notification.py
index 61586208c69..bd58add14a0 100644
--- a/backend/danswer/db/notification.py
+++ b/backend/danswer/db/notification.py
@@ -1,3 +1,5 @@
+from uuid import UUID
+
 from sqlalchemy import select
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import func
@@ -8,16 +10,37 @@
 
 
 def create_notification(
-    user: User | None,
+    user_id: UUID | None,
     notif_type: NotificationType,
     db_session: Session,
+    additional_data: dict | None = None,
 ) -> Notification:
+    # Check if an undismissed notification of the same type and data exists
+    existing_notification = (
+        db_session.query(Notification)
+        .filter_by(
+            user_id=user_id,
+            notif_type=notif_type,
+            dismissed=False,
+        )
+        .filter(Notification.additional_data == additional_data)
+        .first()
+    )
+
+    if existing_notification:
+        # Update the last_shown timestamp
+        existing_notification.last_shown = func.now()
+        db_session.commit()
+        return existing_notification
+
+    # Create a new notification if none exists
     notification = Notification(
-        user_id=user.id if user else None,
+        user_id=user_id,
         notif_type=notif_type,
         dismissed=False,
         last_shown=func.now(),
         first_shown=func.now(),
+        additional_data=additional_data,
     )
     db_session.add(notification)
     db_session.commit()
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index 9db1b9cd35c..fe563c7c695 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -57,6 +57,7 @@
     admin_router as admin_input_prompt_router,
 )
 from danswer.server.features.input_prompt.api import basic_router as input_prompt_router
+from danswer.server.features.notifications.api import router as notification_router
 from danswer.server.features.persona.api import admin_router as admin_persona_router
 from danswer.server.features.persona.api import basic_router as persona_router
 from danswer.server.features.prompt.api import basic_router as prompt_router
@@ -246,6 +247,7 @@ def get_application() -> FastAPI:
     include_router_with_global_prefix_prepended(application, admin_persona_router)
     include_router_with_global_prefix_prepended(application, input_prompt_router)
     include_router_with_global_prefix_prepended(application, admin_input_prompt_router)
+    include_router_with_global_prefix_prepended(application, notification_router)
     include_router_with_global_prefix_prepended(application, prompt_router)
     include_router_with_global_prefix_prepended(application, tool_router)
     include_router_with_global_prefix_prepended(application, admin_tool_router)
diff --git a/backend/danswer/server/features/notifications/api.py b/backend/danswer/server/features/notifications/api.py
new file mode 100644
index 00000000000..a4f5415a6a1
--- /dev/null
+++ b/backend/danswer/server/features/notifications/api.py
@@ -0,0 +1,47 @@
+from fastapi import APIRouter
+from fastapi import Depends
+from fastapi import HTTPException
+from sqlalchemy.orm import Session
+
+from danswer.auth.users import current_user
+from danswer.db.engine import get_session
+from danswer.db.models import User
+from danswer.db.notification import dismiss_notification
+from danswer.db.notification import get_notification_by_id
+from danswer.db.notification import get_notifications
+from danswer.server.settings.models import Notification as NotificationModel
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+router = APIRouter(prefix="/notifications")
+
+
+@router.get("")
+def get_notifications_api(
+    user: User = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> list[NotificationModel]:
+    notifications = [
+        NotificationModel.from_model(notif)
+        for notif in get_notifications(user, db_session, include_dismissed=False)
+    ]
+    return notifications
+
+
+@router.post("/{notification_id}/dismiss")
+def dismiss_notification_endpoint(
+    notification_id: int,
+    user: User | None = Depends(current_user),
+    db_session: Session = Depends(get_session),
+) -> None:
+    try:
+        notification = get_notification_by_id(notification_id, user, db_session)
+    except PermissionError:
+        raise HTTPException(
+            status_code=403, detail="Not authorized to dismiss this notification"
+        )
+    except ValueError:
+        raise HTTPException(status_code=404, detail="Notification not found")
+
+    dismiss_notification(notification, db_session)
diff --git a/backend/danswer/server/features/persona/api.py b/backend/danswer/server/features/persona/api.py
index 8b4305755dc..8c0e1943861 100644
--- a/backend/danswer/server/features/persona/api.py
+++ b/backend/danswer/server/features/persona/api.py
@@ -13,8 +13,10 @@
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
 from danswer.configs.constants import FileOrigin
+from danswer.configs.constants import NotificationType
 from danswer.db.engine import get_session
 from danswer.db.models import User
+from danswer.db.notification import create_notification
 from danswer.db.persona import create_update_persona
 from danswer.db.persona import get_persona_by_id
 from danswer.db.persona import get_personas
@@ -28,6 +30,7 @@
 from danswer.file_store.models import ChatFileType
 from danswer.llm.answering.prompts.utils import build_dummy_prompt
 from danswer.server.features.persona.models import CreatePersonaRequest
+from danswer.server.features.persona.models import PersonaSharedNotificationData
 from danswer.server.features.persona.models import PersonaSnapshot
 from danswer.server.features.persona.models import PromptTemplateResponse
 from danswer.server.models import DisplayPriorityRequest
@@ -183,11 +186,12 @@ class PersonaShareRequest(BaseModel):
     user_ids: list[UUID]
 
 
+# We notify each user when a user is shared with them
 @basic_router.patch("/{persona_id}/share")
 def share_persona(
     persona_id: int,
     persona_share_request: PersonaShareRequest,
-    user: User | None = Depends(current_user),
+    user: User = Depends(current_user),
     db_session: Session = Depends(get_session),
 ) -> None:
     update_persona_shared_users(
@@ -197,6 +201,18 @@ def share_persona(
         db_session=db_session,
     )
 
+    for user_id in persona_share_request.user_ids:
+        # Don't notify the user that they have access to their own persona
+        if user_id != user.id:
+            create_notification(
+                user_id=user_id,
+                notif_type=NotificationType.PERSONA_SHARED,
+                db_session=db_session,
+                additional_data=PersonaSharedNotificationData(
+                    persona_id=persona_id,
+                ).model_dump(),
+            )
+
 
 @basic_router.delete("/{persona_id}")
 def delete_persona(
@@ -216,23 +232,31 @@ def list_personas(
     user: User | None = Depends(current_user),
     db_session: Session = Depends(get_session),
     include_deleted: bool = False,
+    persona_ids: list[int] = Query(None),
 ) -> list[PersonaSnapshot]:
-    return [
-        PersonaSnapshot.from_model(persona)
-        for persona in get_personas(
-            user=user,
-            include_deleted=include_deleted,
-            db_session=db_session,
-            get_editable=False,
-            joinedload_all=True,
-        )
-        # If the persona has an image generation tool and it's not available, don't include it
+    personas = get_personas(
+        user=user,
+        include_deleted=include_deleted,
+        db_session=db_session,
+        get_editable=False,
+        joinedload_all=True,
+    )
+
+    if persona_ids:
+        personas = [p for p in personas if p.id in persona_ids]
+
+    # Filter out personas with unavailable tools
+    personas = [
+        p
+        for p in personas
         if not (
-            any(tool.in_code_tool_id == "ImageGenerationTool" for tool in persona.tools)
+            any(tool.in_code_tool_id == "ImageGenerationTool" for tool in p.tools)
             and not is_image_generation_available(db_session=db_session)
         )
     ]
 
+    return [PersonaSnapshot.from_model(p) for p in personas]
+
 
 @basic_router.get("/{persona_id}")
 def get_persona(
diff --git a/backend/danswer/server/features/persona/models.py b/backend/danswer/server/features/persona/models.py
index 016defda369..866d70f11bb 100644
--- a/backend/danswer/server/features/persona/models.py
+++ b/backend/danswer/server/features/persona/models.py
@@ -120,3 +120,7 @@ def from_model(
 
 class PromptTemplateResponse(BaseModel):
     final_prompt_template: str
+
+
+class PersonaSharedNotificationData(BaseModel):
+    persona_id: int
diff --git a/backend/danswer/server/settings/api.py b/backend/danswer/server/settings/api.py
index 48157253c9a..35ab0b12c6d 100644
--- a/backend/danswer/server/settings/api.py
+++ b/backend/danswer/server/settings/api.py
@@ -15,8 +15,6 @@
 from danswer.db.models import User
 from danswer.db.notification import create_notification
 from danswer.db.notification import dismiss_all_notifications
-from danswer.db.notification import dismiss_notification
-from danswer.db.notification import get_notification_by_id
 from danswer.db.notification import get_notifications
 from danswer.db.notification import update_notification_last_shown
 from danswer.key_value_store.factory import get_kv_store
@@ -55,7 +53,7 @@ def fetch_settings(
     """Settings and notifications are stuffed into this single endpoint to reduce number of
     Postgres calls"""
     general_settings = load_settings()
-    user_notifications = get_user_notifications(user, db_session)
+    user_notifications = get_reindex_notification(user, db_session)
 
     try:
         kv_store = get_kv_store()
@@ -70,25 +68,7 @@ def fetch_settings(
     )
 
 
-@basic_router.post("/notifications/{notification_id}/dismiss")
-def dismiss_notification_endpoint(
-    notification_id: int,
-    user: User | None = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> None:
-    try:
-        notification = get_notification_by_id(notification_id, user, db_session)
-    except PermissionError:
-        raise HTTPException(
-            status_code=403, detail="Not authorized to dismiss this notification"
-        )
-    except ValueError:
-        raise HTTPException(status_code=404, detail="Notification not found")
-
-    dismiss_notification(notification, db_session)
-
-
-def get_user_notifications(
+def get_reindex_notification(
     user: User | None, db_session: Session
 ) -> list[Notification]:
     """Get notifications for the user, currently the logic is very specific to the reindexing flag"""
@@ -121,7 +101,7 @@ def get_user_notifications(
 
         if not reindex_notifs:
             notif = create_notification(
-                user=user,
+                user_id=user.id if user else None,
                 notif_type=NotificationType.REINDEX,
                 db_session=db_session,
             )
diff --git a/backend/danswer/server/settings/models.py b/backend/danswer/server/settings/models.py
index 6713f7f67e8..af93595501d 100644
--- a/backend/danswer/server/settings/models.py
+++ b/backend/danswer/server/settings/models.py
@@ -24,6 +24,7 @@ class Notification(BaseModel):
     dismissed: bool
     last_shown: datetime
     first_shown: datetime
+    additional_data: dict | None = None
 
     @classmethod
     def from_model(cls, notif: NotificationDBModel) -> "Notification":
@@ -33,6 +34,7 @@ def from_model(cls, notif: NotificationDBModel) -> "Notification":
             dismissed=notif.dismissed,
             last_shown=notif.last_shown,
             first_shown=notif.first_shown,
+            additional_data=notif.additional_data,
         )
 
 
diff --git a/web/src/app/admin/settings/interfaces.ts b/web/src/app/admin/settings/interfaces.ts
index 2df8b5c26b2..2c18a2c8262 100644
--- a/web/src/app/admin/settings/interfaces.ts
+++ b/web/src/app/admin/settings/interfaces.ts
@@ -15,12 +15,20 @@ export interface Settings {
   product_gating: GatingType;
 }
 
+export enum NotificationType {
+  PERSONA_SHARED = "persona_shared",
+  REINDEX_NEEDED = "reindex_needed",
+}
+
 export interface Notification {
   id: number;
   notif_type: string;
+  time_created: string;
   dismissed: boolean;
-  last_shown: string;
-  first_shown: string;
+  additional_data?: {
+    persona_id?: number;
+    [key: string]: any;
+  };
 }
 
 export interface NavigationItem {
diff --git a/web/src/app/assistants/SidebarWrapper.tsx b/web/src/app/assistants/SidebarWrapper.tsx
index 9a1d320d735..58190228bbd 100644
--- a/web/src/app/assistants/SidebarWrapper.tsx
+++ b/web/src/app/assistants/SidebarWrapper.tsx
@@ -26,14 +26,9 @@ interface SidebarWrapperProps<T extends object> {
   folders?: Folder[];
   initiallyToggled: boolean;
   openedFolders?: { [key: number]: boolean };
-  content: (props: T) => ReactNode;
-  headerProps: {
-    page: pageType;
-    user: User | null;
-  };
-  contentProps: T;
   page: pageType;
   size?: "sm" | "lg";
+  children: ReactNode;
 }
 
 export default function SidebarWrapper<T extends object>({
@@ -42,10 +37,8 @@ export default function SidebarWrapper<T extends object>({
   folders,
   openedFolders,
   page,
-  headerProps,
-  contentProps,
-  content,
   size = "sm",
+  children,
 }: SidebarWrapperProps<T>) {
   const [toggledSidebar, setToggledSidebar] = useState(initiallyToggled);
   const [showDocSidebar, setShowDocSidebar] = useState(false); // State to track if sidebar is open
@@ -144,7 +137,6 @@ export default function SidebarWrapper<T extends object>({
           sidebarToggled={toggledSidebar}
           toggleSidebar={toggleSidebar}
           page="assistants"
-          user={headerProps.user}
         />
         <div className="w-full flex">
           <div
@@ -163,7 +155,7 @@ export default function SidebarWrapper<T extends object>({
           <div
             className={`mt-4 w-full ${size == "lg" ? "max-w-4xl" : "max-w-3xl"} mx-auto`}
           >
-            {content(contentProps)}
+            {children}
           </div>
         </div>
       </div>
diff --git a/web/src/app/assistants/gallery/AssistantsGallery.tsx b/web/src/app/assistants/gallery/AssistantsGallery.tsx
index 635bb80cf54..62b9f3906df 100644
--- a/web/src/app/assistants/gallery/AssistantsGallery.tsx
+++ b/web/src/app/assistants/gallery/AssistantsGallery.tsx
@@ -15,6 +15,8 @@ import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { useRouter } from "next/navigation";
 import { AssistantTools } from "../ToolsDisplay";
 import { classifyAssistants } from "@/lib/assistants/utils";
+import { useAssistants } from "@/components/context/AssistantsContext";
+import { useUser } from "@/components/user/UserProvider";
 export function AssistantGalleryCard({
   assistant,
   user,
@@ -26,6 +28,7 @@ export function AssistantGalleryCard({
   setPopup: (popup: PopupSpec) => void;
   selectedAssistant: boolean;
 }) {
+  const { refreshUser } = useUser();
   const router = useRouter();
   return (
     <div
@@ -80,7 +83,7 @@ export function AssistantGalleryCard({
                       message: `"${assistant.name}" has been removed from your list.`,
                       type: "success",
                     });
-                    router.refresh();
+                    await refreshUser();
                   } else {
                     setPopup({
                       message: `"${assistant.name}" could not be removed from your list.`,
@@ -108,7 +111,7 @@ export function AssistantGalleryCard({
                       message: `"${assistant.name}" has been added to your list.`,
                       type: "success",
                     });
-                    router.refresh();
+                    await refreshUser();
                   } else {
                     setPopup({
                       message: `"${assistant.name}" could not be added to your list.`,
@@ -136,14 +139,10 @@ export function AssistantGalleryCard({
     </div>
   );
 }
-export function AssistantsGallery({
-  assistants,
-  user,
-}: {
-  assistants: Persona[];
+export function AssistantsGallery() {
+  const { assistants } = useAssistants();
+  const { user } = useUser();
 
-  user: User | null;
-}) {
   const router = useRouter();
 
   const [searchQuery, setSearchQuery] = useState("");
diff --git a/web/src/app/assistants/gallery/WrappedAssistantsGallery.tsx b/web/src/app/assistants/gallery/WrappedAssistantsGallery.tsx
index f70ef0f7b8b..afb5f6a7a60 100644
--- a/web/src/app/assistants/gallery/WrappedAssistantsGallery.tsx
+++ b/web/src/app/assistants/gallery/WrappedAssistantsGallery.tsx
@@ -12,15 +12,11 @@ export default function WrappedAssistantsGallery({
   initiallyToggled,
   folders,
   openedFolders,
-  user,
-  assistants,
 }: {
   chatSessions: ChatSession[];
   folders: Folder[];
   initiallyToggled: boolean;
   openedFolders?: { [key: number]: boolean };
-  user: User | null;
-  assistants: Persona[];
 }) {
   return (
     <SidebarWrapper
@@ -29,17 +25,8 @@ export default function WrappedAssistantsGallery({
       chatSessions={chatSessions}
       folders={folders}
       openedFolders={openedFolders}
-      headerProps={{ user, page: "chat" }}
-      contentProps={{
-        assistants: assistants,
-        user: user,
-      }}
-      content={(contentProps) => (
-        <AssistantsGallery
-          assistants={contentProps.assistants}
-          user={contentProps.user}
-        />
-      )}
-    />
+    >
+      <AssistantsGallery />
+    </SidebarWrapper>
   );
 }
diff --git a/web/src/app/assistants/gallery/page.tsx b/web/src/app/assistants/gallery/page.tsx
index e955eb0117e..24b36cd2bcc 100644
--- a/web/src/app/assistants/gallery/page.tsx
+++ b/web/src/app/assistants/gallery/page.tsx
@@ -4,6 +4,7 @@ import { fetchChatData } from "@/lib/chat/fetchChatData";
 import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
 import WrappedAssistantsGallery from "./WrappedAssistantsGallery";
+import { AssistantsProvider } from "@/components/context/AssistantsContext";
 
 export default async function GalleryPage({
   searchParams,
@@ -26,22 +27,28 @@ export default async function GalleryPage({
     openedFolders,
     shouldShowWelcomeModal,
     toggleSidebar,
+    hasAnyConnectors,
+    hasImageCompatibleModel,
   } = data;
 
   return (
     <>
-      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
+      <AssistantsProvider
+        initialAssistants={assistants}
+        hasAnyConnectors={hasAnyConnectors}
+        hasImageCompatibleModel={hasImageCompatibleModel}
+      >
+        {shouldShowWelcomeModal && <WelcomeModal user={user} />}
 
-      <InstantSSRAutoRefresh />
+        <InstantSSRAutoRefresh />
 
-      <WrappedAssistantsGallery
-        initiallyToggled={toggleSidebar}
-        chatSessions={chatSessions}
-        folders={folders}
-        openedFolders={openedFolders}
-        user={user}
-        assistants={assistants}
-      />
+        <WrappedAssistantsGallery
+          initiallyToggled={toggleSidebar}
+          chatSessions={chatSessions}
+          folders={folders}
+          openedFolders={openedFolders}
+        />
+      </AssistantsProvider>
     </>
   );
 }
diff --git a/web/src/app/assistants/mine/AssistantSharingModal.tsx b/web/src/app/assistants/mine/AssistantSharingModal.tsx
index a30f3ce088d..96f9aeab923 100644
--- a/web/src/app/assistants/mine/AssistantSharingModal.tsx
+++ b/web/src/app/assistants/mine/AssistantSharingModal.tsx
@@ -16,6 +16,7 @@ import { Bubble } from "@/components/Bubble";
 import { useRouter } from "next/navigation";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 import { Spinner } from "@/components/Spinner";
+import { useAssistants } from "@/components/context/AssistantsContext";
 
 interface AssistantSharingModalProps {
   assistant: Persona;
@@ -32,7 +33,7 @@ export function AssistantSharingModal({
   show,
   onClose,
 }: AssistantSharingModalProps) {
-  const router = useRouter();
+  const { refreshAssistants } = useAssistants();
   const { popup, setPopup } = usePopup();
   const [isUpdating, setIsUpdating] = useState(false);
   const [selectedUsers, setSelectedUsers] = useState<MinimalUserSnapshot[]>([]);
@@ -54,7 +55,7 @@ export function AssistantSharingModal({
       assistant,
       selectedUsers.map((user) => user.id)
     );
-    router.refresh();
+    await refreshAssistants();
 
     const elapsedTime = Date.now() - startTime;
     const remainingTime = Math.max(0, 1000 - elapsedTime);
@@ -96,7 +97,7 @@ export function AssistantSharingModal({
                   assistant,
                   [u.id]
                 );
-                router.refresh();
+                await refreshAssistants();
 
                 const elapsedTime = Date.now() - startTime;
                 const remainingTime = Math.max(0, 1000 - elapsedTime);
@@ -138,7 +139,6 @@ export function AssistantSharingModal({
         onOutsideClick={onClose}
       >
         <div>
-          {isUpdating && <Spinner />}
           <p className="text-text-600 text-lg mb-6">
             Manage access to this assistant by sharing it with other users.
           </p>
@@ -225,6 +225,7 @@ export function AssistantSharingModal({
           )}
         </div>
       </Modal>
+      {isUpdating && <Spinner />}
     </>
   );
 }
diff --git a/web/src/app/assistants/mine/AssistantsList.tsx b/web/src/app/assistants/mine/AssistantsList.tsx
index 84eca4d13cf..26e081d94a3 100644
--- a/web/src/app/assistants/mine/AssistantsList.tsx
+++ b/web/src/app/assistants/mine/AssistantsList.tsx
@@ -55,12 +55,9 @@ import {
 } from "@/app/admin/assistants/lib";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
 import { MakePublicAssistantModal } from "@/app/chat/modal/MakePublicAssistantModal";
-import {
-  classifyAssistants,
-  getUserCreatedAssistants,
-  orderAssistantsForUser,
-} from "@/lib/assistants/utils";
 import { CustomTooltip } from "@/components/tooltip/CustomTooltip";
+import { useAssistants } from "@/components/context/AssistantsContext";
+import { useUser } from "@/components/user/UserProvider";
 
 function DraggableAssistantListItem(props: any) {
   const {
@@ -112,6 +109,7 @@ function AssistantListItem({
   setPopup: (popupSpec: PopupSpec | null) => void;
   isDragging?: boolean;
 }) {
+  const { refreshUser } = useUser();
   const router = useRouter();
   const [showSharingModal, setShowSharingModal] = useState(false);
 
@@ -206,7 +204,7 @@ function AssistantListItem({
                           message: `"${assistant.name}" has been removed from your list.`,
                           type: "success",
                         });
-                        router.refresh();
+                        await refreshUser();
                       } else {
                         setPopup({
                           message: `"${assistant.name}" could not be removed from your list.`,
@@ -229,7 +227,7 @@ function AssistantListItem({
                           message: `"${assistant.name}" has been added to your list.`,
                           type: "success",
                         });
-                        router.refresh();
+                        await refreshUser();
                       } else {
                         setPopup({
                           message: `"${assistant.name}" could not be added to your list.`,
@@ -284,32 +282,20 @@ function AssistantListItem({
     </>
   );
 }
-export function AssistantsList({
-  user,
-  assistants,
-}: {
-  user: User | null;
-  assistants: Persona[];
-}) {
-  // Define the distinct groups of assistants
-  const { visibleAssistants, hiddenAssistants } = classifyAssistants(
-    user,
-    assistants
-  );
+export function AssistantsList() {
+  const {
+    assistants,
+    ownedButHiddenAssistants,
+    finalAssistants,
+    refreshAssistants,
+  } = useAssistants();
 
-  const [currentlyVisibleAssistants, setCurrentlyVisibleAssistants] = useState<
-    Persona[]
-  >([]);
+  const [currentlyVisibleAssistants, setCurrentlyVisibleAssistants] =
+    useState(finalAssistants);
 
   useEffect(() => {
-    const orderedAssistants = orderAssistantsForUser(visibleAssistants, user);
-    setCurrentlyVisibleAssistants(orderedAssistants);
-  }, [assistants, user]);
-
-  const ownedButHiddenAssistants = getUserCreatedAssistants(
-    user,
-    hiddenAssistants
-  );
+    setCurrentlyVisibleAssistants(finalAssistants);
+  }, [finalAssistants]);
 
   const allAssistantIds = assistants.map((assistant) =>
     assistant.id.toString()
@@ -320,6 +306,8 @@ export function AssistantsList({
     null
   );
 
+  const { refreshUser, user } = useUser();
+
   const { popup, setPopup } = usePopup();
   const router = useRouter();
   const { data: users } = useSWR<MinimalUserSnapshot[]>(
@@ -338,18 +326,22 @@ export function AssistantsList({
     const { active, over } = event;
 
     if (over && active.id !== over.id) {
-      setCurrentlyVisibleAssistants((assistants) => {
-        const oldIndex = assistants.findIndex(
-          (a) => a.id.toString() === active.id
-        );
-        const newIndex = assistants.findIndex(
-          (a) => a.id.toString() === over.id
-        );
-        const newAssistants = arrayMove(assistants, oldIndex, newIndex);
-
-        updateUserAssistantList(newAssistants.map((a) => a.id));
-        return newAssistants;
-      });
+      const oldIndex = currentlyVisibleAssistants.findIndex(
+        (item) => item.id.toString() === active.id
+      );
+      const newIndex = currentlyVisibleAssistants.findIndex(
+        (item) => item.id.toString() === over.id
+      );
+      const updatedAssistants = arrayMove(
+        currentlyVisibleAssistants,
+        oldIndex,
+        newIndex
+      );
+
+      setCurrentlyVisibleAssistants(updatedAssistants);
+      await updateUserAssistantList(updatedAssistants.map((a) => a.id));
+      await refreshUser();
+      await refreshAssistants();
     }
   }
 
@@ -368,7 +360,7 @@ export function AssistantsList({
                 message: `"${deletingPersona.name}" has been deleted.`,
                 type: "success",
               });
-              router.refresh();
+              await refreshUser();
             } else {
               setPopup({
                 message: `"${deletingPersona.name}" could not be deleted.`,
@@ -389,7 +381,7 @@ export function AssistantsList({
               makePublicPersona.id,
               newPublicStatus
             );
-            router.refresh();
+            await refreshAssistants();
           }}
         />
       )}
diff --git a/web/src/app/assistants/mine/WrappedAssistantsMine.tsx b/web/src/app/assistants/mine/WrappedAssistantsMine.tsx
index 2e65d321e18..3e54662b8cb 100644
--- a/web/src/app/assistants/mine/WrappedAssistantsMine.tsx
+++ b/web/src/app/assistants/mine/WrappedAssistantsMine.tsx
@@ -3,23 +3,17 @@ import { AssistantsList } from "./AssistantsList";
 import SidebarWrapper from "../SidebarWrapper";
 import { ChatSession } from "@/app/chat/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
-import { Persona } from "@/app/admin/assistants/interfaces";
-import { User } from "@/lib/types";
 
 export default function WrappedAssistantsMine({
   chatSessions,
   initiallyToggled,
   folders,
   openedFolders,
-  user,
-  assistants,
 }: {
   chatSessions: ChatSession[];
   folders: Folder[];
   initiallyToggled: boolean;
   openedFolders?: { [key: number]: boolean };
-  user: User | null;
-  assistants: Persona[];
 }) {
   return (
     <SidebarWrapper
@@ -28,17 +22,8 @@ export default function WrappedAssistantsMine({
       chatSessions={chatSessions}
       folders={folders}
       openedFolders={openedFolders}
-      headerProps={{ user, page: "chat" }}
-      contentProps={{
-        assistants: assistants,
-        user: user,
-      }}
-      content={(contentProps) => (
-        <AssistantsList
-          assistants={contentProps.assistants}
-          user={contentProps.user}
-        />
-      )}
-    />
+    >
+      <AssistantsList />
+    </SidebarWrapper>
   );
 }
diff --git a/web/src/app/assistants/mine/WrappedInputPrompts.tsx b/web/src/app/assistants/mine/WrappedInputPrompts.tsx
index 31bcd28d97e..e39e695366c 100644
--- a/web/src/app/assistants/mine/WrappedInputPrompts.tsx
+++ b/web/src/app/assistants/mine/WrappedInputPrompts.tsx
@@ -2,7 +2,6 @@
 import SidebarWrapper from "../SidebarWrapper";
 import { ChatSession } from "@/app/chat/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
-import { Persona } from "@/app/admin/assistants/interfaces";
 import { User } from "@/lib/types";
 
 import { AssistantsPageTitle } from "../AssistantsPageTitle";
@@ -14,15 +13,11 @@ export default function WrappedPrompts({
   initiallyToggled,
   folders,
   openedFolders,
-  user,
-  assistants,
 }: {
   chatSessions: ChatSession[];
   folders: Folder[];
   initiallyToggled: boolean;
   openedFolders?: { [key: number]: boolean };
-  user: User | null;
-  assistants: Persona[];
 }) {
   const {
     data: promptLibrary,
@@ -39,24 +34,18 @@ export default function WrappedPrompts({
       chatSessions={chatSessions}
       folders={folders}
       openedFolders={openedFolders}
-      headerProps={{ user, page: "chat" }}
-      contentProps={{
-        assistants: assistants,
-        user: user,
-      }}
-      content={(contentProps) => (
-        <div className="mx-auto w-searchbar-xs 2xl:w-searchbar-sm 3xl:w-searchbar">
-          <AssistantsPageTitle>Prompt Gallery</AssistantsPageTitle>
-          <PromptSection
-            promptLibrary={promptLibrary || []}
-            isLoading={promptLibraryIsLoading}
-            error={promptLibraryError}
-            refreshPrompts={refreshPrompts}
-            isPublic={false}
-            centering
-          />
-        </div>
-      )}
-    />
+    >
+      <div className="mx-auto w-searchbar-xs 2xl:w-searchbar-sm 3xl:w-searchbar">
+        <AssistantsPageTitle>Prompt Gallery</AssistantsPageTitle>
+        <PromptSection
+          promptLibrary={promptLibrary || []}
+          isLoading={promptLibraryIsLoading}
+          error={promptLibraryError}
+          refreshPrompts={refreshPrompts}
+          isPublic={false}
+          centering
+        />
+      </div>
+    </SidebarWrapper>
   );
 }
diff --git a/web/src/app/assistants/mine/page.tsx b/web/src/app/assistants/mine/page.tsx
index 33e8ed1cc0e..a2655593e1b 100644
--- a/web/src/app/assistants/mine/page.tsx
+++ b/web/src/app/assistants/mine/page.tsx
@@ -4,6 +4,7 @@ import { fetchChatData } from "@/lib/chat/fetchChatData";
 import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
 import WrappedAssistantsMine from "./WrappedAssistantsMine";
+import { AssistantsProvider } from "@/components/context/AssistantsContext";
 
 export default async function GalleryPage({
   searchParams,
@@ -21,27 +22,30 @@ export default async function GalleryPage({
   const {
     user,
     chatSessions,
-    assistants,
     folders,
+    assistants,
     openedFolders,
     shouldShowWelcomeModal,
     toggleSidebar,
+    hasAnyConnectors,
+    hasImageCompatibleModel,
   } = data;
 
   return (
-    <>
+    <AssistantsProvider
+      initialAssistants={assistants}
+      hasAnyConnectors={hasAnyConnectors}
+      hasImageCompatibleModel={hasImageCompatibleModel}
+    >
       {shouldShowWelcomeModal && <WelcomeModal user={user} />}
 
       <InstantSSRAutoRefresh />
-
       <WrappedAssistantsMine
         initiallyToggled={toggleSidebar}
         chatSessions={chatSessions}
         folders={folders}
         openedFolders={openedFolders}
-        user={user}
-        assistants={assistants}
       />
-    </>
+    </AssistantsProvider>
   );
 }
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index bed27f6a19d..b67ec473026 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -101,12 +101,9 @@ import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
 import { SEARCH_TOOL_NAME } from "./tools/constants";
 import { useUser } from "@/components/user/UserProvider";
 import { ApiKeyModal } from "@/components/llm/ApiKeyModal";
-import {
-  classifyAssistants,
-  orderAssistantsForUser,
-} from "@/lib/assistants/utils";
 import BlurBackground from "./shared_chat_search/BlurBackground";
 import { NoAssistantModal } from "@/components/modals/NoAssistantModal";
+import { useAssistants } from "@/components/context/AssistantsContext";
 
 const TEMP_USER_MESSAGE_ID = -1;
 const TEMP_ASSISTANT_MESSAGE_ID = -2;
@@ -128,7 +125,6 @@ export function ChatPage({
     chatSessions,
     availableSources,
     availableDocumentSets,
-    availableAssistants,
     llmProviders,
     folders,
     openedFolders,
@@ -138,9 +134,11 @@ export function ChatPage({
     refreshChatSessions,
   } = useChatContext();
 
+  const { assistants: availableAssistants, finalAssistants } = useAssistants();
+
   const [showApiKeyModal, setShowApiKeyModal] = useState(true);
 
-  const { user, refreshUser, isAdmin, isLoadingUser } = useUser();
+  const { user, isAdmin, isLoadingUser } = useUser();
 
   const existingChatIdRaw = searchParams.get("chatId");
   const currentPersonaId = searchParams.get(SEARCH_PARAM_NAMES.PERSONA_ID);
@@ -157,18 +155,6 @@ export function ChatPage({
   // Useful for determining which session has been loaded (i.e. still on `new, empty session` or `previous session`)
   const loadedIdSessionRef = useRef<string | null>(existingChatSessionId);
 
-  // Assistants in order
-  const { finalAssistants } = useMemo(() => {
-    const { visibleAssistants, hiddenAssistants: _ } = classifyAssistants(
-      user,
-      availableAssistants
-    );
-    const finalAssistants = user
-      ? orderAssistantsForUser(visibleAssistants, user)
-      : visibleAssistants;
-    return { finalAssistants };
-  }, [user, availableAssistants]);
-
   const existingChatSessionAssistantId = selectedChatSession?.persona_id;
   const [selectedAssistant, setSelectedAssistant] = useState<
     Persona | undefined
@@ -1833,7 +1819,6 @@ export function ChatPage({
           setPopup={setPopup}
           setLlmOverride={llmOverrideManager.setGlobalDefault}
           defaultModel={user?.preferences.default_model!}
-          refreshUser={refreshUser}
           llmProviders={llmProviders}
           onClose={() => setSettingsToggled(false)}
         />
@@ -1953,7 +1938,6 @@ export function ChatPage({
                       : undefined
                   }
                   toggleSidebar={toggleSidebar}
-                  user={user}
                   currentChatSession={selectedChatSession}
                 />
               )}
@@ -2438,7 +2422,6 @@ export function ChatPage({
                               showDocs={() => setDocumentSelection(true)}
                               selectedDocuments={selectedDocuments}
                               // assistant stuff
-                              assistantOptions={finalAssistants}
                               selectedAssistant={liveAssistant}
                               setSelectedAssistant={onAssistantChange}
                               setAlternativeAssistant={setAlternativeAssistant}
@@ -2454,7 +2437,6 @@ export function ChatPage({
                               handleFileUpload={handleImageUpload}
                               textAreaRef={textAreaRef}
                               chatSessionId={chatSessionIdRef.current!}
-                              refreshUser={refreshUser}
                             />
 
                             {enterpriseSettings &&
diff --git a/web/src/app/chat/input/ChatInputBar.tsx b/web/src/app/chat/input/ChatInputBar.tsx
index e6e220f9ff1..78afa5381d6 100644
--- a/web/src/app/chat/input/ChatInputBar.tsx
+++ b/web/src/app/chat/input/ChatInputBar.tsx
@@ -34,6 +34,7 @@ import { Hoverable } from "@/components/Hoverable";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { ChatState } from "../types";
 import UnconfiguredProviderText from "@/components/chat_search/UnconfiguredProviderText";
+import { useAssistants } from "@/components/context/AssistantsContext";
 
 const MAX_INPUT_HEIGHT = 200;
 
@@ -52,7 +53,6 @@ export function ChatInputBar({
 
   // assistants
   selectedAssistant,
-  assistantOptions,
   setSelectedAssistant,
   setAlternativeAssistant,
 
@@ -63,7 +63,6 @@ export function ChatInputBar({
   alternativeAssistant,
   chatSessionId,
   inputPrompts,
-  refreshUser,
 }: {
   showConfigureAPIKey: () => void;
   openModelSettings: () => void;
@@ -71,7 +70,6 @@ export function ChatInputBar({
   stopGenerating: () => void;
   showDocs: () => void;
   selectedDocuments: DanswerDocument[];
-  assistantOptions: Persona[];
   setAlternativeAssistant: (alternativeAssistant: Persona | null) => void;
   setSelectedAssistant: (assistant: Persona) => void;
   inputPrompts: InputPrompt[];
@@ -87,7 +85,6 @@ export function ChatInputBar({
   handleFileUpload: (files: File[]) => void;
   textAreaRef: React.RefObject<HTMLTextAreaElement>;
   chatSessionId?: string;
-  refreshUser: () => void;
 }) {
   useEffect(() => {
     const textarea = textAreaRef.current;
@@ -118,6 +115,7 @@ export function ChatInputBar({
   };
 
   const settings = useContext(SettingsContext);
+  const { finalAssistants: assistantOptions } = useAssistants();
 
   const { llmProviders } = useChatContext();
   const [_, llmName] = getFinalLLM(llmProviders, selectedAssistant, null);
@@ -527,14 +525,12 @@ export function ChatInputBar({
                 removePadding
                 content={(close) => (
                   <AssistantsTab
-                    availableAssistants={assistantOptions}
                     llmProviders={llmProviders}
                     selectedAssistant={selectedAssistant}
                     onSelect={(assistant) => {
                       setSelectedAssistant(assistant);
                       close();
                     }}
-                    refreshUser={refreshUser}
                   />
                 )}
                 flexPriority="shrink"
diff --git a/web/src/app/chat/modal/SetDefaultModelModal.tsx b/web/src/app/chat/modal/SetDefaultModelModal.tsx
index 46482a9ee72..5a47d9e66f2 100644
--- a/web/src/app/chat/modal/SetDefaultModelModal.tsx
+++ b/web/src/app/chat/modal/SetDefaultModelModal.tsx
@@ -8,6 +8,7 @@ import { destructureValue, structureValue } from "@/lib/llm/utils";
 import { setUserDefaultModel } from "@/lib/users/UserSettings";
 import { useRouter } from "next/navigation";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
+import { useUser } from "@/components/user/UserProvider";
 
 export function SetDefaultModelModal({
   setPopup,
@@ -15,15 +16,14 @@ export function SetDefaultModelModal({
   onClose,
   setLlmOverride,
   defaultModel,
-  refreshUser,
 }: {
   setPopup: (popupSpec: PopupSpec | null) => void;
   llmProviders: LLMProviderDescriptor[];
   setLlmOverride: Dispatch<SetStateAction<LlmOverride>>;
   onClose: () => void;
   defaultModel: string | null;
-  refreshUser: () => void;
 }) {
+  const { refreshUser } = useUser();
   const containerRef = useRef<HTMLDivElement>(null);
   const messageRef = useRef<HTMLDivElement>(null);
 
diff --git a/web/src/app/chat/modal/configuration/AssistantsTab.tsx b/web/src/app/chat/modal/configuration/AssistantsTab.tsx
index fca691d8fd2..c62fab443ba 100644
--- a/web/src/app/chat/modal/configuration/AssistantsTab.tsx
+++ b/web/src/app/chat/modal/configuration/AssistantsTab.tsx
@@ -16,25 +16,29 @@ import {
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { getFinalLLM } from "@/lib/llm/utils";
-import React, { useState } from "react";
+import React, { useEffect, useState } from "react";
 import { updateUserAssistantList } from "@/lib/assistants/updateAssistantPreferences";
 import { DraggableAssistantCard } from "@/components/assistants/AssistantCards";
+import { useAssistants } from "@/components/context/AssistantsContext";
+import { useUser } from "@/components/user/UserProvider";
 
 export function AssistantsTab({
   selectedAssistant,
-  availableAssistants,
   llmProviders,
   onSelect,
-  refreshUser,
 }: {
   selectedAssistant: Persona;
-  availableAssistants: Persona[];
   llmProviders: LLMProviderDescriptor[];
   onSelect: (assistant: Persona) => void;
-  refreshUser: () => void;
 }) {
+  const { refreshUser } = useUser();
   const [_, llmName] = getFinalLLM(llmProviders, null, null);
-  const [assistants, setAssistants] = useState(availableAssistants);
+  const { finalAssistants, refreshAssistants } = useAssistants();
+  const [assistants, setAssistants] = useState(finalAssistants);
+
+  useEffect(() => {
+    setAssistants(finalAssistants);
+  }, [finalAssistants]);
 
   const sensors = useSensors(
     useSensor(PointerSensor),
@@ -57,7 +61,8 @@ export function AssistantsTab({
 
       setAssistants(updatedAssistants);
       await updateUserAssistantList(updatedAssistants.map((a) => a.id));
-      refreshUser();
+      await refreshUser();
+      await refreshAssistants();
     }
   }
 
diff --git a/web/src/app/chat/page.tsx b/web/src/app/chat/page.tsx
index c192c55aa38..3c872390c1b 100644
--- a/web/src/app/chat/page.tsx
+++ b/web/src/app/chat/page.tsx
@@ -5,6 +5,7 @@ import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrap
 import { ChatProvider } from "@/components/context/ChatContext";
 import { fetchChatData } from "@/lib/chat/fetchChatData";
 import WrappedChat from "./WrappedChat";
+import { AssistantsProvider } from "@/components/context/AssistantsContext";
 
 export default async function Page({
   searchParams,
@@ -24,7 +25,6 @@ export default async function Page({
     chatSessions,
     availableSources,
     documentSets,
-    assistants,
     tags,
     llmProviders,
     folders,
@@ -32,31 +32,38 @@ export default async function Page({
     openedFolders,
     defaultAssistantId,
     shouldShowWelcomeModal,
+    assistants,
     userInputPrompts,
+    hasAnyConnectors,
+    hasImageCompatibleModel,
   } = data;
 
   return (
     <>
       <InstantSSRAutoRefresh />
       {shouldShowWelcomeModal && <WelcomeModal user={user} />}
-
-      <ChatProvider
-        value={{
-          chatSessions,
-          availableSources,
-          availableDocumentSets: documentSets,
-          availableAssistants: assistants,
-          availableTags: tags,
-          llmProviders,
-          folders,
-          openedFolders,
-          userInputPrompts,
-          shouldShowWelcomeModal,
-          defaultAssistantId,
-        }}
+      <AssistantsProvider
+        initialAssistants={assistants}
+        hasAnyConnectors={hasAnyConnectors}
+        hasImageCompatibleModel={hasImageCompatibleModel}
       >
-        <WrappedChat initiallyToggled={toggleSidebar} />
-      </ChatProvider>
+        <ChatProvider
+          value={{
+            chatSessions,
+            availableSources,
+            availableDocumentSets: documentSets,
+            availableTags: tags,
+            llmProviders,
+            folders,
+            openedFolders,
+            userInputPrompts,
+            shouldShowWelcomeModal,
+            defaultAssistantId,
+          }}
+        >
+          <WrappedChat initiallyToggled={toggleSidebar} />
+        </ChatProvider>
+      </AssistantsProvider>
     </>
   );
 }
diff --git a/web/src/app/chat/shared/[chatId]/page.tsx b/web/src/app/chat/shared/[chatId]/page.tsx
index e620e85cded..27728123262 100644
--- a/web/src/app/chat/shared/[chatId]/page.tsx
+++ b/web/src/app/chat/shared/[chatId]/page.tsx
@@ -67,7 +67,7 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   return (
     <div>
       <div className="absolute top-0 z-40 w-full">
-        <FunctionalHeader page="shared" user={user} />
+        <FunctionalHeader page="shared" />
       </div>
 
       <div className="flex relative bg-background text-default overflow-hidden pt-16 h-screen">
diff --git a/web/src/app/prompts/page.tsx b/web/src/app/prompts/page.tsx
index c3933e8ab10..20c9038ac2e 100644
--- a/web/src/app/prompts/page.tsx
+++ b/web/src/app/prompts/page.tsx
@@ -16,14 +16,7 @@ export default async function GalleryPage({
     redirect(data.redirect);
   }
 
-  const {
-    user,
-    chatSessions,
-    assistants,
-    folders,
-    openedFolders,
-    toggleSidebar,
-  } = data;
+  const { chatSessions, folders, openedFolders, toggleSidebar } = data;
 
   return (
     <WrappedPrompts
@@ -31,8 +24,6 @@ export default async function GalleryPage({
       chatSessions={chatSessions}
       folders={folders}
       openedFolders={openedFolders}
-      user={user}
-      assistants={assistants}
     />
   );
 }
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index 3d26053dd41..d515a1fb8df 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -36,6 +36,7 @@ import WrappedSearch from "./WrappedSearch";
 import { SearchProvider } from "@/components/context/SearchContext";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "../admin/configuration/llm/interfaces";
+import { AssistantsProvider } from "@/components/context/AssistantsContext";
 import { headers } from "next/headers";
 
 export default async function Home({
@@ -193,36 +194,39 @@ export default async function Home({
       <HealthCheckBanner />
       {shouldShowWelcomeModal && <WelcomeModal user={user} />}
       <InstantSSRAutoRefresh />
-
       {shouldDisplayNoSourcesModal && <NoSourcesModal />}
-
       {shouldDisplaySourcesIncompleteModal && (
         <NoCompleteSourcesModal ccPairs={ccPairs} />
       )}
-
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 
       Only used in the EE version of the app. */}
       <ChatPopup />
-
-      <SearchProvider
-        value={{
-          querySessions,
-          ccPairs,
-          documentSets,
-          assistants,
-          tags,
-          agenticSearchEnabled,
-          disabledAgentic: DISABLE_LLM_DOC_RELEVANCE,
-          initiallyToggled: toggleSidebar,
-          shouldShowWelcomeModal,
-          shouldDisplayNoSources: shouldDisplayNoSourcesModal,
-        }}
+      <AssistantsProvider
+        initialAssistants={assistants}
+        hasAnyConnectors={hasAnyConnectors}
+        hasImageCompatibleModel={false}
       >
-        <WrappedSearch
-          initiallyToggled={toggleSidebar}
-          searchTypeDefault={searchTypeDefault}
-        />
-      </SearchProvider>
+        <SearchProvider
+          value={{
+            querySessions,
+            ccPairs,
+            documentSets,
+            assistants,
+            tags,
+            agenticSearchEnabled,
+            disabledAgentic: DISABLE_LLM_DOC_RELEVANCE,
+            initiallyToggled: toggleSidebar,
+            shouldShowWelcomeModal,
+            shouldDisplayNoSources: shouldDisplayNoSourcesModal,
+          }}
+        >
+          <WrappedSearch
+            initiallyToggled={toggleSidebar}
+            searchTypeDefault={searchTypeDefault}
+          />
+        </SearchProvider>
+      </AssistantsProvider>
+      s
     </>
   );
 }
diff --git a/web/src/components/UserDropdown.tsx b/web/src/components/UserDropdown.tsx
index 00c3b83c1fa..80c6be1785f 100644
--- a/web/src/components/UserDropdown.tsx
+++ b/web/src/components/UserDropdown.tsx
@@ -4,19 +4,20 @@ import { useState, useRef, useContext, useEffect, useMemo } from "react";
 import { FiLogOut } from "react-icons/fi";
 import Link from "next/link";
 import { useRouter, usePathname, useSearchParams } from "next/navigation";
-import { User, UserRole } from "@/lib/types";
+import { UserRole } from "@/lib/types";
 import { checkUserIsNoAuthUser, logout } from "@/lib/user";
 import { Popover } from "./popover/Popover";
 import { LOGOUT_DISABLED } from "@/lib/constants";
 import { SettingsContext } from "./settings/SettingsProvider";
-import {
-  AssistantsIconSkeleton,
-  LightSettingsIcon,
-  UsersIcon,
-} from "./icons/icons";
+import { BellIcon, LightSettingsIcon } from "./icons/icons";
 import { pageType } from "@/app/chat/sessionSidebar/types";
-import { NavigationItem } from "@/app/admin/settings/interfaces";
+import { NavigationItem, Notification } from "@/app/admin/settings/interfaces";
 import DynamicFaIcon, { preloadIcons } from "./icons/DynamicFaIcon";
+import { useUser } from "./user/UserProvider";
+import { usePaidEnterpriseFeaturesEnabled } from "./settings/usePaidEnterpriseFeaturesEnabled";
+import { Notifications } from "./chat_search/Notifications";
+import useSWR from "swr";
+import { errorHandlingFetcher } from "@/lib/fetcher";
 
 interface DropdownOptionProps {
   href?: string;
@@ -55,24 +56,25 @@ const DropdownOption: React.FC<DropdownOptionProps> = ({
   }
 };
 
-export function UserDropdown({
-  user,
-  page,
-}: {
-  user: User | null;
-  page?: pageType;
-}) {
+export function UserDropdown({ page }: { page?: pageType }) {
+  const { user } = useUser();
   const [userInfoVisible, setUserInfoVisible] = useState(false);
   const userInfoRef = useRef<HTMLDivElement>(null);
   const router = useRouter();
   const pathname = usePathname();
   const searchParams = useSearchParams();
+  const [showNotifications, setShowNotifications] = useState(false);
 
   const combinedSettings = useContext(SettingsContext);
   const customNavItems: NavigationItem[] = useMemo(
     () => combinedSettings?.enterpriseSettings?.custom_nav_items || [],
     [combinedSettings]
   );
+  const {
+    data: notifications,
+    error,
+    mutate: refreshNotifications,
+  } = useSWR<Notification[]>("/api/notifications", errorHandlingFetcher);
 
   useEffect(() => {
     const iconNames = customNavItems
@@ -110,15 +112,20 @@ export function UserDropdown({
   const showLogout =
     user && !checkUserIsNoAuthUser(user.id) && !LOGOUT_DISABLED;
 
+  const onOpenChange = (open: boolean) => {
+    setUserInfoVisible(open);
+    setShowNotifications(false);
+  };
+
   return (
     <div className="group relative" ref={userInfoRef}>
       <Popover
         open={userInfoVisible}
-        onOpenChange={setUserInfoVisible}
+        onOpenChange={onOpenChange}
         content={
           <div
             onClick={() => setUserInfoVisible(!userInfoVisible)}
-            className="flex cursor-pointer"
+            className="flex relative cursor-pointer"
           >
             <div
               className="
@@ -138,6 +145,9 @@ export function UserDropdown({
             >
               {user && user.email ? user.email[0].toUpperCase() : "A"}
             </div>
+            {notifications && notifications.length > 0 && (
+              <div className="absolute right-0 top-0 w-2 h-2 bg-red-500 rounded-full"></div>
+            )}
           </div>
         }
         popover={
@@ -161,14 +171,22 @@ export function UserDropdown({
                 overscroll-contain
               `}
           >
-            {customNavItems.map((item, i) => (
-              <DropdownOption
-                key={i}
-                href={item.link}
-                icon={
-                  item.svg_logo ? (
-                    <div
-                      className="
+            {page != "admin" && showNotifications ? (
+              <Notifications
+                navigateToDropdown={() => setShowNotifications(false)}
+                notifications={notifications || []}
+                refreshNotifications={refreshNotifications}
+              />
+            ) : (
+              <>
+                {customNavItems.map((item, i) => (
+                  <DropdownOption
+                    key={i}
+                    href={item.link}
+                    icon={
+                      item.svg_logo ? (
+                        <div
+                          className="
                         h-4
                         w-4
                         my-auto
@@ -178,57 +196,72 @@ export function UserDropdown({
                         items-center
                         justify-center
                       "
-                      aria-label={item.title}
-                    >
-                      <svg
-                        viewBox="0 0 24 24"
-                        width="100%"
-                        height="100%"
-                        preserveAspectRatio="xMidYMid meet"
-                        dangerouslySetInnerHTML={{ __html: item.svg_logo }}
-                      />
-                    </div>
-                  ) : (
-                    <DynamicFaIcon
-                      name={item.icon!}
-                      className="h-4 w-4 my-auto mr-2"
+                          aria-label={item.title}
+                        >
+                          <svg
+                            viewBox="0 0 24 24"
+                            width="100%"
+                            height="100%"
+                            preserveAspectRatio="xMidYMid meet"
+                            dangerouslySetInnerHTML={{ __html: item.svg_logo }}
+                          />
+                        </div>
+                      ) : (
+                        <DynamicFaIcon
+                          name={item.icon!}
+                          className="h-4 w-4 my-auto mr-2"
+                        />
+                      )
+                    }
+                    label={item.title}
+                    openInNewTab
+                  />
+                ))}
+
+                {showAdminPanel ? (
+                  <DropdownOption
+                    href="/admin/indexing/status"
+                    icon={
+                      <LightSettingsIcon className="h-5 w-5 my-auto mr-2" />
+                    }
+                    label="Admin Panel"
+                  />
+                ) : (
+                  showCuratorPanel && (
+                    <DropdownOption
+                      href="/admin/indexing/status"
+                      icon={
+                        <LightSettingsIcon className="h-5 w-5 my-auto mr-2" />
+                      }
+                      label="Curator Panel"
                     />
                   )
-                }
-                label={item.title}
-                openInNewTab
-              />
-            ))}
+                )}
 
-            {showAdminPanel ? (
-              <DropdownOption
-                href="/admin/indexing/status"
-                icon={<LightSettingsIcon className="h-5 w-5 my-auto mr-2" />}
-                label="Admin Panel"
-              />
-            ) : (
-              showCuratorPanel && (
                 <DropdownOption
-                  href="/admin/indexing/status"
-                  icon={<LightSettingsIcon className="h-5 w-5 my-auto mr-2" />}
-                  label="Curator Panel"
+                  onClick={() => {
+                    setUserInfoVisible(true);
+                    setShowNotifications(true);
+                  }}
+                  icon={<BellIcon className="h-5 w-5 my-auto mr-2" />}
+                  label={`Notifications ${notifications && notifications.length > 0 ? `(${notifications.length})` : ""}`}
                 />
-              )
-            )}
 
-            {showLogout &&
-              (showCuratorPanel ||
-                showAdminPanel ||
-                customNavItems.length > 0) && (
-                <div className="border-t border-border my-1" />
-              )}
-
-            {showLogout && (
-              <DropdownOption
-                onClick={handleLogout}
-                icon={<FiLogOut className="my-auto mr-2 text-lg" />}
-                label="Log out"
-              />
+                {showLogout &&
+                  (showCuratorPanel ||
+                    showAdminPanel ||
+                    customNavItems.length > 0) && (
+                    <div className="border-t border-border my-1" />
+                  )}
+
+                {showLogout && (
+                  <DropdownOption
+                    onClick={handleLogout}
+                    icon={<FiLogOut className="my-auto mr-2 text-lg" />}
+                    label="Log out"
+                  />
+                )}
+              </>
             )}
           </div>
         }
diff --git a/web/src/components/admin/ClientLayout.tsx b/web/src/components/admin/ClientLayout.tsx
index 68154c90387..16500b5e0eb 100644
--- a/web/src/components/admin/ClientLayout.tsx
+++ b/web/src/components/admin/ClientLayout.tsx
@@ -418,7 +418,7 @@ export function ClientLayout({
         </div>
         <div className="pb-8 relative h-full overflow-y-auto w-full">
           <div className="fixed bg-background left-0 gap-x-4 mb-8 px-4 py-2 w-full items-center flex justify-end">
-            <UserDropdown user={user} />
+            <UserDropdown />
           </div>
           <div className="pt-20 flex overflow-y-auto h-full px-4 md:px-12">
             {children}
diff --git a/web/src/components/chat_search/Header.tsx b/web/src/components/chat_search/Header.tsx
index c19d46ce874..f884d11f2e4 100644
--- a/web/src/components/chat_search/Header.tsx
+++ b/web/src/components/chat_search/Header.tsx
@@ -11,9 +11,9 @@ import { pageType } from "@/app/chat/sessionSidebar/types";
 import { useRouter } from "next/navigation";
 import { ChatBanner } from "@/app/chat/ChatBanner";
 import LogoType from "../header/LogoType";
+import { useUser } from "../user/UserProvider";
 
 export default function FunctionalHeader({
-  user,
   page,
   currentChatSession,
   setSharingModalVisible,
@@ -23,7 +23,6 @@ export default function FunctionalHeader({
 }: {
   reset?: () => void;
   page: pageType;
-  user: User | null;
   sidebarToggled?: boolean;
   currentChatSession?: ChatSession | null | undefined;
   setSharingModalVisible?: (value: SetStateAction<boolean>) => void;
@@ -109,7 +108,7 @@ export default function FunctionalHeader({
           )}
 
           <div className="mobile:hidden flex my-auto">
-            <UserDropdown user={user} />
+            <UserDropdown />
           </div>
           <Link
             className="desktop:hidden my-auto"
diff --git a/web/src/components/chat_search/Notifications.tsx b/web/src/components/chat_search/Notifications.tsx
new file mode 100644
index 00000000000..31853060f41
--- /dev/null
+++ b/web/src/components/chat_search/Notifications.tsx
@@ -0,0 +1,231 @@
+import React, { useEffect, useState } from "react";
+import useSWR from "swr";
+import { Persona } from "@/app/admin/assistants/interfaces";
+import {
+  Notification,
+  NotificationType,
+} from "@/app/admin/settings/interfaces";
+import { errorHandlingFetcher } from "@/lib/fetcher";
+import { AssistantIcon } from "@/components/assistants/AssistantIcon";
+import { addAssistantToList } from "@/lib/assistants/updateAssistantPreferences";
+import { useAssistants } from "../context/AssistantsContext";
+import { useUser } from "../user/UserProvider";
+import { XIcon } from "../icons/icons";
+import { Spinner } from "@phosphor-icons/react";
+
+export const Notifications = ({
+  notifications,
+  refreshNotifications,
+  navigateToDropdown,
+}: {
+  notifications: Notification[];
+  refreshNotifications: () => void;
+  navigateToDropdown: () => void;
+}) => {
+  const [showDropdown, setShowDropdown] = useState(false);
+
+  const { refreshAssistants } = useAssistants();
+
+  const { refreshUser } = useUser();
+  const [personas, setPersonas] = useState<Record<number, Persona> | undefined>(
+    undefined
+  );
+
+  useEffect(() => {
+    const fetchPersonas = async () => {
+      if (notifications) {
+        const personaIds = notifications
+          .filter(
+            (n) =>
+              n.notif_type.toLowerCase() === "persona_shared" &&
+              n.additional_data?.persona_id !== undefined
+          )
+          .map((n) => n.additional_data!.persona_id!);
+
+        if (personaIds.length > 0) {
+          const queryParams = personaIds
+            .map((id) => `persona_ids=${id}`)
+            .join("&");
+          try {
+            const response = await fetch(`/api/persona?${queryParams}`);
+
+            if (!response.ok) {
+              throw new Error(
+                `Error fetching personas: ${response.statusText}`
+              );
+            }
+            const personasData: Persona[] = await response.json();
+            setPersonas(
+              personasData.reduce(
+                (acc, persona) => {
+                  acc[persona.id] = persona;
+                  return acc;
+                },
+                {} as Record<number, Persona>
+              )
+            );
+          } catch (err) {
+            console.error("Failed to fetch personas:", err);
+          }
+        }
+      }
+    };
+
+    fetchPersonas();
+  }, [notifications]);
+
+  const dismissNotification = async (notificationId: number) => {
+    try {
+      await fetch(`/api/notifications/${notificationId}/dismiss`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+      });
+      refreshNotifications();
+    } catch (error) {
+      console.error("Error dismissing notification:", error);
+    }
+  };
+
+  const handleAssistantShareAcceptance = async (
+    notification: Notification,
+    persona: Persona
+  ) => {
+    addAssistantToList(persona.id);
+    await dismissNotification(notification.id);
+    await refreshUser();
+    await refreshAssistants();
+  };
+
+  const sortedNotifications = notifications
+    ? notifications
+        .filter((notification) => {
+          const personaId = notification.additional_data?.persona_id;
+          return (
+            personaId !== undefined &&
+            personas &&
+            personas[personaId] !== undefined
+          );
+        })
+        .sort(
+          (a, b) =>
+            new Date(b.time_created).getTime() -
+            new Date(a.time_created).getTime()
+        )
+    : [];
+
+  useEffect(() => {
+    const handleClickOutside = (event: MouseEvent) => {
+      if (
+        showDropdown &&
+        !(event.target as Element).closest(".notification-dropdown")
+      ) {
+        setShowDropdown(false);
+      }
+    };
+    document.addEventListener("mousedown", handleClickOutside);
+
+    return () => {
+      document.removeEventListener("mousedown", handleClickOutside);
+    };
+  }, [showDropdown]);
+  return (
+    <div className="w-full">
+      <button
+        onClick={navigateToDropdown}
+        className="absolute right-2 text-background-600 hover:text-background-900 transition-colors duration-150 ease-in-out rounded-full focus:outline-none focus:ring-2 focus:ring-blue-500"
+        aria-label="Back"
+      >
+        <XIcon className="w-5 h-5" />
+      </button>
+
+      {notifications && notifications.length > 0 ? (
+        sortedNotifications.length > 0 && personas ? (
+          sortedNotifications
+            .filter(
+              (notification) =>
+                notification.notif_type === NotificationType.PERSONA_SHARED
+            )
+            .map((notification) => {
+              const persona = notification.additional_data?.persona_id
+                ? personas[notification.additional_data.persona_id]
+                : null;
+
+              return (
+                <div
+                  key={notification.id}
+                  className="w-72 px-4 py-3 border-b last:border-b-0 hover:bg-gray-50 transition duration-150 ease-in-out"
+                >
+                  <div className="flex items-start">
+                    {persona && (
+                      <div className="mt-2 flex-shrink-0 mr-3">
+                        <AssistantIcon assistant={persona} size="small" />
+                      </div>
+                    )}
+                    <div className="flex-grow">
+                      <p className="font-semibold text-sm text-gray-800">
+                        New Assistant Shared: {persona?.name}
+                      </p>
+                      {persona?.description && (
+                        <p className="text-xs text-gray-600 mt-1">
+                          {persona.description}
+                        </p>
+                      )}
+                      {persona && (
+                        <div className="mt-2">
+                          {persona.tools.length > 0 && (
+                            <p className="text-xs text-gray-500">
+                              Tools:{" "}
+                              {persona.tools
+                                .map((tool) => tool.name)
+                                .join(", ")}
+                            </p>
+                          )}
+                          {persona.document_sets.length > 0 && (
+                            <p className="text-xs text-gray-500">
+                              Document Sets:{" "}
+                              {persona.document_sets
+                                .map((set) => set.name)
+                                .join(", ")}
+                            </p>
+                          )}
+                          {persona.llm_model_version_override && (
+                            <p className="text-xs text-gray-500">
+                              Model: {persona.llm_model_version_override}
+                            </p>
+                          )}
+                        </div>
+                      )}
+                    </div>
+                  </div>
+                  <div className="flex justify-end mt-2 space-x-2">
+                    <button
+                      onClick={() =>
+                        handleAssistantShareAcceptance(notification, persona!)
+                      }
+                      className="px-3 py-1 text-sm font-medium text-blue-600 hover:text-blue-800 transition duration-150 ease-in-out"
+                    >
+                      Accept
+                    </button>
+                    <button
+                      onClick={() => dismissNotification(notification.id)}
+                      className="px-3 py-1 text-sm font-medium text-gray-600 hover:text-gray-800 transition duration-150 ease-in-out"
+                    >
+                      Dismiss
+                    </button>
+                  </div>
+                </div>
+              );
+            })
+        ) : (
+          <div className="flex h-20 justify-center items-center w-72">
+            <Spinner size={20} />
+          </div>
+        )
+      ) : (
+        <div className="px-4 py-3 text-center text-gray-600">
+          No new notifications
+        </div>
+      )}
+    </div>
+  );
+};
diff --git a/web/src/components/context/AssistantsContext.tsx b/web/src/components/context/AssistantsContext.tsx
new file mode 100644
index 00000000000..30503a29c67
--- /dev/null
+++ b/web/src/components/context/AssistantsContext.tsx
@@ -0,0 +1,119 @@
+"use client";
+import React, { createContext, useState, useContext, useMemo } from "react";
+import { Persona } from "@/app/admin/assistants/interfaces";
+import {
+  classifyAssistants,
+  orderAssistantsForUser,
+  getUserCreatedAssistants,
+} from "@/lib/assistants/utils";
+import { useUser } from "../user/UserProvider";
+
+interface AssistantsContextProps {
+  assistants: Persona[];
+  visibleAssistants: Persona[];
+  hiddenAssistants: Persona[];
+  finalAssistants: Persona[];
+  ownedButHiddenAssistants: Persona[];
+  refreshAssistants: () => Promise<void>;
+}
+
+const AssistantsContext = createContext<AssistantsContextProps | undefined>(
+  undefined
+);
+
+export const AssistantsProvider: React.FC<{
+  children: React.ReactNode;
+  initialAssistants: Persona[];
+  hasAnyConnectors: boolean;
+  hasImageCompatibleModel: boolean;
+}> = ({
+  children,
+  initialAssistants,
+  hasAnyConnectors,
+  hasImageCompatibleModel,
+}) => {
+  const [assistants, setAssistants] = useState<Persona[]>(
+    initialAssistants || []
+  );
+  const { user } = useUser();
+
+  const refreshAssistants = async () => {
+    try {
+      const response = await fetch("/api/persona", {
+        method: "GET",
+        headers: {
+          "Content-Type": "application/json",
+        },
+      });
+      if (!response.ok) throw new Error("Failed to fetch assistants");
+      let assistants: Persona[] = await response.json();
+      if (!hasImageCompatibleModel) {
+        assistants = assistants.filter(
+          (assistant) =>
+            !assistant.tools.some(
+              (tool) => tool.in_code_tool_id === "ImageGenerationTool"
+            )
+        );
+      }
+      if (!hasAnyConnectors) {
+        assistants = assistants.filter(
+          (assistant) => assistant.num_chunks === 0
+        );
+      }
+      setAssistants(assistants);
+    } catch (error) {
+      console.error("Error refreshing assistants:", error);
+    }
+  };
+
+  const {
+    visibleAssistants,
+    hiddenAssistants,
+    finalAssistants,
+    ownedButHiddenAssistants,
+  } = useMemo(() => {
+    const { visibleAssistants, hiddenAssistants } = classifyAssistants(
+      user,
+      assistants
+    );
+
+    const finalAssistants = user
+      ? orderAssistantsForUser(visibleAssistants, user)
+      : visibleAssistants;
+
+    const ownedButHiddenAssistants = getUserCreatedAssistants(
+      user,
+      hiddenAssistants
+    );
+
+    return {
+      visibleAssistants,
+      hiddenAssistants,
+      finalAssistants,
+      ownedButHiddenAssistants,
+    };
+  }, [user, assistants]);
+
+  return (
+    <AssistantsContext.Provider
+      value={{
+        assistants,
+        visibleAssistants,
+        hiddenAssistants,
+        finalAssistants,
+        ownedButHiddenAssistants,
+        refreshAssistants,
+      }}
+    >
+      {children}
+    </AssistantsContext.Provider>
+  );
+};
+
+export const useAssistants = (): AssistantsContextProps => {
+  const context = useContext(AssistantsContext);
+  if (!context) {
+    throw new Error("useAssistants must be used within an AssistantsProvider");
+  }
+  return context;
+};
diff --git a/web/src/components/context/ChatContext.tsx b/web/src/components/context/ChatContext.tsx
index 06a63c903cc..9ecd0dcefc0 100644
--- a/web/src/components/context/ChatContext.tsx
+++ b/web/src/components/context/ChatContext.tsx
@@ -7,12 +7,12 @@ import { Persona } from "@/app/admin/assistants/interfaces";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
 import { InputPrompt } from "@/app/admin/prompt-library/interfaces";
+import { personaComparator } from "@/app/admin/assistants/lib";
 
 interface ChatContextProps {
   chatSessions: ChatSession[];
   availableSources: ValidSources[];
   availableDocumentSets: DocumentSet[];
-  availableAssistants: Persona[];
   availableTags: Tag[];
   llmProviders: LLMProviderDescriptor[];
   folders: Folder[];
@@ -29,7 +29,10 @@ const ChatContext = createContext<ChatContextProps | undefined>(undefined);
 // We use Omit to exclude 'refreshChatSessions' from the value prop type
 // because we're defining it within the component
 export const ChatProvider: React.FC<{
-  value: Omit<ChatContextProps, "refreshChatSessions">;
+  value: Omit<
+    ChatContextProps,
+    "refreshChatSessions" | "refreshAvailableAssistants"
+  >;
   children: React.ReactNode;
 }> = ({ value, children }) => {
   const [chatSessions, setChatSessions] = useState(value?.chatSessions || []);
@@ -47,7 +50,11 @@ export const ChatProvider: React.FC<{
 
   return (
     <ChatContext.Provider
-      value={{ ...value, chatSessions, refreshChatSessions }}
+      value={{
+        ...value,
+        chatSessions,
+        refreshChatSessions,
+      }}
     >
       {children}
     </ChatContext.Provider>
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 9295a08c083..8a7179a0640 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -959,6 +959,29 @@ export const SearchIcon = ({
   );
 };
 
+export const BellIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <svg
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+      xmlns="http://www.w3.org/2000/svg"
+      width="200"
+      height="200"
+      viewBox="0 0 24 24"
+    >
+      <path
+        fill="currentColor"
+        fill-rule="evenodd"
+        d="M12 1.25A7.75 7.75 0 0 0 4.25 9v.704a3.53 3.53 0 0 1-.593 1.958L2.51 13.385c-1.334 2-.316 4.718 2.003 5.35c.755.206 1.517.38 2.284.523l.002.005C7.567 21.315 9.622 22.75 12 22.75s4.433-1.435 5.202-3.487l.002-.005a28.472 28.472 0 0 0 2.284-.523c2.319-.632 3.337-3.35 2.003-5.35l-1.148-1.723a3.53 3.53 0 0 1-.593-1.958V9A7.75 7.75 0 0 0 12 1.25Zm3.376 18.287a28.46 28.46 0 0 1-6.753 0c.711 1.021 1.948 1.713 3.377 1.713c1.429 0 2.665-.692 3.376-1.713ZM5.75 9a6.25 6.25 0 1 1 12.5 0v.704c0 .993.294 1.964.845 2.79l1.148 1.723a2.02 2.02 0 0 1-1.15 3.071a26.96 26.96 0 0 1-14.187 0a2.021 2.021 0 0 1-1.15-3.07l1.15-1.724a5.03 5.03 0 0 0 .844-2.79V9Z"
+        clip-rule="evenodd"
+      />
+    </svg>
+  );
+};
+
 export const LightSettingsIcon = ({
   size = 16,
   className = defaultTailwindCSS,
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index b1215a3708a..b8c13be918d 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -709,7 +709,6 @@ export const SearchSection = ({
             reset={() => setQuery("")}
             toggleSidebar={toggleSidebar}
             page="search"
-            user={user}
           />
           <div className="w-full flex">
             <div
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index 82b10887dfa..8c2f0fb5a3d 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -47,6 +47,8 @@ interface FetchChatDataResult {
   finalDocumentSidebarInitialWidth?: number;
   shouldShowWelcomeModal: boolean;
   userInputPrompts: InputPrompt[];
+  hasAnyConnectors: boolean;
+  hasImageCompatibleModel: boolean;
 }
 
 export async function fetchChatData(searchParams: {
@@ -251,5 +253,7 @@ export async function fetchChatData(searchParams: {
     toggleSidebar,
     shouldShowWelcomeModal,
     userInputPrompts,
+    hasAnyConnectors,
+    hasImageCompatibleModel,
   };
 }

From f7d77a3c7627aaf1f055133c7ad3ed95f27f23c9 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 10:55:39 -0700
Subject: [PATCH 158/376] Empty embedding fix (#2853)

* account for malformed urls

* fix

* k
---
 backend/danswer/connectors/web/connector.py   |  3 +++
 .../search_nlp_models.py                      | 19 +++++++++++--------
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/backend/danswer/connectors/web/connector.py b/backend/danswer/connectors/web/connector.py
index bb1f64efdfe..9e0671ea248 100644
--- a/backend/danswer/connectors/web/connector.py
+++ b/backend/danswer/connectors/web/connector.py
@@ -128,6 +128,9 @@ def get_internal_links(
         if not href:
             continue
 
+        # Account for malformed backslashes in URLs
+        href = href.replace("\\", "/")
+
         if should_ignore_pound and "#" in href:
             href = href.split("#")[0]
 
diff --git a/backend/danswer/natural_language_processing/search_nlp_models.py b/backend/danswer/natural_language_processing/search_nlp_models.py
index 700c8c08cf0..d75fce304d6 100644
--- a/backend/danswer/natural_language_processing/search_nlp_models.py
+++ b/backend/danswer/natural_language_processing/search_nlp_models.py
@@ -50,23 +50,26 @@ def clean_model_name(model_str: str) -> str:
     return model_str.replace("/", "_").replace("-", "_").replace(".", "_")
 
 
-_WHITELIST = set(
-    " !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\n\t"
-)
 _INITIAL_FILTER = re.compile(
     "["
-    "\U00000080-\U0000FFFF"  # All Unicode characters beyond ASCII
-    "\U00010000-\U0010FFFF"  # All Unicode characters in supplementary planes
+    "\U0000FFF0-\U0000FFFF"  # Specials
+    "\U0001F000-\U0001F9FF"  # Emoticons
+    "\U00002000-\U0000206F"  # General Punctuation
+    "\U00002190-\U000021FF"  # Arrows
+    "\U00002700-\U000027BF"  # Dingbats
     "]+",
     flags=re.UNICODE,
 )
 
 
 def clean_openai_text(text: str) -> str:
-    # First, remove all weird characters
+    # Remove specific Unicode ranges that might cause issues
     cleaned = _INITIAL_FILTER.sub("", text)
-    # Then, keep only whitelisted characters
-    return "".join(char for char in cleaned if char in _WHITELIST)
+
+    # Remove any control characters except for newline and tab
+    cleaned = "".join(ch for ch in cleaned if ch >= " " or ch in "\n\t")
+
+    return cleaned
 
 
 def build_model_server_url(

From 2c77ad2aab5e33eb4a071aa13980d2dfaa4aa33e Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 12:11:46 -0700
Subject: [PATCH 159/376] Add errors to search (#2854)

* minor - add errors to search

* k
---
 .../server/query_and_chat/query_backend.py    | 23 +++++++++++++------
 web/src/components/search/SearchSection.tsx   |  5 ----
 2 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/backend/danswer/server/query_and_chat/query_backend.py b/backend/danswer/server/query_and_chat/query_backend.py
index 703ef2c0475..1b8d5dc4b5e 100644
--- a/backend/danswer/server/query_and_chat/query_backend.py
+++ b/backend/danswer/server/query_and_chat/query_backend.py
@@ -1,3 +1,5 @@
+import json
+from collections.abc import Generator
 from uuid import UUID
 
 from fastapi import APIRouter
@@ -267,10 +269,17 @@ def get_answer_with_quote(
 
     logger.notice(f"Received query for one shot answer with quotes: {query}")
 
-    packets = stream_search_answer(
-        query_req=query_request,
-        user=user,
-        max_document_tokens=None,
-        max_history_tokens=0,
-    )
-    return StreamingResponse(packets, media_type="application/json")
+    def stream_generator() -> Generator[str, None, None]:
+        try:
+            for packet in stream_search_answer(
+                query_req=query_request,
+                user=user,
+                max_document_tokens=None,
+                max_history_tokens=0,
+            ):
+                yield json.dumps(packet) if isinstance(packet, dict) else packet
+        except Exception as e:
+            logger.exception(f"Error in search answer streaming: {e}")
+            yield json.dumps({"error": str(e)})
+
+    return StreamingResponse(stream_generator(), media_type="application/json")
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index b8c13be918d..9f49e4715cf 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -59,11 +59,6 @@ const SEARCH_DEFAULT_OVERRIDES_START: SearchDefaultOverrides = {
   offset: 0,
 };
 
-const VALID_QUESTION_RESPONSE_DEFAULT: ValidQuestionResponse = {
-  reasoning: null,
-  error: null,
-};
-
 interface SearchSectionProps {
   toggle: () => void;
   defaultSearchType: SearchType;

From ee1cb084ac2ebb7670aee7d9843600542bf72788 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 12:12:42 -0700
Subject: [PATCH 160/376] modify default (#2856)

---
 backend/Dockerfile              | 2 +-
 backend/Dockerfile.model_server | 2 +-
 web/Dockerfile                  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/Dockerfile b/backend/Dockerfile
index 9bcd71952d7..f8feefdda76 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -8,7 +8,7 @@ Edition features outside of personal development or testing purposes. Please rea
 founders@danswer.ai for more information. Please visit https://github.com/danswer-ai/danswer"
 
 # Default DANSWER_VERSION, typically overriden during builds by GitHub Actions.
-ARG DANSWER_VERSION=0.3-dev
+ARG DANSWER_VERSION=0.8-dev
 ENV DANSWER_VERSION=${DANSWER_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true"
 
diff --git a/backend/Dockerfile.model_server b/backend/Dockerfile.model_server
index 05a284a2baa..c7b6d2006d0 100644
--- a/backend/Dockerfile.model_server
+++ b/backend/Dockerfile.model_server
@@ -7,7 +7,7 @@ You can find it at https://hub.docker.com/r/danswer/danswer-model-server. For mo
 visit https://github.com/danswer-ai/danswer."
 
 # Default DANSWER_VERSION, typically overriden during builds by GitHub Actions.
-ARG DANSWER_VERSION=0.3-dev
+ARG DANSWER_VERSION=0.8-dev
 ENV DANSWER_VERSION=${DANSWER_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true"
 
diff --git a/web/Dockerfile b/web/Dockerfile
index 48c13f57be1..3cfd1a0f3e4 100644
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -8,7 +8,7 @@ Edition features outside of personal development or testing purposes. Please rea
 founders@danswer.ai for more information. Please visit https://github.com/danswer-ai/danswer"
 
 # Default DANSWER_VERSION, typically overriden during builds by GitHub Actions.
-ARG DANSWER_VERSION=0.3-dev
+ARG DANSWER_VERSION=0.8-dev
 ENV DANSWER_VERSION=${DANSWER_VERSION}
 RUN echo "DANSWER_VERSION: ${DANSWER_VERSION}"
 

From 8f9d4335ceab2366f55373ff30b26fa9f99f1228 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 12:13:21 -0700
Subject: [PATCH 161/376] (minor) search memoization + context (#2732)

* add markdown blocks to search

* nit

* k
---
 backend/danswer/db/engine.py                  |  1 +
 .../chat_search/MinimalMarkdown.tsx           | 32 +++++++++++--------
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index a1f2335d348..1c6a6a3a329 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -346,6 +346,7 @@ def get_session() -> Generator[Session, None, None]:
         raise HTTPException(status_code=401, detail="User must authenticate")
 
     engine = get_sqlalchemy_engine()
+
     with Session(engine, expire_on_commit=False) as session:
         if MULTI_TENANT:
             if not is_valid_schema_name(tenant_id):
diff --git a/web/src/components/chat_search/MinimalMarkdown.tsx b/web/src/components/chat_search/MinimalMarkdown.tsx
index 4731e2de9c3..401f0adf977 100644
--- a/web/src/components/chat_search/MinimalMarkdown.tsx
+++ b/web/src/components/chat_search/MinimalMarkdown.tsx
@@ -4,7 +4,7 @@ import {
   MemoizedLink,
   MemoizedParagraph,
 } from "@/app/chat/message/MemoizedTextComponents";
-import React from "react";
+import React, { useMemo } from "react";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
 
@@ -17,22 +17,26 @@ export const MinimalMarkdown: React.FC<MinimalMarkdownProps> = ({
   content,
   className = "",
 }) => {
+  const markdownComponents = useMemo(
+    () => ({
+      a: MemoizedLink,
+      p: MemoizedParagraph,
+      code: ({ node, inline, className, children, ...props }: any) => {
+        const codeText = extractCodeText(node, content, children);
+        return (
+          <CodeBlock className={className} codeText={codeText}>
+            {children}
+          </CodeBlock>
+        );
+      },
+    }),
+    [content]
+  );
+
   return (
     <ReactMarkdown
       className={`w-full text-wrap break-word ${className}`}
-      components={{
-        a: MemoizedLink,
-        p: MemoizedParagraph,
-        code: ({ node, inline, className, children, ...props }: any) => {
-          const codeText = extractCodeText(node, content, children);
-
-          return (
-            <CodeBlock className={className} codeText={codeText}>
-              {children}
-            </CodeBlock>
-          );
-        },
-      }}
+      components={markdownComponents}
       remarkPlugins={[remarkGfm]}
     >
       {content}

From 2fb1d06fbf40d331d0b3d08417eb217f66c07c71 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 14:03:04 -0700
Subject: [PATCH 162/376] update google sites + formik (#2834)

* update google sites + formik

* nit

* k
---
 .../[connector]/AddConnectorPage.tsx          |   1 +
 .../pages/ConnectorInput/FileInput.tsx        |  16 ++-
 .../pages/DynamicConnectorCreationForm.tsx    |   2 -
 .../[connector]/pages/utils/google_site.ts    |   9 +-
 .../sets/DocumentSetCreationForm.tsx          |   1 +
 .../embeddings/pages/EmbeddingFormPage.tsx    | 123 ++++++++++--------
 6 files changed, 87 insertions(+), 65 deletions(-)

diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index ce846851820..cc8173082da 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -270,6 +270,7 @@ export default function AddConnector({
             advancedConfiguration.pruneFreq,
             advancedConfiguration.indexingStart,
             values.access_type == "public",
+            groups,
             name
           );
           if (response) {
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/FileInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/FileInput.tsx
index 50af2dfff70..63e530658a4 100644
--- a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/FileInput.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/FileInput.tsx
@@ -1,3 +1,4 @@
+import { useField } from "formik";
 import { FileUpload } from "@/components/admin/connectors/FileUpload";
 import CredentialSubText from "@/components/credentials/CredentialFields";
 
@@ -6,8 +7,6 @@ interface FileInputProps {
   label: string;
   optional?: boolean;
   description?: string;
-  selectedFiles: File[];
-  setSelectedFiles: (files: File[]) => void;
 }
 
 export default function FileInput({
@@ -15,9 +14,9 @@ export default function FileInput({
   label,
   optional = false,
   description,
-  selectedFiles,
-  setSelectedFiles,
 }: FileInputProps) {
+  const [field, meta, helpers] = useField(name);
+
   return (
     <>
       <label
@@ -29,9 +28,14 @@ export default function FileInput({
       </label>
       {description && <CredentialSubText>{description}</CredentialSubText>}
       <FileUpload
-        selectedFiles={selectedFiles}
-        setSelectedFiles={setSelectedFiles}
+        selectedFiles={field.value ? [field.value] : []}
+        setSelectedFiles={(files: File[]) => {
+          helpers.setValue(files[0] || null);
+        }}
       />
+      {meta.touched && meta.error && (
+        <div className="text-red-500 text-sm mt-1">{meta.error}</div>
+      )}
     </>
   );
 }
diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
index f02111d262c..85237df2c7d 100644
--- a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
@@ -45,8 +45,6 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
           label={field.label}
           optional={field.optional}
           description={field.description}
-          selectedFiles={selectedFiles}
-          setSelectedFiles={setSelectedFiles}
         />
       ) : field.type === "list" ? (
         <ListInput field={field} />
diff --git a/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts b/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts
index abc1097cc36..45b713fe786 100644
--- a/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts
+++ b/web/src/app/admin/connectors/[connector]/pages/utils/google_site.ts
@@ -11,6 +11,7 @@ export const submitGoogleSite = async (
   pruneFreq: number,
   indexingStart: Date,
   is_public: boolean,
+  groups: number[],
   name?: string
 ) => {
   const uploadCreateAndTriggerConnector = async () => {
@@ -56,7 +57,13 @@ export const submitGoogleSite = async (
       return false;
     }
 
-    const credentialResponse = await linkCredential(connector.id, 0, base_url);
+    const credentialResponse = await linkCredential(
+      connector.id,
+      0,
+      base_url,
+      undefined,
+      groups
+    );
     if (!credentialResponse.ok) {
       const credentialResponseJson = await credentialResponse.json();
       setPopup({
diff --git a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
index 8da5ee0a8b8..6c61953fb94 100644
--- a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
+++ b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
@@ -125,6 +125,7 @@ export const DocumentSetCreationForm = ({
                 placeholder="Describe what the document set represents"
                 autoCompleteDisabled={true}
               />
+
               {isPaidEnterpriseFeaturesEnabled && (
                 <IsPublicGroupSelector
                   formikProps={props}
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index 1b4852ed4d8..90f1b70fa04 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -3,7 +3,7 @@ import { usePopup } from "@/components/admin/connectors/Popup";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 
 import { EmbeddingModelSelection } from "../EmbeddingModelSelectionForm";
-import { useEffect, useState } from "react";
+import { useEffect, useMemo, useState } from "react";
 import { Button, Card, Text } from "@tremor/react";
 import { ArrowLeft, ArrowRight, WarningCircle } from "@phosphor-icons/react";
 import {
@@ -152,6 +152,68 @@ export default function EmbeddingForm() {
     }
   }, [currentEmbeddingModel]);
 
+  const handleReindex = async () => {
+    const update = await updateSearch();
+    if (update) {
+      await onConfirm();
+    }
+  };
+
+  const needsReIndex =
+    currentEmbeddingModel != selectedProvider ||
+    searchSettings?.multipass_indexing !=
+      advancedEmbeddingDetails.multipass_indexing;
+
+  const ReIndexingButton = useMemo(() => {
+    const ReIndexingButtonComponent = ({
+      needsReIndex,
+    }: {
+      needsReIndex: boolean;
+    }) => {
+      return needsReIndex ? (
+        <div className="flex mx-auto gap-x-1 ml-auto items-center">
+          <button
+            className="enabled:cursor-pointer disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
+            onClick={handleReindex}
+          >
+            Re-index
+          </button>
+          <div className="relative group">
+            <WarningCircle
+              className="text-text-800 cursor-help"
+              size={20}
+              weight="fill"
+            />
+            <div className="absolute z-10 invisible group-hover:visible bg-background-800 text-text-200 text-sm rounded-md shadow-md p-2 right-0 mt-1 w-64">
+              <p className="font-semibold mb-2">Needs re-indexing due to:</p>
+              <ul className="list-disc pl-5">
+                {currentEmbeddingModel != selectedProvider && (
+                  <li>Changed embedding provider</li>
+                )}
+                {searchSettings?.multipass_indexing !=
+                  advancedEmbeddingDetails.multipass_indexing && (
+                  <li>Multipass indexing modification</li>
+                )}
+              </ul>
+            </div>
+          </div>
+        </div>
+      ) : (
+        <button
+          className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
+          onClick={async () => {
+            updateSearch();
+            navigateToEmbeddingPage("search settings");
+          }}
+        >
+          Update Search
+        </button>
+      );
+    };
+    ReIndexingButtonComponent.displayName = "ReIndexingButton";
+    return ReIndexingButtonComponent;
+  }, [needsReIndex]);
+
   if (!selectedProvider) {
     return <ThreeDotsLoader />;
   }
@@ -196,12 +258,13 @@ export default function EmbeddingForm() {
 
     // We use a spread operation to merge properties from multiple objects into a single object.
     // Advanced embedding details may update default values.
+    // Do NOT modify the order unless you are positive the new hierarchy is correct.
     if (selectedProvider.provider_type != null) {
       // This is a cloud model
       newModel = {
+        ...selectedProvider,
         ...rerankingDetails,
         ...advancedEmbeddingDetails,
-        ...selectedProvider,
         provider_type:
           (selectedProvider.provider_type
             ?.toLowerCase()
@@ -213,10 +276,10 @@ export default function EmbeddingForm() {
         ...selectedProvider,
         ...rerankingDetails,
         ...advancedEmbeddingDetails,
-        ...selectedProvider,
         provider_type: null,
       };
     }
+
     newModel.index_name = null;
 
     const response = await fetch(
@@ -239,58 +302,6 @@ export default function EmbeddingForm() {
     }
   };
 
-  const needsReIndex =
-    currentEmbeddingModel != selectedProvider ||
-    searchSettings?.multipass_indexing !=
-      advancedEmbeddingDetails.multipass_indexing;
-
-  const ReIndexingButton = ({ needsReIndex }: { needsReIndex: boolean }) => {
-    return needsReIndex ? (
-      <div className="flex mx-auto gap-x-1 ml-auto items-center">
-        <button
-          className="enabled:cursor-pointer disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
-          onClick={async () => {
-            const update = await updateSearch();
-            if (update) {
-              await onConfirm();
-            }
-          }}
-        >
-          Re-index
-        </button>
-        <div className="relative group">
-          <WarningCircle
-            className="text-text-800 cursor-help"
-            size={20}
-            weight="fill"
-          />
-          <div className="absolute z-10 invisible group-hover:visible bg-background-800 text-text-200 text-sm rounded-md shadow-md p-2 right-0 mt-1 w-64">
-            <p className="font-semibold mb-2">Needs re-indexing due to:</p>
-            <ul className="list-disc pl-5">
-              {currentEmbeddingModel != selectedProvider && (
-                <li>Changed embedding provider</li>
-              )}
-              {searchSettings?.multipass_indexing !=
-                advancedEmbeddingDetails.multipass_indexing && (
-                <li>Multipass indexing modification</li>
-              )}
-            </ul>
-          </div>
-        </div>
-      </div>
-    ) : (
-      <button
-        className="enabled:cursor-pointer ml-auto disabled:bg-accent/50 disabled:cursor-not-allowed bg-accent flex mx-auto gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
-        onClick={async () => {
-          updateSearch();
-          navigateToEmbeddingPage("search settings");
-        }}
-      >
-        Update Search
-      </button>
-    );
-  };
-
   return (
     <div className="mx-auto mb-8 w-full">
       {popup}
@@ -391,7 +402,7 @@ export default function EmbeddingForm() {
               />
             </Card>
 
-            <div className={` mt-4 w-full grid grid-cols-3`}>
+            <div className={`mt-4 w-full grid grid-cols-3`}>
               <button
                 className="border-border-dark mr-auto border flex gap-x-1 items-center text-text p-2.5 text-sm font-regular rounded-sm "
                 onClick={() => prevFormStep()}

From 457e7992a40faaaad067da487c36356981ed7c23 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Sat, 19 Oct 2024 14:10:42 -0700
Subject: [PATCH 163/376] missing tenant_id as optional param (#2851)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 backend/ee/danswer/background/task_name_builders.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/ee/danswer/background/task_name_builders.py b/backend/ee/danswer/background/task_name_builders.py
index 7a8eee0cd70..aea6648a02d 100644
--- a/backend/ee/danswer/background/task_name_builders.py
+++ b/backend/ee/danswer/background/task_name_builders.py
@@ -1,4 +1,4 @@
-def name_chat_ttl_task(retention_limit_days: int) -> str:
+def name_chat_ttl_task(retention_limit_days: int, tenant_id: str | None = None) -> str:
     return f"chat_ttl_{retention_limit_days}_days"
 
 

From eaaa135f90881c5938592df670554f80008b0dc4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 16:43:26 -0700
Subject: [PATCH 164/376] push vespa managed service configs (#2857)

* push vespa managed service configs

* organize

* k

* k

* k

* nit

* k

* minor cleanup

* ensure no unnecessary timeout
---
 backend/danswer/configs/app_configs.py        | 10 ++++++++
 backend/danswer/connectors/file/connector.py  |  5 ++++
 .../document_index/vespa/chunk_retrieval.py   | 16 +++++++------
 backend/danswer/document_index/vespa/index.py | 18 +++++++-------
 .../vespa/shared_utils/utils.py               | 24 +++++++++++++++++++
 .../danswer/document_index/vespa_constants.py | 11 +++++++--
 backend/danswer/setup.py                      |  4 +++-
 7 files changed, 70 insertions(+), 18 deletions(-)

diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 0d8b7da7010..08caf7181f8 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -115,10 +115,16 @@
 VESPA_CONFIG_SERVER_HOST = os.environ.get("VESPA_CONFIG_SERVER_HOST") or VESPA_HOST
 VESPA_PORT = os.environ.get("VESPA_PORT") or "8081"
 VESPA_TENANT_PORT = os.environ.get("VESPA_TENANT_PORT") or "19071"
+
+VESPA_CLOUD_URL = os.environ.get("VESPA_CLOUD_URL", "")
+
 # The default below is for dockerized deployment
 VESPA_DEPLOYMENT_ZIP = (
     os.environ.get("VESPA_DEPLOYMENT_ZIP") or "/app/danswer/vespa-app.zip"
 )
+VESPA_CLOUD_CERT_PATH = os.environ.get("VESPA_CLOUD_CERT_PATH")
+VESPA_CLOUD_KEY_PATH = os.environ.get("VESPA_CLOUD_KEY_PATH")
+
 # Number of documents in a batch during indexing (further batching done by chunks before passing to bi-encoder)
 try:
     INDEX_BATCH_SIZE = int(os.environ.get("INDEX_BATCH_SIZE", 16))
@@ -428,6 +434,10 @@
 
 # Multi-tenancy configuration
 MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
+
+# Use managed Vespa (Vespa Cloud). If set, must also set VESPA_CLOUD_URL, VESPA_CLOUD_CERT_PATH and VESPA_CLOUD_KEY_PATH
+MANAGED_VESPA = os.environ.get("MANAGED_VESPA", "").lower() == "true"
+
 ENABLE_EMAIL_INVITES = os.environ.get("ENABLE_EMAIL_INVITES", "").lower() == "true"
 
 # Security and authentication
diff --git a/backend/danswer/connectors/file/connector.py b/backend/danswer/connectors/file/connector.py
index 106fed8b2af..9992159eb35 100644
--- a/backend/danswer/connectors/file/connector.py
+++ b/backend/danswer/connectors/file/connector.py
@@ -28,6 +28,7 @@
 from danswer.file_processing.extract_file_text import read_text_file
 from danswer.file_store.file_store import get_default_file_store
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import current_tenant_id
 
 logger = setup_logger()
 
@@ -174,6 +175,8 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
 
     def load_from_state(self) -> GenerateDocumentsOutput:
         documents: list[Document] = []
+        token = current_tenant_id.set(self.tenant_id)
+
         with get_session_with_tenant(self.tenant_id) as db_session:
             for file_path in self.file_locations:
                 current_datetime = datetime.now(timezone.utc)
@@ -196,6 +199,8 @@ def load_from_state(self) -> GenerateDocumentsOutput:
             if documents:
                 yield documents
 
+        current_tenant_id.reset(token)
+
 
 if __name__ == "__main__":
     connector = LocalFileConnector(file_locations=[os.environ["TEST_FILE"]])
diff --git a/backend/danswer/document_index/vespa/chunk_retrieval.py b/backend/danswer/document_index/vespa/chunk_retrieval.py
index e4b2ad83ce2..3de64cbc840 100644
--- a/backend/danswer/document_index/vespa/chunk_retrieval.py
+++ b/backend/danswer/document_index/vespa/chunk_retrieval.py
@@ -7,11 +7,13 @@
 from typing import Any
 from typing import cast
 
+import httpx
 import requests
 from retry import retry
 
 from danswer.configs.app_configs import LOG_VESPA_TIMING_INFORMATION
 from danswer.document_index.interfaces import VespaChunkRequest
+from danswer.document_index.vespa.shared_utils.utils import get_vespa_http_client
 from danswer.document_index.vespa.shared_utils.vespa_request_builders import (
     build_vespa_filters,
 )
@@ -293,13 +295,12 @@ def query_vespa(
         if LOG_VESPA_TIMING_INFORMATION
         else {},
     )
+
     try:
-        response = requests.post(
-            SEARCH_ENDPOINT,
-            json=params,
-        )
-        response.raise_for_status()
-    except requests.HTTPError as e:
+        with get_vespa_http_client() as http_client:
+            response = http_client.post(SEARCH_ENDPOINT, json=params)
+            response.raise_for_status()
+    except httpx.HTTPError as e:
         request_info = f"Headers: {response.request.headers}\nPayload: {params}"
         response_info = (
             f"Status Code: {response.status_code}\n"
@@ -312,9 +313,10 @@ def query_vespa(
             f"{response_info}\n"
             f"Exception: {e}"
         )
-        raise requests.HTTPError(error_base) from e
+        raise httpx.HTTPError(error_base) from e
 
     response_json: dict[str, Any] = response.json()
+
     if LOG_VESPA_TIMING_INFORMATION:
         logger.debug("Vespa timing info: %s", response_json.get("timing"))
     hits = response_json["root"].get("children", [])
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index d71d198aea7..86bc481e573 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -18,7 +18,6 @@
 
 from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
 from danswer.configs.app_configs import MULTI_TENANT
-from danswer.configs.app_configs import VESPA_REQUEST_TIMEOUT
 from danswer.configs.chat_configs import DOC_TIME_DECAY
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
 from danswer.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -43,6 +42,7 @@
 from danswer.document_index.vespa.indexing_utils import (
     get_existing_documents_from_chunks,
 )
+from danswer.document_index.vespa.shared_utils.utils import get_vespa_http_client
 from danswer.document_index.vespa.shared_utils.utils import (
     replace_invalid_doc_id_characters,
 )
@@ -133,6 +133,7 @@ def __init__(
         self.index_name = index_name
         self.secondary_index_name = secondary_index_name
         self.multitenant = multitenant
+        self.http_client = get_vespa_http_client()
 
     def ensure_indices_exist(
         self,
@@ -319,7 +320,7 @@ def index(
         # indexing / updates / deletes since we have to make a large volume of requests.
         with (
             concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor,
-            httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client,
+            get_vespa_http_client() as http_client,
         ):
             # Check for existing documents, existing documents need to have all of their chunks deleted
             # prior to indexing as the document size (num chunks) may have shrunk
@@ -382,9 +383,10 @@ def _update_chunk(
 
         # NOTE: using `httpx` here since `requests` doesn't support HTTP2. This is beneficient for
         # indexing / updates / deletes since we have to make a large volume of requests.
+
         with (
             concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor,
-            httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client,
+            get_vespa_http_client() as http_client,
         ):
             for update_batch in batch_generator(updates, batch_size):
                 future_to_document_id = {
@@ -528,7 +530,7 @@ def update_single(self, doc_id: str, fields: VespaDocumentFields) -> int:
         if self.secondary_index_name:
             index_names.append(self.secondary_index_name)
 
-        with httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client:
+        with get_vespa_http_client() as http_client:
             for index_name in index_names:
                 params = httpx.QueryParams(
                     {
@@ -584,7 +586,7 @@ def delete(self, doc_ids: list[str]) -> None:
 
         # NOTE: using `httpx` here since `requests` doesn't support HTTP2. This is beneficial for
         # indexing / updates / deletes since we have to make a large volume of requests.
-        with httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client:
+        with get_vespa_http_client() as http_client:
             index_names = [self.index_name]
             if self.secondary_index_name:
                 index_names.append(self.secondary_index_name)
@@ -612,7 +614,7 @@ def delete_single(self, doc_id: str) -> int:
         if self.secondary_index_name:
             index_names.append(self.secondary_index_name)
 
-        with httpx.Client(http2=True, timeout=VESPA_REQUEST_TIMEOUT) as http_client:
+        with get_vespa_http_client() as http_client:
             for index_name in index_names:
                 params = httpx.QueryParams(
                     {
@@ -822,7 +824,7 @@ def _get_all_document_ids_by_tenant_id(
                 f"Querying for document IDs with tenant_id: {tenant_id}, offset: {offset}"
             )
 
-            with httpx.Client(http2=True) as http_client:
+            with get_vespa_http_client(no_timeout=True) as http_client:
                 response = http_client.get(url, params=query_params)
                 response.raise_for_status()
 
@@ -871,7 +873,7 @@ def _delete_document(
         logger.debug(f"Starting batch deletion for {len(delete_requests)} documents")
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=NUM_THREADS) as executor:
-            with httpx.Client(http2=True) as http_client:
+            with get_vespa_http_client(no_timeout=True) as http_client:
                 for batch_start in range(0, len(delete_requests), batch_size):
                     batch = delete_requests[batch_start : batch_start + batch_size]
 
diff --git a/backend/danswer/document_index/vespa/shared_utils/utils.py b/backend/danswer/document_index/vespa/shared_utils/utils.py
index c74afc9a629..49fdd680198 100644
--- a/backend/danswer/document_index/vespa/shared_utils/utils.py
+++ b/backend/danswer/document_index/vespa/shared_utils/utils.py
@@ -1,4 +1,12 @@
 import re
+from typing import cast
+
+import httpx
+
+from danswer.configs.app_configs import MANAGED_VESPA
+from danswer.configs.app_configs import VESPA_CLOUD_CERT_PATH
+from danswer.configs.app_configs import VESPA_CLOUD_KEY_PATH
+from danswer.configs.app_configs import VESPA_REQUEST_TIMEOUT
 
 # NOTE: This does not seem to be used in reality despite the Vespa Docs pointing to this code
 # See here for reference: https://docs.vespa.ai/en/documents.html
@@ -45,3 +53,19 @@ def remove_invalid_unicode_chars(text: str) -> str:
         "[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]"
     )
     return _illegal_xml_chars_RE.sub("", text)
+
+
+def get_vespa_http_client(no_timeout: bool = False) -> httpx.Client:
+    """
+    Configure and return an HTTP client for communicating with Vespa,
+    including authentication if needed.
+    """
+
+    return httpx.Client(
+        cert=cast(tuple[str, str], (VESPA_CLOUD_CERT_PATH, VESPA_CLOUD_KEY_PATH))
+        if MANAGED_VESPA
+        else None,
+        verify=False if not MANAGED_VESPA else True,
+        timeout=None if no_timeout else VESPA_REQUEST_TIMEOUT,
+        http2=True,
+    )
diff --git a/backend/danswer/document_index/vespa_constants.py b/backend/danswer/document_index/vespa_constants.py
index a4d6aa52e2f..d4a36ef9725 100644
--- a/backend/danswer/document_index/vespa_constants.py
+++ b/backend/danswer/document_index/vespa_constants.py
@@ -1,3 +1,4 @@
+from danswer.configs.app_configs import VESPA_CLOUD_URL
 from danswer.configs.app_configs import VESPA_CONFIG_SERVER_HOST
 from danswer.configs.app_configs import VESPA_HOST
 from danswer.configs.app_configs import VESPA_PORT
@@ -18,15 +19,21 @@
             attribute: fast-search
         }"""
 # config server
-VESPA_CONFIG_SERVER_URL = f"http://{VESPA_CONFIG_SERVER_HOST}:{VESPA_TENANT_PORT}"
+
+
+VESPA_CONFIG_SERVER_URL = (
+    VESPA_CLOUD_URL or f"http://{VESPA_CONFIG_SERVER_HOST}:{VESPA_TENANT_PORT}"
+)
 VESPA_APPLICATION_ENDPOINT = f"{VESPA_CONFIG_SERVER_URL}/application/v2"
 
 # main search application
-VESPA_APP_CONTAINER_URL = f"http://{VESPA_HOST}:{VESPA_PORT}"
+VESPA_APP_CONTAINER_URL = VESPA_CLOUD_URL or f"http://{VESPA_HOST}:{VESPA_PORT}"
+
 # danswer_chunk below is defined in vespa/app_configs/schemas/danswer_chunk.sd
 DOCUMENT_ID_ENDPOINT = (
     f"{VESPA_APP_CONTAINER_URL}/document/v1/default/{{index_name}}/docid"
 )
+
 SEARCH_ENDPOINT = f"{VESPA_APP_CONTAINER_URL}/search/"
 
 NUM_THREADS = (
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index 84f0382f9ef..747fe809451 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -4,6 +4,7 @@
 
 from danswer.chat.load_yamls import load_chat_yamls
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+from danswer.configs.app_configs import MANAGED_VESPA
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.configs.constants import KV_SEARCH_SETTINGS
@@ -310,7 +311,8 @@ def update_default_multipass_indexing(db_session: Session) -> None:
 
 
 def setup_multitenant_danswer() -> None:
-    setup_vespa_multitenant(SUPPORTED_EMBEDDING_MODELS)
+    if not MANAGED_VESPA:
+        setup_vespa_multitenant(SUPPORTED_EMBEDDING_MODELS)
 
 
 def setup_vespa_multitenant(supported_indices: list[SupportedEmbeddingModel]) -> bool:

From f745ca1e0320090f36c4d165c5b287b81f00d3bc Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 19 Oct 2024 16:44:11 -0700
Subject: [PATCH 165/376] ensure **all** sharp-related packages installed
 (#2855)

---
 web/package-lock.json | 438 +++++++++++++++++++++++++++++++++++++++++-
 web/package.json      |   1 +
 2 files changed, 436 insertions(+), 3 deletions(-)

diff --git a/web/package-lock.json b/web/package-lock.json
index 4401222f737..ffb68fc16bd 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -44,6 +44,7 @@
         "rehype-prism-plus": "^2.0.0",
         "remark-gfm": "^4.0.0",
         "semver": "^7.5.4",
+        "sharp": "^0.33.5",
         "stripe": "^17.0.0",
         "swr": "^2.1.5",
         "tailwindcss": "^3.3.1",
@@ -551,6 +552,15 @@
         "react": ">=16.8.0"
       }
     },
+    "node_modules/@emnapi/runtime": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz",
+      "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==",
+      "optional": true,
+      "dependencies": {
+        "tslib": "^2.4.0"
+      }
+    },
     "node_modules/@emotion/babel-plugin": {
       "version": "11.11.0",
       "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz",
@@ -853,6 +863,348 @@
       "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==",
       "dev": true
     },
+    "node_modules/@img/sharp-darwin-arm64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz",
+      "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-darwin-arm64": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-darwin-x64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz",
+      "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-darwin-x64": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-libvips-darwin-arm64": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz",
+      "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-darwin-x64": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz",
+      "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-arm": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz",
+      "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==",
+      "cpu": [
+        "arm"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-arm64": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz",
+      "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-s390x": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz",
+      "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==",
+      "cpu": [
+        "s390x"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linux-x64": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz",
+      "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linuxmusl-arm64": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz",
+      "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-libvips-linuxmusl-x64": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz",
+      "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-linux-arm": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz",
+      "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==",
+      "cpu": [
+        "arm"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-arm": "1.0.5"
+      }
+    },
+    "node_modules/@img/sharp-linux-arm64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz",
+      "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-arm64": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-linux-s390x": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz",
+      "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==",
+      "cpu": [
+        "s390x"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-s390x": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-linux-x64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz",
+      "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linux-x64": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-linuxmusl-arm64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz",
+      "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==",
+      "cpu": [
+        "arm64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linuxmusl-arm64": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-linuxmusl-x64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz",
+      "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "linux"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-libvips-linuxmusl-x64": "1.0.4"
+      }
+    },
+    "node_modules/@img/sharp-wasm32": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz",
+      "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==",
+      "cpu": [
+        "wasm32"
+      ],
+      "optional": true,
+      "dependencies": {
+        "@emnapi/runtime": "^1.2.0"
+      },
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-ia32": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz",
+      "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==",
+      "cpu": [
+        "ia32"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
+    "node_modules/@img/sharp-win32-x64": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz",
+      "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==",
+      "cpu": [
+        "x64"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      }
+    },
     "node_modules/@isaacs/cliui": {
       "version": "8.0.2",
       "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
@@ -4071,6 +4423,18 @@
         "node": ">=6"
       }
     },
+    "node_modules/color": {
+      "version": "4.2.3",
+      "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz",
+      "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==",
+      "dependencies": {
+        "color-convert": "^2.0.1",
+        "color-string": "^1.9.0"
+      },
+      "engines": {
+        "node": ">=12.5.0"
+      }
+    },
     "node_modules/color-convert": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -4087,6 +4451,15 @@
       "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
       "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
     },
+    "node_modules/color-string": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz",
+      "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==",
+      "dependencies": {
+        "color-name": "^1.0.0",
+        "simple-swizzle": "^0.2.2"
+      }
+    },
     "node_modules/comma-separated-tokens": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz",
@@ -4454,6 +4827,14 @@
         "node": ">=6"
       }
     },
+    "node_modules/detect-libc": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz",
+      "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==",
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/detect-node-es": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz",
@@ -6047,6 +6428,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-arrayish": {
+      "version": "0.3.2",
+      "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz",
+      "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="
+    },
     "node_modules/is-async-function": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz",
@@ -11527,9 +11913,9 @@
       }
     },
     "node_modules/semver": {
-      "version": "7.6.2",
-      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz",
-      "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==",
+      "version": "7.6.3",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz",
+      "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==",
       "bin": {
         "semver": "bin/semver.js"
       },
@@ -11582,6 +11968,44 @@
       "resolved": "https://registry.npmjs.org/shallowequal/-/shallowequal-1.1.0.tgz",
       "integrity": "sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ=="
     },
+    "node_modules/sharp": {
+      "version": "0.33.5",
+      "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz",
+      "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==",
+      "hasInstallScript": true,
+      "dependencies": {
+        "color": "^4.2.3",
+        "detect-libc": "^2.0.3",
+        "semver": "^7.6.3"
+      },
+      "engines": {
+        "node": "^18.17.0 || ^20.3.0 || >=21.0.0"
+      },
+      "funding": {
+        "url": "https://opencollective.com/libvips"
+      },
+      "optionalDependencies": {
+        "@img/sharp-darwin-arm64": "0.33.5",
+        "@img/sharp-darwin-x64": "0.33.5",
+        "@img/sharp-libvips-darwin-arm64": "1.0.4",
+        "@img/sharp-libvips-darwin-x64": "1.0.4",
+        "@img/sharp-libvips-linux-arm": "1.0.5",
+        "@img/sharp-libvips-linux-arm64": "1.0.4",
+        "@img/sharp-libvips-linux-s390x": "1.0.4",
+        "@img/sharp-libvips-linux-x64": "1.0.4",
+        "@img/sharp-libvips-linuxmusl-arm64": "1.0.4",
+        "@img/sharp-libvips-linuxmusl-x64": "1.0.4",
+        "@img/sharp-linux-arm": "0.33.5",
+        "@img/sharp-linux-arm64": "0.33.5",
+        "@img/sharp-linux-s390x": "0.33.5",
+        "@img/sharp-linux-x64": "0.33.5",
+        "@img/sharp-linuxmusl-arm64": "0.33.5",
+        "@img/sharp-linuxmusl-x64": "0.33.5",
+        "@img/sharp-wasm32": "0.33.5",
+        "@img/sharp-win32-ia32": "0.33.5",
+        "@img/sharp-win32-x64": "0.33.5"
+      }
+    },
     "node_modules/shebang-command": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -11634,6 +12058,14 @@
         "url": "https://github.com/sponsors/isaacs"
       }
     },
+    "node_modules/simple-swizzle": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz",
+      "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==",
+      "dependencies": {
+        "is-arrayish": "^0.3.1"
+      }
+    },
     "node_modules/slash": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz",
diff --git a/web/package.json b/web/package.json
index c3efcd78d3f..e7e2d4098ff 100644
--- a/web/package.json
+++ b/web/package.json
@@ -45,6 +45,7 @@
     "rehype-prism-plus": "^2.0.0",
     "remark-gfm": "^4.0.0",
     "semver": "^7.5.4",
+    "sharp": "^0.33.5",
     "stripe": "^17.0.0",
     "swr": "^2.1.5",
     "tailwindcss": "^3.3.1",

From dd2551040f2aa9c5859ed189345be00932a57e72 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 20 Oct 2024 15:31:08 -0700
Subject: [PATCH 166/376] Docstring Update for Docs (#2863)

---
 .../server/query_and_chat/chat_backend.py     | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index 6e2d3c40988..ac64231dcf2 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -311,13 +311,26 @@ def handle_new_chat_message(
     _: None = Depends(check_token_rate_limits),
     is_disconnected_func: Callable[[], bool] = Depends(is_disconnected),
 ) -> StreamingResponse:
-    """This endpoint is both used for all the following purposes:
+    """
+    This endpoint is both used for all the following purposes:
     - Sending a new message in the session
     - Regenerating a message in the session (just send the same one again)
     - Editing a message (similar to regenerating but sending a different message)
     - Kicking off a seeded chat session (set `use_existing_user_message`)
-    To avoid extra overhead/latency, this assumes (and checks) that previous messages on the path
-    have already been set as latest"""
+
+    Assumes that previous messages have been set as the latest to minimize overhead.
+
+    Args:
+        chat_message_req (CreateChatMessageRequest): Details about the new chat message.
+        request (Request): The current HTTP request context.
+        user (User | None): The current user, obtained via dependency injection.
+        _ (None): Rate limit check is run if user/group/global rate limits are enabled.
+        is_disconnected_func (Callable[[], bool]): Function to check client disconnection,
+            used to stop the streaming response if the client disconnects.
+
+    Returns:
+        StreamingResponse: Streams the response to the new chat message.
+    """
     logger.debug(f"Received new chat message: {chat_message_req.message}")
 
     if (

From 7ab0063dc651d011697a5f2ff92d78b5585e46e2 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 20 Oct 2024 16:31:18 -0700
Subject: [PATCH 167/376] (minor) quote overflow (#2862)

* k

* k
---
 web/src/components/search/results/QuotesSection.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/components/search/results/QuotesSection.tsx b/web/src/components/search/results/QuotesSection.tsx
index 16f1324f213..0ea2072707f 100644
--- a/web/src/components/search/results/QuotesSection.tsx
+++ b/web/src/components/search/results/QuotesSection.tsx
@@ -18,7 +18,7 @@ const QuoteDisplay = ({ quoteInfo }: { quoteInfo: Quote }) => {
     >
       {detailIsOpen && (
         <div className="absolute top-0 mt-9 pt-2 z-50">
-          <div className="flex flex-shrink-0 rounded-lg w-96 bg-background border border-border shadow p-3 text-sm leading-relaxed">
+          <div className="flex flex-shrink-0 rounded-lg break-words hyphens-auto w-96 bg-background border border-border shadow p-3 text-sm leading-relaxed overflow-hidden">
             <div>
               <b>Quote:</b> <i>{quoteInfo.quote}</i>
             </div>

From a24b465663a1f92ac0e6d5ac34b9f2c673f757cf Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 20 Oct 2024 16:48:00 -0700
Subject: [PATCH 168/376] Minor tenant ID improvements (#2850)

* add migration dockerfile

* address edge case

* k

* k

* k

* nit

* k

* k

* k

* k

* remove

* k

* add comment
---
 backend/alembic/env.py                        |   4 +-
 backend/danswer/auth/invited_users.py         |   1 +
 backend/danswer/auth/users.py                 |   1 +
 .../danswer/background/celery/celery_app.py   |   2 +-
 .../slack/handlers/handle_regular_answer.py   |   1 +
 backend/danswer/danswerbot/slack/listener.py  | 104 ++++++++++--------
 backend/danswer/db/engine.py                  |  29 ++++-
 backend/danswer/db/llm.py                     |   3 +-
 backend/danswer/key_value_store/store.py      |   1 -
 .../danswer/background/celery/celery_app.py   |   2 +-
 deployment/kubernetes/env-configmap.yaml      |   3 +-
 11 files changed, 98 insertions(+), 53 deletions(-)

diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index c89d3455227..b4b0ecb4665 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -13,7 +13,7 @@
 from danswer.db.engine import build_connection_string
 from danswer.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore
-from danswer.background.celery.celery_app import get_all_tenant_ids
+from danswer.db.engine import get_all_tenant_ids
 
 # Alembic Config object
 config = context.config
@@ -61,7 +61,7 @@ def get_schema_options() -> tuple[str, bool, bool]:
     create_schema = x_args.get("create_schema", "true").lower() == "true"
     upgrade_all_tenants = x_args.get("upgrade_all_tenants", "false").lower() == "true"
 
-    if MULTI_TENANT and schema_name == "public":
+    if MULTI_TENANT and schema_name == "public" and not upgrade_all_tenants:
         raise ValueError(
             "Cannot run default migrations in public schema when multi-tenancy is enabled. "
             "Please specify a tenant-specific schema."
diff --git a/backend/danswer/auth/invited_users.py b/backend/danswer/auth/invited_users.py
index 2bbf79050ca..15ec9abf50e 100644
--- a/backend/danswer/auth/invited_users.py
+++ b/backend/danswer/auth/invited_users.py
@@ -9,6 +9,7 @@
 def get_invited_users() -> list[str]:
     try:
         store = get_kv_store()
+
         return cast(list, store.load(KV_USER_STORE_KEY))
     except KvKeyNotFoundError:
         return list()
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index f8b07d15b5a..5abf2ac8116 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -316,6 +316,7 @@ async def oauth_callback(
 
             verify_email_in_whitelist(account_email, tenant_id)
             verify_email_domain(account_email)
+
             if MULTI_TENANT:
                 tenant_user_db = SQLAlchemyUserAdminDB(db_session, User, OAuthAccount)
                 self.user_db = tenant_user_db
diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
index 702ab520589..ee59b8d50fd 100644
--- a/backend/danswer/background/celery/celery_app.py
+++ b/backend/danswer/background/celery/celery_app.py
@@ -28,7 +28,6 @@
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
-from danswer.background.celery.celery_utils import get_all_tenant_ids
 from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerRedisLocks
@@ -37,6 +36,7 @@
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_APP_NAME
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_LIGHT_APP_NAME
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
+from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import SqlEngine
 from danswer.db.search_settings import get_current_search_settings
diff --git a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
index 66f9ba54601..9dadf5614cb 100644
--- a/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
+++ b/backend/danswer/danswerbot/slack/handlers/handle_regular_answer.py
@@ -211,6 +211,7 @@ def _get_answer(new_message_request: DirectQARequest) -> OneShotQAResponse | Non
                 use_citations=use_citations,
                 danswerbot_flow=True,
             )
+
             if not answer.error_msg:
                 return answer
             else:
diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index 86e59708820..e3b2d213e83 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -7,7 +7,6 @@
 from slack_sdk.socket_mode.request import SocketModeRequest
 from slack_sdk.socket_mode.response import SocketModeResponse
 
-from danswer.background.celery.celery_app import get_all_tenant_ids
 from danswer.configs.constants import MessageType
 from danswer.configs.danswerbot_configs import DANSWER_BOT_REPHRASE_MESSAGE
 from danswer.configs.danswerbot_configs import DANSWER_BOT_RESPOND_EVERY_CHANNEL
@@ -47,6 +46,7 @@
 from danswer.danswerbot.slack.utils import rephrase_slack_message
 from danswer.danswerbot.slack.utils import respond_in_thread
 from danswer.danswerbot.slack.utils import TenantSocketModeClient
+from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.search_settings import get_current_search_settings
 from danswer.key_value_store.interface import KvKeyNotFoundError
@@ -57,6 +57,7 @@
 from danswer.server.manage.models import SlackBotTokens
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
+from shared_configs.configs import current_tenant_id
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
 from shared_configs.configs import SLACK_CHANNEL_ID
@@ -345,7 +346,9 @@ def process_message(
     respond_every_channel: bool = DANSWER_BOT_RESPOND_EVERY_CHANNEL,
     notify_no_answer: bool = NOTIFY_SLACKBOT_NO_ANSWER,
 ) -> None:
-    logger.debug(f"Received Slack request of type: '{req.type}'")
+    logger.debug(
+        f"Received Slack request of type: '{req.type}' for tenant, {client.tenant_id}"
+    )
 
     # Throw out requests that can't or shouldn't be handled
     if not prefilter_requests(req, client):
@@ -357,51 +360,59 @@ def process_message(
         client=client.web_client, channel_id=channel
     )
 
-    with get_session_with_tenant(client.tenant_id) as db_session:
-        slack_bot_config = get_slack_bot_config_for_channel(
-            channel_name=channel_name, db_session=db_session
-        )
+    # Set the current tenant ID at the beginning for all DB calls within this thread
+    if client.tenant_id:
+        logger.info(f"Setting tenant ID to {client.tenant_id}")
+        token = current_tenant_id.set(client.tenant_id)
+    try:
+        with get_session_with_tenant(client.tenant_id) as db_session:
+            slack_bot_config = get_slack_bot_config_for_channel(
+                channel_name=channel_name, db_session=db_session
+            )
 
-        # Be careful about this default, don't want to accidentally spam every channel
-        # Users should be able to DM slack bot in their private channels though
-        if (
-            slack_bot_config is None
-            and not respond_every_channel
-            # Can't have configs for DMs so don't toss them out
-            and not is_dm
-            # If /DanswerBot (is_bot_msg) or @DanswerBot (bypass_filters)
-            # always respond with the default configs
-            and not (details.is_bot_msg or details.bypass_filters)
-        ):
-            return
+            # Be careful about this default, don't want to accidentally spam every channel
+            # Users should be able to DM slack bot in their private channels though
+            if (
+                slack_bot_config is None
+                and not respond_every_channel
+                # Can't have configs for DMs so don't toss them out
+                and not is_dm
+                # If /DanswerBot (is_bot_msg) or @DanswerBot (bypass_filters)
+                # always respond with the default configs
+                and not (details.is_bot_msg or details.bypass_filters)
+            ):
+                return
 
-        follow_up = bool(
-            slack_bot_config
-            and slack_bot_config.channel_config
-            and slack_bot_config.channel_config.get("follow_up_tags") is not None
-        )
-        feedback_reminder_id = schedule_feedback_reminder(
-            details=details, client=client.web_client, include_followup=follow_up
-        )
+            follow_up = bool(
+                slack_bot_config
+                and slack_bot_config.channel_config
+                and slack_bot_config.channel_config.get("follow_up_tags") is not None
+            )
+            feedback_reminder_id = schedule_feedback_reminder(
+                details=details, client=client.web_client, include_followup=follow_up
+            )
 
-        failed = handle_message(
-            message_info=details,
-            slack_bot_config=slack_bot_config,
-            client=client.web_client,
-            feedback_reminder_id=feedback_reminder_id,
-            tenant_id=client.tenant_id,
-        )
+            failed = handle_message(
+                message_info=details,
+                slack_bot_config=slack_bot_config,
+                client=client.web_client,
+                feedback_reminder_id=feedback_reminder_id,
+                tenant_id=client.tenant_id,
+            )
 
-        if failed:
-            if feedback_reminder_id:
-                remove_scheduled_feedback_reminder(
-                    client=client.web_client,
-                    channel=details.sender,
-                    msg_id=feedback_reminder_id,
-                )
-            # Skipping answering due to pre-filtering is not considered a failure
-            if notify_no_answer:
-                apologize_for_fail(details, client)
+            if failed:
+                if feedback_reminder_id:
+                    remove_scheduled_feedback_reminder(
+                        client=client.web_client,
+                        channel=details.sender,
+                        msg_id=feedback_reminder_id,
+                    )
+                # Skipping answering due to pre-filtering is not considered a failure
+                if notify_no_answer:
+                    apologize_for_fail(details, client)
+    finally:
+        if client.tenant_id:
+            current_tenant_id.reset(token)
 
 
 def acknowledge_message(req: SocketModeRequest, client: TenantSocketModeClient) -> None:
@@ -499,7 +510,9 @@ def _initialize_socket_client(socket_client: TenantSocketModeClient) -> None:
             for tenant_id in tenant_ids:
                 with get_session_with_tenant(tenant_id) as db_session:
                     try:
+                        token = current_tenant_id.set(tenant_id or "public")
                         latest_slack_bot_tokens = fetch_tokens()
+                        current_tenant_id.reset(token)
 
                         if (
                             tenant_id not in slack_bot_tokens
@@ -533,6 +546,11 @@ def _initialize_socket_client(socket_client: TenantSocketModeClient) -> None:
                             socket_client = _get_socket_client(
                                 latest_slack_bot_tokens, tenant_id
                             )
+
+                            # Initialize socket client for this tenant. Each tenant has its own
+                            # socket client, allowing for multiple concurrent connections (one
+                            # per tenant) with the tenant ID wrapped in the socket model client.
+                            # Each `connect` stores websocket connection in a separate thread.
                             _initialize_socket_client(socket_client)
 
                             socket_clients[tenant_id] = socket_client
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 1c6a6a3a329..625c36435cd 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -38,6 +38,7 @@
 from danswer.configs.app_configs import SECRET_JWT_KEY
 from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
+from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import current_tenant_id
 
@@ -188,6 +189,29 @@ def get_app_name(cls) -> str:
         return cls._app_name
 
 
+def get_all_tenant_ids() -> list[str] | list[None]:
+    if not MULTI_TENANT:
+        return [None]
+    with get_session_with_tenant(tenant_id="public") as session:
+        result = session.execute(
+            text(
+                """
+            SELECT schema_name
+            FROM information_schema.schemata
+            WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
+            )
+        )
+        tenant_ids = [row[0] for row in result]
+
+    valid_tenants = [
+        tenant
+        for tenant in tenant_ids
+        if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
+    ]
+
+    return valid_tenants
+
+
 def build_connection_string(
     *,
     db_api: str = ASYNC_DB_API,
@@ -332,9 +356,8 @@ def get_session_with_tenant(
                         cursor.close()
 
 
-def get_session_generator_with_tenant(
-    tenant_id: str | None = None,
-) -> Generator[Session, None, None]:
+def get_session_generator_with_tenant() -> Generator[Session, None, None]:
+    tenant_id = current_tenant_id.get()
     with get_session_with_tenant(tenant_id) as session:
         yield session
 
diff --git a/backend/danswer/db/llm.py b/backend/danswer/db/llm.py
index c03ed99e412..b01fd81079c 100644
--- a/backend/danswer/db/llm.py
+++ b/backend/danswer/db/llm.py
@@ -95,10 +95,11 @@ def upsert_llm_provider(
         group_ids=llm_provider.groups,
         db_session=db_session,
     )
+    full_llm_provider = FullLLMProvider.from_model(existing_llm_provider)
 
     db_session.commit()
 
-    return FullLLMProvider.from_model(existing_llm_provider)
+    return full_llm_provider
 
 
 def fetch_existing_embedding_providers(
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index 240ff355b5b..98f3d7ec1cb 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -18,7 +18,6 @@
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import current_tenant_id
 
-
 logger = setup_logger()
 
 
diff --git a/backend/ee/danswer/background/celery/celery_app.py b/backend/ee/danswer/background/celery/celery_app.py
index afc77c1466d..4010b8b3998 100644
--- a/backend/ee/danswer/background/celery/celery_app.py
+++ b/backend/ee/danswer/background/celery/celery_app.py
@@ -1,11 +1,11 @@
 from datetime import timedelta
 
 from danswer.background.celery.celery_app import celery_app
-from danswer.background.celery.celery_utils import get_all_tenant_ids
 from danswer.background.task_utils import build_celery_task_wrapper
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.chat import delete_chat_sessions_older_than
+from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import get_session_with_tenant
 from danswer.server.settings.store import load_settings
 from danswer.utils.logger import setup_logger
diff --git a/deployment/kubernetes/env-configmap.yaml b/deployment/kubernetes/env-configmap.yaml
index b833e0791ec..1d4bf1cffd7 100644
--- a/deployment/kubernetes/env-configmap.yaml
+++ b/deployment/kubernetes/env-configmap.yaml
@@ -2,7 +2,8 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: env-configmap
-data:
+data: 
+
   # Auth Setting, also check the secrets file
   AUTH_TYPE: "disabled"  # Change this for production uses unless Danswer is only accessible behind VPN
   ENCRYPTION_KEY_SECRET: ""  # This should not be specified directly in the yaml, this is just for reference

From cee68106ef8a1992666b3944e705494936b930d3 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 20 Oct 2024 17:41:18 -0700
Subject: [PATCH 169/376] Minor vespa standardization (#2861)

* minor additional standardization

* nit: typo

* k

* account for malformed params
---
 .../document_index/vespa/chunk_retrieval.py   | 34 ++++++++-----------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/backend/danswer/document_index/vespa/chunk_retrieval.py b/backend/danswer/document_index/vespa/chunk_retrieval.py
index 3de64cbc840..3622b9a3e02 100644
--- a/backend/danswer/document_index/vespa/chunk_retrieval.py
+++ b/backend/danswer/document_index/vespa/chunk_retrieval.py
@@ -8,7 +8,6 @@
 from typing import cast
 
 import httpx
-import requests
 from retry import retry
 
 from danswer.configs.app_configs import LOG_VESPA_TIMING_INFORMATION
@@ -194,20 +193,21 @@ def _get_chunks_via_visit_api(
 
     document_chunks: list[dict] = []
     while True:
-        response = requests.get(url, params=params)
         try:
-            response.raise_for_status()
-        except requests.HTTPError as e:
-            request_info = f"Headers: {response.request.headers}\nPayload: {params}"
-            response_info = f"Status Code: {response.status_code}\nResponse Content: {response.text}"
-            error_base = f"Error occurred getting chunk by Document ID {chunk_request.document_id}"
+            filtered_params = {k: v for k, v in params.items() if v is not None}
+            with get_vespa_http_client() as http_client:
+                response = http_client.get(url, params=filtered_params)
+                response.raise_for_status()
+        except httpx.HTTPError as e:
+            error_base = "Failed to query Vespa"
             logger.error(
                 f"{error_base}:\n"
-                f"{request_info}\n"
-                f"{response_info}\n"
-                f"Exception: {e}"
+                f"Request URL: {e.request.url}\n"
+                f"Request Headers: {e.request.headers}\n"
+                f"Request Payload: {params}\n"
+                f"Exception: {str(e)}"
             )
-            raise requests.HTTPError(error_base) from e
+            raise httpx.HTTPError(error_base) from e
 
         # Check if the response contains any documents
         response_data = response.json()
@@ -301,17 +301,13 @@ def query_vespa(
             response = http_client.post(SEARCH_ENDPOINT, json=params)
             response.raise_for_status()
     except httpx.HTTPError as e:
-        request_info = f"Headers: {response.request.headers}\nPayload: {params}"
-        response_info = (
-            f"Status Code: {response.status_code}\n"
-            f"Response Content: {response.text}"
-        )
         error_base = "Failed to query Vespa"
         logger.error(
             f"{error_base}:\n"
-            f"{request_info}\n"
-            f"{response_info}\n"
-            f"Exception: {e}"
+            f"Request URL: {e.request.url}\n"
+            f"Request Headers: {e.request.headers}\n"
+            f"Request Payload: {params}\n"
+            f"Exception: {str(e)}"
         )
         raise httpx.HTTPError(error_base) from e
 

From 45d852a9db986181b8208a4371aa2d062833c7a8 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 20 Oct 2024 20:42:26 -0700
Subject: [PATCH 170/376] modal onboarding clarity (#2780)

---
 .../llm/CustomLLMProviderUpdateForm.tsx       |  7 ++-
 .../llm/LLMProviderUpdateForm.tsx             |  4 +-
 .../initialSetup/search/NoSourcesModal.tsx    |  4 +-
 .../initialSetup/welcome/WelcomeModal.tsx     | 13 ++---
 web/src/components/llm/ApiKeyForm.tsx         |  4 ++
 web/src/components/llm/ApiKeyModal.tsx        | 47 +++++++++----------
 6 files changed, 37 insertions(+), 42 deletions(-)

diff --git a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
index 1b86f9ef8fa..cc059ebe3d0 100644
--- a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
@@ -38,11 +38,13 @@ export function CustomLLMProviderUpdateForm({
   existingLlmProvider,
   shouldMarkAsDefault,
   setPopup,
+  hideSuccess,
 }: {
   onClose: () => void;
   existingLlmProvider?: FullLLMProvider;
   shouldMarkAsDefault?: boolean;
   setPopup?: (popup: PopupSpec) => void;
+  hideSuccess?: boolean;
 }) {
   const { mutate } = useSWRConfig();
 
@@ -108,9 +110,6 @@ export function CustomLLMProviderUpdateForm({
           return;
         }
 
-        // don't set groups if marked as public
-        const groups = values.is_public ? [] : values.groups;
-
         // test the configuration
         if (!isEqual(values, initialValues)) {
           setIsTesting(true);
@@ -190,7 +189,7 @@ export function CustomLLMProviderUpdateForm({
         const successMsg = existingLlmProvider
           ? "Provider updated successfully!"
           : "Provider enabled successfully!";
-        if (setPopup) {
+        if (!hideSuccess && setPopup) {
           setPopup({
             type: "success",
             message: successMsg,
diff --git a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
index b072083662f..80c7ce0f3b3 100644
--- a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
@@ -25,6 +25,7 @@ export function LLMProviderUpdateForm({
   shouldMarkAsDefault,
   setPopup,
   hideAdvanced,
+  hideSuccess,
 }: {
   llmProviderDescriptor: WellKnownLLMProviderDescriptor;
   onClose: () => void;
@@ -32,6 +33,7 @@ export function LLMProviderUpdateForm({
   shouldMarkAsDefault?: boolean;
   hideAdvanced?: boolean;
   setPopup?: (popup: PopupSpec) => void;
+  hideSuccess?: boolean;
 }) {
   const { mutate } = useSWRConfig();
 
@@ -202,7 +204,7 @@ export function LLMProviderUpdateForm({
         const successMsg = existingLlmProvider
           ? "Provider updated successfully!"
           : "Provider enabled successfully!";
-        if (setPopup) {
+        if (!hideSuccess && setPopup) {
           setPopup({
             type: "success",
             message: successMsg,
diff --git a/web/src/components/initialSetup/search/NoSourcesModal.tsx b/web/src/components/initialSetup/search/NoSourcesModal.tsx
index f220e274924..b4f6ec004ab 100644
--- a/web/src/components/initialSetup/search/NoSourcesModal.tsx
+++ b/web/src/components/initialSetup/search/NoSourcesModal.tsx
@@ -10,7 +10,7 @@ import { SettingsContext } from "@/components/settings/SettingsProvider";
 export function NoSourcesModal() {
   const settings = useContext(SettingsContext);
   const [isHidden, setIsHidden] = useState(
-    !settings?.settings.search_page_enabled ?? false
+    !settings?.settings.search_page_enabled
   );
 
   if (isHidden) {
@@ -19,7 +19,7 @@ export function NoSourcesModal() {
 
   return (
     <Modal
-      className="max-w-4xl"
+      width="max-w-3xl w-full"
       title="🧐 No sources connected"
       onOutsideClick={() => setIsHidden(true)}
     >
diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index 1c94ae22961..81496edb8f9 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -27,7 +27,6 @@ export function _CompletedWelcomeFlowDummyComponent() {
 export function _WelcomeModal({ user }: { user: User | null }) {
   const router = useRouter();
 
-  const [canBegin, setCanBegin] = useState(false);
   const [providerOptions, setProviderOptions] = useState<
     WellKnownLLMProviderDescriptor[]
   >([]);
@@ -75,19 +74,13 @@ export function _WelcomeModal({ user }: { user: User | null }) {
 
           <div className="max-h-[900px] overflow-y-scroll">
             <ApiKeyForm
+              // Don't show success message on initial setup
+              hideSuccess
               setPopup={setPopup}
-              onSuccess={() => {
-                router.refresh();
-                refreshProviderInfo();
-                setCanBegin(true);
-              }}
+              onSuccess={clientSetWelcomeFlowComplete}
               providerOptions={providerOptions}
             />
           </div>
-          <Divider />
-          <Button disabled={!canBegin} onClick={clientSetWelcomeFlowComplete}>
-            Get Started
-          </Button>
         </div>
       </Modal>
     </>
diff --git a/web/src/components/llm/ApiKeyForm.tsx b/web/src/components/llm/ApiKeyForm.tsx
index 189d1fc5738..276bcc36e6c 100644
--- a/web/src/components/llm/ApiKeyForm.tsx
+++ b/web/src/components/llm/ApiKeyForm.tsx
@@ -9,10 +9,12 @@ export const ApiKeyForm = ({
   onSuccess,
   providerOptions,
   setPopup,
+  hideSuccess,
 }: {
   onSuccess: () => void;
   providerOptions: WellKnownLLMProviderDescriptor[];
   setPopup: (popup: PopupSpec) => void;
+  hideSuccess?: boolean;
 }) => {
   const defaultProvider = providerOptions[0]?.name;
   const providerNameToIndexMap = new Map<string, number>();
@@ -56,6 +58,7 @@ export const ApiKeyForm = ({
                   onClose={() => onSuccess()}
                   shouldMarkAsDefault
                   setPopup={setPopup}
+                  hideSuccess={hideSuccess}
                 />
               </TabPanel>
             );
@@ -65,6 +68,7 @@ export const ApiKeyForm = ({
               onClose={() => onSuccess()}
               shouldMarkAsDefault
               setPopup={setPopup}
+              hideSuccess={hideSuccess}
             />
           </TabPanel>
         </TabPanels>
diff --git a/web/src/components/llm/ApiKeyModal.tsx b/web/src/components/llm/ApiKeyModal.tsx
index d1301311c1c..d401036e493 100644
--- a/web/src/components/llm/ApiKeyModal.tsx
+++ b/web/src/components/llm/ApiKeyModal.tsx
@@ -24,38 +24,35 @@ export const ApiKeyModal = ({
   if (!shouldShowConfigurationNeeded) {
     return null;
   }
-
   return (
     <Modal
       title="Set an API Key!"
       width="max-w-3xl w-full"
       onOutsideClick={() => hide()}
     >
-      <div className="max-h-[75vh] overflow-y-auto flex flex-col">
-        <div>
-          <div className="mb-5 text-sm">
-            Please provide an API Key below in order to start using
-            Danswer – you can always change this later.
-            <br />
-            If you&apos;d rather look around first, you can
-            <strong onClick={() => hide()} className="text-link cursor-pointer">
-              {" "}
-              skip this step
-            </strong>
-            .
-          </div>
-
-          <ApiKeyForm
-            setPopup={setPopup}
-            onSuccess={() => {
-              router.refresh();
-              refreshProviderInfo();
-              hide();
-            }}
-            providerOptions={providerOptions}
-          />
+      <>
+        <div className="mb-5 text-sm text-gray-700">
+          Please provide an API Key below in order to start using Danswer – you
+          can always change this later.
+          <br />
+          If you&apos;d rather look around first, you can
+          <strong onClick={() => hide()} className="text-link cursor-pointer">
+            {" "}
+            skip this step
+          </strong>
+          .
         </div>
-      </div>
+
+        <ApiKeyForm
+          setPopup={setPopup}
+          onSuccess={() => {
+            router.refresh();
+            refreshProviderInfo();
+            hide();
+          }}
+          providerOptions={providerOptions}
+        />
+      </>
     </Modal>
   );
 };

From c516f3541caa5afcdf9b1e2dc763380dd582317f Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Mon, 21 Oct 2024 11:51:53 -0700
Subject: [PATCH 171/376] Make it so you can update model providers (#2866)

---
 web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
index 80c7ce0f3b3..7968fc5d7bf 100644
--- a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
@@ -104,7 +104,7 @@ export function LLMProviderUpdateForm({
       : {}),
     deployment_name: llmProviderDescriptor.deployment_name_required
       ? Yup.string().required("Deployment Name is required")
-      : Yup.string(),
+      : Yup.string().nullable(),
     default_model_name: Yup.string().required("Model name is required"),
     fast_default_model_name: Yup.string().nullable(),
     // EE Only

From 802086ee57af2951654a37f3e4f0418d5e04f146 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Mon, 21 Oct 2024 16:03:40 -0700
Subject: [PATCH 172/376] Refactored Confluence Connector (#2859)

* Refactored Confluence Connector

* rename metadataconnector to slimconnector

Finish rename

* danswer->onyx

* added rec

* typo

* refactored doc_sync for confluence

* mypy + enable tests

* tested and fixed for confluence cloud

* fixed all server syncing

* fixed connector test

* mypy+connector test fixes

* addressed richards comments

* minor fix
---
 .../danswer/background/celery/celery_utils.py |  34 +-
 .../background/celery/tasks/pruning/tasks.py  |   2 +-
 backend/danswer/configs/app_configs.py        |   6 -
 backend/danswer/connectors/README.md          |   4 +-
 .../connectors/confluence/confluence_utils.py |  32 -
 .../connectors/confluence/connector.py        | 948 ++++--------------
 ...te_limit_handler.py => onyx_confluence.py} | 266 +++--
 .../danswer/connectors/confluence/utils.py    | 176 ++++
 backend/danswer/connectors/factory.py         |   2 +-
 backend/danswer/connectors/interfaces.py      |   6 +-
 backend/danswer/connectors/models.py          |   7 +-
 .../connectors/salesforce/connector.py        |  19 +-
 backend/danswer/connectors/slack/connector.py |  26 +-
 .../confluence/doc_sync.py                    | 265 ++---
 .../confluence/group_sync.py                  |  10 +-
 .../confluence/sync_utils.py                  |  14 +-
 .../external_permissions/permission_sync.py   |   3 +-
 .../external_permissions/slack/doc_sync.py    |  45 +-
 .../confluence/test_confluence_basic.py       |  33 +-
 .../slack/test_permission_sync.py             |   4 +-
 .../connector_job_tests/slack/test_prune.py   |   4 +-
 .../confluence/test_rate_limit_handler.py     |   6 +-
 22 files changed, 754 insertions(+), 1158 deletions(-)
 delete mode 100644 backend/danswer/connectors/confluence/confluence_utils.py
 rename backend/danswer/connectors/confluence/{rate_limit_handler.py => onyx_confluence.py} (71%)
 create mode 100644 backend/danswer/connectors/confluence/utils.py

diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index b76e148e237..4b499268cb4 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -14,9 +14,9 @@
     rate_limit_builder,
 )
 from danswer.connectors.interfaces import BaseConnector
-from danswer.connectors.interfaces import IdConnector
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import Document
 from danswer.db.connector_credential_pair import get_connector_credential_pair
 from danswer.db.engine import get_session_with_tenant
@@ -67,7 +67,9 @@ def get_deletion_attempt_snapshot(
     )
 
 
-def document_batch_to_ids(doc_batch: list[Document]) -> set[str]:
+def document_batch_to_ids(
+    doc_batch: list[Document],
+) -> set[str]:
     return {doc.id for doc in doc_batch}
 
 
@@ -83,10 +85,13 @@ def extract_ids_from_runnable_connector(
     """
     all_connector_doc_ids: set[str] = set()
 
+    if isinstance(runnable_connector, SlimConnector):
+        for metadata_batch in runnable_connector.retrieve_all_slim_documents():
+            all_connector_doc_ids.update({doc.id for doc in metadata_batch})
+
     doc_batch_generator = None
-    if isinstance(runnable_connector, IdConnector):
-        all_connector_doc_ids = runnable_connector.retrieve_all_source_ids()
-    elif isinstance(runnable_connector, LoadConnector):
+
+    if isinstance(runnable_connector, LoadConnector):
         doc_batch_generator = runnable_connector.load_from_state()
     elif isinstance(runnable_connector, PollConnector):
         start = datetime(1970, 1, 1, tzinfo=timezone.utc).timestamp()
@@ -95,16 +100,15 @@ def extract_ids_from_runnable_connector(
     else:
         raise RuntimeError("Pruning job could not find a valid runnable_connector.")
 
-    if doc_batch_generator:
-        doc_batch_processing_func = document_batch_to_ids
-        if MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE:
-            doc_batch_processing_func = rate_limit_builder(
-                max_calls=MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE, period=60
-            )(document_batch_to_ids)
-        for doc_batch in doc_batch_generator:
-            if progress_callback:
-                progress_callback(len(doc_batch))
-            all_connector_doc_ids.update(doc_batch_processing_func(doc_batch))
+    doc_batch_processing_func = document_batch_to_ids
+    if MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE:
+        doc_batch_processing_func = rate_limit_builder(
+            max_calls=MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE, period=60
+        )(document_batch_to_ids)
+    for doc_batch in doc_batch_generator:
+        if progress_callback:
+            progress_callback(len(doc_batch))
+        all_connector_doc_ids.update(doc_batch_processing_func(doc_batch))
 
     return all_connector_doc_ids
 
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index ee5adfd10c0..4bfde82292a 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -242,7 +242,7 @@ def redis_increment_callback(amount: int) -> None:
             runnable_connector = instantiate_connector(
                 db_session,
                 cc_pair.connector.source,
-                InputType.PRUNE,
+                InputType.SLIM_RETRIEVAL,
                 cc_pair.connector.connector_specific_config,
                 cc_pair.credential,
             )
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 08caf7181f8..d53fb0b12ea 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -253,12 +253,6 @@
     os.environ.get("CONFLUENCE_CONNECTOR_INDEX_ARCHIVED_PAGES", "").lower() == "true"
 )
 
-# Save pages labels as Danswer metadata tags
-# The reason to skip this would be to reduce the number of calls to Confluence due to rate limit concerns
-CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING = (
-    os.environ.get("CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING", "").lower() == "true"
-)
-
 # Attachments exceeding this size will not be retrieved (in bytes)
 CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD = int(
     os.environ.get("CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD", 10 * 1024 * 1024)
diff --git a/backend/danswer/connectors/README.md b/backend/danswer/connectors/README.md
index ef6c63d2697..bb7f5a5fe4f 100644
--- a/backend/danswer/connectors/README.md
+++ b/backend/danswer/connectors/README.md
@@ -13,8 +13,8 @@ Connectors come in 3 different flows:
   documents via a connector's API or loads the documents from some sort of a dump file.
 - Poll connector:
   - Incrementally updates documents based on a provided time range. It is used by the background job to pull the latest
-  changes additions and changes since the last round of polling. This connector helps keep the document index up to date
-  without needing to fetch/embed/index every document which generally be too slow to do frequently on large sets of
+  changes and additions since the last round of polling. This connector helps keep the document index up to date
+  without needing to fetch/embed/index every document which would be too slow to do frequently on large sets of
   documents.
 - Event Based connectors:
   - Connectors that listen to events and update documents accordingly.
diff --git a/backend/danswer/connectors/confluence/confluence_utils.py b/backend/danswer/connectors/confluence/confluence_utils.py
deleted file mode 100644
index 927e989bf3f..00000000000
--- a/backend/danswer/connectors/confluence/confluence_utils.py
+++ /dev/null
@@ -1,32 +0,0 @@
-import bs4
-
-
-def build_confluence_document_id(base_url: str, content_url: str) -> str:
-    """For confluence, the document id is the page url for a page based document
-        or the attachment download url for an attachment based document
-
-    Args:
-        base_url (str): The base url of the Confluence instance
-        content_url (str): The url of the page or attachment download url
-
-    Returns:
-        str: The document id
-    """
-    return f"{base_url}{content_url}"
-
-
-def get_used_attachments(text: str) -> list[str]:
-    """Parse a Confluence html page to generate a list of current
-        attachment in used
-
-    Args:
-        text (str): The page content
-
-    Returns:
-        list[str]: List of filenames currently in use by the page text
-    """
-    files_in_used = []
-    soup = bs4.BeautifulSoup(text, "html.parser")
-    for attachment in soup.findAll("ri:attachment"):
-        files_in_used.append(attachment.attrs["ri:filename"])
-    return files_in_used
diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index 03b91fa29d4..92d981855f1 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -1,44 +1,26 @@
-import io
-import os
-from collections.abc import Callable
-from collections.abc import Collection
 from datetime import datetime
 from datetime import timezone
-from functools import lru_cache
 from typing import Any
-from urllib.parse import parse_qs
-from urllib.parse import urlparse
-
-import bs4
-from atlassian import Confluence  # type:ignore
-from requests import HTTPError
-
-from danswer.configs.app_configs import (
-    CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD,
-)
-from danswer.configs.app_configs import CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD
-from danswer.configs.app_configs import CONFLUENCE_CONNECTOR_INDEX_ARCHIVED_PAGES
+
 from danswer.configs.app_configs import CONFLUENCE_CONNECTOR_LABELS_TO_SKIP
-from danswer.configs.app_configs import CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING
 from danswer.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
-from danswer.connectors.confluence.confluence_utils import (
-    build_confluence_document_id,
-)
-from danswer.connectors.confluence.confluence_utils import get_used_attachments
-from danswer.connectors.confluence.rate_limit_handler import (
-    make_confluence_call_handle_rate_limit,
-)
+from danswer.connectors.confluence.onyx_confluence import OnyxConfluence
+from danswer.connectors.confluence.utils import attachment_to_content
+from danswer.connectors.confluence.utils import build_confluence_document_id
+from danswer.connectors.confluence.utils import datetime_from_string
+from danswer.connectors.confluence.utils import extract_text_from_confluence_html
 from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import ConnectorMissingCredentialError
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
-from danswer.file_processing.extract_file_text import extract_file_text
-from danswer.file_processing.html_utils import format_document_soup
+from danswer.connectors.models import SlimDocument
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -47,250 +29,27 @@
 # 1. Include attachments, etc
 # 2. Segment into Sections for more accurate linking, can split by headers but make sure no text/ordering is lost
 
-
-NO_PERMISSIONS_TO_VIEW_ATTACHMENTS_ERROR_STR = (
-    "User not permitted to view attachments on content"
-)
-NO_PARENT_OR_NO_PERMISSIONS_ERROR_STR = (
-    "No parent or not permitted to view content with id"
-)
-
-
-class DanswerConfluence(Confluence):
-    """
-    This is a custom Confluence class that overrides the default Confluence class to add a custom CQL method.
-    This is necessary because the default Confluence class does not properly support cql expansions.
-    """
-
-    def __init__(self, url: str, *args: Any, **kwargs: Any) -> None:
-        super(DanswerConfluence, self).__init__(url, *args, **kwargs)
-
-    def danswer_cql(
-        self,
-        cql: str,
-        expand: str | None = None,
-        cursor: str | None = None,
-        limit: int = 500,
-        include_archived_spaces: bool = False,
-    ) -> dict[str, Any]:
-        url_suffix = f"rest/api/content/search?cql={cql}"
-        if expand:
-            url_suffix += f"&expand={expand}"
-        if cursor:
-            url_suffix += f"&cursor={cursor}"
-        url_suffix += f"&limit={limit}"
-        if include_archived_spaces:
-            url_suffix += "&includeArchivedSpaces=true"
-        try:
-            response = self.get(url_suffix)
-            return response
-        except Exception as e:
-            raise e
-
-
-@lru_cache()
-def _get_user(user_id: str, confluence_client: DanswerConfluence) -> str:
-    """Get Confluence Display Name based on the account-id or userkey value
-
-    Args:
-        user_id (str): The user id (i.e: the account-id or userkey)
-        confluence_client (Confluence): The Confluence Client
-
-    Returns:
-        str: The User Display Name. 'Unknown User' if the user is deactivated or not found
-    """
-    user_not_found = "Unknown User"
-
-    get_user_details_by_accountid = make_confluence_call_handle_rate_limit(
-        confluence_client.get_user_details_by_accountid
-    )
-    try:
-        logger.info(f"_get_user - get_user_details_by_accountid: id={user_id}")
-        return get_user_details_by_accountid(user_id).get("displayName", user_not_found)
-    except Exception as e:
-        logger.warning(
-            f"Unable to get the User Display Name with the id: '{user_id}' - {e}"
-        )
-    return user_not_found
-
-
-def parse_html_page(text: str, confluence_client: DanswerConfluence) -> str:
-    """Parse a Confluence html page and replace the 'user Id' by the real
-        User Display Name
-
-    Args:
-        text (str): The page content
-        confluence_client (Confluence): Confluence client
-
-    Returns:
-        str: loaded and formated Confluence page
-    """
-    soup = bs4.BeautifulSoup(text, "html.parser")
-    for user in soup.findAll("ri:user"):
-        user_id = (
-            user.attrs["ri:account-id"]
-            if "ri:account-id" in user.attrs
-            else user.get("ri:userkey")
-        )
-        if not user_id:
-            logger.warning(
-                "ri:userkey not found in ri:user element. " f"Found attrs: {user.attrs}"
-            )
-            continue
-        # Include @ sign for tagging, more clear for LLM
-        user.replaceWith("@" + _get_user(user_id, confluence_client))
-    return format_document_soup(soup)
-
-
-def _comment_dfs(
-    comments_str: str,
-    comment_pages: Collection[dict[str, Any]],
-    confluence_client: DanswerConfluence,
-) -> str:
-    get_page_child_by_type = make_confluence_call_handle_rate_limit(
-        confluence_client.get_page_child_by_type
-    )
-
-    for comment_page in comment_pages:
-        comment_html = comment_page["body"]["storage"]["value"]
-        comments_str += "\nComment:\n" + parse_html_page(
-            comment_html, confluence_client
-        )
-        try:
-            logger.info(
-                f"_comment_dfs - get_page_by_child_type: id={comment_page['id']}"
-            )
-            child_comment_pages = get_page_child_by_type(
-                comment_page["id"],
-                type="comment",
-                start=None,
-                limit=None,
-                expand="body.storage.value",
-            )
-            comments_str = _comment_dfs(
-                comments_str, child_comment_pages, confluence_client
-            )
-        except HTTPError as e:
-            # not the cleanest, but I'm not aware of a nicer way to check the error
-            if NO_PARENT_OR_NO_PERMISSIONS_ERROR_STR not in str(e):
-                raise
-
-    return comments_str
-
-
-def _datetime_from_string(datetime_string: str) -> datetime:
-    datetime_object = datetime.fromisoformat(datetime_string)
-
-    if datetime_object.tzinfo is None:
-        # If no timezone info, assume it is UTC
-        datetime_object = datetime_object.replace(tzinfo=timezone.utc)
-    else:
-        # If not in UTC, translate it
-        datetime_object = datetime_object.astimezone(timezone.utc)
-
-    return datetime_object
-
-
-class RecursiveIndexer:
-    def __init__(
-        self,
-        batch_size: int,
-        confluence_client: Confluence,
-        index_recursively: bool,
-        origin_page_id: str,
-    ) -> None:
-        self.batch_size = batch_size
-        self.confluence_client = confluence_client
-        self.index_recursively = index_recursively
-        self.origin_page_id = origin_page_id
-        self.pages = self.recurse_children_pages(self.origin_page_id)
-
-    def get_origin_page(self) -> list[dict[str, Any]]:
-        return [self._fetch_origin_page()]
-
-    def get_pages(self) -> list[dict[str, Any]]:
-        return self.pages
-
-    def _fetch_origin_page(self) -> dict[str, Any]:
-        get_page_by_id = make_confluence_call_handle_rate_limit(
-            self.confluence_client.get_page_by_id
-        )
-        try:
-            logger.info(
-                f"_fetch_origin_page - get_page_by_id: id={self.origin_page_id}"
-            )
-            origin_page = get_page_by_id(
-                self.origin_page_id, expand="body.storage.value,version,space"
-            )
-            return origin_page
-        except Exception:
-            logger.exception(
-                f"Appending origin page with id {self.origin_page_id} failed."
-            )
-            return {}
-
-    def recurse_children_pages(
-        self,
-        page_id: str,
-    ) -> list[dict[str, Any]]:
-        pages: list[dict[str, Any]] = []
-        queue: list[str] = [page_id]
-        visited_pages: set[str] = set()
-
-        get_page_by_id = make_confluence_call_handle_rate_limit(
-            self.confluence_client.get_page_by_id
-        )
-
-        get_page_child_by_type = make_confluence_call_handle_rate_limit(
-            self.confluence_client.get_page_child_by_type
-        )
-
-        while queue:
-            current_page_id = queue.pop(0)
-            if current_page_id in visited_pages:
-                continue
-            visited_pages.add(current_page_id)
-
-            try:
-                # Fetch the page itself
-                logger.info(
-                    f"recurse_children_pages - get_page_by_id: id={current_page_id}"
-                )
-                page = get_page_by_id(
-                    current_page_id, expand="body.storage.value,version,space"
-                )
-                pages.append(page)
-            except Exception:
-                logger.exception(f"Failed to fetch page {current_page_id}.")
-                continue
-
-            if not self.index_recursively:
-                continue
-
-            # Fetch child pages
-            start = 0
-            while True:
-                logger.info(
-                    f"recurse_children_pages - get_page_by_child_type: id={current_page_id}"
-                )
-                child_pages_response = get_page_child_by_type(
-                    current_page_id,
-                    type="page",
-                    start=start,
-                    limit=self.batch_size,
-                    expand="",
-                )
-                if not child_pages_response:
-                    break
-                for child_page in child_pages_response:
-                    child_page_id = child_page["id"]
-                    queue.append(child_page_id)
-                start += len(child_pages_response)
-
-        return pages
-
-
-class ConfluenceConnector(LoadConnector, PollConnector):
+_COMMENT_EXPANSION_FIELDS = ["body.storage.value"]
+_PAGE_EXPANSION_FIELDS = [
+    "body.storage.value",
+    "version",
+    "space",
+    "metadata.labels",
+]
+_ATTACHMENT_EXPANSION_FIELDS = [
+    "version",
+    "space",
+    "metadata.labels",
+]
+
+_RESTRICTIONS_EXPANSION_FIELDS = [
+    "space",
+    "restrictions.read.restrictions.user",
+    "restrictions.read.restrictions.group",
+]
+
+
+class ConfluenceConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
         wiki_base: str,
@@ -298,44 +57,43 @@ def __init__(
         space: str = "",
         page_id: str = "",
         index_recursively: bool = True,
+        cql_query: str | None = None,
         batch_size: int = INDEX_BATCH_SIZE,
         continue_on_failure: bool = CONTINUE_ON_CONNECTOR_FAILURE,
         # if a page has one of the labels specified in this list, we will just
         # skip it. This is generally used to avoid indexing extra sensitive
         # pages.
         labels_to_skip: list[str] = CONFLUENCE_CONNECTOR_LABELS_TO_SKIP,
-        cql_query: str | None = None,
     ) -> None:
         self.batch_size = batch_size
         self.continue_on_failure = continue_on_failure
-        self.labels_to_skip = set(labels_to_skip)
-        self.recursive_indexer: RecursiveIndexer | None = None
-        self.index_recursively = False if cql_query else index_recursively
+        self.confluence_client: OnyxConfluence | None = None
+        self.is_cloud = is_cloud
 
         # Remove trailing slash from wiki_base if present
         self.wiki_base = wiki_base.rstrip("/")
-        self.page_id = "" if cql_query else page_id
-        self.space_level_scan = bool(not self.page_id)
-
-        self.is_cloud = is_cloud
-
-        self.confluence_client: DanswerConfluence | None = None
 
-        # if a cql_query is provided, we will use it to fetch the pages
-        # if no cql_query is provided, we will use the space to fetch the pages
-        # if no space is provided and no cql_query, we will default to fetching all pages, regardless of space
+        # if nothing is provided, we will fetch all pages
+        self.cql_page_query = "type=page"
         if cql_query:
-            self.cql_query = cql_query
+            # if a cql_query is provided, we will use it to fetch the pages
+            self.cql_page_query = cql_query
         elif space:
-            self.cql_query = f"type=page and space='{space}'"
-        else:
-            self.cql_query = "type=page"
-
-        logger.info(
-            f"wiki_base: {self.wiki_base}, space: {space}, page_id: {self.page_id},"
-            + f" space_level_scan: {self.space_level_scan}, index_recursively: {self.index_recursively},"
-            + f" cql_query: {self.cql_query}"
-        )
+            # if no cql_query is provided, we will use the space to fetch the pages
+            self.cql_page_query += f" and space='{space}'"
+        elif page_id:
+            if index_recursively:
+                self.cql_page_query += f" and ancestor='{page_id}'"
+            else:
+                # if neither a space nor a cql_query is provided, we will use the page_id to fetch the page
+                self.cql_page_query += f" and id='{page_id}'"
+
+        self.cql_label_filter = ""
+        self.cql_time_filter = ""
+        if labels_to_skip:
+            labels_to_skip = list(set(labels_to_skip))
+            comma_separated_labels = ",".join(labels_to_skip)
+            self.cql_label_filter = f"&label not in ({comma_separated_labels})"
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         username = credentials["confluence_username"]
@@ -343,7 +101,7 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
 
         # see https://github.com/atlassian-api/atlassian-python-api/blob/master/atlassian/rest_client.py
         # for a list of other hidden constructor args
-        self.confluence_client = DanswerConfluence(
+        self.confluence_client = OnyxConfluence(
             url=self.wiki_base,
             username=username if self.is_cloud else None,
             password=access_token if self.is_cloud else None,
@@ -354,474 +112,188 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
         )
         return None
 
-    def _fetch_pages(
-        self,
-        cursor: str | None,
-    ) -> tuple[list[dict[str, Any]], str | None]:
+    def _get_comment_string_for_page_id(self, page_id: str) -> str:
         if self.confluence_client is None:
-            raise Exception("Confluence client is not initialized")
-
-        def _fetch_space(
-            cursor: str | None, batch_size: int
-        ) -> tuple[list[dict[str, Any]], str | None]:
-            if not self.confluence_client:
-                raise Exception("Confluence client is not initialized")
-            get_all_pages = make_confluence_call_handle_rate_limit(
-                self.confluence_client.danswer_cql
-            )
+            raise ConnectorMissingCredentialError("Confluence")
 
-            include_archived_spaces = (
-                CONFLUENCE_CONNECTOR_INDEX_ARCHIVED_PAGES
-                if not self.is_cloud
-                else False
-            )
+        comment_string = ""
 
-            try:
-                logger.info(
-                    f"_fetch_space - get_all_pages: cursor={cursor} limit={batch_size}"
-                )
-                response = get_all_pages(
-                    cql=self.cql_query,
-                    cursor=cursor,
-                    limit=batch_size,
-                    expand="body.storage.value,version,space",
-                    include_archived_spaces=include_archived_spaces,
-                )
-                pages = response.get("results", [])
-                next_cursor = None
-                if "_links" in response and "next" in response["_links"]:
-                    next_link = response["_links"]["next"]
-                    parsed_url = urlparse(next_link)
-                    query_params = parse_qs(parsed_url.query)
-                    cursor_list = query_params.get("cursor", [])
-                    if cursor_list:
-                        next_cursor = cursor_list[0]
-                return pages, next_cursor
-            except Exception:
-                logger.warning(
-                    f"Batch failed with cql {self.cql_query} with cursor {cursor} "
-                    f"and size {batch_size}, processing pages individually..."
-                )
+        comment_cql = f"type=comment and container='{page_id}'"
+        comment_cql += self.cql_label_filter
 
-                view_pages: list[dict[str, Any]] = []
-                for _ in range(self.batch_size):
-                    try:
-                        logger.info(
-                            f"_fetch_space - get_all_pages: cursor={cursor} limit=1"
-                        )
-                        response = get_all_pages(
-                            cql=self.cql_query,
-                            cursor=cursor,
-                            limit=1,
-                            expand="body.view.value,version,space",
-                            include_archived_spaces=include_archived_spaces,
-                        )
-                        pages = response.get("results", [])
-                        view_pages.extend(pages)
-                        if "_links" in response and "next" in response["_links"]:
-                            next_link = response["_links"]["next"]
-                            parsed_url = urlparse(next_link)
-                            query_params = parse_qs(parsed_url.query)
-                            cursor_list = query_params.get("cursor", [])
-                            if cursor_list:
-                                cursor = cursor_list[0]
-                            else:
-                                cursor = None
-                        else:
-                            cursor = None
-                            break
-                    except HTTPError as e:
-                        logger.warning(
-                            f"Page failed with cql {self.cql_query} with cursor {cursor}, "
-                            f"trying alternative expand option: {e}"
-                        )
-                        logger.info(
-                            f"_fetch_space - get_all_pages - trying alternative expand: cursor={cursor} limit=1"
-                        )
-                        response = get_all_pages(
-                            cql=self.cql_query,
-                            cursor=cursor,
-                            limit=1,
-                            expand="body.view.value,version,space",
-                        )
-                        pages = response.get("results", [])
-                        view_pages.extend(pages)
-                        if "_links" in response and "next" in response["_links"]:
-                            next_link = response["_links"]["next"]
-                            parsed_url = urlparse(next_link)
-                            query_params = parse_qs(parsed_url.query)
-                            cursor_list = query_params.get("cursor", [])
-                            if cursor_list:
-                                cursor = cursor_list[0]
-                            else:
-                                cursor = None
-                        else:
-                            cursor = None
-                            break
-
-                return view_pages, cursor
-
-        def _fetch_page() -> tuple[list[dict[str, Any]], str | None]:
-            if self.confluence_client is None:
-                raise Exception("Confluence client is not initialized")
-
-            if self.recursive_indexer is None:
-                self.recursive_indexer = RecursiveIndexer(
-                    origin_page_id=self.page_id,
-                    batch_size=self.batch_size,
-                    confluence_client=self.confluence_client,
-                    index_recursively=self.index_recursively,
+        expand = ",".join(_COMMENT_EXPANSION_FIELDS)
+        for comments in self.confluence_client.paginated_cql_page_retrieval(
+            cql=comment_cql,
+            expand=expand,
+        ):
+            for comment in comments:
+                comment_string += "\nComment:\n"
+                comment_string += extract_text_from_confluence_html(
+                    confluence_client=self.confluence_client, confluence_object=comment
                 )
 
-            pages = self.recursive_indexer.get_pages()
-            return pages, None  # Since we fetched all pages, no cursor
-
-        try:
-            pages, next_cursor = (
-                _fetch_space(cursor, self.batch_size)
-                if self.space_level_scan
-                else _fetch_page()
-            )
-            return pages, next_cursor
-        except Exception as e:
-            if not self.continue_on_failure:
-                raise e
-
-            logger.exception("Ran into exception when fetching pages from Confluence")
-            return [], None
-
-    def _fetch_comments(self, confluence_client: Confluence, page_id: str) -> str:
-        get_page_child_by_type = make_confluence_call_handle_rate_limit(
-            confluence_client.get_page_child_by_type
-        )
-
-        try:
-            logger.info(f"_fetch_comments - get_page_child_by_type: id={page_id}")
-            comment_pages = list(
-                get_page_child_by_type(
-                    page_id,
-                    type="comment",
-                    start=None,
-                    limit=None,
-                    expand="body.storage.value",
-                )
-            )
-            return _comment_dfs("", comment_pages, confluence_client)
-        except Exception as e:
-            if not self.continue_on_failure:
-                raise e
+        return comment_string
 
-            logger.exception("Fetching comments from Confluence exceptioned")
-            return ""
+    def _convert_object_to_document(
+        self, confluence_object: dict[str, Any]
+    ) -> Document | None:
+        """
+        Takes in a confluence object, extracts all metadata, and converts it into a document.
+        If its a page, it extracts the text, adds the comments for the document text.
+        If its an attachment, it just downloads the attachment and converts that into a document.
+        """
+        if self.confluence_client is None:
+            raise ConnectorMissingCredentialError("Confluence")
 
-    def _fetch_labels(self, confluence_client: Confluence, page_id: str) -> list[str]:
-        get_page_labels = make_confluence_call_handle_rate_limit(
-            confluence_client.get_page_labels
+        # The url and the id are the same
+        object_url = build_confluence_document_id(
+            self.wiki_base, confluence_object["_links"]["webui"]
         )
-        try:
-            logger.info(f"_fetch_labels - get_page_labels: id={page_id}")
-            labels_response = get_page_labels(page_id)
-            return [label["name"] for label in labels_response["results"]]
-        except Exception as e:
-            if not self.continue_on_failure:
-                raise e
-
-            logger.exception("Fetching labels from Confluence exceptioned")
-            return []
-
-    @classmethod
-    def _attachment_to_download_link(
-        cls, confluence_client: Confluence, attachment: dict[str, Any]
-    ) -> str:
-        return confluence_client.url + attachment["_links"]["download"]
-
-    @classmethod
-    def _attachment_to_content(
-        cls,
-        confluence_client: Confluence,
-        attachment: dict[str, Any],
-    ) -> str | None:
-        """If it returns None, assume that we should skip this attachment."""
-        if attachment["metadata"]["mediaType"] in [
-            "image/jpeg",
-            "image/png",
-            "image/gif",
-            "image/svg+xml",
-            "video/mp4",
-            "video/quicktime",
-        ]:
-            return None
 
-        download_link = cls._attachment_to_download_link(confluence_client, attachment)
-
-        attachment_size = attachment["extensions"]["fileSize"]
-        if attachment_size > CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD:
-            logger.warning(
-                f"Skipping {download_link} due to size. "
-                f"size={attachment_size} "
-                f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD}"
+        object_text = None
+        # Extract text from page
+        if confluence_object["type"] == "page":
+            object_text = extract_text_from_confluence_html(
+                self.confluence_client, confluence_object
             )
-            return None
-
-        logger.info(f"_attachment_to_content - _session.get: link={download_link}")
-        response = confluence_client._session.get(download_link)
-        if response.status_code != 200:
-            logger.warning(
-                f"Failed to fetch {download_link} with invalid status code {response.status_code}"
+            # Add comments to text
+            object_text += self._get_comment_string_for_page_id(confluence_object["id"])
+        elif confluence_object["type"] == "attachment":
+            object_text = attachment_to_content(
+                self.confluence_client, confluence_object
             )
-            return None
 
-        extracted_text = extract_file_text(
-            io.BytesIO(response.content),
-            file_name=attachment["title"],
-            break_on_unprocessable=False,
-        )
-        if len(extracted_text) > CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD:
-            logger.warning(
-                f"Skipping {download_link} due to char count. "
-                f"char count={len(extracted_text)} "
-                f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD}"
-            )
+        if object_text is None:
             return None
 
-        return extracted_text
-
-    def _fetch_attachments(
-        self, confluence_client: Confluence, page_id: str, files_in_use: list[str]
-    ) -> tuple[str, list[dict[str, Any]]]:
-        unused_attachments: list[dict[str, Any]] = []
-        files_attachment_content: list[str] = []
+        # Get space name
+        doc_metadata: dict[str, str | list[str]] = {
+            "Wiki Space Name": confluence_object["space"]["name"]
+        }
 
-        get_attachments_from_content = make_confluence_call_handle_rate_limit(
-            confluence_client.get_attachments_from_content
+        # Get labels
+        label_dicts = confluence_object["metadata"]["labels"]["results"]
+        page_labels = [label["name"] for label in label_dicts]
+        if page_labels:
+            doc_metadata["labels"] = page_labels
+
+        # Get last modified and author email
+        last_modified = datetime_from_string(confluence_object["version"]["when"])
+        author_email = confluence_object["version"].get("by", {}).get("email")
+
+        return Document(
+            id=object_url,
+            sections=[Section(link=object_url, text=object_text)],
+            source=DocumentSource.CONFLUENCE,
+            semantic_identifier=confluence_object["title"],
+            doc_updated_at=last_modified,
+            primary_owners=(
+                [BasicExpertInfo(email=author_email)] if author_email else None
+            ),
+            metadata=doc_metadata,
         )
 
-        try:
-            expand = "history.lastUpdated,metadata.labels"
-            attachments_container = get_attachments_from_content(
-                page_id, start=None, limit=None, expand=expand
-            )
-            for attachment in attachments_container.get("results", []):
-                if attachment["title"] not in files_in_use:
-                    unused_attachments.append(attachment)
-                    continue
-
-                attachment_content = self._attachment_to_content(
-                    confluence_client, attachment
-                )
-                if attachment_content:
-                    files_attachment_content.append(attachment_content)
-
-        except Exception as e:
-            if isinstance(
-                e, HTTPError
-            ) and NO_PERMISSIONS_TO_VIEW_ATTACHMENTS_ERROR_STR in str(e):
-                logger.warning(
-                    f"User does not have access to attachments on page '{page_id}'"
-                )
-                return "", []
-            if not self.continue_on_failure:
-                raise e
-            logger.exception("Fetching attachments from Confluence exceptioned.")
-
-        return "\n".join(files_attachment_content), unused_attachments
-
-    def _get_doc_batch(
-        self, cursor: str | None
-    ) -> tuple[list[Any], str | None, list[dict[str, Any]]]:
+    def _fetch_document_batches(self) -> GenerateDocumentsOutput:
         if self.confluence_client is None:
-            raise Exception("Confluence client is not initialized")
-
-        doc_batch: list[Any] = []
-        unused_attachments: list[dict[str, Any]] = []
-
-        batch, next_cursor = self._fetch_pages(cursor)
-
-        for page in batch:
-            last_modified = _datetime_from_string(page["version"]["when"])
-            author = page["version"].get("by", {}).get("email")
+            raise ConnectorMissingCredentialError("Confluence")
 
-            page_id = page["id"]
+        doc_batch: list[Document] = []
+        confluence_page_ids: list[str] = []
+
+        page_query = self.cql_page_query + self.cql_label_filter + self.cql_time_filter
+        # Fetch pages as Documents
+        for pages in self.confluence_client.paginated_cql_page_retrieval(
+            cql=page_query,
+            expand=",".join(_PAGE_EXPANSION_FIELDS),
+            limit=self.batch_size,
+        ):
+            for page in pages:
+                confluence_page_ids.append(page["id"])
+                doc = self._convert_object_to_document(page)
+                if doc is not None:
+                    doc_batch.append(doc)
+                if len(doc_batch) >= self.batch_size:
+                    yield doc_batch
+                    doc_batch = []
+
+        # Fetch attachments as Documents
+        for confluence_page_id in confluence_page_ids:
+            attachment_cql = f"type=attachment and container='{confluence_page_id}'"
+            attachment_cql += self.cql_label_filter
+            # TODO: maybe should add time filter as well?
+            for attachments in self.confluence_client.paginated_cql_page_retrieval(
+                cql=attachment_cql,
+                expand=",".join(_ATTACHMENT_EXPANSION_FIELDS),
+            ):
+                for attachment in attachments:
+                    doc = self._convert_object_to_document(attachment)
+                    if doc is not None:
+                        doc_batch.append(doc)
+                    if len(doc_batch) >= self.batch_size:
+                        yield doc_batch
+                        doc_batch = []
+
+        if doc_batch:
+            yield doc_batch
 
-            if self.labels_to_skip or not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING:
-                page_labels = self._fetch_labels(self.confluence_client, page_id)
-            else:
-                page_labels = []
-
-            # check disallowed labels
-            if self.labels_to_skip:
-                label_intersection = self.labels_to_skip.intersection(page_labels)
-                if label_intersection:
-                    logger.info(
-                        f"Page with ID '{page_id}' has a label which has been "
-                        f"designated as disallowed: {label_intersection}. Skipping."
-                    )
-                    continue
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self._fetch_document_batches()
 
-            page_html = (
-                page["body"].get("storage", page["body"].get("view", {})).get("value")
-            )
-            # The url and the id are the same
-            page_url = build_confluence_document_id(
-                self.wiki_base, page["_links"]["webui"]
-            )
-            if not page_html:
-                logger.debug("Page is empty, skipping: %s", page_url)
-                continue
-            page_text = parse_html_page(page_html, self.confluence_client)
-
-            files_in_use = get_used_attachments(page_html)
-            attachment_text, unused_page_attachments = self._fetch_attachments(
-                self.confluence_client, page_id, files_in_use
-            )
-            unused_attachments.extend(unused_page_attachments)
-
-            page_text += "\n" + attachment_text if attachment_text else ""
-            comments_text = self._fetch_comments(self.confluence_client, page_id)
-            page_text += comments_text
-            doc_metadata: dict[str, str | list[str]] = {
-                "Wiki Space Name": page["space"]["name"]
-            }
-            if not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING and page_labels:
-                doc_metadata["labels"] = page_labels
-
-            doc_batch.append(
-                Document(
-                    id=page_url,
-                    sections=[Section(link=page_url, text=page_text)],
-                    source=DocumentSource.CONFLUENCE,
-                    semantic_identifier=page["title"],
-                    doc_updated_at=last_modified,
-                    primary_owners=(
-                        [BasicExpertInfo(email=author)] if author else None
-                    ),
-                    metadata=doc_metadata,
-                )
-            )
-        return (
-            doc_batch,
-            next_cursor,
-            unused_attachments,
+    def poll_source(self, start: float, end: float) -> GenerateDocumentsOutput:
+        # Add time filters
+        formatted_start_time = datetime.fromtimestamp(start, tz=timezone.utc).strftime(
+            "%Y-%m-%d %H:%M"
         )
+        formatted_end_time = datetime.fromtimestamp(end, tz=timezone.utc).strftime(
+            "%Y-%m-%d %H:%M"
+        )
+        self.cql_time_filter = f" and lastmodified >= '{formatted_start_time}'"
+        self.cql_time_filter += f" and lastmodified <= '{formatted_end_time}'"
+        return self._fetch_document_batches()
 
-    def _get_attachment_batch(
-        self,
-        start_ind: int,
-        attachments: list[dict[str, Any]],
-        time_filter: Callable[[datetime], bool] | None = None,
-    ) -> tuple[list[Any], int]:
-        doc_batch: list[Any] = []
-
+    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
         if self.confluence_client is None:
             raise ConnectorMissingCredentialError("Confluence")
 
-        end_ind = min(start_ind + self.batch_size, len(attachments))
-
-        for attachment in attachments[start_ind:end_ind]:
-            last_updated = _datetime_from_string(
-                attachment["history"]["lastUpdated"]["when"]
-            )
-
-            if time_filter and not time_filter(last_updated):
-                continue
-
-            # The url and the id are the same
-            attachment_url = build_confluence_document_id(
-                self.wiki_base, attachment["_links"]["download"]
-            )
-            attachment_content = self._attachment_to_content(
-                self.confluence_client, attachment
-            )
-            if attachment_content is None:
-                continue
-
-            creator_email = attachment["history"]["createdBy"].get("email")
-
-            comment = attachment["metadata"].get("comment", "")
-            doc_metadata: dict[str, Any] = {"comment": comment}
-
-            attachment_labels: list[str] = []
-            if not CONFLUENCE_CONNECTOR_SKIP_LABEL_INDEXING:
-                for label in attachment["metadata"]["labels"]["results"]:
-                    attachment_labels.append(label["name"])
-
-            doc_metadata["labels"] = attachment_labels
-
-            doc_batch.append(
-                Document(
-                    id=attachment_url,
-                    sections=[Section(link=attachment_url, text=attachment_content)],
-                    source=DocumentSource.CONFLUENCE,
-                    semantic_identifier=attachment["title"],
-                    doc_updated_at=last_updated,
-                    primary_owners=(
-                        [BasicExpertInfo(email=creator_email)]
-                        if creator_email
-                        else None
-                    ),
-                    metadata=doc_metadata,
+        doc_metadata_list: list[SlimDocument] = []
+
+        restrictions_expand = ",".join(_RESTRICTIONS_EXPANSION_FIELDS)
+
+        page_query = self.cql_page_query + self.cql_label_filter
+        for pages in self.confluence_client.cql_paginate_all_expansions(
+            cql=page_query,
+            expand=restrictions_expand,
+        ):
+            for page in pages:
+                # If the page has restrictions, add them to the perm_sync_data
+                # These will be used by doc_sync.py to sync permissions
+                perm_sync_data = {
+                    "restrictions": page.get("restrictions", {}),
+                    "space_key": page.get("space", {}).get("key"),
+                }
+
+                doc_metadata_list.append(
+                    SlimDocument(
+                        id=build_confluence_document_id(
+                            self.wiki_base, page["_links"]["webui"]
+                        ),
+                        perm_sync_data=perm_sync_data,
+                    )
                 )
-            )
-
-        return doc_batch, end_ind - start_ind
-
-    def _handle_batch_retrieval(
-        self,
-        start: float | None = None,
-        end: float | None = None,
-    ) -> GenerateDocumentsOutput:
-        start_time = datetime.fromtimestamp(start, tz=timezone.utc) if start else None
-        end_time = datetime.fromtimestamp(end, tz=timezone.utc) if end else None
-
-        unused_attachments: list[dict[str, Any]] = []
-        cursor = None
-        while True:
-            doc_batch, cursor, new_unused_attachments = self._get_doc_batch(cursor)
-            unused_attachments.extend(new_unused_attachments)
-            if doc_batch:
-                yield doc_batch
-
-            if not cursor:
-                break
-
-        # Process attachments if any
-        start_ind = 0
-        while True:
-            attachment_batch, num_attachments = self._get_attachment_batch(
-                start_ind=start_ind,
-                attachments=unused_attachments,
-                time_filter=(lambda t: start_time <= t <= end_time)
-                if start_time and end_time
-                else None,
-            )
-
-            start_ind += num_attachments
-            if attachment_batch:
-                yield attachment_batch
-
-            if num_attachments < self.batch_size:
-                break
-
-    def load_from_state(self) -> GenerateDocumentsOutput:
-        return self._handle_batch_retrieval()
-
-    def poll_source(self, start: float, end: float) -> GenerateDocumentsOutput:
-        return self._handle_batch_retrieval(start=start, end=end)
-
-
-if __name__ == "__main__":
-    connector = ConfluenceConnector(
-        wiki_base=os.environ["CONFLUENCE_TEST_SPACE_URL"],
-        space=os.environ["CONFLUENCE_TEST_SPACE"],
-        is_cloud=os.environ.get("CONFLUENCE_IS_CLOUD", "true").lower() == "true",
-        page_id=os.environ.get("CONFLUENCE_TEST_PAGE_ID", ""),
-        index_recursively=True,
-    )
-    connector.load_credentials(
-        {
-            "confluence_username": os.environ["CONFLUENCE_USER_NAME"],
-            "confluence_access_token": os.environ["CONFLUENCE_ACCESS_TOKEN"],
-        }
-    )
-    document_batches = connector.load_from_state()
-    print(next(document_batches))
+                attachment_cql = f"type=attachment and container='{page['id']}'"
+                attachment_cql += self.cql_label_filter
+                for attachments in self.confluence_client.cql_paginate_all_expansions(
+                    cql=attachment_cql,
+                    expand=restrictions_expand,
+                ):
+                    for attachment in attachments:
+                        doc_metadata_list.append(
+                            SlimDocument(
+                                id=build_confluence_document_id(
+                                    self.wiki_base, attachment["_links"]["webui"]
+                                ),
+                                perm_sync_data=perm_sync_data,
+                            )
+                        )
+                yield doc_metadata_list
+                doc_metadata_list = []
diff --git a/backend/danswer/connectors/confluence/rate_limit_handler.py b/backend/danswer/connectors/confluence/onyx_confluence.py
similarity index 71%
rename from backend/danswer/connectors/confluence/rate_limit_handler.py
rename to backend/danswer/connectors/confluence/onyx_confluence.py
index 8dbdeba1ab6..4aea49bc655 100644
--- a/backend/danswer/connectors/confluence/rate_limit_handler.py
+++ b/backend/danswer/connectors/confluence/onyx_confluence.py
@@ -1,10 +1,12 @@
 import math
 import time
 from collections.abc import Callable
+from collections.abc import Iterator
 from typing import Any
 from typing import cast
 from typing import TypeVar
 
+from atlassian import Confluence  # type:ignore
 from requests import HTTPError
 
 from danswer.utils.logger import setup_logger
@@ -22,6 +24,183 @@ class ConfluenceRateLimitError(Exception):
     pass
 
 
+def _handle_http_error(e: HTTPError, attempt: int) -> int:
+    MIN_DELAY = 2
+    MAX_DELAY = 60
+    STARTING_DELAY = 5
+    BACKOFF = 2
+
+    # Check if the response or headers are None to avoid potential AttributeError
+    if e.response is None or e.response.headers is None:
+        logger.warning("HTTPError with `None` as response or as headers")
+        raise e
+
+    if (
+        e.response.status_code != 429
+        and RATE_LIMIT_MESSAGE_LOWERCASE not in e.response.text.lower()
+    ):
+        raise e
+
+    retry_after = None
+
+    retry_after_header = e.response.headers.get("Retry-After")
+    if retry_after_header is not None:
+        try:
+            retry_after = int(retry_after_header)
+            if retry_after > MAX_DELAY:
+                logger.warning(
+                    f"Clamping retry_after from {retry_after} to {MAX_DELAY} seconds..."
+                )
+                retry_after = MAX_DELAY
+            if retry_after < MIN_DELAY:
+                retry_after = MIN_DELAY
+        except ValueError:
+            pass
+
+    if retry_after is not None:
+        logger.warning(
+            f"Rate limiting with retry header. Retrying after {retry_after} seconds..."
+        )
+        delay = retry_after
+    else:
+        logger.warning(
+            "Rate limiting without retry header. Retrying with exponential backoff..."
+        )
+        delay = min(STARTING_DELAY * (BACKOFF**attempt), MAX_DELAY)
+
+    delay_until = math.ceil(time.monotonic() + delay)
+    return delay_until
+
+
+# https://developer.atlassian.com/cloud/confluence/rate-limiting/
+# this uses the native rate limiting option provided by the
+# confluence client and otherwise applies a simpler set of error handling
+def handle_confluence_rate_limit(confluence_call: F) -> F:
+    def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
+        MAX_RETRIES = 5
+
+        TIMEOUT = 3600
+        timeout_at = time.monotonic() + TIMEOUT
+
+        for attempt in range(MAX_RETRIES):
+            if time.monotonic() > timeout_at:
+                raise TimeoutError(
+                    f"Confluence call attempts took longer than {TIMEOUT} seconds."
+                )
+
+            try:
+                # we're relying more on the client to rate limit itself
+                # and applying our own retries in a more specific set of circumstances
+                return confluence_call(*args, **kwargs)
+            except HTTPError as e:
+                delay_until = _handle_http_error(e, attempt)
+                while time.monotonic() < delay_until:
+                    # in the future, check a signal here to exit
+                    time.sleep(1)
+            except AttributeError as e:
+                # Some error within the Confluence library, unclear why it fails.
+                # Users reported it to be intermittent, so just retry
+                if attempt == MAX_RETRIES - 1:
+                    raise e
+
+                logger.exception(
+                    "Confluence Client raised an AttributeError. Retrying..."
+                )
+                time.sleep(5)
+
+    return cast(F, wrapped_call)
+
+
+_PAGINATION_LIMIT = 100
+
+
+class OnyxConfluence(Confluence):
+    """
+    This is a custom Confluence class that overrides the default Confluence class to add a custom CQL method.
+    This is necessary because the default Confluence class does not properly support cql expansions.
+    All methods are automatically wrapped with handle_confluence_rate_limit.
+    """
+
+    def __init__(self, url: str, *args: Any, **kwargs: Any) -> None:
+        super(OnyxConfluence, self).__init__(url, *args, **kwargs)
+        self._wrap_methods()
+
+    def _wrap_methods(self) -> None:
+        """
+        For each attribute that is callable (i.e., a method) and doesn't start with an underscore,
+        wrap it with handle_confluence_rate_limit.
+        """
+        for attr_name in dir(self):
+            if callable(getattr(self, attr_name)) and not attr_name.startswith("_"):
+                setattr(
+                    self,
+                    attr_name,
+                    handle_confluence_rate_limit(getattr(self, attr_name)),
+                )
+
+    def paginated_cql_page_retrieval(
+        self,
+        cql: str,
+        expand: str | None = None,
+        limit: int | None = None,
+    ) -> Iterator[list[dict[str, Any]]]:
+        """
+        This will paginate through the top level query.
+        """
+        url_suffix = f"rest/api/content/search?cql={cql}"
+        if expand:
+            url_suffix += f"&expand={expand}"
+        if not limit:
+            limit = _PAGINATION_LIMIT
+        url_suffix += f"&limit={limit}"
+
+        while True:
+            try:
+                response = self.get(url_suffix)
+                results = response["results"]
+            except Exception as e:
+                logger.exception("Error in danswer_cql: \n")
+                raise e
+
+            yield results
+
+            url_suffix = response.get("_links", {}).get("next")
+            if not url_suffix:
+                break
+
+    def cql_paginate_all_expansions(
+        self,
+        cql: str,
+        expand: str | None = None,
+        limit: int | None = None,
+    ) -> Iterator[list[dict[str, Any]]]:
+        """
+        This function will paginate through the top level query first, then
+        paginate through all of the expansions.
+        The limit only applies to the top level query.
+        All expansion paginations use default pagination limit (defined by Atlassian).
+        """
+
+        def _traverse_and_update(data: dict | list) -> None:
+            if isinstance(data, dict):
+                next_url = data.get("_links", {}).get("next")
+                if next_url and "results" in data:
+                    while next_url:
+                        next_response = self.get(next_url)
+                        data["results"].extend(next_response.get("results", []))
+                        next_url = next_response.get("_links", {}).get("next")
+
+                for value in data.values():
+                    _traverse_and_update(value)
+            elif isinstance(data, list):
+                for item in data:
+                    _traverse_and_update(item)
+
+        for results in self.paginated_cql_page_retrieval(cql, expand, limit):
+            _traverse_and_update(results)
+            yield results
+
+
 # commenting out while we try using confluence's rate limiter instead
 # # https://developer.atlassian.com/cloud/confluence/rate-limiting/
 # def make_confluence_call_handle_rate_limit(confluence_call: F) -> F:
@@ -130,90 +309,3 @@ class ConfluenceRateLimitError(Exception):
 #                     raise e
 
 #     return cast(F, wrapped_call)
-
-
-def _handle_http_error(e: HTTPError, attempt: int) -> int:
-    MIN_DELAY = 2
-    MAX_DELAY = 60
-    STARTING_DELAY = 5
-    BACKOFF = 2
-
-    # Check if the response or headers are None to avoid potential AttributeError
-    if e.response is None or e.response.headers is None:
-        logger.warning("HTTPError with `None` as response or as headers")
-        raise e
-
-    if (
-        e.response.status_code != 429
-        and RATE_LIMIT_MESSAGE_LOWERCASE not in e.response.text.lower()
-    ):
-        raise e
-
-    retry_after = None
-
-    retry_after_header = e.response.headers.get("Retry-After")
-    if retry_after_header is not None:
-        try:
-            retry_after = int(retry_after_header)
-            if retry_after > MAX_DELAY:
-                logger.warning(
-                    f"Clamping retry_after from {retry_after} to {MAX_DELAY} seconds..."
-                )
-                retry_after = MAX_DELAY
-            if retry_after < MIN_DELAY:
-                retry_after = MIN_DELAY
-        except ValueError:
-            pass
-
-    if retry_after is not None:
-        logger.warning(
-            f"Rate limiting with retry header. Retrying after {retry_after} seconds..."
-        )
-        delay = retry_after
-    else:
-        logger.warning(
-            "Rate limiting without retry header. Retrying with exponential backoff..."
-        )
-        delay = min(STARTING_DELAY * (BACKOFF**attempt), MAX_DELAY)
-
-    delay_until = math.ceil(time.monotonic() + delay)
-    return delay_until
-
-
-# https://developer.atlassian.com/cloud/confluence/rate-limiting/
-# this uses the native rate limiting option provided by the
-# confluence client and otherwise applies a simpler set of error handling
-def make_confluence_call_handle_rate_limit(confluence_call: F) -> F:
-    def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
-        MAX_RETRIES = 5
-
-        TIMEOUT = 3600
-        timeout_at = time.monotonic() + TIMEOUT
-
-        for attempt in range(MAX_RETRIES):
-            if time.monotonic() > timeout_at:
-                raise TimeoutError(
-                    f"Confluence call attempts took longer than {TIMEOUT} seconds."
-                )
-
-            try:
-                # we're relying more on the client to rate limit itself
-                # and applying our own retries in a more specific set of circumstances
-                return confluence_call(*args, **kwargs)
-            except HTTPError as e:
-                delay_until = _handle_http_error(e, attempt)
-                while time.monotonic() < delay_until:
-                    # in the future, check a signal here to exit
-                    time.sleep(1)
-            except AttributeError as e:
-                # Some error within the Confluence library, unclear why it fails.
-                # Users reported it to be intermittent, so just retry
-                if attempt == MAX_RETRIES - 1:
-                    raise e
-
-                logger.exception(
-                    "Confluence Client raised an AttributeError. Retrying..."
-                )
-                time.sleep(5)
-
-    return cast(F, wrapped_call)
diff --git a/backend/danswer/connectors/confluence/utils.py b/backend/danswer/connectors/confluence/utils.py
new file mode 100644
index 00000000000..05ed9586c2b
--- /dev/null
+++ b/backend/danswer/connectors/confluence/utils.py
@@ -0,0 +1,176 @@
+import io
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+
+import bs4
+
+from danswer.configs.app_configs import (
+    CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD,
+)
+from danswer.configs.app_configs import CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD
+from danswer.connectors.confluence.onyx_confluence import (
+    OnyxConfluence,
+)
+from danswer.file_processing.extract_file_text import extract_file_text
+from danswer.file_processing.html_utils import format_document_soup
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+_USER_NOT_FOUND = "Unknown User"
+_USER_ID_TO_DISPLAY_NAME_CACHE: dict[str, str] = {}
+
+
+def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str:
+    """Get Confluence Display Name based on the account-id or userkey value
+
+    Args:
+        user_id (str): The user id (i.e: the account-id or userkey)
+        confluence_client (Confluence): The Confluence Client
+
+    Returns:
+        str: The User Display Name. 'Unknown User' if the user is deactivated or not found
+    """
+    # Cache hit
+    if user_id in _USER_ID_TO_DISPLAY_NAME_CACHE:
+        return _USER_ID_TO_DISPLAY_NAME_CACHE[user_id]
+
+    try:
+        result = confluence_client.get_user_details_by_accountid(user_id)
+        if found_display_name := result.get("displayName"):
+            _USER_ID_TO_DISPLAY_NAME_CACHE[user_id] = found_display_name
+    except Exception:
+        # may need to just not log this error but will leave here for now
+        logger.exception(
+            f"Unable to get the User Display Name with the id: '{user_id}'"
+        )
+
+    return _USER_ID_TO_DISPLAY_NAME_CACHE.get(user_id, _USER_NOT_FOUND)
+
+
+def extract_text_from_confluence_html(
+    confluence_client: OnyxConfluence, confluence_object: dict[str, Any]
+) -> str:
+    """Parse a Confluence html page and replace the 'user Id' by the real
+        User Display Name
+
+    Args:
+        confluence_object (dict): The confluence object as a dict
+        confluence_client (Confluence): Confluence client
+
+    Returns:
+        str: loaded and formated Confluence page
+    """
+    body = confluence_object["body"]
+    object_html = body.get("storage", body.get("view", {})).get("value")
+
+    soup = bs4.BeautifulSoup(object_html, "html.parser")
+    for user in soup.findAll("ri:user"):
+        user_id = (
+            user.attrs["ri:account-id"]
+            if "ri:account-id" in user.attrs
+            else user.get("ri:userkey")
+        )
+        if not user_id:
+            logger.warning(
+                "ri:userkey not found in ri:user element. " f"Found attrs: {user.attrs}"
+            )
+            continue
+        # Include @ sign for tagging, more clear for LLM
+        user.replaceWith("@" + _get_user(confluence_client, user_id))
+    return format_document_soup(soup)
+
+
+def attachment_to_content(
+    confluence_client: OnyxConfluence,
+    attachment: dict[str, Any],
+) -> str | None:
+    """If it returns None, assume that we should skip this attachment."""
+    if attachment["metadata"]["mediaType"] in [
+        "image/jpeg",
+        "image/png",
+        "image/gif",
+        "image/svg+xml",
+        "video/mp4",
+        "video/quicktime",
+    ]:
+        return None
+
+    download_link = confluence_client.url + attachment["_links"]["download"]
+
+    attachment_size = attachment["extensions"]["fileSize"]
+    if attachment_size > CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD:
+        logger.warning(
+            f"Skipping {download_link} due to size. "
+            f"size={attachment_size} "
+            f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_SIZE_THRESHOLD}"
+        )
+        return None
+
+    logger.info(f"_attachment_to_content - _session.get: link={download_link}")
+    response = confluence_client._session.get(download_link)
+    if response.status_code != 200:
+        logger.warning(
+            f"Failed to fetch {download_link} with invalid status code {response.status_code}"
+        )
+        return None
+
+    extracted_text = extract_file_text(
+        io.BytesIO(response.content),
+        file_name=attachment["title"],
+        break_on_unprocessable=False,
+    )
+    if len(extracted_text) > CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD:
+        logger.warning(
+            f"Skipping {download_link} due to char count. "
+            f"char count={len(extracted_text)} "
+            f"threshold={CONFLUENCE_CONNECTOR_ATTACHMENT_CHAR_COUNT_THRESHOLD}"
+        )
+        return None
+
+    return extracted_text
+
+
+def build_confluence_document_id(base_url: str, content_url: str) -> str:
+    """For confluence, the document id is the page url for a page based document
+        or the attachment download url for an attachment based document
+
+    Args:
+        base_url (str): The base url of the Confluence instance
+        content_url (str): The url of the page or attachment download url
+
+    Returns:
+        str: The document id
+    """
+    return f"{base_url}{content_url}"
+
+
+def extract_referenced_attachment_names(page_text: str) -> list[str]:
+    """Parse a Confluence html page to generate a list of current
+        attachments in use
+
+    Args:
+        text (str): The page content
+
+    Returns:
+        list[str]: List of filenames currently in use by the page text
+    """
+    referenced_attachment_filenames = []
+    soup = bs4.BeautifulSoup(page_text, "html.parser")
+    for attachment in soup.findAll("ri:attachment"):
+        referenced_attachment_filenames.append(attachment.attrs["ri:filename"])
+    return referenced_attachment_filenames
+
+
+def datetime_from_string(datetime_string: str) -> datetime:
+    datetime_object = datetime.fromisoformat(datetime_string)
+
+    if datetime_object.tzinfo is None:
+        # If no timezone info, assume it is UTC
+        datetime_object = datetime_object.replace(tzinfo=timezone.utc)
+    else:
+        # If not in UTC, translate it
+        datetime_object = datetime_object.astimezone(timezone.utc)
+
+    return datetime_object
diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py
index 52fb0194aa6..05f201a45c2 100644
--- a/backend/danswer/connectors/factory.py
+++ b/backend/danswer/connectors/factory.py
@@ -64,7 +64,7 @@ def identify_connector_class(
         DocumentSource.SLACK: {
             InputType.LOAD_STATE: SlackLoadConnector,
             InputType.POLL: SlackPollConnector,
-            InputType.PRUNE: SlackPollConnector,
+            InputType.SLIM_RETRIEVAL: SlackPollConnector,
         },
         DocumentSource.GITHUB: GithubConnector,
         DocumentSource.GMAIL: GmailConnector,
diff --git a/backend/danswer/connectors/interfaces.py b/backend/danswer/connectors/interfaces.py
index c5b4850d9b0..4734212147e 100644
--- a/backend/danswer/connectors/interfaces.py
+++ b/backend/danswer/connectors/interfaces.py
@@ -3,11 +3,13 @@
 from typing import Any
 
 from danswer.connectors.models import Document
+from danswer.connectors.models import SlimDocument
 
 
 SecondsSinceUnixEpoch = float
 
 GenerateDocumentsOutput = Iterator[list[Document]]
+GenerateSlimDocumentOutput = Iterator[list[SlimDocument]]
 
 
 class BaseConnector(abc.ABC):
@@ -52,9 +54,9 @@ def poll_source(
         raise NotImplementedError
 
 
-class IdConnector(BaseConnector):
+class SlimConnector(BaseConnector):
     @abc.abstractmethod
-    def retrieve_all_source_ids(self) -> set[str]:
+    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
         raise NotImplementedError
 
 
diff --git a/backend/danswer/connectors/models.py b/backend/danswer/connectors/models.py
index 7d86d21980d..ba1368dc944 100644
--- a/backend/danswer/connectors/models.py
+++ b/backend/danswer/connectors/models.py
@@ -14,7 +14,7 @@ class InputType(str, Enum):
     LOAD_STATE = "load_state"  # e.g. loading a current full state or a save state, such as from a file
     POLL = "poll"  # e.g. calling an API to get all documents in the last hour
     EVENT = "event"  # e.g. registered an endpoint as a listener, and processing connector events
-    PRUNE = "prune"
+    SLIM_RETRIEVAL = "slim_retrieval"
 
 
 class ConnectorMissingCredentialError(PermissionError):
@@ -169,6 +169,11 @@ def from_base(cls, base: DocumentBase) -> "Document":
         )
 
 
+class SlimDocument(BaseModel):
+    id: str
+    perm_sync_data: Any | None = None
+
+
 class DocumentErrorSummary(BaseModel):
     id: str
     semantic_id: str
diff --git a/backend/danswer/connectors/salesforce/connector.py b/backend/danswer/connectors/salesforce/connector.py
index 03326df4efd..9c9fa5e9956 100644
--- a/backend/danswer/connectors/salesforce/connector.py
+++ b/backend/danswer/connectors/salesforce/connector.py
@@ -11,14 +11,16 @@
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
 from danswer.connectors.interfaces import GenerateDocumentsOutput
-from danswer.connectors.interfaces import IdConnector
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import ConnectorMissingCredentialError
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
+from danswer.connectors.models import SlimDocument
 from danswer.connectors.salesforce.utils import extract_dict_text
 from danswer.utils.logger import setup_logger
 
@@ -29,7 +31,7 @@
 logger = setup_logger()
 
 
-class SalesforceConnector(LoadConnector, PollConnector, IdConnector):
+class SalesforceConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
         batch_size: int = INDEX_BATCH_SIZE,
@@ -243,19 +245,22 @@ def poll_source(
         end_datetime = datetime.utcfromtimestamp(end)
         return self._fetch_from_salesforce(start=start_datetime, end=end_datetime)
 
-    def retrieve_all_source_ids(self) -> set[str]:
+    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
         if self.sf_client is None:
             raise ConnectorMissingCredentialError("Salesforce")
-        all_retrieved_ids: set[str] = set()
+        doc_metadata_list: list[SlimDocument] = []
         for parent_object_type in self.parent_object_list:
             query = f"SELECT Id FROM {parent_object_type}"
             query_result = self.sf_client.query_all(query)
-            all_retrieved_ids.update(
-                f"{ID_PREFIX}{instance_dict.get('Id', '')}"
+            doc_metadata_list.extend(
+                SlimDocument(
+                    id=f"{ID_PREFIX}{instance_dict.get('Id', '')}",
+                    perm_sync_data={},
+                )
                 for instance_dict in query_result["records"]
             )
 
-        return all_retrieved_ids
+        yield doc_metadata_list
 
 
 if __name__ == "__main__":
diff --git a/backend/danswer/connectors/slack/connector.py b/backend/danswer/connectors/slack/connector.py
index 4a9249e5157..f5728950e4f 100644
--- a/backend/danswer/connectors/slack/connector.py
+++ b/backend/danswer/connectors/slack/connector.py
@@ -13,13 +13,15 @@
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.interfaces import GenerateDocumentsOutput
-from danswer.connectors.interfaces import IdConnector
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import ConnectorMissingCredentialError
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
+from danswer.connectors.models import SlimDocument
 from danswer.connectors.slack.utils import expert_info_from_slack_id
 from danswer.connectors.slack.utils import get_message_link
 from danswer.connectors.slack.utils import make_paginated_slack_api_call_w_retries
@@ -326,7 +328,7 @@ def _get_all_doc_ids(
     channels: list[str] | None = None,
     channel_name_regex_enabled: bool = False,
     msg_filter_func: Callable[[MessageType], bool] = default_msg_filter,
-) -> set[str]:
+) -> GenerateSlimDocumentOutput:
     """
     Get all document ids in the workspace, channel by channel
     This is pretty identical to get_all_docs, but it returns a set of ids instead of documents
@@ -338,13 +340,14 @@ def _get_all_doc_ids(
         all_channels, channels, channel_name_regex_enabled
     )
 
-    all_doc_ids = set()
     for channel in filtered_channels:
+        channel_id = channel["id"]
         channel_message_batches = get_channel_messages(
             client=client,
             channel=channel,
         )
 
+        message_ts_set: set[str] = set()
         for message_batch in channel_message_batches:
             for message in message_batch:
                 if msg_filter_func(message):
@@ -353,12 +356,21 @@ def _get_all_doc_ids(
                 # The document id is the channel id and the ts of the first message in the thread
                 # Since we already have the first message of the thread, we dont have to
                 # fetch the thread for id retrieval, saving time and API calls
-                all_doc_ids.add(f"{channel['id']}__{message['ts']}")
+                message_ts_set.add(message["ts"])
+
+        channel_metadata_list: list[SlimDocument] = []
+        for message_ts in message_ts_set:
+            channel_metadata_list.append(
+                SlimDocument(
+                    id=f"{channel_id}__{message_ts}",
+                    perm_sync_data={"channel_id": channel_id},
+                )
+            )
 
-    return all_doc_ids
+        yield channel_metadata_list
 
 
-class SlackPollConnector(PollConnector, IdConnector):
+class SlackPollConnector(PollConnector, SlimConnector):
     def __init__(
         self,
         workspace: str,
@@ -379,7 +391,7 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
         self.client = WebClient(token=bot_token)
         return None
 
-    def retrieve_all_source_ids(self) -> set[str]:
+    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
         if self.client is None:
             raise ConnectorMissingCredentialError("Slack")
 
diff --git a/backend/ee/danswer/external_permissions/confluence/doc_sync.py b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
index 1330655ec61..94874d36a68 100644
--- a/backend/ee/danswer/external_permissions/confluence/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
@@ -3,26 +3,17 @@
 https://confluence.atlassian.com/conf85/check-who-can-view-a-page-1283360557.html
 """
 from typing import Any
-from urllib.parse import parse_qs
-from urllib.parse import urlparse
 
 from sqlalchemy.orm import Session
 
 from danswer.access.models import ExternalAccess
-from danswer.connectors.confluence.confluence_utils import (
-    build_confluence_document_id,
-)
-from danswer.connectors.confluence.connector import DanswerConfluence
-from danswer.connectors.confluence.rate_limit_handler import (
-    make_confluence_call_handle_rate_limit,
-)
+from danswer.connectors.confluence.connector import ConfluenceConnector
+from danswer.connectors.confluence.connector import OnyxConfluence
+from danswer.connectors.models import SlimDocument
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.document import upsert_document_external_perms__no_commit
-from ee.danswer.external_permissions.confluence.sync_utils import (
-    build_confluence_client,
-)
 from ee.danswer.external_permissions.confluence.sync_utils import (
     get_user_email_from_username__server,
 )
@@ -34,16 +25,12 @@
 
 
 def _get_server_space_permissions(
-    confluence_client: DanswerConfluence, space_key: str
+    confluence_client: OnyxConfluence, space_key: str
 ) -> ExternalAccess:
-    get_space_permissions = make_confluence_call_handle_rate_limit(
-        confluence_client.get_space_permissions
-    )
-
-    permissions = get_space_permissions(space_key)
+    space_permissions = confluence_client.get_space_permissions(space_key=space_key)
 
     viewspace_permissions = []
-    for permission_category in permissions:
+    for permission_category in space_permissions:
         if permission_category.get("type") == _VIEWSPACE_PERMISSION_TYPE:
             viewspace_permissions.extend(
                 permission_category.get("spacePermissions", [])
@@ -77,12 +64,9 @@ def _get_server_space_permissions(
 
 
 def _get_cloud_space_permissions(
-    confluence_client: DanswerConfluence, space_key: str
+    confluence_client: OnyxConfluence, space_key: str
 ) -> ExternalAccess:
-    get_space_permissions = make_confluence_call_handle_rate_limit(
-        confluence_client.get_space
-    )
-    space_permissions_result = get_space_permissions(
+    space_permissions_result = confluence_client.get_space(
         space_key=space_key, expand="permissions"
     )
     space_permissions = space_permissions_result.get("permissions", [])
@@ -115,17 +99,16 @@ def _get_cloud_space_permissions(
 
 
 def _get_space_permissions(
-    confluence_client: DanswerConfluence,
+    confluence_client: OnyxConfluence,
     is_cloud: bool,
 ) -> dict[str, ExternalAccess]:
     # Gets all the spaces in the Confluence instance
-    get_all_spaces = make_confluence_call_handle_rate_limit(
-        confluence_client.get_all_spaces
-    )
     all_space_keys = []
     start = 0
     while True:
-        spaces_batch = get_all_spaces(start=start, limit=_REQUEST_PAGINATION_LIMIT)
+        spaces_batch = confluence_client.get_all_spaces(
+            start=start, limit=_REQUEST_PAGINATION_LIMIT
+        )
         for space in spaces_batch.get("results", []):
             all_space_keys.append(space.get("key"))
 
@@ -153,13 +136,11 @@ def _get_space_permissions(
 
 
 def _extract_read_access_restrictions(
-    restrictions: dict[str, Any]
-) -> tuple[list[str], list[str]]:
+    confluence_client: OnyxConfluence, restrictions: dict[str, Any]
+) -> ExternalAccess | None:
     """
-    WARNING: This function includes no paginated retrieval. So if a page is private
-    within the space and has over 200 users or over 200 groups with explicitly read
-    access, this function will leave out some users or groups.
-    200 is a large amount so it is unlikely, but just be aware.
+    Converts a page's restrictions dict into an ExternalAccess object.
+    If there are no restrictions, then return None
     """
     read_access = restrictions.get("read", {})
     read_access_restrictions = read_access.get("restrictions", {})
@@ -167,9 +148,24 @@ def _extract_read_access_restrictions(
     # Extract the users with read access
     read_access_user = read_access_restrictions.get("user", {})
     read_access_user_jsons = read_access_user.get("results", [])
-    read_access_user_emails = [
-        user["email"] for user in read_access_user_jsons if user.get("email")
-    ]
+    read_access_user_emails = []
+    for user in read_access_user_jsons:
+        # If the user has an email, then add it to the list
+        if user.get("email"):
+            read_access_user_emails.append(user["email"])
+        # If the user has a username and not an email, then get the email from Confluence
+        elif user.get("username"):
+            email = get_user_email_from_username__server(
+                confluence_client=confluence_client, user_name=user["username"]
+            )
+            if email:
+                read_access_user_emails.append(email)
+            else:
+                logger.warning(
+                    f"Email for user {user['username']} not found in Confluence"
+                )
+        else:
+            logger.warning(f"User {user} does not have an email or username")
 
     # Extract the groups with read access
     read_access_group = read_access_restrictions.get("group", {})
@@ -178,180 +174,52 @@ def _extract_read_access_restrictions(
         group["name"] for group in read_access_group_jsons if group.get("name")
     ]
 
-    return read_access_user_emails, read_access_group_names
-
-
-def _get_page_specific_restrictions(
-    page: dict[str, Any],
-) -> ExternalAccess | None:
-    user_emails, group_names = _extract_read_access_restrictions(
-        restrictions=page.get("restrictions", {})
-    )
-
     # If there are no restrictions found, then the page
     # inherits the space's restrictions so return None
-    is_space_public = user_emails == [] and group_names == []
+    is_space_public = read_access_user_emails == [] and read_access_group_names == []
     if is_space_public:
         return None
 
     return ExternalAccess(
-        external_user_emails=set(user_emails),
-        external_user_group_ids=set(group_names),
+        external_user_emails=set(read_access_user_emails),
+        external_user_group_ids=set(read_access_group_names),
         # there is no way for a page to be individually public if the space isn't public
         is_public=False,
     )
 
 
-def _fetch_attachment_document_ids_for_page_paginated(
-    confluence_client: DanswerConfluence, page: dict[str, Any]
-) -> list[str]:
-    """
-    Starts by just extracting the first page of attachments from
-    the page. If all attachments are in the first page, then
-    no calls to the api are made from this function.
-    """
-    get_attachments_from_content = make_confluence_call_handle_rate_limit(
-        confluence_client.get_attachments_from_content
-    )
-
-    attachment_doc_ids = []
-    attachments_dict = page["children"]["attachment"]
-    start = 0
-
-    while True:
-        attachments_list = attachments_dict["results"]
-        attachment_doc_ids.extend(
-            [
-                build_confluence_document_id(
-                    base_url=confluence_client.url,
-                    content_url=attachment["_links"]["download"],
-                )
-                for attachment in attachments_list
-            ]
-        )
-
-        if "next" not in attachments_dict["_links"]:
-            break
-
-        start += len(attachments_list)
-        attachments_dict = get_attachments_from_content(
-            page_id=page["id"],
-            start=start,
-            limit=_REQUEST_PAGINATION_LIMIT,
-        )
-
-    return attachment_doc_ids
-
-
-def _fetch_all_pages_paginated(
-    confluence_client: DanswerConfluence,
-    cql_query: str,
-) -> list[dict[str, Any]]:
-    get_all_pages = make_confluence_call_handle_rate_limit(
-        confluence_client.danswer_cql
-    )
-
-    # For each page, this fetches the page's attachments and restrictions.
-    expansion_strings = [
-        "children.attachment",
-        "restrictions.read.restrictions.user",
-        "restrictions.read.restrictions.group",
-        "space",
-    ]
-    expansion_string = ",".join(expansion_strings)
-
-    all_pages: list[dict[str, Any]] = []
-    cursor = None
-    while True:
-        response = get_all_pages(
-            cql=cql_query,
-            expand=expansion_string,
-            cursor=cursor,
-            limit=_REQUEST_PAGINATION_LIMIT,
-        )
-
-        all_pages.extend(response.get("results", []))
-
-        # Handle pagination
-        next_cursor = response.get("_links", {}).get("next", "")
-        cursor = parse_qs(urlparse(next_cursor).query).get("cursor", [None])[0]
-
-        if not cursor:
-            break
-
-    return all_pages
-
-
 def _fetch_all_page_restrictions_for_space(
-    confluence_client: DanswerConfluence,
-    cql_query: str,
+    confluence_client: OnyxConfluence,
+    slim_docs: list[SlimDocument],
     space_permissions_by_space_key: dict[str, ExternalAccess],
 ) -> dict[str, ExternalAccess]:
-    all_pages = _fetch_all_pages_paginated(
-        confluence_client=confluence_client,
-        cql_query=cql_query,
-    )
-
+    """
+    For all pages, if a page has restrictions, then use those restrictions.
+    Otherwise, use the space's restrictions.
+    """
     document_restrictions: dict[str, ExternalAccess] = {}
-    for page in all_pages:
-        """
-        This assigns the same permissions to all attachments of a page and
-        the page itself.
-        This is because the attachments are stored in the same Confluence space as the page.
-        WARNING: We create a dbDocument entry for all attachments, even though attachments
-        may not be their own standalone documents. This is likely fine as we just upsert a
-        document with just permissions.
-        """
-        document_ids = []
-
-        # Add the page's document id
-        document_ids.append(
-            build_confluence_document_id(
-                base_url=confluence_client.url,
-                content_url=page["_links"]["webui"],
-            )
-        )
 
-        # Add the page's attachments document ids
-        document_ids.extend(
-            _fetch_attachment_document_ids_for_page_paginated(
-                confluence_client=confluence_client, page=page
+    for slim_doc in slim_docs:
+        if slim_doc.perm_sync_data is None:
+            raise ValueError(
+                f"No permission sync data found for document {slim_doc.id}"
             )
+        restrictions = _extract_read_access_restrictions(
+            confluence_client=confluence_client,
+            restrictions=slim_doc.perm_sync_data.get("restrictions", {}),
         )
-
-        # Get the page's specific restrictions
-        page_permissions = _get_page_specific_restrictions(
-            page=page,
-        )
-
-        if not page_permissions:
-            # If there are no page specific restrictions,
-            # the page inherits the space's restrictions
-            page_permissions = space_permissions_by_space_key.get(page["space"]["key"])
-            if not page_permissions:
-                # If nothing is in the dict, then the space has not been indexed, so we move on
-                continue
-
-        # Apply the page's specific restrictions to the page and its attachments
-        for document_id in document_ids:
-            document_restrictions[document_id] = page_permissions
+        if restrictions:
+            document_restrictions[slim_doc.id] = restrictions
+        else:
+            space_key = slim_doc.perm_sync_data.get("space_key")
+            if space_permissions := space_permissions_by_space_key.get(space_key):
+                document_restrictions[slim_doc.id] = space_permissions
+            else:
+                logger.warning(f"No permissions found for document {slim_doc.id}")
 
     return document_restrictions
 
 
-def _build_cql_query_from_connector_config(
-    cc_pair: ConnectorCredentialPair,
-) -> str:
-    cql_query = cc_pair.connector.connector_specific_config.get("cql_query")
-    if cql_query:
-        return cql_query
-
-    space_id = cc_pair.connector.connector_specific_config.get("space")
-    if space_id:
-        return f"type=page and space='{space_id}'"
-    return "type=page"
-
-
 def confluence_doc_sync(
     db_session: Session,
     cc_pair: ConnectorCredentialPair,
@@ -362,22 +230,29 @@ def confluence_doc_sync(
     it in postgres so that when it gets created later, the permissions are
     already populated
     """
-    confluence_client = build_confluence_client(
-        connector_specific_config=cc_pair.connector.connector_specific_config,
-        credentials_json=cc_pair.credential.credential_json,
+    confluence_connector = ConfluenceConnector(
+        **cc_pair.connector.connector_specific_config
     )
+    confluence_connector.load_credentials(cc_pair.credential.credential_json)
+    if confluence_connector.confluence_client is None:
+        raise ValueError("Failed to load credentials")
+    confluence_client = confluence_connector.confluence_client
 
-    cql_query = _build_cql_query_from_connector_config(cc_pair)
     is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
 
     space_permissions_by_space_key = _get_space_permissions(
         confluence_client=confluence_client,
         is_cloud=is_cloud,
     )
+    slim_docs = [
+        slim_doc
+        for doc_batch in confluence_connector.retrieve_all_slim_documents()
+        for slim_doc in doc_batch
+    ]
 
     permissions_by_doc_id = _fetch_all_page_restrictions_for_space(
         confluence_client=confluence_client,
-        cql_query=cql_query,
+        slim_docs=slim_docs,
         space_permissions_by_space_key=space_permissions_by_space_key,
     )
 
diff --git a/backend/ee/danswer/external_permissions/confluence/group_sync.py b/backend/ee/danswer/external_permissions/confluence/group_sync.py
index 5c729f51faf..241f57d241f 100644
--- a/backend/ee/danswer/external_permissions/confluence/group_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/group_sync.py
@@ -4,8 +4,8 @@
 from requests import HTTPError
 from sqlalchemy.orm import Session
 
-from danswer.connectors.confluence.rate_limit_handler import (
-    make_confluence_call_handle_rate_limit,
+from danswer.connectors.confluence.onyx_confluence import (
+    handle_confluence_rate_limit,
 )
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
@@ -28,9 +28,7 @@
 def _get_confluence_group_names_paginated(
     confluence_client: Confluence,
 ) -> Iterator[str]:
-    get_all_groups = make_confluence_call_handle_rate_limit(
-        confluence_client.get_all_groups
-    )
+    get_all_groups = handle_confluence_rate_limit(confluence_client.get_all_groups)
 
     start = 0
     while True:
@@ -55,7 +53,7 @@ def _get_group_members_email_paginated(
     group_name: str,
     is_cloud: bool,
 ) -> set[str]:
-    get_group_members = make_confluence_call_handle_rate_limit(
+    get_group_members = handle_confluence_rate_limit(
         confluence_client.get_group_members
     )
     group_member_emails: set[str] = set()
diff --git a/backend/ee/danswer/external_permissions/confluence/sync_utils.py b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
index d6eb225007a..f2b451ca33d 100644
--- a/backend/ee/danswer/external_permissions/confluence/sync_utils.py
+++ b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
@@ -1,8 +1,8 @@
 from typing import Any
 
-from danswer.connectors.confluence.connector import DanswerConfluence
-from danswer.connectors.confluence.rate_limit_handler import (
-    make_confluence_call_handle_rate_limit,
+from danswer.connectors.confluence.connector import OnyxConfluence
+from danswer.connectors.confluence.onyx_confluence import (
+    handle_confluence_rate_limit,
 )
 
 _USER_EMAIL_CACHE: dict[str, str | None] = {}
@@ -10,9 +10,9 @@
 
 def build_confluence_client(
     connector_specific_config: dict[str, Any], credentials_json: dict[str, Any]
-) -> DanswerConfluence:
+) -> OnyxConfluence:
     is_cloud = connector_specific_config.get("is_cloud", False)
-    return DanswerConfluence(
+    return OnyxConfluence(
         api_version="cloud" if is_cloud else "latest",
         # Remove trailing slash from wiki_base if present
         url=connector_specific_config["wiki_base"].rstrip("/"),
@@ -27,10 +27,10 @@ def build_confluence_client(
 
 
 def get_user_email_from_username__server(
-    confluence_client: DanswerConfluence, user_name: str
+    confluence_client: OnyxConfluence, user_name: str
 ) -> str | None:
     global _USER_EMAIL_CACHE
-    get_user_info = make_confluence_call_handle_rate_limit(
+    get_user_info = handle_confluence_rate_limit(
         confluence_client.get_mobile_parameters
     )
     if _USER_EMAIL_CACHE.get(user_name) is None:
diff --git a/backend/ee/danswer/external_permissions/permission_sync.py b/backend/ee/danswer/external_permissions/permission_sync.py
index 3f5e66f875c..ba5bbbd4921 100644
--- a/backend/ee/danswer/external_permissions/permission_sync.py
+++ b/backend/ee/danswer/external_permissions/permission_sync.py
@@ -70,7 +70,7 @@ def run_external_doc_permission_sync(
         # - the user_email <-> document mapping
         # - the external_user_group_id <-> document mapping
         # in postgres without committing
-        logger.debug(f"Syncing docs for {source_type}")
+        logger.info(f"Syncing docs for {source_type}")
         doc_sync_func(
             db_session,
             cc_pair,
@@ -107,6 +107,7 @@ def run_external_doc_permission_sync(
 
         # update postgres
         db_session.commit()
+        logger.info(f"Successfully synced docs for {source_type}")
     except Exception:
         logger.exception("Error Syncing Document Permissions")
         db_session.rollback()
diff --git a/backend/ee/danswer/external_permissions/slack/doc_sync.py b/backend/ee/danswer/external_permissions/slack/doc_sync.py
index fe731746a44..b5f6e9695db 100644
--- a/backend/ee/danswer/external_permissions/slack/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/slack/doc_sync.py
@@ -3,7 +3,7 @@
 
 from danswer.access.models import ExternalAccess
 from danswer.connectors.factory import instantiate_connector
-from danswer.connectors.interfaces import IdConnector
+from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import InputType
 from danswer.connectors.slack.connector import get_channels
 from danswer.connectors.slack.connector import make_paginated_slack_api_call_w_retries
@@ -17,28 +17,6 @@
 logger = setup_logger()
 
 
-def _extract_channel_id_from_doc_id(doc_id: str) -> str:
-    """
-    Extracts the channel ID from a document ID string.
-
-    The document ID is expected to be in the format: "{channel_id}__{message_ts}"
-
-    Args:
-        doc_id (str): The document ID string.
-
-    Returns:
-        str: The extracted channel ID.
-
-    Raises:
-        ValueError: If the doc_id doesn't contain the expected separator.
-    """
-    try:
-        channel_id, _ = doc_id.split("__", 1)
-        return channel_id
-    except ValueError:
-        raise ValueError(f"Invalid doc_id format: {doc_id}")
-
-
 def _get_slack_document_ids_and_channels(
     db_session: Session,
     cc_pair: ConnectorCredentialPair,
@@ -47,24 +25,27 @@ def _get_slack_document_ids_and_channels(
     runnable_connector = instantiate_connector(
         db_session=db_session,
         source=cc_pair.connector.source,
-        input_type=InputType.PRUNE,
+        input_type=InputType.SLIM_RETRIEVAL,
         connector_specific_config=cc_pair.connector.connector_specific_config,
         credential=cc_pair.credential,
     )
 
-    assert isinstance(runnable_connector, IdConnector)
+    assert isinstance(runnable_connector, SlimConnector)
 
     channel_doc_map: dict[str, list[str]] = {}
-    for doc_id in runnable_connector.retrieve_all_source_ids():
-        channel_id = _extract_channel_id_from_doc_id(doc_id)
-        if channel_id not in channel_doc_map:
-            channel_doc_map[channel_id] = []
-        channel_doc_map[channel_id].append(doc_id)
+    for doc_metadata_batch in runnable_connector.retrieve_all_slim_documents():
+        for doc_metadata in doc_metadata_batch:
+            if doc_metadata.perm_sync_data is None:
+                continue
+            channel_id = doc_metadata.perm_sync_data["channel_id"]
+            if channel_id not in channel_doc_map:
+                channel_doc_map[channel_id] = []
+            channel_doc_map[channel_id].append(doc_metadata.id)
 
     return channel_doc_map
 
 
-def _fetch_worspace_permissions(
+def _fetch_workspace_permissions(
     db_session: Session,
     user_id_to_email_map: dict[str, str],
 ) -> ExternalAccess:
@@ -167,7 +148,7 @@ def slack_doc_sync(
         db_session=db_session,
         cc_pair=cc_pair,
     )
-    workspace_permissions = _fetch_worspace_permissions(
+    workspace_permissions = _fetch_workspace_permissions(
         db_session=db_session,
         user_id_to_email_map=user_id_to_email_map,
     )
diff --git a/backend/tests/daily/connectors/confluence/test_confluence_basic.py b/backend/tests/daily/connectors/confluence/test_confluence_basic.py
index a791b1eab3b..d4287298bdb 100644
--- a/backend/tests/daily/connectors/confluence/test_confluence_basic.py
+++ b/backend/tests/daily/connectors/confluence/test_confluence_basic.py
@@ -39,18 +39,33 @@ def test_confluence_connector_basic(
     with pytest.raises(StopIteration):
         next(doc_batch_generator)
 
-    assert len(doc_batch) == 1
+    assert len(doc_batch) == 2
 
-    doc = doc_batch[0]
-    assert doc.semantic_identifier == "DailyConnectorTestSpace Home"
-    assert doc.metadata["labels"] == ["testlabel"]
-    assert doc.primary_owners
-    assert doc.primary_owners[0].email == "chris@danswer.ai"
-    assert len(doc.sections) == 1
+    for doc in doc_batch:
+        if doc.semantic_identifier == "DailyConnectorTestSpace Home":
+            page_doc = doc
+        elif ".txt" in doc.semantic_identifier:
+            txt_doc = doc
 
-    section = doc.sections[0]
-    assert section.text == "test123\nsmall"
+    assert page_doc.semantic_identifier == "DailyConnectorTestSpace Home"
+    assert page_doc.metadata["labels"] == ["testlabel"]
+    assert page_doc.primary_owners
+    assert page_doc.primary_owners[0].email == "chris@danswer.ai"
+    assert len(page_doc.sections) == 1
+
+    section = page_doc.sections[0]
+    assert section.text == "test123"
     assert (
         section.link
         == "https://danswerai.atlassian.net/wiki/spaces/DailyConne/overview"
     )
+
+    assert txt_doc.semantic_identifier == "small-file.txt"
+    assert len(txt_doc.sections) == 1
+    assert txt_doc.sections[0].text == "small"
+    assert txt_doc.primary_owners
+    assert txt_doc.primary_owners[0].email == "chris@danswer.ai"
+    assert (
+        txt_doc.sections[0].link
+        == "https://danswerai.atlassian.net/wiki/pages/viewpageattachments.action?pageId=52494430&preview=%2F52494430%2F52527123%2Fsmall-file.txt"
+    )
diff --git a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
index 7fdb1a0428f..d64986ea826 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_permission_sync.py
@@ -3,8 +3,6 @@
 from datetime import timezone
 from typing import Any
 
-import pytest
-
 from danswer.connectors.models import InputType
 from danswer.db.enums import AccessType
 from danswer.server.documents.models import DocumentSource
@@ -24,7 +22,7 @@
 from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
 
 
-@pytest.mark.xfail(reason="flaky - see DAN-789 for example", strict=False)
+# @pytest.mark.xfail(reason="flaky - see DAN-789 for example", strict=False)
 def test_slack_permission_sync(
     reset: None,
     vespa_client: vespa_fixture,
diff --git a/backend/tests/integration/connector_job_tests/slack/test_prune.py b/backend/tests/integration/connector_job_tests/slack/test_prune.py
index 3abf7bd6fb0..bcef148a2a0 100644
--- a/backend/tests/integration/connector_job_tests/slack/test_prune.py
+++ b/backend/tests/integration/connector_job_tests/slack/test_prune.py
@@ -3,8 +3,6 @@
 from datetime import timezone
 from typing import Any
 
-import pytest
-
 from danswer.connectors.models import InputType
 from danswer.db.enums import AccessType
 from danswer.server.documents.models import DocumentSource
@@ -24,7 +22,7 @@
 from tests.integration.connector_job_tests.slack.slack_api_utils import SlackManager
 
 
-@pytest.mark.xfail(reason="flaky - see DAN-835 for example", strict=False)
+# @pytest.mark.xfail(reason="flaky - see DAN-835 for example", strict=False)
 def test_slack_prune(
     reset: None,
     vespa_client: vespa_fixture,
diff --git a/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py b/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py
index 1779e8b1c33..d1f263a7793 100644
--- a/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py
+++ b/backend/tests/unit/danswer/connectors/confluence/test_rate_limit_handler.py
@@ -3,8 +3,8 @@
 import pytest
 from requests import HTTPError
 
-from danswer.connectors.confluence.rate_limit_handler import (
-    make_confluence_call_handle_rate_limit,
+from danswer.connectors.confluence.onyx_confluence import (
+    handle_confluence_rate_limit,
 )
 
 
@@ -55,7 +55,7 @@ def test_non_rate_limit_error(mock_confluence_call: Mock) -> None:
         response=Mock(status_code=500, text="Internal Server Error")
     )
 
-    handled_call = make_confluence_call_handle_rate_limit(mock_confluence_call)
+    handled_call = handle_confluence_rate_limit(mock_confluence_call)
 
     with pytest.raises(HTTPError):
         handled_call()

From e4779c29a7497827ebf3a440859c4596a1f12e14 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 21 Oct 2024 16:46:23 -0700
Subject: [PATCH 173/376] tighter signaling to prevent indexing cleanup from
 hitting tasks that are just starting (#2867)

* better indexing synchronization

* add logging for fence wait

* handle the task not creating

* add more logging

* add more logging

* raise retry count
---
 .../background/celery/tasks/indexing/tasks.py | 52 +++++++++++++++++--
 .../background/celery/tasks/shared/tasks.py   |  4 +-
 .../background/celery/tasks/vespa/tasks.py    |  4 ++
 backend/danswer/connectors/web/connector.py   |  2 +-
 .../document_index/vespa/indexing_utils.py    |  2 +-
 .../common_utils/managers/cc_pair.py          |  6 +--
 6 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index fefbae03220..0e8e59bf5a6 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -255,7 +255,19 @@ def try_creating_indexing_task(
 
         custom_task_id = f"{rci.generator_task_id_prefix}_{uuid4()}"
 
-        # create the index attempt ... just for tracking purposes
+        # set a basic fence to start
+        fence_value = RedisConnectorIndexingFenceData(
+            index_attempt_id=None,
+            started=None,
+            submitted=datetime.now(timezone.utc),
+            celery_task_id=None,
+        )
+        r.set(rci.fence_key, fence_value.model_dump_json())
+
+        # create the index attempt for tracking purposes
+        # code elsewhere checks for index attempts without an associated redis key
+        # and cleans them up
+        # therefore we must create the attempt and the task after the fence goes up
         index_attempt_id = create_index_attempt(
             cc_pair.id,
             search_settings.id,
@@ -276,17 +288,19 @@ def try_creating_indexing_task(
             priority=DanswerCeleryPriority.MEDIUM,
         )
         if not result:
-            return None
+            raise RuntimeError("send_task for connector_indexing_proxy_task failed.")
 
-        # set this only after all tasks have been added
+        # now fill out the fence with the rest of the data
         fence_value = RedisConnectorIndexingFenceData(
             index_attempt_id=index_attempt_id,
             started=None,
             submitted=datetime.now(timezone.utc),
             celery_task_id=result.id,
         )
+
         r.set(rci.fence_key, fence_value.model_dump_json())
     except Exception:
+        r.delete(rci.fence_key)
         task_logger.exception("Unexpected exception")
         return None
     finally:
@@ -371,6 +385,38 @@ def connector_indexing_task(
 
     rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
 
+    while True:
+        # read related data and evaluate/print task progress
+        fence_value = cast(bytes, r.get(rci.fence_key))
+        if fence_value is None:
+            task_logger.info(
+                f"connector_indexing_task: fence_value not found: fence={rci.fence_key}"
+            )
+            raise
+
+        try:
+            fence_json = fence_value.decode("utf-8")
+            fence_data = RedisConnectorIndexingFenceData.model_validate_json(
+                cast(str, fence_json)
+            )
+        except ValueError:
+            task_logger.exception(
+                f"connector_indexing_task: fence_data not decodeable: fence={rci.fence_key}"
+            )
+            raise
+
+        if fence_data.index_attempt_id is None or fence_data.celery_task_id is None:
+            task_logger.info(
+                f"connector_indexing_task - Waiting for fence: fence={rci.fence_key}"
+            )
+            sleep(1)
+            continue
+
+        task_logger.info(
+            f"connector_indexing_task - Fence found, continuing...: fence={rci.fence_key}"
+        )
+        break
+
     lock = r.lock(
         rci.generator_lock_key,
         timeout=CELERY_INDEXING_LOCK_TIMEOUT,
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 6fc0b5f1f67..26f9d1aac10 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -21,10 +21,10 @@
 
 
 class RedisConnectorIndexingFenceData(BaseModel):
-    index_attempt_id: int
+    index_attempt_id: int | None
     started: datetime | None
     submitted: datetime
-    celery_task_id: str
+    celery_task_id: str | None
 
 
 @shared_task(
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 9830d71f778..2d79045c44f 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -574,6 +574,10 @@ def monitor_ccpair_indexing_taskset(
                 "monitor_ccpair_indexing_taskset: generator_progress_value is not an integer."
             )
 
+    if fence_data.index_attempt_id is None or fence_data.celery_task_id is None:
+        # the task is still setting up
+        return
+
     # Read result state BEFORE generator_complete_key to avoid a race condition
     result: AsyncResult = AsyncResult(fence_data.celery_task_id)
     result_state = result.state
diff --git a/backend/danswer/connectors/web/connector.py b/backend/danswer/connectors/web/connector.py
index 9e0671ea248..9e406b71674 100644
--- a/backend/danswer/connectors/web/connector.py
+++ b/backend/danswer/connectors/web/connector.py
@@ -373,7 +373,7 @@ def load_from_state(self) -> GenerateDocumentsOutput:
                 page.close()
             except Exception as e:
                 last_error = f"Failed to fetch '{current_url}': {e}"
-                logger.error(last_error)
+                logger.exception(last_error)
                 playwright.stop()
                 restart_playwright = True
                 continue
diff --git a/backend/danswer/document_index/vespa/indexing_utils.py b/backend/danswer/document_index/vespa/indexing_utils.py
index 28ff31c8071..8ecdc22672b 100644
--- a/backend/danswer/document_index/vespa/indexing_utils.py
+++ b/backend/danswer/document_index/vespa/indexing_utils.py
@@ -118,7 +118,7 @@ def get_existing_documents_from_chunks(
     return document_ids
 
 
-@retry(tries=3, delay=1, backoff=2)
+@retry(tries=5, delay=1, backoff=2)
 def _index_vespa_chunk(
     chunk: DocMetadataAwareIndexChunk,
     index_name: str,
diff --git a/backend/tests/integration/common_utils/managers/cc_pair.py b/backend/tests/integration/common_utils/managers/cc_pair.py
index 99d8a82a2b3..94114298507 100644
--- a/backend/tests/integration/common_utils/managers/cc_pair.py
+++ b/backend/tests/integration/common_utils/managers/cc_pair.py
@@ -246,17 +246,17 @@ def wait_for_indexing(
                     fetched_cc_pair.last_success
                     and fetched_cc_pair.last_success > after
                 ):
-                    print(f"CC pair {cc_pair.id} indexing complete.")
+                    print(f"Indexing complete: cc_pair={cc_pair.id}")
                     return
 
             elapsed = time.monotonic() - start
             if elapsed > timeout:
                 raise TimeoutError(
-                    f"CC pair {cc_pair.id} indexing was not completed within {timeout} seconds"
+                    f"Indexing wait timed out: cc_pair={cc_pair.id} timeout={timeout}s"
                 )
 
             print(
-                f"CC pair {cc_pair.id} indexing to complete. elapsed={elapsed:.2f} timeout={timeout}"
+                f"Indexing wait for completion: cc_pair={cc_pair.id} elapsed={elapsed:.2f} timeout={timeout}s"
             )
             time.sleep(5)
 

From 6e9b6a1075c7ed880659a0dfd62f778dad4324b7 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Mon, 21 Oct 2024 22:27:26 -0700
Subject: [PATCH 174/376] Handle models like openai/bedrock/claude-3.5-...
 (#2869)

* Handle models like openai/bedrock/claude-3.5-...

* Fix log statement
---
 backend/danswer/llm/utils.py | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/llm/utils.py b/backend/danswer/llm/utils.py
index 3a5e40875f1..bad18214b95 100644
--- a/backend/danswer/llm/utils.py
+++ b/backend/danswer/llm/utils.py
@@ -342,12 +342,26 @@ def get_llm_max_tokens(
 
     try:
         model_obj = model_map.get(f"{model_provider}/{model_name}")
-        if not model_obj:
-            model_obj = model_map[model_name]
-            logger.debug(f"Using model object for {model_name}")
-        else:
+        if model_obj:
             logger.debug(f"Using model object for {model_provider}/{model_name}")
 
+        if not model_obj:
+            model_obj = model_map.get(model_name)
+            if model_obj:
+                logger.debug(f"Using model object for {model_name}")
+
+        if not model_obj:
+            model_name_split = model_name.split("/")
+            if len(model_name_split) > 1:
+                model_obj = model_map.get(model_name_split[1])
+            if model_obj:
+                logger.debug(f"Using model object for {model_name_split[1]}")
+
+        if not model_obj:
+            raise RuntimeError(
+                f"No litellm entry found for {model_provider}/{model_name}"
+            )
+
         if "max_input_tokens" in model_obj:
             max_tokens = model_obj["max_input_tokens"]
             logger.info(

From 8f236a12886295bff9e72e41d69808f07732d339 Mon Sep 17 00:00:00 2001
From: YASH <139299779+Yash-2707@users.noreply.github.com>
Date: Tue, 22 Oct 2024 17:37:07 +0530
Subject: [PATCH 175/376] Update reset_indexes.py

Error Handling: Add more specific error handling to make it easier to debug issues.
Configuration Management: Use environment variables or a configuration file for settings like DOCUMENT_INDEX_NAME and DOCUMENT_ID_ENDPOINT.
Logging: Improve logging to include more details about the operations.
Retry Mechanism: Add a retry mechanism for network requests to handle transient errors.
Testing: Add unit tests for the functions to ensure they work as expected
---
 backend/scripts/reset_indexes.py | 48 +++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 10 deletions(-)

diff --git a/backend/scripts/reset_indexes.py b/backend/scripts/reset_indexes.py
index 4ec8d9bf312..a8c22bf1f31 100644
--- a/backend/scripts/reset_indexes.py
+++ b/backend/scripts/reset_indexes.py
@@ -1,9 +1,12 @@
 # This file is purely for development use, not included in any builds
 import os
 import sys
-
+import logging
 import requests
 
+from requests.exceptions import RequestException
+from time import sleep
+
 # makes it so `PYTHONPATH=.` is not required when running this script
 parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 sys.path.append(parent_dir)
@@ -14,23 +17,48 @@
 
 logger = setup_logger()
 
-
 def wipe_vespa_index() -> None:
+    """
+    Wipes the Vespa index by deleting all documents.
+    """
     continuation = None
     should_continue = True
+    retries = 3
+
     while should_continue:
         params = {"selection": "true", "cluster": DOCUMENT_INDEX_NAME}
         if continuation:
-            params = {**params, "continuation": continuation}
-        response = requests.delete(DOCUMENT_ID_ENDPOINT, params=params)
-        response.raise_for_status()
+            params["continuation"] = continuation
+
+        for attempt in range(retries):
+            try:
+                response = requests.delete(DOCUMENT_ID_ENDPOINT, params=params)
+                response.raise_for_status()
+
+                response_json = response.json()
+                logger.info(f"Response: {response_json}")
 
-        response_json = response.json()
-        print(response_json)
+                continuation = response_json.get("continuation")
+                should_continue = bool(continuation)
+                break  # Exit the retry loop if the request is successful
 
-        continuation = response_json.get("continuation")
-        should_continue = bool(continuation)
+            except RequestException as e:
+                logger.error(f"Request failed: {e}")
+                sleep(2 ** attempt)  # Exponential backoff
+        else:
+            logger.error("Max retries exceeded. Exiting.")
+            sys.exit(1)
 
+def main():
+    """
+    Main function to execute the script.
+    """
+    try:
+        wipe_vespa_index()
+        logger.info("Vespa index wiped successfully.")
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+        sys.exit(1)
 
 if __name__ == "__main__":
-    wipe_vespa_index()
+    main()

From bae794706c73999c418c3dbc2ef2775152ee366b Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Tue, 22 Oct 2024 09:46:14 -0700
Subject: [PATCH 176/376] add stale issues and pr's cron

---
 .../workflows/nightly-close-stale-issues.yml   | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 .github/workflows/nightly-close-stale-issues.yml

diff --git a/.github/workflows/nightly-close-stale-issues.yml b/.github/workflows/nightly-close-stale-issues.yml
new file mode 100644
index 00000000000..393d3ec950f
--- /dev/null
+++ b/.github/workflows/nightly-close-stale-issues.yml
@@ -0,0 +1,18 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '0 11 * * *' # Runs every day at 3 AM PST / 4 AM PDT / 11 AM UTC
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v9
+        with:
+          stale-issue-message: 'This issue is stale because it has been open 75 days with no activity. Remove stale label or comment or this will be closed in 15 days.'
+          stale-pr-message: 'This PR is stale because it has been open 75 days with no activity. Remove stale label or comment or this will be closed in 15 days.'
+          close-issue-message: 'This issue was closed because it has been stalled for 90 days with no activity.'
+          close-pr-message: 'This PR was closed because it has been stalled for 90 days with no activity.'
+          days-before-stale: 75
+#           days-before-close: 90  # uncomment after we test stale behavior
+          
\ No newline at end of file

From e031576c871000971070faed5f3cc86e4c4945b9 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Tue, 22 Oct 2024 10:05:28 -0700
Subject: [PATCH 177/376] Salesforce Connector Note (#2872)

---
 .../versions/6756efa39ada_id_uuid_for_chat_session.py       | 6 +++---
 backend/danswer/connectors/salesforce/connector.py          | 6 ++++++
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py b/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py
index 057521d0f9c..083fececd87 100644
--- a/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py
+++ b/backend/alembic/versions/6756efa39ada_id_uuid_for_chat_session.py
@@ -1,7 +1,9 @@
-"""
+"""Migrate chat_session and chat_message tables to use UUID primary keys
+
 Revision ID: 6756efa39ada
 Revises: 5d12a446f5c0
 Create Date: 2024-10-15 17:47:44.108537
+
 """
 from alembic import op
 import sqlalchemy as sa
@@ -12,8 +14,6 @@
 depends_on = None
 
 """
-Migrate chat_session and chat_message tables to use UUID primary keys.
-
 This script:
 1. Adds UUID columns to chat_session and chat_message
 2. Populates new columns with UUIDs
diff --git a/backend/danswer/connectors/salesforce/connector.py b/backend/danswer/connectors/salesforce/connector.py
index 9c9fa5e9956..78d73d44766 100644
--- a/backend/danswer/connectors/salesforce/connector.py
+++ b/backend/danswer/connectors/salesforce/connector.py
@@ -24,6 +24,12 @@
 from danswer.connectors.salesforce.utils import extract_dict_text
 from danswer.utils.logger import setup_logger
 
+
+# TODO: this connector does not work well at large scales
+# the large query against a large Salesforce instance has been reported to take 1.5 hours.
+# Additionally it seems to eat up more memory over time if the connection is long running (again a scale issue).
+
+
 DEFAULT_PARENT_OBJECT_TYPES = ["Account"]
 MAX_QUERY_LENGTH = 10000  # max query length is 20,000 characters
 ID_PREFIX = "SALESFORCE_"

From 914da2e4cb7600e573a8ef89401f4967c70cc3a6 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Tue, 22 Oct 2024 13:41:47 -0700
Subject: [PATCH 178/376] Confluence polish (#2874)

---
 .../connectors/confluence/connector.py        |  32 ++--
 .../connectors/confluence/onyx_confluence.py  | 179 +++++-------------
 .../danswer/connectors/confluence/utils.py    |  66 +++++--
 .../confluence/doc_sync.py                    |  15 +-
 .../confluence/group_sync.py                  | 105 +++-------
 .../confluence/sync_utils.py                  |  43 -----
 6 files changed, 150 insertions(+), 290 deletions(-)
 delete mode 100644 backend/ee/danswer/external_permissions/confluence/sync_utils.py

diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index 92d981855f1..fe52862982d 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -1,6 +1,7 @@
 from datetime import datetime
 from datetime import timezone
 from typing import Any
+from urllib.parse import quote
 
 from danswer.configs.app_configs import CONFLUENCE_CONNECTOR_LABELS_TO_SKIP
 from danswer.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
@@ -8,6 +9,7 @@
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.confluence.onyx_confluence import OnyxConfluence
 from danswer.connectors.confluence.utils import attachment_to_content
+from danswer.connectors.confluence.utils import build_confluence_client
 from danswer.connectors.confluence.utils import build_confluence_document_id
 from danswer.connectors.confluence.utils import datetime_from_string
 from danswer.connectors.confluence.utils import extract_text_from_confluence_html
@@ -74,20 +76,21 @@ def __init__(
         self.wiki_base = wiki_base.rstrip("/")
 
         # if nothing is provided, we will fetch all pages
-        self.cql_page_query = "type=page"
+        cql_page_query = "type=page"
         if cql_query:
             # if a cql_query is provided, we will use it to fetch the pages
-            self.cql_page_query = cql_query
+            cql_page_query = cql_query
         elif space:
             # if no cql_query is provided, we will use the space to fetch the pages
-            self.cql_page_query += f" and space='{space}'"
+            cql_page_query += f" and space='{quote(space)}'"
         elif page_id:
             if index_recursively:
-                self.cql_page_query += f" and ancestor='{page_id}'"
+                cql_page_query += f" and ancestor='{page_id}'"
             else:
                 # if neither a space nor a cql_query is provided, we will use the page_id to fetch the page
-                self.cql_page_query += f" and id='{page_id}'"
+                cql_page_query += f" and id='{page_id}'"
 
+        self.cql_page_query = cql_page_query
         self.cql_label_filter = ""
         self.cql_time_filter = ""
         if labels_to_skip:
@@ -96,19 +99,12 @@ def __init__(
             self.cql_label_filter = f"&label not in ({comma_separated_labels})"
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
-        username = credentials["confluence_username"]
-        access_token = credentials["confluence_access_token"]
-
         # see https://github.com/atlassian-api/atlassian-python-api/blob/master/atlassian/rest_client.py
         # for a list of other hidden constructor args
-        self.confluence_client = OnyxConfluence(
-            url=self.wiki_base,
-            username=username if self.is_cloud else None,
-            password=access_token if self.is_cloud else None,
-            token=access_token if not self.is_cloud else None,
-            backoff_and_retry=True,
-            max_backoff_retries=60,
-            max_backoff_seconds=60,
+        self.confluence_client = build_confluence_client(
+            credentials_json=credentials,
+            is_cloud=self.is_cloud,
+            wiki_base=self.wiki_base,
         )
         return None
 
@@ -202,12 +198,12 @@ def _fetch_document_batches(self) -> GenerateDocumentsOutput:
 
         page_query = self.cql_page_query + self.cql_label_filter + self.cql_time_filter
         # Fetch pages as Documents
-        for pages in self.confluence_client.paginated_cql_page_retrieval(
+        for page_batch in self.confluence_client.paginated_cql_page_retrieval(
             cql=page_query,
             expand=",".join(_PAGE_EXPANSION_FIELDS),
             limit=self.batch_size,
         ):
-            for page in pages:
+            for page in page_batch:
                 confluence_page_ids.append(page["id"])
                 doc = self._convert_object_to_document(page)
                 if doc is not None:
diff --git a/backend/danswer/connectors/confluence/onyx_confluence.py b/backend/danswer/connectors/confluence/onyx_confluence.py
index 4aea49bc655..c01f45dea6a 100644
--- a/backend/danswer/connectors/confluence/onyx_confluence.py
+++ b/backend/danswer/connectors/confluence/onyx_confluence.py
@@ -5,6 +5,7 @@
 from typing import Any
 from typing import cast
 from typing import TypeVar
+from urllib.parse import quote
 
 from atlassian import Confluence  # type:ignore
 from requests import HTTPError
@@ -111,7 +112,7 @@ def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
     return cast(F, wrapped_call)
 
 
-_PAGINATION_LIMIT = 100
+_DEFAULT_PAGINATION_LIMIT = 100
 
 
 class OnyxConfluence(Confluence):
@@ -138,35 +139,62 @@ def _wrap_methods(self) -> None:
                     handle_confluence_rate_limit(getattr(self, attr_name)),
                 )
 
-    def paginated_cql_page_retrieval(
-        self,
-        cql: str,
-        expand: str | None = None,
-        limit: int | None = None,
+    def _paginate_url(
+        self, url_suffix: str, limit: int | None = None
     ) -> Iterator[list[dict[str, Any]]]:
         """
         This will paginate through the top level query.
         """
-        url_suffix = f"rest/api/content/search?cql={cql}"
-        if expand:
-            url_suffix += f"&expand={expand}"
         if not limit:
-            limit = _PAGINATION_LIMIT
-        url_suffix += f"&limit={limit}"
+            limit = _DEFAULT_PAGINATION_LIMIT
+
+        connection_char = "&" if "?" in url_suffix else "?"
+        url_suffix += f"{connection_char}limit={limit}"
 
-        while True:
+        while url_suffix:
             try:
-                response = self.get(url_suffix)
-                results = response["results"]
+                next_response = self.get(url_suffix)
             except Exception as e:
                 logger.exception("Error in danswer_cql: \n")
                 raise e
+            yield next_response.get("results", [])
+            url_suffix = next_response.get("_links", {}).get("next")
 
-            yield results
+    def paginated_groups_retrieval(
+        self,
+        limit: int | None = None,
+    ) -> Iterator[list[dict[str, Any]]]:
+        return self._paginate_url("rest/api/group", limit)
+
+    def paginated_group_members_retrieval(
+        self,
+        group_name: str,
+        limit: int | None = None,
+    ) -> Iterator[list[dict[str, Any]]]:
+        group_name = quote(group_name)
+        return self._paginate_url(f"rest/api/group/{group_name}/member", limit)
+
+    def paginated_cql_user_retrieval(
+        self,
+        cql: str,
+        expand: str | None = None,
+        limit: int | None = None,
+    ) -> Iterator[list[dict[str, Any]]]:
+        expand_string = f"&expand={expand}" if expand else ""
+        return self._paginate_url(
+            f"rest/api/search/user?cql={cql}{expand_string}", limit
+        )
 
-            url_suffix = response.get("_links", {}).get("next")
-            if not url_suffix:
-                break
+    def paginated_cql_page_retrieval(
+        self,
+        cql: str,
+        expand: str | None = None,
+        limit: int | None = None,
+    ) -> Iterator[list[dict[str, Any]]]:
+        expand_string = f"&expand={expand}" if expand else ""
+        return self._paginate_url(
+            f"rest/api/content/search?cql={cql}{expand_string}", limit
+        )
 
     def cql_paginate_all_expansions(
         self,
@@ -185,10 +213,7 @@ def _traverse_and_update(data: dict | list) -> None:
             if isinstance(data, dict):
                 next_url = data.get("_links", {}).get("next")
                 if next_url and "results" in data:
-                    while next_url:
-                        next_response = self.get(next_url)
-                        data["results"].extend(next_response.get("results", []))
-                        next_url = next_response.get("_links", {}).get("next")
+                    data["results"].extend(self._paginate_url(next_url))
 
                 for value in data.values():
                     _traverse_and_update(value)
@@ -199,113 +224,3 @@ def _traverse_and_update(data: dict | list) -> None:
         for results in self.paginated_cql_page_retrieval(cql, expand, limit):
             _traverse_and_update(results)
             yield results
-
-
-# commenting out while we try using confluence's rate limiter instead
-# # https://developer.atlassian.com/cloud/confluence/rate-limiting/
-# def make_confluence_call_handle_rate_limit(confluence_call: F) -> F:
-#     def wrapped_call(*args: list[Any], **kwargs: Any) -> Any:
-#         max_retries = 5
-#         starting_delay = 5
-#         backoff = 2
-
-#         # max_delay is used when the server doesn't hand back "Retry-After"
-#         # and we have to decide the retry delay ourselves
-#         max_delay = 30  # Atlassian uses max_delay = 30 in their examples
-
-#         # max_retry_after is used when we do get a "Retry-After" header
-#         max_retry_after = 300  # should we really cap the maximum retry delay?
-
-#         NEXT_RETRY_KEY = BaseConnector.REDIS_KEY_PREFIX + "confluence_next_retry"
-
-#         # for testing purposes, rate limiting is written to fall back to a simpler
-#         # rate limiting approach when redis is not available
-#         r = get_redis_client()
-
-#         for attempt in range(max_retries):
-#             try:
-#                 # if multiple connectors are waiting for the next attempt, there could be an issue
-#                 # where many connectors are "released" onto the server at the same time.
-#                 # That's not ideal ... but coming up with a mechanism for queueing
-#                 # all of these connectors is a bigger problem that we want to take on
-#                 # right now
-#                 try:
-#                     next_attempt = r.get(NEXT_RETRY_KEY)
-#                     if next_attempt is None:
-#                         next_attempt = 0
-#                     else:
-#                         next_attempt = int(cast(int, next_attempt))
-
-#                     # TODO: all connectors need to be interruptible moving forward
-#                     while time.monotonic() < next_attempt:
-#                         time.sleep(1)
-#                 except ConnectionError:
-#                     pass
-
-#                 return confluence_call(*args, **kwargs)
-#             except HTTPError as e:
-#                 # Check if the response or headers are None to avoid potential AttributeError
-#                 if e.response is None or e.response.headers is None:
-#                     logger.warning("HTTPError with `None` as response or as headers")
-#                     raise e
-
-#                 retry_after_header = e.response.headers.get("Retry-After")
-#                 if (
-#                     e.response.status_code == 429
-#                     or RATE_LIMIT_MESSAGE_LOWERCASE in e.response.text.lower()
-#                 ):
-#                     retry_after = None
-#                     if retry_after_header is not None:
-#                         try:
-#                             retry_after = int(retry_after_header)
-#                         except ValueError:
-#                             pass
-
-#                     if retry_after is not None:
-#                         if retry_after > max_retry_after:
-#                             logger.warning(
-#                                 f"Clamping retry_after from {retry_after} to {max_delay} seconds..."
-#                             )
-#                             retry_after = max_delay
-
-#                         logger.warning(
-#                             f"Rate limit hit. Retrying after {retry_after} seconds..."
-#                         )
-#                         try:
-#                             r.set(
-#                                 NEXT_RETRY_KEY,
-#                                 math.ceil(time.monotonic() + retry_after),
-#                             )
-#                         except ConnectionError:
-#                             pass
-#                     else:
-#                         logger.warning(
-#                             "Rate limit hit. Retrying with exponential backoff..."
-#                         )
-#                         delay = min(starting_delay * (backoff**attempt), max_delay)
-#                         delay_until = math.ceil(time.monotonic() + delay)
-
-#                         try:
-#                             r.set(NEXT_RETRY_KEY, delay_until)
-#                         except ConnectionError:
-#                             while time.monotonic() < delay_until:
-#                                 time.sleep(1)
-#                 else:
-#                     # re-raise, let caller handle
-#                     raise
-#             except AttributeError as e:
-#                 # Some error within the Confluence library, unclear why it fails.
-#                 # Users reported it to be intermittent, so just retry
-#                 logger.warning(f"Confluence Internal Error, retrying... {e}")
-#                 delay = min(starting_delay * (backoff**attempt), max_delay)
-#                 delay_until = math.ceil(time.monotonic() + delay)
-#                 try:
-#                     r.set(NEXT_RETRY_KEY, delay_until)
-#                 except ConnectionError:
-#                     while time.monotonic() < delay_until:
-#                         time.sleep(1)
-
-#                 if attempt == max_retries - 1:
-#                     raise e
-
-#     return cast(F, wrapped_call)
diff --git a/backend/danswer/connectors/confluence/utils.py b/backend/danswer/connectors/confluence/utils.py
index 05ed9586c2b..029e35e6538 100644
--- a/backend/danswer/connectors/confluence/utils.py
+++ b/backend/danswer/connectors/confluence/utils.py
@@ -18,7 +18,25 @@
 
 logger = setup_logger()
 
-_USER_NOT_FOUND = "Unknown User"
+
+_USER_EMAIL_CACHE: dict[str, str | None] = {}
+
+
+def get_user_email_from_username__server(
+    confluence_client: OnyxConfluence, user_name: str
+) -> str | None:
+    global _USER_EMAIL_CACHE
+    if _USER_EMAIL_CACHE.get(user_name) is None:
+        try:
+            response = confluence_client.get_mobile_parameters(user_name)
+            email = response.get("email")
+        except Exception:
+            email = None
+        _USER_EMAIL_CACHE[user_name] = email
+    return _USER_EMAIL_CACHE[user_name]
+
+
+_USER_NOT_FOUND = "Unknown Confluence User"
 _USER_ID_TO_DISPLAY_NAME_CACHE: dict[str, str] = {}
 
 
@@ -32,19 +50,22 @@ def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str:
     Returns:
         str: The User Display Name. 'Unknown User' if the user is deactivated or not found
     """
-    # Cache hit
-    if user_id in _USER_ID_TO_DISPLAY_NAME_CACHE:
-        return _USER_ID_TO_DISPLAY_NAME_CACHE[user_id]
-
-    try:
-        result = confluence_client.get_user_details_by_accountid(user_id)
-        if found_display_name := result.get("displayName"):
-            _USER_ID_TO_DISPLAY_NAME_CACHE[user_id] = found_display_name
-    except Exception:
-        # may need to just not log this error but will leave here for now
-        logger.exception(
-            f"Unable to get the User Display Name with the id: '{user_id}'"
-        )
+    global _USER_ID_TO_DISPLAY_NAME_CACHE
+    if _USER_ID_TO_DISPLAY_NAME_CACHE.get(user_id) is None:
+        try:
+            result = confluence_client.get_user_details_by_userkey(user_id)
+            found_display_name = result.get("displayName")
+        except Exception:
+            found_display_name = None
+
+        if not found_display_name:
+            try:
+                result = confluence_client.get_user_details_by_accountid(user_id)
+                found_display_name = result.get("displayName")
+            except Exception:
+                found_display_name = None
+
+        _USER_ID_TO_DISPLAY_NAME_CACHE[user_id] = found_display_name
 
     return _USER_ID_TO_DISPLAY_NAME_CACHE.get(user_id, _USER_NOT_FOUND)
 
@@ -174,3 +195,20 @@ def datetime_from_string(datetime_string: str) -> datetime:
         datetime_object = datetime_object.astimezone(timezone.utc)
 
     return datetime_object
+
+
+def build_confluence_client(
+    credentials_json: dict[str, Any], is_cloud: bool, wiki_base: str
+) -> OnyxConfluence:
+    return OnyxConfluence(
+        api_version="cloud" if is_cloud else "latest",
+        # Remove trailing slash from wiki_base if present
+        url=wiki_base.rstrip("/"),
+        # passing in username causes issues for Confluence data center
+        username=credentials_json["confluence_username"] if is_cloud else None,
+        password=credentials_json["confluence_access_token"] if is_cloud else None,
+        token=credentials_json["confluence_access_token"] if not is_cloud else None,
+        backoff_and_retry=True,
+        max_backoff_retries=60,
+        max_backoff_seconds=60,
+    )
diff --git a/backend/ee/danswer/external_permissions/confluence/doc_sync.py b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
index 94874d36a68..a7bc898b8b7 100644
--- a/backend/ee/danswer/external_permissions/confluence/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/doc_sync.py
@@ -8,15 +8,13 @@
 
 from danswer.access.models import ExternalAccess
 from danswer.connectors.confluence.connector import ConfluenceConnector
-from danswer.connectors.confluence.connector import OnyxConfluence
+from danswer.connectors.confluence.onyx_confluence import OnyxConfluence
+from danswer.connectors.confluence.utils import get_user_email_from_username__server
 from danswer.connectors.models import SlimDocument
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.document import upsert_document_external_perms__no_commit
-from ee.danswer.external_permissions.confluence.sync_utils import (
-    get_user_email_from_username__server,
-)
 
 logger = setup_logger()
 
@@ -244,11 +242,10 @@ def confluence_doc_sync(
         confluence_client=confluence_client,
         is_cloud=is_cloud,
     )
-    slim_docs = [
-        slim_doc
-        for doc_batch in confluence_connector.retrieve_all_slim_documents()
-        for slim_doc in doc_batch
-    ]
+
+    slim_docs = []
+    for doc_batch in confluence_connector.retrieve_all_slim_documents():
+        slim_docs.extend(doc_batch)
 
     permissions_by_doc_id = _fetch_all_page_restrictions_for_space(
         confluence_client=confluence_client,
diff --git a/backend/ee/danswer/external_permissions/confluence/group_sync.py b/backend/ee/danswer/external_permissions/confluence/group_sync.py
index 241f57d241f..a55bb777bc5 100644
--- a/backend/ee/danswer/external_permissions/confluence/group_sync.py
+++ b/backend/ee/danswer/external_permissions/confluence/group_sync.py
@@ -1,91 +1,40 @@
-from collections.abc import Iterator
+from typing import Any
 
-from atlassian import Confluence  # type:ignore
-from requests import HTTPError
 from sqlalchemy.orm import Session
 
-from danswer.connectors.confluence.onyx_confluence import (
-    handle_confluence_rate_limit,
-)
+from danswer.connectors.confluence.onyx_confluence import OnyxConfluence
+from danswer.connectors.confluence.utils import build_confluence_client
+from danswer.connectors.confluence.utils import get_user_email_from_username__server
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.external_perm import ExternalUserGroup
 from ee.danswer.db.external_perm import replace_user__ext_group_for_cc_pair__no_commit
-from ee.danswer.external_permissions.confluence.sync_utils import (
-    build_confluence_client,
-)
-from ee.danswer.external_permissions.confluence.sync_utils import (
-    get_user_email_from_username__server,
-)
 
 
 logger = setup_logger()
 
-_PAGE_SIZE = 100
-
-
-def _get_confluence_group_names_paginated(
-    confluence_client: Confluence,
-) -> Iterator[str]:
-    get_all_groups = handle_confluence_rate_limit(confluence_client.get_all_groups)
-
-    start = 0
-    while True:
-        try:
-            groups = get_all_groups(start=start, limit=_PAGE_SIZE)
-        except HTTPError as e:
-            if e.response.status_code in (403, 404):
-                return
-            raise e
-
-        for group in groups:
-            if group_name := group.get("name"):
-                yield group_name
-
-        if len(groups) < _PAGE_SIZE:
-            break
-        start += _PAGE_SIZE
-
 
 def _get_group_members_email_paginated(
-    confluence_client: Confluence,
+    confluence_client: OnyxConfluence,
     group_name: str,
-    is_cloud: bool,
 ) -> set[str]:
-    get_group_members = handle_confluence_rate_limit(
-        confluence_client.get_group_members
-    )
-    group_member_emails: set[str] = set()
-    start = 0
-    while True:
-        try:
-            members = get_group_members(
-                group_name=group_name, start=start, limit=_PAGE_SIZE
-            )
-        except HTTPError as e:
-            if e.response.status_code == 403 or e.response.status_code == 404:
-                return group_member_emails
-            raise e
+    members: list[dict[str, Any]] = []
+    for member_batch in confluence_client.paginated_group_members_retrieval(group_name):
+        members.extend(member_batch)
 
-        for member in members:
-            if is_cloud:
-                email = member.get("email")
-            elif user_name := member.get("username"):
+    group_member_emails: set[str] = set()
+    for member in members:
+        email = member.get("email")
+        if not email:
+            user_name = member.get("username")
+            if user_name:
                 email = get_user_email_from_username__server(
-                    confluence_client, user_name
+                    confluence_client=confluence_client,
+                    user_name=user_name,
                 )
-            else:
-                logger.warning(f"Member has no email or username: {member}")
-                email = None
-
-            if email:
-                group_member_emails.add(email)
-
-        if len(members) < _PAGE_SIZE:
-            break
-
-        start += _PAGE_SIZE
+        if email:
+            group_member_emails.add(email)
 
     return group_member_emails
 
@@ -94,17 +43,25 @@ def confluence_group_sync(
     db_session: Session,
     cc_pair: ConnectorCredentialPair,
 ) -> None:
+    is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
     confluence_client = build_confluence_client(
-        connector_specific_config=cc_pair.connector.connector_specific_config,
         credentials_json=cc_pair.credential.credential_json,
+        is_cloud=is_cloud,
+        wiki_base=cc_pair.connector.connector_specific_config["wiki_base"],
     )
 
+    # Get all group names
+    group_names: list[str] = []
+    for group_batch in confluence_client.paginated_groups_retrieval():
+        for group in group_batch:
+            if group_name := group.get("name"):
+                group_names.append(group_name)
+
+    # For each group name, get all members and create a danswer group
     danswer_groups: list[ExternalUserGroup] = []
-    is_cloud = cc_pair.connector.connector_specific_config.get("is_cloud", False)
-    # Confluence enforces that group names are unique
-    for group_name in _get_confluence_group_names_paginated(confluence_client):
+    for group_name in group_names:
         group_member_emails = _get_group_members_email_paginated(
-            confluence_client, group_name, is_cloud
+            confluence_client, group_name
         )
         group_members = batch_add_non_web_user_if_not_exists__no_commit(
             db_session=db_session, emails=list(group_member_emails)
diff --git a/backend/ee/danswer/external_permissions/confluence/sync_utils.py b/backend/ee/danswer/external_permissions/confluence/sync_utils.py
deleted file mode 100644
index f2b451ca33d..00000000000
--- a/backend/ee/danswer/external_permissions/confluence/sync_utils.py
+++ /dev/null
@@ -1,43 +0,0 @@
-from typing import Any
-
-from danswer.connectors.confluence.connector import OnyxConfluence
-from danswer.connectors.confluence.onyx_confluence import (
-    handle_confluence_rate_limit,
-)
-
-_USER_EMAIL_CACHE: dict[str, str | None] = {}
-
-
-def build_confluence_client(
-    connector_specific_config: dict[str, Any], credentials_json: dict[str, Any]
-) -> OnyxConfluence:
-    is_cloud = connector_specific_config.get("is_cloud", False)
-    return OnyxConfluence(
-        api_version="cloud" if is_cloud else "latest",
-        # Remove trailing slash from wiki_base if present
-        url=connector_specific_config["wiki_base"].rstrip("/"),
-        # passing in username causes issues for Confluence data center
-        username=credentials_json["confluence_username"] if is_cloud else None,
-        password=credentials_json["confluence_access_token"] if is_cloud else None,
-        token=credentials_json["confluence_access_token"] if not is_cloud else None,
-        backoff_and_retry=True,
-        max_backoff_retries=60,
-        max_backoff_seconds=60,
-    )
-
-
-def get_user_email_from_username__server(
-    confluence_client: OnyxConfluence, user_name: str
-) -> str | None:
-    global _USER_EMAIL_CACHE
-    get_user_info = handle_confluence_rate_limit(
-        confluence_client.get_mobile_parameters
-    )
-    if _USER_EMAIL_CACHE.get(user_name) is None:
-        try:
-            response = get_user_info(user_name)
-            email = response.get("email")
-        except Exception:
-            email = None
-        _USER_EMAIL_CACHE[user_name] = email
-    return _USER_EMAIL_CACHE[user_name]

From eccec6ab7c1b19fce610c8d4fc021f745cff485e Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Tue, 22 Oct 2024 14:10:31 -0700
Subject: [PATCH 179/376] Notion Fix Nested Properties (#2877)

---
 .../danswer/connectors/notion/connector.py    | 49 ++++++++++++-------
 backend/scripts/dev_run_background_jobs.py    |  4 --
 2 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/backend/danswer/connectors/notion/connector.py b/backend/danswer/connectors/notion/connector.py
index 13cad489d80..d4de5eaebd2 100644
--- a/backend/danswer/connectors/notion/connector.py
+++ b/backend/danswer/connectors/notion/connector.py
@@ -241,24 +241,29 @@ def _recurse_properties(inner_dict: dict[str, Any]) -> str | None:
                     )
 
             # TODO there may be more types to handle here
-            if "name" in inner_dict:
-                return inner_dict["name"]
-            if "content" in inner_dict:
-                return inner_dict["content"]
-            start = inner_dict.get("start")
-            end = inner_dict.get("end")
-            if start is not None:
-                if end is not None:
-                    return f"{start} - {end}"
-                return start
-            elif end is not None:
-                return f"Until {end}"
-
-            if "id" in inner_dict:
-                # This is not useful to index, it's a reference to another Notion object
-                # and this ID value in plaintext is useless outside of the Notion context
-                logger.debug("Skipping Notion object id field property")
-                return None
+            if isinstance(inner_dict, str):
+                # For some objects the innermost value could just be a string, not sure what causes this
+                return inner_dict
+
+            elif isinstance(inner_dict, dict):
+                if "name" in inner_dict:
+                    return inner_dict["name"]
+                if "content" in inner_dict:
+                    return inner_dict["content"]
+                start = inner_dict.get("start")
+                end = inner_dict.get("end")
+                if start is not None:
+                    if end is not None:
+                        return f"{start} - {end}"
+                    return start
+                elif end is not None:
+                    return f"Until {end}"
+
+                if "id" in inner_dict:
+                    # This is not useful to index, it's a reference to another Notion object
+                    # and this ID value in plaintext is useless outside of the Notion context
+                    logger.debug("Skipping Notion object id field property")
+                    return None
 
             logger.debug(f"Unreadable property from innermost prop: {inner_dict}")
             return None
@@ -268,7 +273,13 @@ def _recurse_properties(inner_dict: dict[str, Any]) -> str | None:
             if not prop:
                 continue
 
-            inner_value = _recurse_properties(prop)
+            try:
+                inner_value = _recurse_properties(prop)
+            except Exception as e:
+                # This is not a critical failure, these properties are not the actual contents of the page
+                # more similar to metadata
+                logger.warning(f"Error recursing properties for {prop_name}: {e}")
+                continue
             # Not a perfect way to format Notion database tables but there's no perfect representation
             # since this must be represented as plaintext
             if inner_value:
diff --git a/backend/scripts/dev_run_background_jobs.py b/backend/scripts/dev_run_background_jobs.py
index 4c87a3d3703..f3a00392465 100644
--- a/backend/scripts/dev_run_background_jobs.py
+++ b/backend/scripts/dev_run_background_jobs.py
@@ -1,4 +1,3 @@
-import argparse
 import subprocess
 import threading
 
@@ -135,7 +134,4 @@ def run_jobs() -> None:
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Run background jobs.")
-    args = parser.parse_args()
-
     run_jobs()

From 9105f95d138a21bca3c5c64722cf93b1c3a09fb9 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 22 Oct 2024 15:57:36 -0700
Subject: [PATCH 180/376] Feature/celery refactor (#2813)

* fresh indexing feature branch

* cherry pick test

* Revert "cherry pick test"

This reverts commit 2a624220687affdda3de347e30f2011136f64bda.

* set multitenant so that vespa fields match when indexing

* cleanup pass

* mypy

* pass through env var to control celery indexing concurrency

* comments on task kickoff and some logging improvements

* disentangle configuration for different workers and beats.

* use get_session_with_tenant

* comment out all of update.py

* rename to RedisConnectorIndexingFenceData

* first check num_indexing_workers

* refactor RedisConnectorIndexingFenceData

* comment out on_worker_process_init

* missed a file

* scope db sessions to short lengths

* update launch.json template

* fix types

* code review
---
 .vscode/launch.template.jsonc                 | 264 +++++++-
 .../background/celery/apps/app_base.py        | 256 ++++++++
 .../danswer/background/celery/apps/beat.py    |  99 +++
 .../danswer/background/celery/apps/heavy.py   |  88 +++
 .../background/celery/apps/indexing.py        | 116 ++++
 .../danswer/background/celery/apps/light.py   |  89 +++
 .../danswer/background/celery/apps/primary.py | 278 ++++++++
 .../background/celery/apps/task_formatters.py |  26 +
 .../danswer/background/celery/celery_app.py   | 601 ------------------
 .../danswer/background/celery/celery_redis.py |   2 +-
 .../danswer/background/celery/celery_utils.py |  29 +-
 .../{celeryconfig.py => configs/base.py}      |  24 +-
 .../danswer/background/celery/configs/beat.py |  14 +
 .../background/celery/configs/heavy.py        |  20 +
 .../background/celery/configs/indexing.py     |  21 +
 .../background/celery/configs/light.py        |  22 +
 .../background/celery/configs/primary.py      |  20 +
 .../celery/tasks/connector_deletion/tasks.py  |  34 +-
 .../background/celery/tasks/indexing/tasks.py |  47 +-
 .../background/celery/tasks/periodic/tasks.py |   2 +-
 .../background/celery/tasks/pruning/tasks.py  |  34 +-
 .../shared/RedisConnectorIndexingFenceData.py |  10 +
 .../background/celery/tasks/shared/tasks.py   |   2 +-
 .../background/celery/tasks/vespa/tasks.py    | 155 +++--
 .../{celery_run.py => versioned_apps/beat.py} |   7 +-
 .../background/celery/versioned_apps/heavy.py |  17 +
 .../celery/versioned_apps/indexing.py         |  17 +
 .../background/celery/versioned_apps/light.py |  17 +
 .../celery/versioned_apps/primary.py          |   8 +
 backend/danswer/configs/app_configs.py        |  35 +
 backend/danswer/server/documents/cc_pair.py   |   6 +-
 backend/danswer/server/documents/connector.py |   2 +
 .../danswer/server/manage/administrative.py   |   4 +-
 .../ee/danswer/background/celery/apps/beat.py |  52 ++
 .../celery/{celery_app.py => apps/primary.py} |  55 +-
 .../background/celery/tasks/vespa/tasks.py    |   2 +-
 backend/scripts/dev_run_background_jobs.py    |  22 +-
 backend/supervisord.conf                      |  34 +-
 38 files changed, 1665 insertions(+), 866 deletions(-)
 create mode 100644 backend/danswer/background/celery/apps/app_base.py
 create mode 100644 backend/danswer/background/celery/apps/beat.py
 create mode 100644 backend/danswer/background/celery/apps/heavy.py
 create mode 100644 backend/danswer/background/celery/apps/indexing.py
 create mode 100644 backend/danswer/background/celery/apps/light.py
 create mode 100644 backend/danswer/background/celery/apps/primary.py
 create mode 100644 backend/danswer/background/celery/apps/task_formatters.py
 delete mode 100644 backend/danswer/background/celery/celery_app.py
 rename backend/danswer/background/celery/{celeryconfig.py => configs/base.py} (95%)
 create mode 100644 backend/danswer/background/celery/configs/beat.py
 create mode 100644 backend/danswer/background/celery/configs/heavy.py
 create mode 100644 backend/danswer/background/celery/configs/indexing.py
 create mode 100644 backend/danswer/background/celery/configs/light.py
 create mode 100644 backend/danswer/background/celery/configs/primary.py
 create mode 100644 backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py
 rename backend/danswer/background/celery/{celery_run.py => versioned_apps/beat.py} (55%)
 create mode 100644 backend/danswer/background/celery/versioned_apps/heavy.py
 create mode 100644 backend/danswer/background/celery/versioned_apps/indexing.py
 create mode 100644 backend/danswer/background/celery/versioned_apps/light.py
 create mode 100644 backend/danswer/background/celery/versioned_apps/primary.py
 create mode 100644 backend/ee/danswer/background/celery/apps/beat.py
 rename backend/ee/danswer/background/celery/{celery_app.py => apps/primary.py} (76%)

diff --git a/.vscode/launch.template.jsonc b/.vscode/launch.template.jsonc
index c733800981c..87875907cd5 100644
--- a/.vscode/launch.template.jsonc
+++ b/.vscode/launch.template.jsonc
@@ -6,19 +6,69 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "compounds": [
+		{
+			// Dummy entry used to label the group
+			"name": "--- Compound ---",
+		  	"configurations": [
+		  		"--- Individual ---"
+		  	],
+			"presentation": {
+				 "group": "1",
+			 }
+		},
         {
             "name": "Run All Danswer Services",
             "configurations": [
                 "Web Server",
                 "Model Server",
                 "API Server",
-                "Indexing",
-                "Background Jobs",
-                "Slack Bot"
-            ]
-        }
+                "Slack Bot",
+		  		"Celery primary", 
+		  		"Celery light", 
+		  		"Celery heavy", 
+		  		"Celery indexing", 
+		  		"Celery beat",
+            ],
+			"presentation": {
+				 "group": "1",
+			 }
+        },
+		{
+			"name": "Web / Model / API",
+		  	"configurations": [
+		  		"Web Server", 
+		  		"Model Server", 
+		  		"API Server",
+		  	],
+			"presentation": {
+				 "group": "1",
+			 }
+		},
+		{
+			"name": "Celery (all)",
+		  	"configurations": [
+		  		"Celery primary", 
+		  		"Celery light", 
+		  		"Celery heavy", 
+		  		"Celery indexing", 
+		  		"Celery beat"
+		  	],
+			"presentation": {
+				 "group": "1",
+			 }
+		}
     ],
     "configurations": [
+		{
+	    	// Dummy entry used to label the group
+			"name": "--- Individual ---",
+			"type": "node",
+			"request": "launch",
+			"presentation": {
+				"group": "2",
+				"order": 0
+			}
+		},
         {
             "name": "Web Server",
             "type": "node",
@@ -29,7 +79,11 @@
             "runtimeArgs": [
                 "run", "dev"
             ],
-            "console": "integratedTerminal"
+            "presentation": {
+				 "group": "2",
+			 },
+            "console": "integratedTerminal",
+            "consoleTitle": "Web Server Console"
         },
         {
             "name": "Model Server",
@@ -48,7 +102,11 @@
                 "--reload",
                 "--port",
                 "9000"
-            ]
+            ],
+            "presentation": {
+				 "group": "2",
+			},
+            "consoleTitle": "Model Server Console"
         },
         {
             "name": "API Server",
@@ -68,57 +126,171 @@
                 "--reload",
                 "--port",
                 "8080"
-            ]
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "API Server Console"
         },
+        // For the listener to access the Slack API,
+        // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
         {
-            "name": "Indexing",
-            "consoleName": "Indexing",
+            "name": "Slack Bot",
+            "consoleName": "Slack Bot",
             "type": "debugpy",
             "request": "launch",
-            "program": "danswer/background/update.py",
+            "program": "danswer/danswerbot/slack/listener.py",
             "cwd": "${workspaceFolder}/backend",
             "envFile": "${workspaceFolder}/.vscode/.env",
             "env": {
-                "ENABLE_MULTIPASS_INDEXING": "false",
                 "LOG_LEVEL": "DEBUG",
                 "PYTHONUNBUFFERED": "1",
                 "PYTHONPATH": "."
-            }
+            },
+            "presentation": {
+				 "group": "2",
+			},
+            "consoleTitle": "Slack Bot Console"
         },
-        // Celery and all async jobs, usually would include indexing as well but this is handled separately above for dev
         {
-            "name": "Background Jobs",
-            "consoleName": "Background Jobs",
+            "name": "Celery primary",
             "type": "debugpy",
             "request": "launch",
-            "program": "scripts/dev_run_background_jobs.py",
+            "module": "celery",
             "cwd": "${workspaceFolder}/backend",
             "envFile": "${workspaceFolder}/.vscode/.env",
             "env": {
-                "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+                "LOG_LEVEL": "INFO",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            },
+            "args": [
+                "-A",
+                "danswer.background.celery.versioned_apps.primary",
+                "worker",
+                "--pool=threads",
+                "--concurrency=4",
+                "--prefetch-multiplier=1",
+                "--loglevel=INFO",
+                "--hostname=primary@%n",
+                "-Q",
+                "celery",
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "Celery primary Console"
+        },
+        {
+            "name": "Celery light",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "celery",
+            "cwd": "${workspaceFolder}/backend",
+            "envFile": "${workspaceFolder}/.vscode/.env",
+            "env": {
+                "LOG_LEVEL": "INFO",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            },
+            "args": [
+                "-A",
+                "danswer.background.celery.versioned_apps.light",
+                "worker",
+                "--pool=threads",
+                "--concurrency=64",
+                "--prefetch-multiplier=8",
+                "--loglevel=INFO",
+                "--hostname=light@%n",
+                "-Q",
+                "vespa_metadata_sync,connector_deletion",
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "Celery light Console"
+        },
+        {
+            "name": "Celery heavy",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "celery",
+            "cwd": "${workspaceFolder}/backend",
+            "envFile": "${workspaceFolder}/.vscode/.env",
+            "env": {
+                "LOG_LEVEL": "INFO",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            },
+            "args": [
+                "-A",
+                "danswer.background.celery.versioned_apps.heavy",
+                "worker",
+                "--pool=threads",
+                "--concurrency=4",
+                "--prefetch-multiplier=1",
+                "--loglevel=INFO",
+                "--hostname=heavy@%n",
+                "-Q",
+                "connector_pruning",
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "Celery heavy Console"
+        },
+        {
+            "name": "Celery indexing",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "celery",
+            "cwd": "${workspaceFolder}/backend",
+            "envFile": "${workspaceFolder}/.vscode/.env",
+            "env": {
+                "ENABLE_MULTIPASS_INDEXING": "false",
                 "LOG_LEVEL": "DEBUG",
                 "PYTHONUNBUFFERED": "1",
                 "PYTHONPATH": "."
             },
             "args": [
-                "--no-indexing"
-            ]
+                "-A",
+                "danswer.background.celery.versioned_apps.indexing",
+                "worker",
+                "--pool=threads",
+                "--concurrency=1",
+                "--prefetch-multiplier=1",
+                "--loglevel=INFO",
+                "--hostname=indexing@%n",
+                "-Q",
+                "connector_indexing",
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "Celery indexing Console"
         },
-        // For the listner to access the Slack API,
-        // DANSWER_BOT_SLACK_APP_TOKEN & DANSWER_BOT_SLACK_BOT_TOKEN need to be set in .env file located in the root of the project
         {
-            "name": "Slack Bot",
-            "consoleName": "Slack Bot",
+            "name": "Celery beat",
             "type": "debugpy",
             "request": "launch",
-            "program": "danswer/danswerbot/slack/listener.py",
+            "module": "celery",
             "cwd": "${workspaceFolder}/backend",
             "envFile": "${workspaceFolder}/.vscode/.env",
             "env": {
                 "LOG_LEVEL": "DEBUG",
                 "PYTHONUNBUFFERED": "1",
                 "PYTHONPATH": "."
-            }
+            },
+            "args": [
+                "-A",
+                "danswer.background.celery.versioned_apps.beat",
+                "beat",
+                "--loglevel=INFO",
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "Celery beat Console"
         },
         {
             "name": "Pytest",
@@ -137,8 +309,22 @@
                 "-v"
                 // Specify a sepcific module/test to run or provide nothing to run all tests
                 //"tests/unit/danswer/llm/answering/test_prune_and_merge.py"
-            ]
+            ],
+            "presentation": {
+				 "group": "2",
+			 },
+            "consoleTitle": "Pytest Console"
         },
+		{
+	    	// Dummy entry used to label the group
+			"name": "--- Tasks ---",
+			"type": "node",
+			"request": "launch",
+			"presentation": {
+				"group": "3",
+				"order": 0
+			}
+		},
         {
             "name": "Clear and Restart External Volumes and Containers",
             "type": "node",
@@ -147,7 +333,27 @@
             "runtimeArgs": ["${workspaceFolder}/backend/scripts/restart_containers.sh"],
             "cwd": "${workspaceFolder}",
             "console": "integratedTerminal",
-            "stopOnEntry": true
-        }
+            "stopOnEntry": true,
+            "presentation": {
+				 "group": "3",
+			 },
+        },
+        {
+	        // Celery jobs launched through a single background script (legacy)
+	        // Recommend using the "Celery (all)" compound launch instead.
+            "name": "Background Jobs",
+            "consoleName": "Background Jobs",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "scripts/dev_run_background_jobs.py",
+            "cwd": "${workspaceFolder}/backend",
+            "envFile": "${workspaceFolder}/.vscode/.env",
+            "env": {
+                "LOG_DANSWER_MODEL_INTERACTIONS": "True",
+                "LOG_LEVEL": "DEBUG",
+                "PYTHONUNBUFFERED": "1",
+                "PYTHONPATH": "."
+            },
+        },
     ]
 }
diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
new file mode 100644
index 00000000000..2a52abde5d1
--- /dev/null
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -0,0 +1,256 @@
+import logging
+import multiprocessing
+import time
+from typing import Any
+
+import sentry_sdk
+from celery import Task
+from celery.exceptions import WorkerShutdown
+from celery.states import READY_STATES
+from celery.utils.log import get_task_logger
+from sentry_sdk.integrations.celery import CeleryIntegration
+
+from danswer.background.celery.apps.task_formatters import CeleryTaskColoredFormatter
+from danswer.background.celery.apps.task_formatters import CeleryTaskPlainFormatter
+from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
+from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorPruning
+from danswer.background.celery.celery_redis import RedisDocumentSet
+from danswer.background.celery.celery_redis import RedisUserGroup
+from danswer.background.celery.celery_utils import celery_is_worker_primary
+from danswer.configs.constants import DanswerRedisLocks
+from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import ColoredFormatter
+from danswer.utils.logger import PlainFormatter
+from danswer.utils.logger import setup_logger
+from shared_configs.configs import SENTRY_DSN
+
+logger = setup_logger()
+
+task_logger = get_task_logger(__name__)
+
+if SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=SENTRY_DSN,
+        integrations=[CeleryIntegration()],
+        traces_sample_rate=0.5,
+    )
+    logger.info("Sentry initialized")
+else:
+    logger.debug("Sentry DSN not provided, skipping Sentry initialization")
+
+
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    pass
+
+
+def on_task_postrun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    retval: Any | None = None,
+    state: str | None = None,
+    **kwds: Any,
+) -> None:
+    """We handle this signal in order to remove completed tasks
+    from their respective tasksets. This allows us to track the progress of document set
+    and user group syncs.
+
+    This function runs after any task completes (both success and failure)
+    Note that this signal does not fire on a task that failed to complete and is going
+    to be retried.
+
+    This also does not fire if a worker with acks_late=False crashes (which all of our
+    long running workers are)
+    """
+    if not task:
+        return
+
+    task_logger.debug(f"Task {task.name} (ID: {task_id}) completed with state: {state}")
+
+    if state not in READY_STATES:
+        return
+
+    if not task_id:
+        return
+
+    r = get_redis_client()
+
+    if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
+        r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)
+        return
+
+    if task_id.startswith(RedisDocumentSet.PREFIX):
+        document_set_id = RedisDocumentSet.get_id_from_task_id(task_id)
+        if document_set_id is not None:
+            rds = RedisDocumentSet(int(document_set_id))
+            r.srem(rds.taskset_key, task_id)
+        return
+
+    if task_id.startswith(RedisUserGroup.PREFIX):
+        usergroup_id = RedisUserGroup.get_id_from_task_id(task_id)
+        if usergroup_id is not None:
+            rug = RedisUserGroup(int(usergroup_id))
+            r.srem(rug.taskset_key, task_id)
+        return
+
+    if task_id.startswith(RedisConnectorDeletion.PREFIX):
+        cc_pair_id = RedisConnectorDeletion.get_id_from_task_id(task_id)
+        if cc_pair_id is not None:
+            rcd = RedisConnectorDeletion(int(cc_pair_id))
+            r.srem(rcd.taskset_key, task_id)
+        return
+
+    if task_id.startswith(RedisConnectorPruning.SUBTASK_PREFIX):
+        cc_pair_id = RedisConnectorPruning.get_id_from_task_id(task_id)
+        if cc_pair_id is not None:
+            rcp = RedisConnectorPruning(int(cc_pair_id))
+            r.srem(rcp.taskset_key, task_id)
+        return
+
+
+def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
+    """The first signal sent on celery worker startup"""
+    multiprocessing.set_start_method("spawn")  # fork is unsafe, set to spawn
+
+
+def wait_for_redis(sender: Any, **kwargs: Any) -> None:
+    r = get_redis_client()
+
+    WAIT_INTERVAL = 5
+    WAIT_LIMIT = 60
+
+    time_start = time.monotonic()
+    logger.info("Redis: Readiness check starting.")
+    while True:
+        try:
+            if r.ping():
+                break
+        except Exception:
+            pass
+
+        time_elapsed = time.monotonic() - time_start
+        logger.info(
+            f"Redis: Ping failed. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
+        )
+        if time_elapsed > WAIT_LIMIT:
+            msg = (
+                f"Redis: Readiness check did not succeed within the timeout "
+                f"({WAIT_LIMIT} seconds). Exiting..."
+            )
+            logger.error(msg)
+            raise WorkerShutdown(msg)
+
+        time.sleep(WAIT_INTERVAL)
+
+    logger.info("Redis: Readiness check succeeded. Continuing...")
+    return
+
+
+def on_secondary_worker_init(sender: Any, **kwargs: Any) -> None:
+    r = get_redis_client()
+
+    WAIT_INTERVAL = 5
+    WAIT_LIMIT = 60
+
+    logger.info("Running as a secondary celery worker.")
+    logger.info("Waiting for primary worker to be ready...")
+    time_start = time.monotonic()
+    while True:
+        if r.exists(DanswerRedisLocks.PRIMARY_WORKER):
+            break
+
+        time.monotonic()
+        time_elapsed = time.monotonic() - time_start
+        logger.info(
+            f"Primary worker is not ready yet. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
+        )
+        if time_elapsed > WAIT_LIMIT:
+            msg = (
+                f"Primary worker was not ready within the timeout. "
+                f"({WAIT_LIMIT} seconds). Exiting..."
+            )
+            logger.error(msg)
+            raise WorkerShutdown(msg)
+
+        time.sleep(WAIT_INTERVAL)
+
+    logger.info("Wait for primary worker completed successfully. Continuing...")
+    return
+
+
+def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    task_logger.info("worker_ready signal received.")
+
+
+def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
+    if not celery_is_worker_primary(sender):
+        return
+
+    if not sender.primary_worker_lock:
+        return
+
+    logger.info("Releasing primary worker lock.")
+    lock = sender.primary_worker_lock
+    if lock.owned():
+        lock.release()
+        sender.primary_worker_lock = None
+
+
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    # TODO: could unhardcode format and colorize and accept these as options from
+    # celery's config
+
+    # reformats the root logger
+    root_logger = logging.getLogger()
+
+    root_handler = logging.StreamHandler()  # Set up a handler for the root logger
+    root_formatter = ColoredFormatter(
+        "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
+        datefmt="%m/%d/%Y %I:%M:%S %p",
+    )
+    root_handler.setFormatter(root_formatter)
+    root_logger.addHandler(root_handler)  # Apply the handler to the root logger
+
+    if logfile:
+        root_file_handler = logging.FileHandler(logfile)
+        root_file_formatter = PlainFormatter(
+            "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
+            datefmt="%m/%d/%Y %I:%M:%S %p",
+        )
+        root_file_handler.setFormatter(root_file_formatter)
+        root_logger.addHandler(root_file_handler)
+
+    root_logger.setLevel(loglevel)
+
+    # reformats celery's task logger
+    task_formatter = CeleryTaskColoredFormatter(
+        "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
+        datefmt="%m/%d/%Y %I:%M:%S %p",
+    )
+    task_handler = logging.StreamHandler()  # Set up a handler for the task logger
+    task_handler.setFormatter(task_formatter)
+    task_logger.addHandler(task_handler)  # Apply the handler to the task logger
+
+    if logfile:
+        task_file_handler = logging.FileHandler(logfile)
+        task_file_formatter = CeleryTaskPlainFormatter(
+            "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
+            datefmt="%m/%d/%Y %I:%M:%S %p",
+        )
+        task_file_handler.setFormatter(task_file_formatter)
+        task_logger.addHandler(task_file_handler)
+
+    task_logger.setLevel(loglevel)
+    task_logger.propagate = False
diff --git a/backend/danswer/background/celery/apps/beat.py b/backend/danswer/background/celery/apps/beat.py
new file mode 100644
index 00000000000..47be61e36be
--- /dev/null
+++ b/backend/danswer/background/celery/apps/beat.py
@@ -0,0 +1,99 @@
+from datetime import timedelta
+from typing import Any
+
+from celery import Celery
+from celery import signals
+from celery.signals import beat_init
+
+import danswer.background.celery.apps.app_base as app_base
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import POSTGRES_CELERY_BEAT_APP_NAME
+from danswer.db.engine import get_all_tenant_ids
+from danswer.db.engine import SqlEngine
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+celery_app = Celery(__name__)
+celery_app.config_from_object("danswer.background.celery.configs.beat")
+
+
+@beat_init.connect
+def on_beat_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("beat_init signal received.")
+    SqlEngine.set_app_name(POSTGRES_CELERY_BEAT_APP_NAME)
+    SqlEngine.init_engine(pool_size=2, max_overflow=0)
+    app_base.wait_for_redis(sender, **kwargs)
+
+
+@signals.setup_logging.connect
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
+
+
+#####
+# Celery Beat (Periodic Tasks) Settings
+#####
+
+tenant_ids = get_all_tenant_ids()
+
+tasks_to_schedule = [
+    {
+        "name": "check-for-vespa-sync",
+        "task": "check_for_vespa_sync_task",
+        "schedule": timedelta(seconds=5),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "check-for-connector-deletion",
+        "task": "check_for_connector_deletion_task",
+        "schedule": timedelta(seconds=60),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "check-for-indexing",
+        "task": "check_for_indexing",
+        "schedule": timedelta(seconds=10),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "check-for-prune",
+        "task": "check_for_pruning",
+        "schedule": timedelta(seconds=10),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "kombu-message-cleanup",
+        "task": "kombu_message_cleanup_task",
+        "schedule": timedelta(seconds=3600),
+        "options": {"priority": DanswerCeleryPriority.LOWEST},
+    },
+    {
+        "name": "monitor-vespa-sync",
+        "task": "monitor_vespa_sync",
+        "schedule": timedelta(seconds=5),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+]
+
+# Build the celery beat schedule dynamically
+beat_schedule = {}
+
+for tenant_id in tenant_ids:
+    for task in tasks_to_schedule:
+        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
+        beat_schedule[task_name] = {
+            "task": task["task"],
+            "schedule": task["schedule"],
+            "options": task["options"],
+            "args": (tenant_id,),  # Must pass tenant_id as an argument
+        }
+
+# Include any existing beat schedules
+existing_beat_schedule = celery_app.conf.beat_schedule or {}
+beat_schedule.update(existing_beat_schedule)
+
+# Update the Celery app configuration once
+celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/danswer/background/celery/apps/heavy.py b/backend/danswer/background/celery/apps/heavy.py
new file mode 100644
index 00000000000..ba53776bedb
--- /dev/null
+++ b/backend/danswer/background/celery/apps/heavy.py
@@ -0,0 +1,88 @@
+import multiprocessing
+from typing import Any
+
+from celery import Celery
+from celery import signals
+from celery import Task
+from celery.signals import celeryd_init
+from celery.signals import worker_init
+from celery.signals import worker_ready
+from celery.signals import worker_shutdown
+
+import danswer.background.celery.apps.app_base as app_base
+from danswer.configs.constants import POSTGRES_CELERY_WORKER_HEAVY_APP_NAME
+from danswer.db.engine import SqlEngine
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
+
+celery_app = Celery(__name__)
+celery_app.config_from_object("danswer.background.celery.configs.heavy")
+
+
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+
+
+@signals.task_postrun.connect
+def on_task_postrun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    retval: Any | None = None,
+    state: str | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+
+
+@celeryd_init.connect
+def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
+    app_base.on_celeryd_init(sender, conf, **kwargs)
+
+
+@worker_init.connect
+def on_worker_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("worker_init signal received.")
+    logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
+
+    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_HEAVY_APP_NAME)
+    SqlEngine.init_engine(pool_size=8, max_overflow=0)
+
+    app_base.wait_for_redis(sender, **kwargs)
+    app_base.on_secondary_worker_init(sender, **kwargs)
+
+
+@worker_ready.connect
+def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_ready(sender, **kwargs)
+
+
+@worker_shutdown.connect
+def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_shutdown(sender, **kwargs)
+
+
+@signals.setup_logging.connect
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
+
+
+celery_app.autodiscover_tasks(
+    [
+        "danswer.background.celery.tasks.pruning",
+    ]
+)
diff --git a/backend/danswer/background/celery/apps/indexing.py b/backend/danswer/background/celery/apps/indexing.py
new file mode 100644
index 00000000000..5e51ebc8c54
--- /dev/null
+++ b/backend/danswer/background/celery/apps/indexing.py
@@ -0,0 +1,116 @@
+import multiprocessing
+from typing import Any
+
+from celery import Celery
+from celery import signals
+from celery import Task
+from celery.signals import celeryd_init
+from celery.signals import worker_init
+from celery.signals import worker_ready
+from celery.signals import worker_shutdown
+from sqlalchemy.orm import Session
+
+import danswer.background.celery.apps.app_base as app_base
+from danswer.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_APP_NAME
+from danswer.db.engine import SqlEngine
+from danswer.db.search_settings import get_current_search_settings
+from danswer.db.swap_index import check_index_swap
+from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
+from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
+from danswer.utils.logger import setup_logger
+from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
+from shared_configs.configs import MODEL_SERVER_PORT
+
+
+logger = setup_logger()
+
+celery_app = Celery(__name__)
+celery_app.config_from_object("danswer.background.celery.configs.indexing")
+
+
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+
+
+@signals.task_postrun.connect
+def on_task_postrun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    retval: Any | None = None,
+    state: str | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+
+
+@celeryd_init.connect
+def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
+    app_base.on_celeryd_init(sender, conf, **kwargs)
+
+
+@worker_init.connect
+def on_worker_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("worker_init signal received.")
+    logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
+
+    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_APP_NAME)
+    SqlEngine.init_engine(pool_size=8, max_overflow=0)
+
+    # TODO: why is this necessary for the indexer to do?
+    engine = SqlEngine.get_engine()
+    with Session(engine) as db_session:
+        check_index_swap(db_session=db_session)
+        search_settings = get_current_search_settings(db_session)
+
+        # So that the first time users aren't surprised by really slow speed of first
+        # batch of documents indexed
+        if search_settings.provider_type is None:
+            logger.notice("Running a first inference to warm up embedding model")
+            embedding_model = EmbeddingModel.from_db_model(
+                search_settings=search_settings,
+                server_host=INDEXING_MODEL_SERVER_HOST,
+                server_port=MODEL_SERVER_PORT,
+            )
+
+            warm_up_bi_encoder(
+                embedding_model=embedding_model,
+            )
+            logger.notice("First inference complete.")
+
+    app_base.wait_for_redis(sender, **kwargs)
+    app_base.on_secondary_worker_init(sender, **kwargs)
+
+
+@worker_ready.connect
+def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_ready(sender, **kwargs)
+
+
+@worker_shutdown.connect
+def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_shutdown(sender, **kwargs)
+
+
+@signals.setup_logging.connect
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
+
+
+celery_app.autodiscover_tasks(
+    [
+        "danswer.background.celery.tasks.indexing",
+    ]
+)
diff --git a/backend/danswer/background/celery/apps/light.py b/backend/danswer/background/celery/apps/light.py
new file mode 100644
index 00000000000..6f39074b601
--- /dev/null
+++ b/backend/danswer/background/celery/apps/light.py
@@ -0,0 +1,89 @@
+import multiprocessing
+from typing import Any
+
+from celery import Celery
+from celery import signals
+from celery import Task
+from celery.signals import celeryd_init
+from celery.signals import worker_init
+from celery.signals import worker_ready
+from celery.signals import worker_shutdown
+
+import danswer.background.celery.apps.app_base as app_base
+from danswer.configs.constants import POSTGRES_CELERY_WORKER_LIGHT_APP_NAME
+from danswer.db.engine import SqlEngine
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
+
+celery_app = Celery(__name__)
+celery_app.config_from_object("danswer.background.celery.configs.light")
+
+
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+
+
+@signals.task_postrun.connect
+def on_task_postrun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    retval: Any | None = None,
+    state: str | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+
+
+@celeryd_init.connect
+def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
+    app_base.on_celeryd_init(sender, conf, **kwargs)
+
+
+@worker_init.connect
+def on_worker_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("worker_init signal received.")
+    logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
+
+    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_LIGHT_APP_NAME)
+    SqlEngine.init_engine(pool_size=sender.concurrency, max_overflow=8)
+
+    app_base.wait_for_redis(sender, **kwargs)
+    app_base.on_secondary_worker_init(sender, **kwargs)
+
+
+@worker_ready.connect
+def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_ready(sender, **kwargs)
+
+
+@worker_shutdown.connect
+def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_shutdown(sender, **kwargs)
+
+
+@signals.setup_logging.connect
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
+
+
+celery_app.autodiscover_tasks(
+    [
+        "danswer.background.celery.tasks.shared",
+        "danswer.background.celery.tasks.vespa",
+    ]
+)
diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
new file mode 100644
index 00000000000..c99607f4bc3
--- /dev/null
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -0,0 +1,278 @@
+import multiprocessing
+from typing import Any
+
+import redis
+from celery import bootsteps  # type: ignore
+from celery import Celery
+from celery import signals
+from celery import Task
+from celery.exceptions import WorkerShutdown
+from celery.signals import celeryd_init
+from celery.signals import worker_init
+from celery.signals import worker_ready
+from celery.signals import worker_shutdown
+from celery.utils.log import get_task_logger
+
+import danswer.background.celery.apps.app_base as app_base
+from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
+from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
+from danswer.background.celery.celery_redis import RedisConnectorPruning
+from danswer.background.celery.celery_redis import RedisDocumentSet
+from danswer.background.celery.celery_redis import RedisUserGroup
+from danswer.background.celery.celery_utils import celery_is_worker_primary
+from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerRedisLocks
+from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
+from danswer.db.engine import SqlEngine
+from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
+
+# use this within celery tasks to get celery task specific logging
+task_logger = get_task_logger(__name__)
+
+celery_app = Celery(__name__)
+celery_app.config_from_object("danswer.background.celery.configs.primary")
+
+
+@signals.task_prerun.connect
+def on_task_prerun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_prerun(sender, task_id, task, args, kwargs, **kwds)
+
+
+@signals.task_postrun.connect
+def on_task_postrun(
+    sender: Any | None = None,
+    task_id: str | None = None,
+    task: Task | None = None,
+    args: tuple | None = None,
+    kwargs: dict | None = None,
+    retval: Any | None = None,
+    state: str | None = None,
+    **kwds: Any,
+) -> None:
+    app_base.on_task_postrun(sender, task_id, task, args, kwargs, retval, state, **kwds)
+
+
+@celeryd_init.connect
+def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
+    app_base.on_celeryd_init(sender, conf, **kwargs)
+
+
+@worker_init.connect
+def on_worker_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("worker_init signal received.")
+    logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
+
+    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
+    SqlEngine.init_engine(pool_size=8, max_overflow=0)
+
+    app_base.wait_for_redis(sender, **kwargs)
+
+    logger.info("Running as the primary celery worker.")
+
+    # This is singleton work that should be done on startup exactly once
+    # by the primary worker
+    r = get_redis_client()
+
+    # For the moment, we're assuming that we are the only primary worker
+    # that should be running.
+    # TODO: maybe check for or clean up another zombie primary worker if we detect it
+    r.delete(DanswerRedisLocks.PRIMARY_WORKER)
+
+    # this process wide lock is taken to help other workers start up in order.
+    # it is planned to use this lock to enforce singleton behavior on the primary
+    # worker, since the primary worker does redis cleanup on startup, but this isn't
+    # implemented yet.
+    lock = r.lock(
+        DanswerRedisLocks.PRIMARY_WORKER,
+        timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
+    )
+
+    logger.info("Primary worker lock: Acquire starting.")
+    acquired = lock.acquire(blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2)
+    if acquired:
+        logger.info("Primary worker lock: Acquire succeeded.")
+    else:
+        logger.error("Primary worker lock: Acquire failed!")
+        raise WorkerShutdown("Primary worker lock could not be acquired!")
+
+    sender.primary_worker_lock = lock
+
+    # As currently designed, when this worker starts as "primary", we reinitialize redis
+    # to a clean state (for our purposes, anyway)
+    r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
+    r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+    r.delete(RedisConnectorCredentialPair.get_taskset_key())
+    r.delete(RedisConnectorCredentialPair.get_fence_key())
+
+    for key in r.scan_iter(RedisDocumentSet.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisUserGroup.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorDeletion.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_COMPLETE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_PROGRESS_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.TASKSET_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_COMPLETE_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_PROGRESS_PREFIX + "*"):
+        r.delete(key)
+
+    for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
+        r.delete(key)
+
+
+# @worker_process_init.connect
+# def on_worker_process_init(sender: Any, **kwargs: Any) -> None:
+#     """This only runs inside child processes when the worker is in pool=prefork mode.
+#     This may be technically unnecessary since we're finding prefork pools to be
+#     unstable and currently aren't planning on using them."""
+#     logger.info("worker_process_init signal received.")
+#     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME)
+#     SqlEngine.init_engine(pool_size=5, max_overflow=0)
+
+#     # https://stackoverflow.com/questions/43944787/sqlalchemy-celery-with-scoped-session-error
+#     SqlEngine.get_engine().dispose(close=False)
+
+
+@worker_ready.connect
+def on_worker_ready(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_ready(sender, **kwargs)
+
+
+@worker_shutdown.connect
+def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
+    app_base.on_worker_shutdown(sender, **kwargs)
+
+
+@signals.setup_logging.connect
+def on_setup_logging(
+    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
+) -> None:
+    app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
+
+
+class HubPeriodicTask(bootsteps.StartStopStep):
+    """Regularly reacquires the primary worker lock outside of the task queue.
+    Use the task_logger in this class to avoid double logging.
+
+    This cannot be done inside a regular beat task because it must run on schedule and
+    a queue of existing work would starve the task from running.
+    """
+
+    # it's unclear to me whether using the hub's timer or the bootstep timer is better
+    requires = {"celery.worker.components:Hub"}
+
+    def __init__(self, worker: Any, **kwargs: Any) -> None:
+        self.interval = CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 8  # Interval in seconds
+        self.task_tref = None
+
+    def start(self, worker: Any) -> None:
+        if not celery_is_worker_primary(worker):
+            return
+
+        # Access the worker's event loop (hub)
+        hub = worker.consumer.controller.hub
+
+        # Schedule the periodic task
+        self.task_tref = hub.call_repeatedly(
+            self.interval, self.run_periodic_task, worker
+        )
+        task_logger.info("Scheduled periodic task with hub.")
+
+    def run_periodic_task(self, worker: Any) -> None:
+        try:
+            if not worker.primary_worker_lock:
+                return
+
+            if not hasattr(worker, "primary_worker_lock"):
+                return
+
+            r = get_redis_client()
+
+            lock: redis.lock.Lock = worker.primary_worker_lock
+
+            if lock.owned():
+                task_logger.debug("Reacquiring primary worker lock.")
+                lock.reacquire()
+            else:
+                task_logger.warning(
+                    "Full acquisition of primary worker lock. "
+                    "Reasons could be computer sleep or a clock change."
+                )
+                lock = r.lock(
+                    DanswerRedisLocks.PRIMARY_WORKER,
+                    timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
+                )
+
+                task_logger.info("Primary worker lock: Acquire starting.")
+                acquired = lock.acquire(
+                    blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2
+                )
+                if acquired:
+                    task_logger.info("Primary worker lock: Acquire succeeded.")
+                else:
+                    task_logger.error("Primary worker lock: Acquire failed!")
+                    raise TimeoutError("Primary worker lock could not be acquired!")
+
+                worker.primary_worker_lock = lock
+        except Exception:
+            task_logger.exception("HubPeriodicTask.run_periodic_task exceptioned.")
+
+    def stop(self, worker: Any) -> None:
+        # Cancel the scheduled task when the worker stops
+        if self.task_tref:
+            self.task_tref.cancel()
+            task_logger.info("Canceled periodic task with hub.")
+
+
+celery_app.steps["worker"].add(HubPeriodicTask)
+
+celery_app.autodiscover_tasks(
+    [
+        "danswer.background.celery.tasks.connector_deletion",
+        "danswer.background.celery.tasks.indexing",
+        "danswer.background.celery.tasks.periodic",
+        "danswer.background.celery.tasks.pruning",
+        "danswer.background.celery.tasks.shared",
+        "danswer.background.celery.tasks.vespa",
+    ]
+)
diff --git a/backend/danswer/background/celery/apps/task_formatters.py b/backend/danswer/background/celery/apps/task_formatters.py
new file mode 100644
index 00000000000..e82b23a5431
--- /dev/null
+++ b/backend/danswer/background/celery/apps/task_formatters.py
@@ -0,0 +1,26 @@
+import logging
+
+from celery import current_task
+
+from danswer.utils.logger import ColoredFormatter
+from danswer.utils.logger import PlainFormatter
+
+
+class CeleryTaskPlainFormatter(PlainFormatter):
+    def format(self, record: logging.LogRecord) -> str:
+        task = current_task
+        if task and task.request:
+            record.__dict__.update(task_id=task.request.id, task_name=task.name)
+            record.msg = f"[{task.name}({task.request.id})] {record.msg}"
+
+        return super().format(record)
+
+
+class CeleryTaskColoredFormatter(ColoredFormatter):
+    def format(self, record: logging.LogRecord) -> str:
+        task = current_task
+        if task and task.request:
+            record.__dict__.update(task_id=task.request.id, task_name=task.name)
+            record.msg = f"[{task.name}({task.request.id})] {record.msg}"
+
+        return super().format(record)
diff --git a/backend/danswer/background/celery/celery_app.py b/backend/danswer/background/celery/celery_app.py
deleted file mode 100644
index ee59b8d50fd..00000000000
--- a/backend/danswer/background/celery/celery_app.py
+++ /dev/null
@@ -1,601 +0,0 @@
-import logging
-import multiprocessing
-import time
-from datetime import timedelta
-from typing import Any
-
-import redis
-import sentry_sdk
-from celery import bootsteps  # type: ignore
-from celery import Celery
-from celery import current_task
-from celery import signals
-from celery import Task
-from celery.exceptions import WorkerShutdown
-from celery.signals import beat_init
-from celery.signals import celeryd_init
-from celery.signals import worker_init
-from celery.signals import worker_ready
-from celery.signals import worker_shutdown
-from celery.states import READY_STATES
-from celery.utils.log import get_task_logger
-from sentry_sdk.integrations.celery import CeleryIntegration
-
-from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.celery_redis import RedisConnectorPruning
-from danswer.background.celery.celery_redis import RedisDocumentSet
-from danswer.background.celery.celery_redis import RedisUserGroup
-from danswer.background.celery.celery_utils import celery_is_worker_primary
-from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
-from danswer.configs.constants import DanswerCeleryPriority
-from danswer.configs.constants import DanswerRedisLocks
-from danswer.configs.constants import POSTGRES_CELERY_BEAT_APP_NAME
-from danswer.configs.constants import POSTGRES_CELERY_WORKER_HEAVY_APP_NAME
-from danswer.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_APP_NAME
-from danswer.configs.constants import POSTGRES_CELERY_WORKER_LIGHT_APP_NAME
-from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
-from danswer.db.engine import get_all_tenant_ids
-from danswer.db.engine import get_session_with_tenant
-from danswer.db.engine import SqlEngine
-from danswer.db.search_settings import get_current_search_settings
-from danswer.db.swap_index import check_index_swap
-from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
-from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
-from danswer.redis.redis_pool import get_redis_client
-from danswer.utils.logger import ColoredFormatter
-from danswer.utils.logger import PlainFormatter
-from danswer.utils.logger import setup_logger
-from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
-from shared_configs.configs import MODEL_SERVER_PORT
-from shared_configs.configs import SENTRY_DSN
-
-logger = setup_logger()
-
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
-
-if SENTRY_DSN:
-    sentry_sdk.init(
-        dsn=SENTRY_DSN,
-        integrations=[CeleryIntegration()],
-        traces_sample_rate=0.5,
-    )
-    logger.info("Sentry initialized")
-else:
-    logger.debug("Sentry DSN not provided, skipping Sentry initialization")
-
-
-celery_app = Celery(__name__)
-celery_app.config_from_object(
-    "danswer.background.celery.celeryconfig"
-)  # Load configuration from 'celeryconfig.py'
-
-
-@signals.task_prerun.connect
-def on_task_prerun(
-    sender: Any | None = None,
-    task_id: str | None = None,
-    task: Task | None = None,
-    args: tuple | None = None,
-    kwargs: dict | None = None,
-    **kwds: Any,
-) -> None:
-    pass
-
-
-@signals.task_postrun.connect
-def on_task_postrun(
-    sender: Any | None = None,
-    task_id: str | None = None,
-    task: Task | None = None,
-    args: tuple | None = None,
-    kwargs: dict | None = None,
-    retval: Any | None = None,
-    state: str | None = None,
-    **kwds: Any,
-) -> None:
-    """We handle this signal in order to remove completed tasks
-    from their respective tasksets. This allows us to track the progress of document set
-    and user group syncs.
-
-    This function runs after any task completes (both success and failure)
-    Note that this signal does not fire on a task that failed to complete and is going
-    to be retried.
-
-    This also does not fire if a worker with acks_late=False crashes (which all of our
-    long running workers are)
-    """
-    if not task:
-        return
-
-    task_logger.debug(f"Task {task.name} (ID: {task_id}) completed with state: {state}")
-
-    if state not in READY_STATES:
-        return
-
-    if not task_id:
-        return
-
-    r = get_redis_client()
-
-    if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
-        r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)
-        return
-
-    if task_id.startswith(RedisDocumentSet.PREFIX):
-        document_set_id = RedisDocumentSet.get_id_from_task_id(task_id)
-        if document_set_id is not None:
-            rds = RedisDocumentSet(int(document_set_id))
-            r.srem(rds.taskset_key, task_id)
-        return
-
-    if task_id.startswith(RedisUserGroup.PREFIX):
-        usergroup_id = RedisUserGroup.get_id_from_task_id(task_id)
-        if usergroup_id is not None:
-            rug = RedisUserGroup(int(usergroup_id))
-            r.srem(rug.taskset_key, task_id)
-        return
-
-    if task_id.startswith(RedisConnectorDeletion.PREFIX):
-        cc_pair_id = RedisConnectorDeletion.get_id_from_task_id(task_id)
-        if cc_pair_id is not None:
-            rcd = RedisConnectorDeletion(int(cc_pair_id))
-            r.srem(rcd.taskset_key, task_id)
-        return
-
-    if task_id.startswith(RedisConnectorPruning.SUBTASK_PREFIX):
-        cc_pair_id = RedisConnectorPruning.get_id_from_task_id(task_id)
-        if cc_pair_id is not None:
-            rcp = RedisConnectorPruning(int(cc_pair_id))
-            r.srem(rcp.taskset_key, task_id)
-        return
-
-
-@celeryd_init.connect
-def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None:
-    """The first signal sent on celery worker startup"""
-    multiprocessing.set_start_method("spawn")  # fork is unsafe, set to spawn
-
-
-@beat_init.connect
-def on_beat_init(sender: Any, **kwargs: Any) -> None:
-    SqlEngine.set_app_name(POSTGRES_CELERY_BEAT_APP_NAME)
-    SqlEngine.init_engine(pool_size=2, max_overflow=0)
-
-
-@worker_init.connect
-def on_worker_init(sender: Any, **kwargs: Any) -> None:
-    logger.info("worker_init signal received.")
-    logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
-
-    # decide some initial startup settings based on the celery worker's hostname
-    # (set at the command line)
-    hostname = sender.hostname
-    if hostname.startswith("light"):
-        SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_LIGHT_APP_NAME)
-        SqlEngine.init_engine(pool_size=sender.concurrency, max_overflow=8)
-    elif hostname.startswith("heavy"):
-        SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_HEAVY_APP_NAME)
-        SqlEngine.init_engine(pool_size=8, max_overflow=0)
-    elif hostname.startswith("indexing"):
-        SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_APP_NAME)
-        SqlEngine.init_engine(pool_size=8, max_overflow=0)
-
-        # TODO: why is this necessary for the indexer to do?
-        with get_session_with_tenant(tenant_id) as db_session:
-            check_index_swap(db_session=db_session)
-            search_settings = get_current_search_settings(db_session)
-
-            # So that the first time users aren't surprised by really slow speed of first
-            # batch of documents indexed
-
-            if search_settings.provider_type is None:
-                logger.notice("Running a first inference to warm up embedding model")
-                embedding_model = EmbeddingModel.from_db_model(
-                    search_settings=search_settings,
-                    server_host=INDEXING_MODEL_SERVER_HOST,
-                    server_port=MODEL_SERVER_PORT,
-                )
-
-                warm_up_bi_encoder(
-                    embedding_model=embedding_model,
-                )
-                logger.notice("First inference complete.")
-    else:
-        SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME)
-        SqlEngine.init_engine(pool_size=8, max_overflow=0)
-
-    r = get_redis_client()
-
-    WAIT_INTERVAL = 5
-    WAIT_LIMIT = 60
-
-    time_start = time.monotonic()
-    logger.info("Redis: Readiness check starting.")
-    while True:
-        try:
-            if r.ping():
-                break
-        except Exception:
-            pass
-
-        time_elapsed = time.monotonic() - time_start
-        logger.info(
-            f"Redis: Ping failed. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
-        )
-        if time_elapsed > WAIT_LIMIT:
-            msg = (
-                f"Redis: Readiness check did not succeed within the timeout "
-                f"({WAIT_LIMIT} seconds). Exiting..."
-            )
-            logger.error(msg)
-            raise WorkerShutdown(msg)
-
-        time.sleep(WAIT_INTERVAL)
-
-    logger.info("Redis: Readiness check succeeded. Continuing...")
-
-    if not celery_is_worker_primary(sender):
-        logger.info("Running as a secondary celery worker.")
-        logger.info("Waiting for primary worker to be ready...")
-        time_start = time.monotonic()
-        while True:
-            if r.exists(DanswerRedisLocks.PRIMARY_WORKER):
-                break
-
-            time.monotonic()
-            time_elapsed = time.monotonic() - time_start
-            logger.info(
-                f"Primary worker is not ready yet. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
-            )
-            if time_elapsed > WAIT_LIMIT:
-                msg = (
-                    f"Primary worker was not ready within the timeout. "
-                    f"({WAIT_LIMIT} seconds). Exiting..."
-                )
-                logger.error(msg)
-                raise WorkerShutdown(msg)
-
-            time.sleep(WAIT_INTERVAL)
-
-        logger.info("Wait for primary worker completed successfully. Continuing...")
-        return
-
-    logger.info("Running as the primary celery worker.")
-
-    # This is singleton work that should be done on startup exactly once
-    # by the primary worker
-    r = get_redis_client()
-
-    # For the moment, we're assuming that we are the only primary worker
-    # that should be running.
-    # TODO: maybe check for or clean up another zombie primary worker if we detect it
-    r.delete(DanswerRedisLocks.PRIMARY_WORKER)
-
-    # this process wide lock is taken to help other workers start up in order.
-    # it is planned to use this lock to enforce singleton behavior on the primary
-    # worker, since the primary worker does redis cleanup on startup, but this isn't
-    # implemented yet.
-    lock = r.lock(
-        DanswerRedisLocks.PRIMARY_WORKER,
-        timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
-    )
-
-    logger.info("Primary worker lock: Acquire starting.")
-    acquired = lock.acquire(blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2)
-    if acquired:
-        logger.info("Primary worker lock: Acquire succeeded.")
-    else:
-        logger.error("Primary worker lock: Acquire failed!")
-        raise WorkerShutdown("Primary worker lock could not be acquired!")
-
-    sender.primary_worker_lock = lock
-
-    # As currently designed, when this worker starts as "primary", we reinitialize redis
-    # to a clean state (for our purposes, anyway)
-    r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
-    r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-    r.delete(RedisConnectorCredentialPair.get_taskset_key())
-    r.delete(RedisConnectorCredentialPair.get_fence_key())
-
-    for key in r.scan_iter(RedisDocumentSet.TASKSET_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisUserGroup.TASKSET_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorDeletion.TASKSET_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorPruning.TASKSET_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_COMPLETE_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_PROGRESS_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorIndexing.TASKSET_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_COMPLETE_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_PROGRESS_PREFIX + "*"):
-        r.delete(key)
-
-    for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
-        r.delete(key)
-
-
-# @worker_process_init.connect
-# def on_worker_process_init(sender: Any, **kwargs: Any) -> None:
-#     """This only runs inside child processes when the worker is in pool=prefork mode.
-#     This may be technically unnecessary since we're finding prefork pools to be
-#     unstable and currently aren't planning on using them."""
-#     logger.info("worker_process_init signal received.")
-#     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME)
-#     SqlEngine.init_engine(pool_size=5, max_overflow=0)
-
-#     # https://stackoverflow.com/questions/43944787/sqlalchemy-celery-with-scoped-session-error
-#     SqlEngine.get_engine().dispose(close=False)
-
-
-@worker_ready.connect
-def on_worker_ready(sender: Any, **kwargs: Any) -> None:
-    task_logger.info("worker_ready signal received.")
-
-
-@worker_shutdown.connect
-def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
-    if not celery_is_worker_primary(sender):
-        return
-
-    if not sender.primary_worker_lock:
-        return
-
-    logger.info("Releasing primary worker lock.")
-    lock = sender.primary_worker_lock
-    if lock.owned():
-        lock.release()
-        sender.primary_worker_lock = None
-
-
-class CeleryTaskPlainFormatter(PlainFormatter):
-    def format(self, record: logging.LogRecord) -> str:
-        task = current_task
-        if task and task.request:
-            record.__dict__.update(task_id=task.request.id, task_name=task.name)
-            record.msg = f"[{task.name}({task.request.id})] {record.msg}"
-
-        return super().format(record)
-
-
-class CeleryTaskColoredFormatter(ColoredFormatter):
-    def format(self, record: logging.LogRecord) -> str:
-        task = current_task
-        if task and task.request:
-            record.__dict__.update(task_id=task.request.id, task_name=task.name)
-            record.msg = f"[{task.name}({task.request.id})] {record.msg}"
-
-        return super().format(record)
-
-
-@signals.setup_logging.connect
-def on_setup_logging(
-    loglevel: Any, logfile: Any, format: Any, colorize: Any, **kwargs: Any
-) -> None:
-    # TODO: could unhardcode format and colorize and accept these as options from
-    # celery's config
-
-    # reformats the root logger
-    root_logger = logging.getLogger()
-
-    root_handler = logging.StreamHandler()  # Set up a handler for the root logger
-    root_formatter = ColoredFormatter(
-        "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
-        datefmt="%m/%d/%Y %I:%M:%S %p",
-    )
-    root_handler.setFormatter(root_formatter)
-    root_logger.addHandler(root_handler)  # Apply the handler to the root logger
-
-    if logfile:
-        root_file_handler = logging.FileHandler(logfile)
-        root_file_formatter = PlainFormatter(
-            "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
-            datefmt="%m/%d/%Y %I:%M:%S %p",
-        )
-        root_file_handler.setFormatter(root_file_formatter)
-        root_logger.addHandler(root_file_handler)
-
-    root_logger.setLevel(loglevel)
-
-    # reformats celery's task logger
-    task_formatter = CeleryTaskColoredFormatter(
-        "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
-        datefmt="%m/%d/%Y %I:%M:%S %p",
-    )
-    task_handler = logging.StreamHandler()  # Set up a handler for the task logger
-    task_handler.setFormatter(task_formatter)
-    task_logger.addHandler(task_handler)  # Apply the handler to the task logger
-
-    if logfile:
-        task_file_handler = logging.FileHandler(logfile)
-        task_file_formatter = CeleryTaskPlainFormatter(
-            "%(asctime)s %(filename)30s %(lineno)4s: %(message)s",
-            datefmt="%m/%d/%Y %I:%M:%S %p",
-        )
-        task_file_handler.setFormatter(task_file_formatter)
-        task_logger.addHandler(task_file_handler)
-
-    task_logger.setLevel(loglevel)
-    task_logger.propagate = False
-
-
-class HubPeriodicTask(bootsteps.StartStopStep):
-    """Regularly reacquires the primary worker lock outside of the task queue.
-    Use the task_logger in this class to avoid double logging.
-
-    This cannot be done inside a regular beat task because it must run on schedule and
-    a queue of existing work would starve the task from running.
-    """
-
-    # it's unclear to me whether using the hub's timer or the bootstep timer is better
-    requires = {"celery.worker.components:Hub"}
-
-    def __init__(self, worker: Any, **kwargs: Any) -> None:
-        self.interval = CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 8  # Interval in seconds
-        self.task_tref = None
-
-    def start(self, worker: Any) -> None:
-        if not celery_is_worker_primary(worker):
-            return
-
-        # Access the worker's event loop (hub)
-        hub = worker.consumer.controller.hub
-
-        # Schedule the periodic task
-        self.task_tref = hub.call_repeatedly(
-            self.interval, self.run_periodic_task, worker
-        )
-        task_logger.info("Scheduled periodic task with hub.")
-
-    def run_periodic_task(self, worker: Any) -> None:
-        try:
-            if not worker.primary_worker_lock:
-                return
-
-            if not hasattr(worker, "primary_worker_lock"):
-                return
-
-            r = get_redis_client()
-
-            lock: redis.lock.Lock = worker.primary_worker_lock
-
-            if lock.owned():
-                task_logger.debug("Reacquiring primary worker lock.")
-                lock.reacquire()
-            else:
-                task_logger.warning(
-                    "Full acquisition of primary worker lock. "
-                    "Reasons could be computer sleep or a clock change."
-                )
-                lock = r.lock(
-                    DanswerRedisLocks.PRIMARY_WORKER,
-                    timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
-                )
-
-                task_logger.info("Primary worker lock: Acquire starting.")
-                acquired = lock.acquire(
-                    blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2
-                )
-                if acquired:
-                    task_logger.info("Primary worker lock: Acquire succeeded.")
-                else:
-                    task_logger.error("Primary worker lock: Acquire failed!")
-                    raise TimeoutError("Primary worker lock could not be acquired!")
-
-                worker.primary_worker_lock = lock
-        except Exception:
-            task_logger.exception("HubPeriodicTask.run_periodic_task exceptioned.")
-
-    def stop(self, worker: Any) -> None:
-        # Cancel the scheduled task when the worker stops
-        if self.task_tref:
-            self.task_tref.cancel()
-            task_logger.info("Canceled periodic task with hub.")
-
-
-celery_app.steps["worker"].add(HubPeriodicTask)
-
-celery_app.autodiscover_tasks(
-    [
-        "danswer.background.celery.tasks.connector_deletion",
-        "danswer.background.celery.tasks.indexing",
-        "danswer.background.celery.tasks.periodic",
-        "danswer.background.celery.tasks.pruning",
-        "danswer.background.celery.tasks.shared",
-        "danswer.background.celery.tasks.vespa",
-    ]
-)
-
-#####
-# Celery Beat (Periodic Tasks) Settings
-#####
-
-tenant_ids = get_all_tenant_ids()
-
-tasks_to_schedule = [
-    {
-        "name": "check-for-vespa-sync",
-        "task": "check_for_vespa_sync_task",
-        "schedule": timedelta(seconds=5),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "check-for-connector-deletion",
-        "task": "check_for_connector_deletion_task",
-        "schedule": timedelta(seconds=60),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "check-for-indexing",
-        "task": "check_for_indexing",
-        "schedule": timedelta(seconds=10),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "check-for-prune",
-        "task": "check_for_pruning",
-        "schedule": timedelta(seconds=10),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "kombu-message-cleanup",
-        "task": "kombu_message_cleanup_task",
-        "schedule": timedelta(seconds=3600),
-        "options": {"priority": DanswerCeleryPriority.LOWEST},
-    },
-    {
-        "name": "monitor-vespa-sync",
-        "task": "monitor_vespa_sync",
-        "schedule": timedelta(seconds=5),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-]
-
-# Build the celery beat schedule dynamically
-beat_schedule = {}
-
-for tenant_id in tenant_ids:
-    for task in tasks_to_schedule:
-        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
-        beat_schedule[task_name] = {
-            "task": task["task"],
-            "schedule": task["schedule"],
-            "options": task["options"],
-            "args": (tenant_id,),  # Must pass tenant_id as an argument
-        }
-
-# Include any existing beat schedules
-existing_beat_schedule = celery_app.conf.beat_schedule or {}
-beat_schedule.update(existing_beat_schedule)
-
-# Update the Celery app configuration once
-celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index 53f20946077..f1a5697e246 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -10,7 +10,7 @@
 from redis import Redis
 from sqlalchemy.orm import Session
 
-from danswer.background.celery.celeryconfig import CELERY_SEPARATOR
+from danswer.background.celery.configs.base import CELERY_SEPARATOR
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DanswerCeleryQueues
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 4b499268cb4..794f89232c5 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -3,13 +3,10 @@
 from datetime import timezone
 from typing import Any
 
-from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
-from danswer.configs.app_configs import MULTI_TENANT
-from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
 )
@@ -19,7 +16,6 @@
 from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import Document
 from danswer.db.connector_credential_pair import get_connector_credential_pair
-from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import TaskStatus
 from danswer.db.models import TaskQueueState
 from danswer.redis.redis_pool import get_redis_client
@@ -129,33 +125,10 @@ def celery_is_listening_to_queue(worker: Any, name: str) -> bool:
 def celery_is_worker_primary(worker: Any) -> bool:
     """There are multiple approaches that could be taken to determine if a celery worker
     is 'primary', as defined by us. But the way we do it is to check the hostname set
-    for the celery worker, which can be done either in celeryconfig.py or on the
+    for the celery worker, which can be done on the
     command line with '--hostname'."""
     hostname = worker.hostname
     if hostname.startswith("primary"):
         return True
 
     return False
-
-
-def get_all_tenant_ids() -> list[str] | list[None]:
-    if not MULTI_TENANT:
-        return [None]
-    with get_session_with_tenant(tenant_id="public") as session:
-        result = session.execute(
-            text(
-                """
-            SELECT schema_name
-            FROM information_schema.schemata
-            WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
-            )
-        )
-        tenant_ids = [row[0] for row in result]
-
-    valid_tenants = [
-        tenant
-        for tenant in tenant_ids
-        if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
-    ]
-
-    return valid_tenants
diff --git a/backend/danswer/background/celery/celeryconfig.py b/backend/danswer/background/celery/configs/base.py
similarity index 95%
rename from backend/danswer/background/celery/celeryconfig.py
rename to backend/danswer/background/celery/configs/base.py
index 3f96364de1e..886fcf545c9 100644
--- a/backend/danswer/background/celery/celeryconfig.py
+++ b/backend/danswer/background/celery/configs/base.py
@@ -31,21 +31,10 @@
     if REDIS_SSL_CA_CERTS:
         SSL_QUERY_PARAMS += f"&ssl_ca_certs={REDIS_SSL_CA_CERTS}"
 
+# region Broker settings
 # example celery_broker_url: "redis://:password@localhost:6379/15"
 broker_url = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY}{SSL_QUERY_PARAMS}"
 
-result_backend = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY_RESULT_BACKEND}{SSL_QUERY_PARAMS}"
-
-# NOTE: prefetch 4 is significantly faster than prefetch 1 for small tasks
-# however, prefetching is bad when tasks are lengthy as those tasks
-# can stall other tasks.
-worker_prefetch_multiplier = 4
-
-# Leaving this to the default of True may cause double logging since both our own app
-# and celery think they are controlling the logger.
-# TODO: Configure celery's logger entirely manually and set this to False
-# worker_hijack_root_logger = False
-
 broker_connection_retry_on_startup = True
 broker_pool_limit = CELERY_BROKER_POOL_LIMIT
 
@@ -60,6 +49,7 @@
     "socket_keepalive": True,
     "socket_keepalive_options": REDIS_SOCKET_KEEPALIVE_OPTIONS,
 }
+# endregion
 
 # redis backend settings
 # https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-backend-settings
@@ -73,10 +63,19 @@
 task_default_priority = DanswerCeleryPriority.MEDIUM
 task_acks_late = True
 
+# region Task result backend settings
 # It's possible we don't even need celery's result backend, in which case all of the optimization below
 # might be irrelevant
+result_backend = f"{REDIS_SCHEME}://{CELERY_PASSWORD_PART}{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB_NUMBER_CELERY_RESULT_BACKEND}{SSL_QUERY_PARAMS}"
 result_expires = CELERY_RESULT_EXPIRES  # 86400 seconds is the default
+# endregion
+
+# Leaving this to the default of True may cause double logging since both our own app
+# and celery think they are controlling the logger.
+# TODO: Configure celery's logger entirely manually and set this to False
+# worker_hijack_root_logger = False
 
+# region Notes on serialization performance
 # Option 0: Defaults (json serializer, no compression)
 # about 1.5 KB per queued task. 1KB in queue, 400B for result, 100 as a child entry in generator result
 
@@ -102,3 +101,4 @@
 # task_serializer = "pickle-bzip2"
 # result_serializer = "pickle-bzip2"
 # accept_content=["pickle", "pickle-bzip2"]
+# endregion
diff --git a/backend/danswer/background/celery/configs/beat.py b/backend/danswer/background/celery/configs/beat.py
new file mode 100644
index 00000000000..ef8b21c386f
--- /dev/null
+++ b/backend/danswer/background/celery/configs/beat.py
@@ -0,0 +1,14 @@
+# docs: https://docs.celeryq.dev/en/stable/userguide/configuration.html
+import danswer.background.celery.configs.base as shared_config
+
+broker_url = shared_config.broker_url
+broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
+broker_pool_limit = shared_config.broker_pool_limit
+broker_transport_options = shared_config.broker_transport_options
+
+redis_socket_keepalive = shared_config.redis_socket_keepalive
+redis_retry_on_timeout = shared_config.redis_retry_on_timeout
+redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
+
+result_backend = shared_config.result_backend
+result_expires = shared_config.result_expires  # 86400 seconds is the default
diff --git a/backend/danswer/background/celery/configs/heavy.py b/backend/danswer/background/celery/configs/heavy.py
new file mode 100644
index 00000000000..2d1c65aa86e
--- /dev/null
+++ b/backend/danswer/background/celery/configs/heavy.py
@@ -0,0 +1,20 @@
+import danswer.background.celery.configs.base as shared_config
+
+broker_url = shared_config.broker_url
+broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
+broker_pool_limit = shared_config.broker_pool_limit
+broker_transport_options = shared_config.broker_transport_options
+
+redis_socket_keepalive = shared_config.redis_socket_keepalive
+redis_retry_on_timeout = shared_config.redis_retry_on_timeout
+redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
+
+result_backend = shared_config.result_backend
+result_expires = shared_config.result_expires  # 86400 seconds is the default
+
+task_default_priority = shared_config.task_default_priority
+task_acks_late = shared_config.task_acks_late
+
+worker_concurrency = 4
+worker_pool = "threads"
+worker_prefetch_multiplier = 1
diff --git a/backend/danswer/background/celery/configs/indexing.py b/backend/danswer/background/celery/configs/indexing.py
new file mode 100644
index 00000000000..d2b1b99baa9
--- /dev/null
+++ b/backend/danswer/background/celery/configs/indexing.py
@@ -0,0 +1,21 @@
+import danswer.background.celery.configs.base as shared_config
+from danswer.configs.app_configs import CELERY_WORKER_INDEXING_CONCURRENCY
+
+broker_url = shared_config.broker_url
+broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
+broker_pool_limit = shared_config.broker_pool_limit
+broker_transport_options = shared_config.broker_transport_options
+
+redis_socket_keepalive = shared_config.redis_socket_keepalive
+redis_retry_on_timeout = shared_config.redis_retry_on_timeout
+redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
+
+result_backend = shared_config.result_backend
+result_expires = shared_config.result_expires  # 86400 seconds is the default
+
+task_default_priority = shared_config.task_default_priority
+task_acks_late = shared_config.task_acks_late
+
+worker_concurrency = CELERY_WORKER_INDEXING_CONCURRENCY
+worker_pool = "threads"
+worker_prefetch_multiplier = 1
diff --git a/backend/danswer/background/celery/configs/light.py b/backend/danswer/background/celery/configs/light.py
new file mode 100644
index 00000000000..f75ddfd0fb5
--- /dev/null
+++ b/backend/danswer/background/celery/configs/light.py
@@ -0,0 +1,22 @@
+import danswer.background.celery.configs.base as shared_config
+from danswer.configs.app_configs import CELERY_WORKER_LIGHT_CONCURRENCY
+from danswer.configs.app_configs import CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER
+
+broker_url = shared_config.broker_url
+broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
+broker_pool_limit = shared_config.broker_pool_limit
+broker_transport_options = shared_config.broker_transport_options
+
+redis_socket_keepalive = shared_config.redis_socket_keepalive
+redis_retry_on_timeout = shared_config.redis_retry_on_timeout
+redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
+
+result_backend = shared_config.result_backend
+result_expires = shared_config.result_expires  # 86400 seconds is the default
+
+task_default_priority = shared_config.task_default_priority
+task_acks_late = shared_config.task_acks_late
+
+worker_concurrency = CELERY_WORKER_LIGHT_CONCURRENCY
+worker_pool = "threads"
+worker_prefetch_multiplier = CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER
diff --git a/backend/danswer/background/celery/configs/primary.py b/backend/danswer/background/celery/configs/primary.py
new file mode 100644
index 00000000000..2d1c65aa86e
--- /dev/null
+++ b/backend/danswer/background/celery/configs/primary.py
@@ -0,0 +1,20 @@
+import danswer.background.celery.configs.base as shared_config
+
+broker_url = shared_config.broker_url
+broker_connection_retry_on_startup = shared_config.broker_connection_retry_on_startup
+broker_pool_limit = shared_config.broker_pool_limit
+broker_transport_options = shared_config.broker_transport_options
+
+redis_socket_keepalive = shared_config.redis_socket_keepalive
+redis_retry_on_timeout = shared_config.redis_retry_on_timeout
+redis_backend_health_check_interval = shared_config.redis_backend_health_check_interval
+
+result_backend = shared_config.result_backend
+result_expires = shared_config.result_expires  # 86400 seconds is the default
+
+task_default_priority = shared_config.task_default_priority
+task_acks_late = shared_config.task_acks_late
+
+worker_concurrency = 4
+worker_pool = "threads"
+worker_prefetch_multiplier = 1
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index b13daff61fc..b3c2eea30b0 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -1,20 +1,20 @@
 import redis
+from celery import Celery
 from celery import shared_task
+from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
 from redis import Redis
 from sqlalchemy.orm import Session
-from sqlalchemy.orm.exc import ObjectDeletedError
 
-from danswer.background.celery.celery_app import celery_app
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
-from danswer.db.models import ConnectorCredentialPair
 from danswer.redis.redis_pool import get_redis_client
 
 
@@ -22,8 +22,9 @@
     name="check_for_connector_deletion_task",
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
+    bind=True,
 )
-def check_for_connector_deletion_task(tenant_id: str | None) -> None:
+def check_for_connector_deletion_task(self: Task, tenant_id: str | None) -> None:
     r = get_redis_client()
 
     lock_beat = r.lock(
@@ -36,11 +37,16 @@ def check_for_connector_deletion_task(tenant_id: str | None) -> None:
         if not lock_beat.acquire(blocking=False):
             return
 
+        cc_pair_ids: list[int] = []
         with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = get_connector_credential_pairs(db_session)
             for cc_pair in cc_pairs:
+                cc_pair_ids.append(cc_pair.id)
+
+        for cc_pair_id in cc_pair_ids:
+            with get_session_with_tenant(tenant_id) as db_session:
                 try_generate_document_cc_pair_cleanup_tasks(
-                    cc_pair, db_session, r, lock_beat, tenant_id
+                    self.app, cc_pair_id, db_session, r, lock_beat, tenant_id
                 )
     except SoftTimeLimitExceeded:
         task_logger.info(
@@ -54,7 +60,8 @@ def check_for_connector_deletion_task(tenant_id: str | None) -> None:
 
 
 def try_generate_document_cc_pair_cleanup_tasks(
-    cc_pair: ConnectorCredentialPair,
+    app: Celery,
+    cc_pair_id: int,
     db_session: Session,
     r: Redis,
     lock_beat: redis.lock.Lock,
@@ -67,18 +74,17 @@ def try_generate_document_cc_pair_cleanup_tasks(
 
     lock_beat.reacquire()
 
-    rcd = RedisConnectorDeletion(cc_pair.id)
+    rcd = RedisConnectorDeletion(cc_pair_id)
 
     # don't generate sync tasks if tasks are still pending
     if r.exists(rcd.fence_key):
         return None
 
-    # we need to refresh the state of the object inside the fence
+    # we need to load the state of the object inside the fence
     # to avoid a race condition with db.commit/fence deletion
     # at the end of this taskset
-    try:
-        db_session.refresh(cc_pair)
-    except ObjectDeletedError:
+    cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session)
+    if not cc_pair:
         return None
 
     if cc_pair.status != ConnectorCredentialPairStatus.DELETING:
@@ -91,9 +97,7 @@ def try_generate_document_cc_pair_cleanup_tasks(
     task_logger.info(
         f"RedisConnectorDeletion.generate_tasks starting. cc_pair_id={cc_pair.id}"
     )
-    tasks_generated = rcd.generate_tasks(
-        celery_app, db_session, r, lock_beat, tenant_id
-    )
+    tasks_generated = rcd.generate_tasks(app, db_session, r, lock_beat, tenant_id)
     if tasks_generated is None:
         return None
 
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 0e8e59bf5a6..ed08787d53e 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -5,15 +5,18 @@
 from typing import cast
 from uuid import uuid4
 
+from celery import Celery
 from celery import shared_task
+from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
 from redis import Redis
 from sqlalchemy.orm import Session
 
-from danswer.background.celery.celery_app import celery_app
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.tasks.shared.tasks import RedisConnectorIndexingFenceData
+from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
+    RedisConnectorIndexingFenceData,
+)
 from danswer.background.indexing.job_client import SimpleJobClient
 from danswer.background.indexing.run_indexing import run_indexing_entrypoint
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
@@ -50,8 +53,9 @@
 @shared_task(
     name="check_for_indexing",
     soft_time_limit=300,
+    bind=True,
 )
-def check_for_indexing(tenant_id: str | None) -> int | None:
+def check_for_indexing(self: Task, tenant_id: str | None) -> int | None:
     tasks_created = 0
 
     r = get_redis_client()
@@ -66,26 +70,37 @@ def check_for_indexing(tenant_id: str | None) -> int | None:
         if not lock_beat.acquire(blocking=False):
             return None
 
+        cc_pair_ids: list[int] = []
         with get_session_with_tenant(tenant_id) as db_session:
-            # Get the primary search settings
-            primary_search_settings = get_current_search_settings(db_session)
-            search_settings = [primary_search_settings]
+            cc_pairs = fetch_connector_credential_pairs(db_session)
+            for cc_pair_entry in cc_pairs:
+                cc_pair_ids.append(cc_pair_entry.id)
 
-            # Check for secondary search settings
-            secondary_search_settings = get_secondary_search_settings(db_session)
-            if secondary_search_settings is not None:
-                # If secondary settings exist, add them to the list
-                search_settings.append(secondary_search_settings)
+        for cc_pair_id in cc_pair_ids:
+            with get_session_with_tenant(tenant_id) as db_session:
+                # Get the primary search settings
+                primary_search_settings = get_current_search_settings(db_session)
+                search_settings = [primary_search_settings]
+
+                # Check for secondary search settings
+                secondary_search_settings = get_secondary_search_settings(db_session)
+                if secondary_search_settings is not None:
+                    # If secondary settings exist, add them to the list
+                    search_settings.append(secondary_search_settings)
 
-            cc_pairs = fetch_connector_credential_pairs(db_session)
-            for cc_pair in cc_pairs:
                 for search_settings_instance in search_settings:
                     rci = RedisConnectorIndexing(
-                        cc_pair.id, search_settings_instance.id
+                        cc_pair_id, search_settings_instance.id
                     )
                     if r.exists(rci.fence_key):
                         continue
 
+                    cc_pair = get_connector_credential_pair_from_id(
+                        cc_pair_id, db_session
+                    )
+                    if not cc_pair:
+                        continue
+
                     last_attempt = get_last_attempt_for_cc_pair(
                         cc_pair.id, search_settings_instance.id, db_session
                     )
@@ -101,6 +116,7 @@ def check_for_indexing(tenant_id: str | None) -> int | None:
                     # using a task queue and only allowing one task per cc_pair/search_setting
                     # prevents us from starving out certain attempts
                     attempt_id = try_creating_indexing_task(
+                        self.app,
                         cc_pair,
                         search_settings_instance,
                         False,
@@ -210,6 +226,7 @@ def _should_index(
 
 
 def try_creating_indexing_task(
+    celery_app: Celery,
     cc_pair: ConnectorCredentialPair,
     search_settings: SearchSettings,
     reindex: bool,
diff --git a/backend/danswer/background/celery/tasks/periodic/tasks.py b/backend/danswer/background/celery/tasks/periodic/tasks.py
index d8da5ba9ca9..20baa7c52fa 100644
--- a/backend/danswer/background/celery/tasks/periodic/tasks.py
+++ b/backend/danswer/background/celery/tasks/periodic/tasks.py
@@ -11,7 +11,7 @@
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import PostgresAdvisoryLocks
 from danswer.db.engine import get_session_with_tenant
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index 4bfde82292a..698c2937299 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -3,13 +3,14 @@
 from datetime import timezone
 from uuid import uuid4
 
+from celery import Celery
 from celery import shared_task
+from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
 from redis import Redis
 from sqlalchemy.orm import Session
 
-from danswer.background.celery.celery_app import celery_app
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector
 from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
@@ -23,6 +24,7 @@
 from danswer.connectors.factory import instantiate_connector
 from danswer.connectors.models import InputType
 from danswer.db.connector_credential_pair import get_connector_credential_pair
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.document import get_documents_for_connector_credential_pair
 from danswer.db.engine import get_session_with_tenant
@@ -37,8 +39,9 @@
 @shared_task(
     name="check_for_pruning",
     soft_time_limit=JOB_TIMEOUT,
+    bind=True,
 )
-def check_for_pruning(tenant_id: str | None) -> None:
+def check_for_pruning(self: Task, tenant_id: str | None) -> None:
     r = get_redis_client()
 
     lock_beat = r.lock(
@@ -51,15 +54,24 @@ def check_for_pruning(tenant_id: str | None) -> None:
         if not lock_beat.acquire(blocking=False):
             return
 
+        cc_pair_ids: list[int] = []
         with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = get_connector_credential_pairs(db_session)
-            for cc_pair in cc_pairs:
-                lock_beat.reacquire()
+            for cc_pair_entry in cc_pairs:
+                cc_pair_ids.append(cc_pair_entry.id)
+
+        for cc_pair_id in cc_pair_ids:
+            lock_beat.reacquire()
+            with get_session_with_tenant(tenant_id) as db_session:
+                cc_pair = get_connector_credential_pair_from_id(cc_pair_id, db_session)
+                if not cc_pair:
+                    continue
+
                 if not is_pruning_due(cc_pair, db_session, r):
                     continue
 
                 tasks_created = try_creating_prune_generator_task(
-                    cc_pair, db_session, r, tenant_id
+                    self.app, cc_pair, db_session, r, tenant_id
                 )
                 if not tasks_created:
                     continue
@@ -118,6 +130,7 @@ def is_pruning_due(
 
 
 def try_creating_prune_generator_task(
+    celery_app: Celery,
     cc_pair: ConnectorCredentialPair,
     db_session: Session,
     r: Redis,
@@ -196,9 +209,14 @@ def try_creating_prune_generator_task(
     soft_time_limit=JOB_TIMEOUT,
     track_started=True,
     trail=False,
+    bind=True,
 )
 def connector_pruning_generator_task(
-    cc_pair_id: int, connector_id: int, credential_id: int, tenant_id: str | None
+    self: Task,
+    cc_pair_id: int,
+    connector_id: int,
+    credential_id: int,
+    tenant_id: str | None,
 ) -> None:
     """connector pruning task. For a cc pair, this task pulls all document IDs from the source
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
@@ -278,7 +296,7 @@ def redis_increment_callback(amount: int) -> None:
                 f"RedisConnectorPruning.generate_tasks starting. cc_pair_id={cc_pair.id}"
             )
             tasks_generated = rcp.generate_tasks(
-                celery_app, db_session, r, None, tenant_id
+                self.app, db_session, r, None, tenant_id
             )
             if tasks_generated is None:
                 return None
diff --git a/backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py b/backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py
new file mode 100644
index 00000000000..224571a4231
--- /dev/null
+++ b/backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py
@@ -0,0 +1,10 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class RedisConnectorIndexingFenceData(BaseModel):
+    index_attempt_id: int | None
+    started: datetime | None
+    submitted: datetime
+    celery_task_id: str | None
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 26f9d1aac10..474a749e786 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -6,7 +6,7 @@
 from pydantic import BaseModel
 
 from danswer.access.access import get_access_for_document
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.db.document import delete_document_by_connector_credential_pair__no_commit
 from danswer.db.document import delete_documents_complete__no_commit
 from danswer.db.document import get_document
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 2d79045c44f..53e26be6954 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -5,6 +5,7 @@
 from typing import cast
 
 import redis
+from celery import Celery
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
@@ -14,8 +15,7 @@
 from sqlalchemy.orm import Session
 
 from danswer.access.access import get_access_for_document
-from danswer.background.celery.celery_app import celery_app
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import celery_get_queue_length
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
@@ -23,7 +23,9 @@
 from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
-from danswer.background.celery.tasks.shared.tasks import RedisConnectorIndexingFenceData
+from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
+    RedisConnectorIndexingFenceData,
+)
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryQueues
@@ -54,7 +56,6 @@
 from danswer.db.index_attempt import mark_attempt_failed
 from danswer.db.models import DocumentSet
 from danswer.db.models import IndexAttempt
-from danswer.db.models import UserGroup
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import VespaDocumentFields
@@ -73,8 +74,9 @@
     name="check_for_vespa_sync_task",
     soft_time_limit=JOB_TIMEOUT,
     trail=False,
+    bind=True,
 )
-def check_for_vespa_sync_task(tenant_id: str | None) -> None:
+def check_for_vespa_sync_task(self: Task, tenant_id: str | None) -> None:
     """Runs periodically to check if any document needs syncing.
     Generates sets of tasks for Celery if syncing is needed."""
 
@@ -91,35 +93,53 @@ def check_for_vespa_sync_task(tenant_id: str | None) -> None:
             return
 
         with get_session_with_tenant(tenant_id) as db_session:
-            try_generate_stale_document_sync_tasks(db_session, r, lock_beat, tenant_id)
+            try_generate_stale_document_sync_tasks(
+                self.app, db_session, r, lock_beat, tenant_id
+            )
 
+        # region document set scan
+        document_set_ids: list[int] = []
+        with get_session_with_tenant(tenant_id) as db_session:
             # check if any document sets are not synced
             document_set_info = fetch_document_sets(
                 user_id=None, db_session=db_session, include_outdated=True
             )
+
             for document_set, _ in document_set_info:
+                document_set_ids.append(document_set.id)
+
+        for document_set_id in document_set_ids:
+            with get_session_with_tenant(tenant_id) as db_session:
                 try_generate_document_set_sync_tasks(
-                    document_set, db_session, r, lock_beat, tenant_id
+                    self.app, document_set_id, db_session, r, lock_beat, tenant_id
                 )
+        # endregion
 
-            # check if any user groups are not synced
-            if global_version.is_ee_version():
-                try:
-                    fetch_user_groups = fetch_versioned_implementation(
-                        "danswer.db.user_group", "fetch_user_groups"
-                    )
-
+        # check if any user groups are not synced
+        if global_version.is_ee_version():
+            try:
+                fetch_user_groups = fetch_versioned_implementation(
+                    "danswer.db.user_group", "fetch_user_groups"
+                )
+            except ModuleNotFoundError:
+                # Always exceptions on the MIT version, which is expected
+                # We shouldn't actually get here if the ee version check works
+                pass
+            else:
+                usergroup_ids: list[int] = []
+                with get_session_with_tenant(tenant_id) as db_session:
                     user_groups = fetch_user_groups(
                         db_session=db_session, only_up_to_date=False
                     )
+
                     for usergroup in user_groups:
+                        usergroup_ids.append(usergroup.id)
+
+                for usergroup_id in usergroup_ids:
+                    with get_session_with_tenant(tenant_id) as db_session:
                         try_generate_user_group_sync_tasks(
-                            usergroup, db_session, r, lock_beat, tenant_id
+                            self.app, usergroup_id, db_session, r, lock_beat, tenant_id
                         )
-                except ModuleNotFoundError:
-                    # Always exceptions on the MIT version, which is expected
-                    # We shouldn't actually get here if the ee version check works
-                    pass
 
     except SoftTimeLimitExceeded:
         task_logger.info(
@@ -133,7 +153,11 @@ def check_for_vespa_sync_task(tenant_id: str | None) -> None:
 
 
 def try_generate_stale_document_sync_tasks(
-    db_session: Session, r: Redis, lock_beat: redis.lock.Lock, tenant_id: str | None
+    celery_app: Celery,
+    db_session: Session,
+    r: Redis,
+    lock_beat: redis.lock.Lock,
+    tenant_id: str | None,
 ) -> int | None:
     # the fence is up, do nothing
     if r.exists(RedisConnectorCredentialPair.get_fence_key()):
@@ -184,7 +208,8 @@ def try_generate_stale_document_sync_tasks(
 
 
 def try_generate_document_set_sync_tasks(
-    document_set: DocumentSet,
+    celery_app: Celery,
+    document_set_id: int,
     db_session: Session,
     r: Redis,
     lock_beat: redis.lock.Lock,
@@ -192,7 +217,7 @@ def try_generate_document_set_sync_tasks(
 ) -> int | None:
     lock_beat.reacquire()
 
-    rds = RedisDocumentSet(document_set.id)
+    rds = RedisDocumentSet(document_set_id)
 
     # don't generate document set sync tasks if tasks are still pending
     if r.exists(rds.fence_key):
@@ -200,7 +225,10 @@ def try_generate_document_set_sync_tasks(
 
     # don't generate sync tasks if we're up to date
     # race condition with the monitor/cleanup function if we use a cached result!
-    db_session.refresh(document_set)
+    document_set = get_document_set_by_id(db_session, document_set_id)
+    if not document_set:
+        return None
+
     if document_set.is_up_to_date:
         return None
 
@@ -235,7 +263,8 @@ def try_generate_document_set_sync_tasks(
 
 
 def try_generate_user_group_sync_tasks(
-    usergroup: UserGroup,
+    celery_app: Celery,
+    usergroup_id: int,
     db_session: Session,
     r: Redis,
     lock_beat: redis.lock.Lock,
@@ -243,14 +272,21 @@ def try_generate_user_group_sync_tasks(
 ) -> int | None:
     lock_beat.reacquire()
 
-    rug = RedisUserGroup(usergroup.id)
+    rug = RedisUserGroup(usergroup_id)
 
     # don't generate sync tasks if tasks are still pending
     if r.exists(rug.fence_key):
         return None
 
     # race condition with the monitor/cleanup function if we use a cached result!
-    db_session.refresh(usergroup)
+    fetch_user_group = fetch_versioned_implementation(
+        "danswer.db.user_group", "fetch_user_group"
+    )
+
+    usergroup = fetch_user_group(db_session, usergroup_id)
+    if not usergroup:
+        return None
+
     if usergroup.is_up_to_date:
         return None
 
@@ -680,36 +716,9 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
             f"pruning={n_pruning}"
         )
 
-        lock_beat.reacquire()
-        if r.exists(RedisConnectorCredentialPair.get_fence_key()):
-            monitor_connector_taskset(r)
-
-        lock_beat.reacquire()
-        for key_bytes in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
-            monitor_connector_deletion_taskset(key_bytes, r, tenant_id)
-
+        # do some cleanup before clearing fences
+        # check the db for any outstanding index attempts
         with get_session_with_tenant(tenant_id) as db_session:
-            lock_beat.reacquire()
-            for key_bytes in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
-                monitor_document_set_taskset(key_bytes, r, db_session)
-
-            lock_beat.reacquire()
-            for key_bytes in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
-                monitor_usergroup_taskset = (
-                    fetch_versioned_implementation_with_fallback(
-                        "danswer.background.celery.tasks.vespa.tasks",
-                        "monitor_usergroup_taskset",
-                        noop_fallback,
-                    )
-                )
-                monitor_usergroup_taskset(key_bytes, r, db_session)
-
-            lock_beat.reacquire()
-            for key_bytes in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
-                monitor_ccpair_pruning_taskset(key_bytes, r, db_session)
-
-            # do some cleanup before clearing fences
-            # check the db for any outstanding index attempts
             attempts: list[IndexAttempt] = []
             attempts.extend(
                 get_all_index_attempts_by_status(IndexingStatus.NOT_STARTED, db_session)
@@ -727,8 +736,42 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
                 if not r.exists(rci.fence_key):
                     mark_attempt_failed(a, db_session, failure_reason=failure_reason)
 
+        lock_beat.reacquire()
+        if r.exists(RedisConnectorCredentialPair.get_fence_key()):
+            monitor_connector_taskset(r)
+
+        lock_beat.reacquire()
+        for key_bytes in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
+            lock_beat.reacquire()
+            monitor_connector_deletion_taskset(key_bytes, r, tenant_id)
+
+        lock_beat.reacquire()
+        for key_bytes in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
+            lock_beat.reacquire()
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_document_set_taskset(key_bytes, r, db_session)
+
+        lock_beat.reacquire()
+        for key_bytes in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
+            lock_beat.reacquire()
+            monitor_usergroup_taskset = fetch_versioned_implementation_with_fallback(
+                "danswer.background.celery.tasks.vespa.tasks",
+                "monitor_usergroup_taskset",
+                noop_fallback,
+            )
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_usergroup_taskset(key_bytes, r, db_session)
+
+        lock_beat.reacquire()
+        for key_bytes in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+            lock_beat.reacquire()
+            with get_session_with_tenant(tenant_id) as db_session:
+                monitor_ccpair_pruning_taskset(key_bytes, r, db_session)
+
+        lock_beat.reacquire()
+        for key_bytes in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
             lock_beat.reacquire()
-            for key_bytes in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
+            with get_session_with_tenant(tenant_id) as db_session:
                 monitor_ccpair_indexing_taskset(key_bytes, r, db_session)
 
         # uncomment for debugging if needed
diff --git a/backend/danswer/background/celery/celery_run.py b/backend/danswer/background/celery/versioned_apps/beat.py
similarity index 55%
rename from backend/danswer/background/celery/celery_run.py
rename to backend/danswer/background/celery/versioned_apps/beat.py
index 0fdb2f044a8..d1b7dc591d9 100644
--- a/backend/danswer/background/celery/celery_run.py
+++ b/backend/danswer/background/celery/versioned_apps/beat.py
@@ -1,9 +1,8 @@
-"""Entry point for running celery worker / celery beat."""
+"""Factory stub for running celery worker / celery beat."""
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 
-
 set_is_ee_based_on_env_variable()
-celery_app = fetch_versioned_implementation(
-    "danswer.background.celery.celery_app", "celery_app"
+app = fetch_versioned_implementation(
+    "danswer.background.celery.apps.beat", "celery_app"
 )
diff --git a/backend/danswer/background/celery/versioned_apps/heavy.py b/backend/danswer/background/celery/versioned_apps/heavy.py
new file mode 100644
index 00000000000..c2b58a53bfc
--- /dev/null
+++ b/backend/danswer/background/celery/versioned_apps/heavy.py
@@ -0,0 +1,17 @@
+"""Factory stub for running celery worker / celery beat.
+This code is different from the primary/beat stubs because there is no EE version to
+fetch. Port over the code in those files if we add an EE version of this worker."""
+from celery import Celery
+
+from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
+
+set_is_ee_based_on_env_variable()
+
+
+def get_app() -> Celery:
+    from danswer.background.celery.apps.heavy import celery_app
+
+    return celery_app
+
+
+app = get_app()
diff --git a/backend/danswer/background/celery/versioned_apps/indexing.py b/backend/danswer/background/celery/versioned_apps/indexing.py
new file mode 100644
index 00000000000..ed26fc548bc
--- /dev/null
+++ b/backend/danswer/background/celery/versioned_apps/indexing.py
@@ -0,0 +1,17 @@
+"""Factory stub for running celery worker / celery beat.
+This code is different from the primary/beat stubs because there is no EE version to
+fetch. Port over the code in those files if we add an EE version of this worker."""
+from celery import Celery
+
+from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
+
+set_is_ee_based_on_env_variable()
+
+
+def get_app() -> Celery:
+    from danswer.background.celery.apps.indexing import celery_app
+
+    return celery_app
+
+
+app = get_app()
diff --git a/backend/danswer/background/celery/versioned_apps/light.py b/backend/danswer/background/celery/versioned_apps/light.py
new file mode 100644
index 00000000000..3d229431ce5
--- /dev/null
+++ b/backend/danswer/background/celery/versioned_apps/light.py
@@ -0,0 +1,17 @@
+"""Factory stub for running celery worker / celery beat.
+This code is different from the primary/beat stubs because there is no EE version to
+fetch. Port over the code in those files if we add an EE version of this worker."""
+from celery import Celery
+
+from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
+
+set_is_ee_based_on_env_variable()
+
+
+def get_app() -> Celery:
+    from danswer.background.celery.apps.light import celery_app
+
+    return celery_app
+
+
+app = get_app()
diff --git a/backend/danswer/background/celery/versioned_apps/primary.py b/backend/danswer/background/celery/versioned_apps/primary.py
new file mode 100644
index 00000000000..2d97caa3da5
--- /dev/null
+++ b/backend/danswer/background/celery/versioned_apps/primary.py
@@ -0,0 +1,8 @@
+"""Factory stub for running celery worker / celery beat."""
+from danswer.utils.variable_functionality import fetch_versioned_implementation
+from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
+
+set_is_ee_based_on_env_variable()
+app = fetch_versioned_implementation(
+    "danswer.background.celery.apps.primary", "celery_app"
+)
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index d53fb0b12ea..caf7a103b94 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -198,6 +198,41 @@
 except ValueError:
     CELERY_BROKER_POOL_LIMIT = CELERY_BROKER_POOL_LIMIT_DEFAULT
 
+CELERY_WORKER_LIGHT_CONCURRENCY_DEFAULT = 24
+try:
+    CELERY_WORKER_LIGHT_CONCURRENCY = int(
+        os.environ.get(
+            "CELERY_WORKER_LIGHT_CONCURRENCY", CELERY_WORKER_LIGHT_CONCURRENCY_DEFAULT
+        )
+    )
+except ValueError:
+    CELERY_WORKER_LIGHT_CONCURRENCY = CELERY_WORKER_LIGHT_CONCURRENCY_DEFAULT
+
+CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER_DEFAULT = 8
+try:
+    CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER = int(
+        os.environ.get(
+            "CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER",
+            CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER_DEFAULT,
+        )
+    )
+except ValueError:
+    CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER = (
+        CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER_DEFAULT
+    )
+
+CELERY_WORKER_INDEXING_CONCURRENCY_DEFAULT = 1
+try:
+    env_value = os.environ.get("CELERY_WORKER_INDEXING_CONCURRENCY")
+    if not env_value:
+        env_value = os.environ.get("NUM_INDEXING_WORKERS")
+
+    if not env_value:
+        env_value = str(CELERY_WORKER_INDEXING_CONCURRENCY_DEFAULT)
+    CELERY_WORKER_INDEXING_CONCURRENCY = int(env_value)
+except ValueError:
+    CELERY_WORKER_INDEXING_CONCURRENCY = CELERY_WORKER_INDEXING_CONCURRENCY_DEFAULT
+
 #####
 # Connector Configs
 #####
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index 9cfe72275af..db35807ad54 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -16,6 +16,7 @@
 from danswer.background.celery.tasks.pruning.tasks import (
     try_creating_prune_generator_task,
 )
+from danswer.background.celery.versioned_apps.primary import app as primary_app
 from danswer.db.connector_credential_pair import add_credential_to_connector
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.connector_credential_pair import remove_credential_from_connector
@@ -49,6 +50,7 @@
 )
 from ee.danswer.db.user_group import validate_user_creation_permissions
 
+
 logger = setup_logger()
 router = APIRouter(prefix="/manage")
 
@@ -261,7 +263,7 @@ def prune_cc_pair(
         f"{cc_pair.connector.name} connector."
     )
     tasks_created = try_creating_prune_generator_task(
-        cc_pair, db_session, r, current_tenant_id.get()
+        primary_app, cc_pair, db_session, r, current_tenant_id.get()
     )
     if not tasks_created:
         raise HTTPException(
@@ -318,7 +320,7 @@ def sync_cc_pair(
     db_session: Session = Depends(get_session),
 ) -> StatusResponse[list[int]]:
     # avoiding circular refs
-    from ee.danswer.background.celery.celery_app import (
+    from ee.danswer.background.celery.apps.primary import (
         sync_external_doc_permissions_task,
     )
 
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index 8de42db3863..54d11e867bd 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -18,6 +18,7 @@
 from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
 from danswer.background.celery.tasks.indexing.tasks import try_creating_indexing_task
+from danswer.background.celery.versioned_apps.primary import app as primary_app
 from danswer.configs.app_configs import ENABLED_CONNECTOR_TYPES
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import FileOrigin
@@ -834,6 +835,7 @@ def connector_run_once(
     for cc_pair in connector_credential_pairs:
         if cc_pair is not None:
             attempt_id = try_creating_indexing_task(
+                primary_app,
                 cc_pair,
                 search_settings,
                 run_info.from_beginning,
diff --git a/backend/danswer/server/manage/administrative.py b/backend/danswer/server/manage/administrative.py
index 7771c1ed824..d16aa59c4cb 100644
--- a/backend/danswer/server/manage/administrative.py
+++ b/backend/danswer/server/manage/administrative.py
@@ -10,7 +10,7 @@
 
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
-from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.versioned_apps.primary import app as primary_app
 from danswer.configs.app_configs import GENERATIVE_MODEL_ACCESS_CHECK_FREQ
 from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import DocumentSource
@@ -195,7 +195,7 @@ def create_deletion_attempt_for_connector_id(
     db_session.commit()
 
     # run the beat task to pick up this deletion from the db immediately
-    celery_app.send_task(
+    primary_app.send_task(
         "check_for_connector_deletion_task",
         priority=DanswerCeleryPriority.HIGH,
         kwargs={"tenant_id": tenant_id},
diff --git a/backend/ee/danswer/background/celery/apps/beat.py b/backend/ee/danswer/background/celery/apps/beat.py
new file mode 100644
index 00000000000..20325e77df6
--- /dev/null
+++ b/backend/ee/danswer/background/celery/apps/beat.py
@@ -0,0 +1,52 @@
+#####
+# Celery Beat (Periodic Tasks) Settings
+#####
+from datetime import timedelta
+
+from danswer.background.celery.apps.beat import celery_app
+from danswer.db.engine import get_all_tenant_ids
+
+
+tenant_ids = get_all_tenant_ids()
+
+tasks_to_schedule = [
+    {
+        "name": "sync-external-doc-permissions",
+        "task": "check_sync_external_doc_permissions_task",
+        "schedule": timedelta(seconds=5),  # TODO: optimize this
+    },
+    {
+        "name": "sync-external-group-permissions",
+        "task": "check_sync_external_group_permissions_task",
+        "schedule": timedelta(seconds=5),  # TODO: optimize this
+    },
+    {
+        "name": "autogenerate_usage_report",
+        "task": "autogenerate_usage_report_task",
+        "schedule": timedelta(days=30),  # TODO: change this to config flag
+    },
+    {
+        "name": "check-ttl-management",
+        "task": "check_ttl_management_task",
+        "schedule": timedelta(hours=1),
+    },
+]
+
+# Build the celery beat schedule dynamically
+beat_schedule = {}
+
+for tenant_id in tenant_ids:
+    for task in tasks_to_schedule:
+        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
+        beat_schedule[task_name] = {
+            "task": task["task"],
+            "schedule": task["schedule"],
+            "args": (tenant_id,),  # Must pass tenant_id as an argument
+        }
+
+# Include any existing beat schedules
+existing_beat_schedule = celery_app.conf.beat_schedule or {}
+beat_schedule.update(existing_beat_schedule)
+
+# Update the Celery app configuration
+celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/ee/danswer/background/celery/celery_app.py b/backend/ee/danswer/background/celery/apps/primary.py
similarity index 76%
rename from backend/ee/danswer/background/celery/celery_app.py
rename to backend/ee/danswer/background/celery/apps/primary.py
index 4010b8b3998..97c5b0221ca 100644
--- a/backend/ee/danswer/background/celery/celery_app.py
+++ b/backend/ee/danswer/background/celery/apps/primary.py
@@ -1,11 +1,8 @@
-from datetime import timedelta
-
-from danswer.background.celery.celery_app import celery_app
+from danswer.background.celery.apps.primary import celery_app
 from danswer.background.task_utils import build_celery_task_wrapper
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.chat import delete_chat_sessions_older_than
-from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import get_session_with_tenant
 from danswer.server.settings.store import load_settings
 from danswer.utils.logger import setup_logger
@@ -138,53 +135,3 @@ def autogenerate_usage_report_task(tenant_id: str | None) -> None:
             user_id=None,
             period=None,
         )
-
-
-#####
-# Celery Beat (Periodic Tasks) Settings
-#####
-
-
-tenant_ids = get_all_tenant_ids()
-
-tasks_to_schedule = [
-    {
-        "name": "sync-external-doc-permissions",
-        "task": "check_sync_external_doc_permissions_task",
-        "schedule": timedelta(seconds=5),  # TODO: optimize this
-    },
-    {
-        "name": "sync-external-group-permissions",
-        "task": "check_sync_external_group_permissions_task",
-        "schedule": timedelta(seconds=5),  # TODO: optimize this
-    },
-    {
-        "name": "autogenerate_usage_report",
-        "task": "autogenerate_usage_report_task",
-        "schedule": timedelta(days=30),  # TODO: change this to config flag
-    },
-    {
-        "name": "check-ttl-management",
-        "task": "check_ttl_management_task",
-        "schedule": timedelta(hours=1),
-    },
-]
-
-# Build the celery beat schedule dynamically
-beat_schedule = {}
-
-for tenant_id in tenant_ids:
-    for task in tasks_to_schedule:
-        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
-        beat_schedule[task_name] = {
-            "task": task["task"],
-            "schedule": task["schedule"],
-            "args": (tenant_id,),  # Must pass tenant_id as an argument
-        }
-
-# Include any existing beat schedules
-existing_beat_schedule = celery_app.conf.beat_schedule or {}
-beat_schedule.update(existing_beat_schedule)
-
-# Update the Celery app configuration
-celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
index 259f2474928..a2b45324d46 100644
--- a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
@@ -3,7 +3,7 @@
 from redis import Redis
 from sqlalchemy.orm import Session
 
-from danswer.background.celery.celery_app import task_logger
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.user_group import delete_user_group
diff --git a/backend/scripts/dev_run_background_jobs.py b/backend/scripts/dev_run_background_jobs.py
index f3a00392465..1ca823e0935 100644
--- a/backend/scripts/dev_run_background_jobs.py
+++ b/backend/scripts/dev_run_background_jobs.py
@@ -20,14 +20,13 @@ def run_jobs() -> None:
     cmd_worker_primary = [
         "celery",
         "-A",
-        "ee.danswer.background.celery.celery_app",
+        "danswer.background.celery.versioned_apps.primary",
         "worker",
         "--pool=threads",
         "--concurrency=6",
         "--prefetch-multiplier=1",
         "--loglevel=INFO",
-        "-n",
-        "primary@%n",
+        "--hostname=primary@%n",
         "-Q",
         "celery",
     ]
@@ -35,14 +34,13 @@ def run_jobs() -> None:
     cmd_worker_light = [
         "celery",
         "-A",
-        "ee.danswer.background.celery.celery_app",
+        "danswer.background.celery.versioned_apps.light",
         "worker",
         "--pool=threads",
         "--concurrency=16",
         "--prefetch-multiplier=8",
         "--loglevel=INFO",
-        "-n",
-        "light@%n",
+        "--hostname=light@%n",
         "-Q",
         "vespa_metadata_sync,connector_deletion",
     ]
@@ -50,14 +48,13 @@ def run_jobs() -> None:
     cmd_worker_heavy = [
         "celery",
         "-A",
-        "ee.danswer.background.celery.celery_app",
+        "danswer.background.celery.versioned_apps.heavy",
         "worker",
         "--pool=threads",
         "--concurrency=6",
         "--prefetch-multiplier=1",
         "--loglevel=INFO",
-        "-n",
-        "heavy@%n",
+        "--hostname=heavy@%n",
         "-Q",
         "connector_pruning",
     ]
@@ -65,21 +62,20 @@ def run_jobs() -> None:
     cmd_worker_indexing = [
         "celery",
         "-A",
-        "ee.danswer.background.celery.celery_app",
+        "danswer.background.celery.versioned_apps.indexing",
         "worker",
         "--pool=threads",
         "--concurrency=1",
         "--prefetch-multiplier=1",
         "--loglevel=INFO",
-        "-n",
-        "indexing@%n",
+        "--hostname=indexing@%n",
         "--queues=connector_indexing",
     ]
 
     cmd_beat = [
         "celery",
         "-A",
-        "ee.danswer.background.celery.celery_app",
+        "danswer.background.celery.versioned_apps.beat",
         "beat",
         "--loglevel=INFO",
     ]
diff --git a/backend/supervisord.conf b/backend/supervisord.conf
index 76026bc5667..93472161854 100644
--- a/backend/supervisord.conf
+++ b/backend/supervisord.conf
@@ -15,10 +15,7 @@ logfile=/var/log/supervisord.log
 # relatively compute-light (e.g. they tend to just make a bunch of requests to 
 # Vespa / Postgres)
 [program:celery_worker_primary]
-command=celery -A danswer.background.celery.celery_run:celery_app worker 
-    --pool=threads
-    --concurrency=4
-    --prefetch-multiplier=1 
+command=celery -A danswer.background.celery.versioned_apps.primary worker 
     --loglevel=INFO 
     --hostname=primary@%%n 
     -Q celery
@@ -33,13 +30,10 @@ stopasgroup=true
 # since this is often the bottleneck for "sync" jobs (e.g. document set syncing, 
 # user group syncing, deletion, etc.) 
 [program:celery_worker_light]
-command=bash -c "celery -A danswer.background.celery.celery_run:celery_app worker \
-    --pool=threads \
-    --concurrency=${CELERY_WORKER_LIGHT_CONCURRENCY:-24} \
-    --prefetch-multiplier=${CELERY_WORKER_LIGHT_PREFETCH_MULTIPLIER:-8} \
-    --loglevel=INFO \
-    --hostname=light@%%n \
-    -Q vespa_metadata_sync,connector_deletion"
+command=celery -A danswer.background.celery.versioned_apps.light worker
+    --loglevel=INFO
+    --hostname=light@%%n
+    -Q vespa_metadata_sync,connector_deletion
 stdout_logfile=/var/log/celery_worker_light.log
 stdout_logfile_maxbytes=16MB
 redirect_stderr=true
@@ -48,10 +42,7 @@ startsecs=10
 stopasgroup=true
 
 [program:celery_worker_heavy]
-command=celery -A danswer.background.celery.celery_run:celery_app worker 
-    --pool=threads
-    --concurrency=4
-    --prefetch-multiplier=1 
+command=celery -A danswer.background.celery.versioned_apps.heavy worker 
     --loglevel=INFO 
     --hostname=heavy@%%n
     -Q connector_pruning
@@ -63,13 +54,10 @@ startsecs=10
 stopasgroup=true
 
 [program:celery_worker_indexing]
-command=bash -c "celery -A danswer.background.celery.celery_run:celery_app worker \
-    --pool=threads \
-    --concurrency=${CELERY_WORKER_INDEXING_CONCURRENCY:-${NUM_INDEXING_WORKERS:-1}} \
-    --prefetch-multiplier=1 \
-    --loglevel=INFO \
-    --hostname=indexing@%%n \
-    -Q connector_indexing"
+command=celery -A danswer.background.celery.versioned_apps.indexing worker
+    --loglevel=INFO
+    --hostname=indexing@%%n
+    -Q connector_indexing
 stdout_logfile=/var/log/celery_worker_indexing.log
 stdout_logfile_maxbytes=16MB
 redirect_stderr=true
@@ -79,7 +67,7 @@ stopasgroup=true
 
 # Job scheduler for periodic tasks
 [program:celery_beat]
-command=celery -A danswer.background.celery.celery_run:celery_app beat
+command=celery -A danswer.background.celery.versioned_apps.beat beat
 stdout_logfile=/var/log/celery_beat.log
 stdout_logfile_maxbytes=16MB
 redirect_stderr=true

From 5703ea47d222b9184a8e74b0cade178ab776b308 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 23 Oct 2024 09:46:30 -0700
Subject: [PATCH 181/376] Auth on main (#2878)

* add cloud auth type

* k

* robustified cloud auth type

* k

* minor typing
---
 backend/danswer/auth/users.py                 | 83 ++++++++++++-------
 backend/danswer/configs/constants.py          |  3 +
 backend/danswer/main.py                       |  4 +-
 web/src/app/auth/login/SignInButton.tsx       |  4 +-
 web/src/app/auth/login/page.tsx               | 22 ++++-
 web/src/app/auth/signup/page.tsx              | 74 ++++++++++-------
 web/src/components/auth/AuthFlowContainer.tsx |  2 +-
 web/src/lib/constants.ts                      |  8 +-
 web/src/lib/userSS.ts                         | 15 +++-
 9 files changed, 146 insertions(+), 69 deletions(-)

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 5abf2ac8116..4565073b6af 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -233,35 +233,60 @@ async def create(
         safe: bool = False,
         request: Optional[Request] = None,
     ) -> User:
-        verify_email_is_invited(user_create.email)
-        verify_email_domain(user_create.email)
-        if hasattr(user_create, "role"):
-            user_count = await get_user_count()
-            if user_count == 0 or user_create.email in get_default_admin_user_emails():
-                user_create.role = UserRole.ADMIN
-            else:
-                user_create.role = UserRole.BASIC
-        user = None
         try:
-            user = await super().create(user_create, safe=safe, request=request)  # type: ignore
-        except exceptions.UserAlreadyExists:
-            user = await self.get_by_email(user_create.email)
-            # Handle case where user has used product outside of web and is now creating an account through web
-            if (
-                not user.has_web_login
-                and hasattr(user_create, "has_web_login")
-                and user_create.has_web_login
-            ):
-                user_update = UserUpdate(
-                    password=user_create.password,
-                    has_web_login=True,
-                    role=user_create.role,
-                    is_verified=user_create.is_verified,
-                )
-                user = await self.update(user_update, user)
-            else:
-                raise exceptions.UserAlreadyExists()
-        return user
+            tenant_id = (
+                get_tenant_id_for_email(user_create.email) if MULTI_TENANT else "public"
+            )
+        except exceptions.UserNotExists:
+            raise HTTPException(status_code=401, detail="User not found")
+
+        if not tenant_id:
+            raise HTTPException(
+                status_code=401, detail="User does not belong to an organization"
+            )
+
+        async with get_async_session_with_tenant(tenant_id) as db_session:
+            token = current_tenant_id.set(tenant_id)
+
+            verify_email_is_invited(user_create.email)
+            verify_email_domain(user_create.email)
+            if MULTI_TENANT:
+                tenant_user_db = SQLAlchemyUserAdminDB(db_session, User, OAuthAccount)
+                self.user_db = tenant_user_db
+                self.database = tenant_user_db
+
+            if hasattr(user_create, "role"):
+                user_count = await get_user_count()
+                if (
+                    user_count == 0
+                    or user_create.email in get_default_admin_user_emails()
+                ):
+                    user_create.role = UserRole.ADMIN
+                else:
+                    user_create.role = UserRole.BASIC
+            user = None
+            try:
+                user = await super().create(user_create, safe=safe, request=request)  # type: ignore
+            except exceptions.UserAlreadyExists:
+                user = await self.get_by_email(user_create.email)
+                # Handle case where user has used product outside of web and is now creating an account through web
+                if (
+                    not user.has_web_login
+                    and hasattr(user_create, "has_web_login")
+                    and user_create.has_web_login
+                ):
+                    user_update = UserUpdate(
+                        password=user_create.password,
+                        has_web_login=True,
+                        role=user_create.role,
+                        is_verified=user_create.is_verified,
+                    )
+                    user = await self.update(user_update, user)
+                else:
+                    raise exceptions.UserAlreadyExists()
+
+            current_tenant_id.reset(token)
+            return user
 
     async def on_after_login(
         self,
@@ -320,7 +345,7 @@ async def oauth_callback(
             if MULTI_TENANT:
                 tenant_user_db = SQLAlchemyUserAdminDB(db_session, User, OAuthAccount)
                 self.user_db = tenant_user_db
-                self.database = tenant_user_db
+                self.database = tenant_user_db  # type: ignore
 
             oauth_account_dict = {
                 "oauth_name": oauth_name,
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 9858f2354b9..3c9a7cffcc2 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -160,6 +160,9 @@ class AuthType(str, Enum):
     OIDC = "oidc"
     SAML = "saml"
 
+    # google auth and basic
+    CLOUD = "cloud"
+
 
 class SessionType(str, Enum):
     CHAT = "Chat"
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index fe563c7c695..a6a338b4c4c 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -269,7 +269,7 @@ def get_application() -> FastAPI:
         # Server logs this during auth setup verification step
         pass
 
-    elif AUTH_TYPE == AuthType.BASIC:
+    if AUTH_TYPE == AuthType.BASIC or AUTH_TYPE == AuthType.CLOUD:
         include_router_with_global_prefix_prepended(
             application,
             fastapi_users.get_auth_router(auth_backend),
@@ -301,7 +301,7 @@ def get_application() -> FastAPI:
             tags=["users"],
         )
 
-    elif AUTH_TYPE == AuthType.GOOGLE_OAUTH:
+    if AUTH_TYPE == AuthType.GOOGLE_OAUTH or AUTH_TYPE == AuthType.CLOUD:
         oauth_client = GoogleOAuth2(OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET)
         include_router_with_global_prefix_prepended(
             application,
diff --git a/web/src/app/auth/login/SignInButton.tsx b/web/src/app/auth/login/SignInButton.tsx
index 9d04321e80a..128f5790c6e 100644
--- a/web/src/app/auth/login/SignInButton.tsx
+++ b/web/src/app/auth/login/SignInButton.tsx
@@ -9,7 +9,7 @@ export function SignInButton({
   authType: AuthType;
 }) {
   let button;
-  if (authType === "google_oauth") {
+  if (authType === "google_oauth" || authType === "cloud") {
     button = (
       <div className="mx-auto flex">
         <div className="my-auto mr-2">
@@ -42,7 +42,7 @@ export function SignInButton({
 
   return (
     <a
-      className="mt-6 py-3 w-72 text-text-100 bg-accent flex rounded cursor-pointer hover:bg-indigo-800"
+      className="mx-auto mt-6 py-3 w-72 text-text-100 bg-accent flex rounded cursor-pointer hover:bg-indigo-800"
       href={authorizeUrl}
     >
       {button}
diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index d7738144af6..244d2429eb4 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -78,7 +78,7 @@ const Page = async ({
         <HealthCheckBanner />
       </div>
 
-      <div>
+      <div className="flex flex-col w-full justify-center">
         {authUrl && authTypeMetadata && (
           <>
             <h2 className="text-center text-xl text-strong font-bold">
@@ -92,6 +92,26 @@ const Page = async ({
           </>
         )}
 
+        {authTypeMetadata?.authType === "cloud" && (
+          <div className="mt-4 w-full justify-center">
+            <div className="flex items-center w-full my-4">
+              <div className="flex-grow border-t border-gray-300"></div>
+              <span className="px-4 text-gray-500">or</span>
+              <div className="flex-grow border-t border-gray-300"></div>
+            </div>
+            <EmailPasswordForm shouldVerify={true} />
+
+            <div className="flex">
+              <Text className="mt-4 mx-auto">
+                Don&apos;t have an account?{" "}
+                <Link href="/auth/signup" className="text-link font-medium">
+                  Create an account
+                </Link>
+              </Text>
+            </div>
+          </div>
+        )}
+
         {authTypeMetadata?.authType === "basic" && (
           <Card className="mt-4 w-96">
             <div className="flex">
diff --git a/web/src/app/auth/signup/page.tsx b/web/src/app/auth/signup/page.tsx
index ec276a09672..f44b53247ec 100644
--- a/web/src/app/auth/signup/page.tsx
+++ b/web/src/app/auth/signup/page.tsx
@@ -4,6 +4,7 @@ import {
   getCurrentUserSS,
   getAuthTypeMetadataSS,
   AuthTypeMetadata,
+  getAuthUrlSS,
 } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 import { EmailPasswordForm } from "../login/EmailPasswordForm";
@@ -11,6 +12,8 @@ import { Card, Title, Text } from "@tremor/react";
 import Link from "next/link";
 import { Logo } from "@/components/Logo";
 import { CLOUD_ENABLED } from "@/lib/constants";
+import { SignInButton } from "../login/SignInButton";
+import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 
 const Page = async () => {
   // catch cases where the backend is completely unreachable here
@@ -26,9 +29,6 @@ const Page = async () => {
   } catch (e) {
     console.log(`Some fetch failed for the login page - ${e}`);
   }
-  if (CLOUD_ENABLED) {
-    return redirect("/auth/login");
-  }
 
   // simply take the user to the home page if Auth is disabled
   if (authTypeMetadata?.authType === "disabled") {
@@ -42,44 +42,56 @@ const Page = async () => {
     }
     return redirect("/auth/waiting-on-verification");
   }
+  const cloud = authTypeMetadata?.authType === "cloud";
 
   // only enable this page if basic login is enabled
-  if (authTypeMetadata?.authType !== "basic") {
+  if (authTypeMetadata?.authType !== "basic" && !cloud) {
     return redirect("/");
   }
 
+  let authUrl: string | null = null;
+  if (cloud && authTypeMetadata) {
+    authUrl = await getAuthUrlSS(authTypeMetadata.authType, null);
+  }
+
   return (
-    <main>
-      <div className="absolute top-10x w-full">
-        <HealthCheckBanner />
-      </div>
-      <div className="min-h-screen flex items-center justify-center py-12 px-4 sm:px-6 lg:px-8">
-        <div>
-          <Logo height={64} width={64} className="mx-auto w-fit" />
+    <AuthFlowContainer>
+      <HealthCheckBanner />
 
-          <Card className="mt-4 w-96">
-            <div className="flex">
-              <Title className="mb-2 mx-auto font-bold">
-                Sign Up for Danswer
-              </Title>
-            </div>
-            <EmailPasswordForm
-              isSignup
-              shouldVerify={authTypeMetadata?.requiresVerification}
-            />
+      <>
+        <div className="absolute top-10x w-full"></div>
+        <div className="flex w-full flex-col justify-center">
+          <h2 className="text-center text-xl text-strong font-bold">
+            {cloud ? "Complete your sign up" : "Sign Up for Danswer"}
+          </h2>
 
-            <div className="flex">
-              <Text className="mt-4 mx-auto">
-                Already have an account?{" "}
-                <Link href="/auth/login" className="text-link font-medium">
-                  Log In
-                </Link>
-              </Text>
+          {cloud && authUrl && (
+            <div className="w-full justify-center">
+              <SignInButton authorizeUrl={authUrl} authType="cloud" />
+              <div className="flex items-center w-full my-4">
+                <div className="flex-grow border-t border-background-300"></div>
+                <span className="px-4 text-gray-500">or</span>
+                <div className="flex-grow border-t border-background-300"></div>
+              </div>
             </div>
-          </Card>
+          )}
+
+          <EmailPasswordForm
+            isSignup
+            shouldVerify={authTypeMetadata?.requiresVerification}
+          />
+
+          <div className="flex">
+            <Text className="mt-4 mx-auto">
+              Already have an account?{" "}
+              <Link href="/auth/login" className="text-link font-medium">
+                Log In
+              </Link>
+            </Text>
+          </div>
         </div>
-      </div>
-    </main>
+      </>
+    </AuthFlowContainer>
   );
 };
 
diff --git a/web/src/components/auth/AuthFlowContainer.tsx b/web/src/components/auth/AuthFlowContainer.tsx
index 35fd3d6f3c3..3be441a0a7b 100644
--- a/web/src/components/auth/AuthFlowContainer.tsx
+++ b/web/src/components/auth/AuthFlowContainer.tsx
@@ -7,7 +7,7 @@ export default function AuthFlowContainer({
 }) {
   return (
     <div className="flex flex-col items-center justify-center min-h-screen bg-background">
-      <div className="w-full max-w-md p-8 gap-y-4 bg-white flex items-center flex-col rounded-xl shadow-lg border border-bacgkround-100">
+      <div className="w-full max-w-md bg-black p-8 mx-4 gap-y-4 bg-white flex items-center flex-col rounded-xl shadow-lg border border-bacgkround-100">
         <Logo width={70} height={70} />
         {children}
       </div>
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index 15e5b5cbcf0..806a1d447a6 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -1,4 +1,10 @@
-export type AuthType = "disabled" | "basic" | "google_oauth" | "oidc" | "saml";
+export type AuthType =
+  | "disabled"
+  | "basic"
+  | "google_oauth"
+  | "oidc"
+  | "saml"
+  | "cloud";
 
 export const HOST_URL = process.env.WEB_DOMAIN || "http://127.0.0.1:3000";
 export const HEADER_HEIGHT = "h-16";
diff --git a/web/src/lib/userSS.ts b/web/src/lib/userSS.ts
index 81261cebea0..8ce25555b2f 100644
--- a/web/src/lib/userSS.ts
+++ b/web/src/lib/userSS.ts
@@ -2,7 +2,7 @@ import { cookies } from "next/headers";
 import { User } from "./types";
 import { buildUrl } from "./utilsSS";
 import { ReadonlyRequestCookies } from "next/dist/server/web/spec-extension/adapters/request-cookies";
-import { AuthType } from "./constants";
+import { AuthType, SERVER_SIDE_ONLY__CLOUD_ENABLED } from "./constants";
 
 export interface AuthTypeMetadata {
   authType: AuthType;
@@ -18,7 +18,15 @@ export const getAuthTypeMetadataSS = async (): Promise<AuthTypeMetadata> => {
 
   const data: { auth_type: string; requires_verification: boolean } =
     await res.json();
-  const authType = data.auth_type as AuthType;
+
+  let authType: AuthType;
+
+  // Override fasapi users auth so we can use both
+  if (SERVER_SIDE_ONLY__CLOUD_ENABLED) {
+    authType = "cloud";
+  } else {
+    authType = data.auth_type as AuthType;
+  }
 
   // for SAML / OIDC, we auto-redirect the user to the IdP when the user visits
   // Danswer in an un-authenticated state
@@ -87,6 +95,9 @@ export const getAuthUrlSS = async (
     case "google_oauth": {
       return await getGoogleOAuthUrlSS();
     }
+    case "cloud": {
+      return await getGoogleOAuthUrlSS();
+    }
     case "saml": {
       return await getSAMLAuthUrlSS();
     }

From a1680fac2f50cc8ae3e96231cbcf3a8f5dbbc18e Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Wed, 23 Oct 2024 12:58:15 -0400
Subject: [PATCH 182/376] Implement freshdesk frontend

---
 backend/danswer/configs/constants.py          |   1 +
 backend/danswer/connectors/factory.py         |   2 +
 .../danswer/connectors/freshdesk/__init__,py  |   0
 .../danswer/connectors/freshdesk/connector.py | 116 ++++++++++++++++++
 backend/danswer/db/models.py                  |   2 +-
 backend/danswer/file_store/models.py          |   2 +-
 backend/model_server/main.py                  |   2 +-
 web/public/Freshdesk.png                      | Bin 0 -> 18386 bytes
 web/src/app/api/[...path]/route.ts            |   6 +-
 web/src/components/icons/icons.tsx            |   8 ++
 web/src/lib/connectors/connectors.tsx         |  11 ++
 web/src/lib/connectors/credentials.ts         |  16 +++
 web/src/lib/sources.ts                        |   7 ++
 web/src/lib/types.ts                          |   1 +
 14 files changed, 170 insertions(+), 4 deletions(-)
 create mode 100644 backend/danswer/connectors/freshdesk/__init__,py
 create mode 100644 backend/danswer/connectors/freshdesk/connector.py
 create mode 100644 web/public/Freshdesk.png

diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 9858f2354b9..bb0674218db 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -128,6 +128,7 @@ class DocumentSource(str, Enum):
     OCI_STORAGE = "oci_storage"
     XENFORO = "xenforo"
     NOT_APPLICABLE = "not_applicable"
+    FRESHDESK = "freshdesk"
 
 
 DocumentSourceRequiringTenantContext: list[DocumentSource] = [DocumentSource.FILE]
diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py
index 52fb0194aa6..a22c320458b 100644
--- a/backend/danswer/connectors/factory.py
+++ b/backend/danswer/connectors/factory.py
@@ -46,6 +46,7 @@
 from danswer.connectors.xenforo.connector import XenforoConnector
 from danswer.connectors.zendesk.connector import ZendeskConnector
 from danswer.connectors.zulip.connector import ZulipConnector
+from danswer.connectors.freshdesk.connector import FreshdeskConnector
 from danswer.db.credentials import backend_update_credential_json
 from danswer.db.models import Credential
 
@@ -101,6 +102,7 @@ def identify_connector_class(
         DocumentSource.GOOGLE_CLOUD_STORAGE: BlobStorageConnector,
         DocumentSource.OCI_STORAGE: BlobStorageConnector,
         DocumentSource.XENFORO: XenforoConnector,
+        DocumentSource.FRESHDESK: FreshdeskConnector,
     }
     connector_by_source = connector_map.get(source, {})
 
diff --git a/backend/danswer/connectors/freshdesk/__init__,py b/backend/danswer/connectors/freshdesk/__init__,py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
new file mode 100644
index 00000000000..be2b1e0e3ea
--- /dev/null
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -0,0 +1,116 @@
+import requests
+import json
+from datetime import datetime, timezone
+from typing import Any, List, Optional
+from bs4 import BeautifulSoup  # Add this import for HTML parsing
+from danswer.configs.app_configs import INDEX_BATCH_SIZE
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector
+from danswer.connectors.models import ConnectorMissingCredentialError, Document, Section
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class FreshdeskConnector(PollConnector):
+    def __init__(self, api_key: str, domain: str, password: str, batch_size: int = INDEX_BATCH_SIZE) -> None:
+        self.api_key = api_key
+        self.domain = domain
+        self.password = password
+        self.batch_size = batch_size
+
+    def ticket_link(self, tid: int) -> str:
+        return f"https://{self.domain}.freshdesk.com/helpdesk/tickets/{tid}"
+
+    def build_doc_sections_from_ticket(self, ticket: dict) -> List[Section]:
+        # Use list comprehension for building sections
+        return [
+            Section(
+                link=self.ticket_link(int(ticket["id"])),
+                text=json.dumps({
+                    key: value
+                    for key, value in ticket.items()
+                    if isinstance(value, str)
+                }, default=str),
+            )
+        ]
+
+    def strip_html_tags(self, html: str) -> str:
+        soup = BeautifulSoup(html, 'html.parser')
+        return soup.get_text()
+
+    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
+        logger.info("Loading credentials")
+        self.api_key = credentials.get("freshdesk_api_key")
+        self.domain = credentials.get("freshdesk_domain")
+        self.password = credentials.get("freshdesk_password")
+        return None
+
+    def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+        logger.info("Processing tickets")
+        if any([self.api_key, self.domain, self.password]) is None:
+            raise ConnectorMissingCredentialError("freshdesk")
+
+        freshdesk_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets?include=description"
+        response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
+        response.raise_for_status()  # raises exception when not a 2xx response
+
+        if response.status_code!= 204:
+            tickets = json.loads(response.content)
+            logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API")
+            doc_batch: List[Document] = []
+
+            for ticket in tickets:
+                # Convert the "created_at", "updated_at", and "due_by" values to ISO 8601 strings
+                for date_field in ["created_at", "updated_at", "due_by"]:
+                    ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")
+
+                # Convert all other values to strings
+                ticket = {
+                    key: str(value) if not isinstance(value, str) else value
+                    for key, value in ticket.items()
+                }
+
+                # Checking for overdue tickets
+                today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
+
+                # Mapping the status field values
+                status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
+                ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
+
+                # Stripping HTML tags from the description field
+                ticket["description"] = self.strip_html_tags(ticket["description"])
+
+                # Remove extra white spaces from the description field
+                ticket["description"] = " ".join(ticket["description"].split())
+
+                # Use list comprehension for building sections
+                sections = self.build_doc_sections_from_ticket(ticket)
+
+                created_at = datetime.fromisoformat(ticket["created_at"])
+                today = datetime.now()
+                if (today - created_at).days / 30.4375 <= 2:
+                    doc = Document(
+                        id=ticket["id"],
+                        sections=sections,
+                        source=DocumentSource.FRESHDESK,
+                        semantic_identifier=ticket["subject"],
+                        metadata={
+                            key: value
+                            for key, value in ticket.items()
+                            if isinstance(value, str) and key not in ["description", "description_text"]
+                        },
+                    )
+
+                    doc_batch.append(doc)
+
+                    if len(doc_batch) >= self.batch_size:
+                        yield doc_batch
+                        doc_batch = []
+
+            if doc_batch:
+                yield doc_batch
+
+    def poll_source(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+        yield from self._process_tickets(start, end)
\ No newline at end of file
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index bee69353437..0ebd7910280 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -3,7 +3,7 @@
 from enum import Enum as PyEnum
 from typing import Any
 from typing import Literal
-from typing import NotRequired
+from typing_extensions import NotRequired
 from typing import Optional
 from uuid import uuid4
 from typing_extensions import TypedDict  # noreorder
diff --git a/backend/danswer/file_store/models.py b/backend/danswer/file_store/models.py
index d944a2fd270..0a3b513bd6a 100644
--- a/backend/danswer/file_store/models.py
+++ b/backend/danswer/file_store/models.py
@@ -1,6 +1,6 @@
 import base64
 from enum import Enum
-from typing import NotRequired
+from typing_extensions import NotRequired
 from typing_extensions import TypedDict  # noreorder
 
 from pydantic import BaseModel
diff --git a/backend/model_server/main.py b/backend/model_server/main.py
index 5505bbc8cdb..50cf9f8b9a5 100644
--- a/backend/model_server/main.py
+++ b/backend/model_server/main.py
@@ -28,7 +28,7 @@
 os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
 
 HF_CACHE_PATH = Path("/root/.cache/huggingface/")
-TEMP_HF_CACHE_PATH = Path("/root/.cache/temp_huggingface/")
+TEMP_HF_CACHE_PATH = Path.home() / ".cache" / "temp_huggingface"
 
 transformer_logging.set_verbosity_error()
 
diff --git a/web/public/Freshdesk.png b/web/public/Freshdesk.png
new file mode 100644
index 0000000000000000000000000000000000000000..a3343ceb01d473601cf3522ca809dfea79d34c97
GIT binary patch
literal 18386
zcmX^-cQ}>r`|mks_R%r3LLobuM^;uP5*gVfd+)4+aI#5KS&=Avlf4y@5i*ZmaS++-
z_Z*+^_4~tB*ZV&2bI<3#@8=%p`F%|_N-}0L2!beYt1CZ%AUOCh93mwK|NQeEIfNj!
z*SD1wb$usr(|*3XOVe_jNB&Q)@+Uk?FTF*3U)k+rwi3PDN24SA=;|sV0|z4nZDAE|
zvZ;BfQ%ElyZ8ZI-3ESkLeGATctw-pw1$9V*f|AncYw3Xv>cnf@3D!@VY@VzSc<p@V
z?(3@?xY1X);@9sNhsLzEm3B6(yG>>^3Eh1f?Q}TK+1RP-g81!(e#~{X6`tf2#s=pm
zORy#UB0>YX!8-`hFiVKr9_)H+pD<ri;Ua}S$CJ#K3G4PXnlK!kZ|AVl%<;ARh6FAv
z<mv<rlWpSN>bU2vb??)p4Q0(7`(;bYuvQ4eB+#Nj@E*=c%Atwk*G3kEna7ImD-*KU
zU%o5(xPLmA^%Kef!YG;d3>p4?E{B??{!;4r!UF9{s>+l&XW3^CIpboKN2G}j$g%P%
zPK|G)-m65rKRK!t^CBjPw4qr3+K*}llVX}NF_OeQ3Ttj6NicKX{f$`OBpgwngX&|&
zpNq-i2|ug7v#jPwXtfz~>G~MquhOnndtK{DBQ%#qBe2c<*|z#rSW)s*wdjiVpV_P*
zZXRcG{y|q}Go6^%^N{t`a4hRWd2q(8^k@vrH)^aGmv@li^05+5B+?4TAu|>Fu~?Kq
zr~9Xf4HK6Q{7JUL^-3#*w!x9I`KfM__=XdX!^{H4@kr!{_=nkq7f!vIl<E@bFVIuF
zZq97=$8uHXC)H98>yZ<;4ItiYv4UyhGZM?PT;38Kuc4$$F@*LFQ5OTDgNJ|LA6pg?
z3P4D@t|s}$_yL@pnsPr;U-<r3VE`xRJvnHvQ_}y`=Npla#IvUhU|rdD`T?mg<D<Fg
zI%I+ltQPErsCqXt?rvq+Kc(_+ikBQxi6hc6VLEv*@GShUV`{JOJR2e$LC|xH@_B5<
zHBJ9gQRJn^u&h#vZ0u*LnwOqU)*?Shg<uJV>CZ9w1r()v9Qr9wzD?7tIh8^8$C<QU
zP11}W^j8Ro{b1ZS*^CL~j1nLrtbNDLe|BK*{EP_Q)(&`82_wiem_IBNUA_+8tmHx<
zh+e?fuN4&Tix>O0vd4_sz&O<I8eMltTdFCMXc32q?N%;xl3hVG@awJ7b-q~7r`~sP
z*kBo)Qw{cX2!1u-Uj-{Bac%Hs156+`aGuJDmP`*^$xvt#B8Q~IUOQO}OaxOI@5Xjv
z>mf@#Jn0C<GSS^SYw7J?f}JD*=)L_&(v*cza~&eV4th|vO@ojgfAUIDM`F2eni<0Y
zWK73IRQCegreWpE1l0gQ*&=1rE8Eu2K>w@Pn`p?P@IFjv!{ZwAvDWcsEay#SWs~Yb
zZ~yYo{+bXuL^S;?)X**^^)-zcWkWzbCV!HmRQBD^7t`tfq@~dP?*UG!XYQ`7L7dqN
z{soog^YtbsJZO&+qyhPd&)12m-q^<+e}(c|Y)LpdqYN1GDAiBYm|Pw7Gx`t^8OCkZ
zrrWo`A;_HKlp$1f!ViPV4o4W9q`W@UwaM&;D@<Q6#(rQcN;%>|Pm!5p@W}Iew$v6r
zw|=o)xY75*0Ls7n;MHtuf-lGV6U-EGRy=-*?GtpBZE5pE@4@4gIJw9b=zd;+^hooD
z%rHkV+I0@Fnuo@`;dZ678c#n7mK5?*2dCdBykJ+xs_zWO_#r(KgNZZ0Wb`3<sn292
zk)#6pKKS~l-Xn`|dmPKc#to5}p1*e*cTX?YEX$4t6p}oexc#N>I@y!-x2C!kN`&OT
zn>?Ik<iFYYfBUfdai~GRNOQ=a1Wj6hF{5z(<W0}X`KMCv)e>{F+t&0IHGdi{`1@O`
zbp0AWOm&w+@74xUBech(%AAg<pfit35;sD$2%ZV1{L6X-+n1m+JeTs1I;#8mzr5ZK
z@(&(BplzQ#F=C2#C~*;A_HOw%7I#ZoImAlf3<pgJLM$-M116OUP06l)5)o|_IndZh
zr?=#iOMU(|Ma>JO3du|+zP<N*_cuhknrH>k%F2ju<&P!M666pyq-}{eZEi)oCCZx)
z&;>kb-pA#4Dz>7j&MuF^FmSwT%^Trfa_)2Csg9EHb2>@jf)I`)ZDu@mmXFq<KCRZX
zlINK0m$RYrjZ&)_f@Wuz)gVsYr`j`}#i`3x=sJY#{zZg}*ej>0VX!MT1>(s+wVGX@
zgFqi<tK!jqC1vF}ZPQ1FqHz5N?Aad@KoEyr^b=F3g9furtd_EJBvaC$i+cZ9a7!=5
z)slKeGLO0WNRD9J1Sg=Zyr@yVU(zu~PdrbmASIR%)7J32diG&om^=c(8jA^F09x8~
zD5eIrjr8n!kkm$Px)6B0!^?<_{6uw+J-O}zFC+!0Nf}~T%2?So&xP>FYDXHT?BGgH
z-*Q0~kCj`!+(h+;90`ue1YDGrM-02tsIg;^FagCSAodK76j`noUpy+v>KNQE#gyl6
zy8pi5&~FOv9M|R8lq;F%L`-uFxvc(#@*oz$B3o5?z9cn@#RunKF~wNlBitk-N8gGS
zN<DBA-C{(;CJccfLK(#n*yoS?tvKw$>zb<t{K(I7!`?%OY;r4*FjXv8XH+*Yxk0b&
zY%X(V_gFjn^Md^?`;w+a!E-vO`}{en&77|$OiJOA#k!B9yc`krDL@rYZ~U7D_Px^l
z6Tg+h9~^NlrSXdp3`VKTl}w>`d@pbBum98lGgkH-Lz5Wu)Pf`7Tj}CY|Muy`_MU`5
zdA4|vw%kVskdi==$i}r^>_gb0J^taZ$LOc@Xqr31X|MTP2q4*WTAp6OE<Lxo?D}Wx
z(#;*pzk8RkJox7rXrv4x{8=0N_G#s#j1`#i`}A-O=2zBLdNc=f#p1H=<i^vBEnm);
zHbv+_tx09RUAMccXW$|RcvT;?pwNd?gvNv#t}sttBgB)osL=+eG5;O?L{lu~L;}r?
zr$8BJ0VB<d$)1u|TGZaWF3-v7E+s_7LQ0%e?Yo@bg{8;wf<>&YFQEEfg(33ot;<g$
zypf!o*>RSzUw9|<n?Q#>wDrdMvh-H?ca=P$=0TlV#<NP%)K7Sn^DBIX(L#(hLEaj-
zid<mm4Ls{P|B%t6HSd@NS%o?@pNS)k3(uDpND+O%ut4VgXVB8m^dgq!9K-2W1e%Ju
zdY^$0KA;q$F@!%BBPJ}cU+79}v8T7AcjVA6=X;V73K4oz+)C+>dPTo2V~#(S*_hT`
zbiC!%;ZM=Orbj9*kN;c9ok8$D10c^QG$00@s-egb2<<F4!Yo1JBsuf%VaAKV406&9
zz@*Yb-ZL!CkYJyl)7RRDK)1ga?mt4IGaXhqz)1npDEY$M_m<MidAMH@Xpf^Y2{tRH
zZOV)ptBGR)tb5l2hd7E{m+9bsQ~~lpckd{lz=U%+e3HsaX%y1%=PmpG^FQ!}GT%{N
ztI1yI-!Ps)<Q;8)S}b1O%AC$n*w#$oAMWeYqw#KIoNbRB{^dJ<+55|fWatpz0(&-5
zg*d!iI0d!!p&>Gx8D~QIQ9MTP`95jFHTe7LP0iT`&mRcdlPNq>G_5vRz6pt*>rbVM
z3XJigl-M-U0aBqHj}j*jX5Ryf5J;!n2$iK~JARJAP(9y=)2vjnL$2lCRQMHc4kV->
z{<XcXgNn~LUc7<ur&5^6ZV*%H<J^1?R4JQF`7w;~AZQ(`A>oH8&JEh4GQx*S0eGF7
z$uOdGy7EmB`jGb{aUWRE2p+-3#|Uwv1nnE!Bwalf6Vl))qzPG)wfqWw&0~-vK#-7+
zSCyXMob%UPDr;;yqONF#kpu%^RL?OU5O?d27Q|zJE*u66d7=b8sfB=$9cS(}a3-|#
zpY14$<zQb0#x5BVm?xYw-<5V$2kP3o2|uS&M$nO5;i<*?02H8xA_1RBk@fPS2Z<_%
z1bR&`WJUh>3DTPj`bktVonkFdbpF|Afi*IP@Q2$rkd5s*Y#R$egK6s93)owb(777V
z2{@9qB;;1iFEs8lLw11iX>D9W6mhGut!|B?A`(nr)|KD;o*p$n_Tij2X5+5A2tm$n
zms{xwp!_}YR><~WO^d^EWYBWeN&b#__UI(~kr{z~1LgRJgR+;y&1;YBOAG3jzb`!J
z4G91CpyjWe*d^J0on&Ulw=pptii~elqbi?;Ffxykge^SH33j9{6x!V{xua<FQ)GO&
zoE`Oc&q8h6bj@}qdMLtVeEfb0>o@;7$?Dxn6=hZ$dM?N+%3Z6HGKF9hm^ZTOkg_jQ
zxnt^SQx1e(SE;(j**==c8ALv+z^NFvJ0m4^vC#ymvT|o~i5qK=`8eD;Rv6IufR*5Z
zylBhE32jX1<(n29{py<CUm8`hBk#4<WA6>dA@AQi)hjHXjTl(1Hn*PE6PRtjKzT}!
zGf>tZ_)Emm$h)U6QhN(UqMg-YBq}9sN7(;65dPtxGz?JWphm{mTvqnZf`@FkSgmzt
z#~?V1g;*6uuRY@C31c)$kZf~#?S*v;;_KUq{r#N$Fyk5iZ#YS{Rj?spiT`1dmw|Bw
z;_Rbo>=p4&+p55cL`-&Q4qg{#{L#{80IEI^Ny4xVi_n@R>@{C^>lKTdHPxOUM{oYN
z&Q$r<Jf)-GnCvN_#Dw;g;gQ{tVcd2BCLn!|mR!2AdoD`R#?t+(0o%X&cl~3}d<yg|
zTG!*tJ_OZ!vt(lydf={`63u4Nue|i|d@kays|%<Tt<LxdJStPS12juNmeH2z21PM=
z3*U;ZQAAN`E4x01sSAG%Bqo*zhsN^;y!9uam57tNM877}@Y3qvj|-0+g=dOPW4n9F
z%hp?{a8k+^-`p|{?83$R84Nb;RygeA)r_TTHhiZ<7Fl98q%}~QBZm#n<ZQEsm^0my
zIT-?F6Kp<Q#Q9_`b@b1hA5a%?VKT;{{Pgmg>CYU9ZL)(klDt`DZ0`=U9oGjSOpA|`
z^=W9lLUYKVk^>%|sIsYA$Clrz(#Mm(7>4hnsH&{=?+UUB3b$0$%UL2g{y^PGbmGPe
zm8<`K3ba{K4?=dnZHZu@C1#Fc^HMP5N869BCzA~GozuNi;tHO~I6jCe+CB3|C_Avc
zw4_46gUOs({rHGGuR3(U6PlRN4X>|mc)s^Z-$CuNJGUAVU8&+{6O|gDmK2#icHw5k
zyibmB`^Vckl{xp(cpaORYz%$qWmJ_zBOP@L8&epHa%UiSt@cJnnVwiBiCFCurON!$
zFzsguab#>Ttz$Wx+9Lk+Xti>IMmqG6$3%6okDZZgz<wp`*JH)aoRQo6OFefJentI=
zyQS#eG8=<O#)e-m^xGvG?PMK6Xkm09&TiUhn}|Zyp&5Y;jOgC$%p5N1=vef`@gglV
z$DCR>?q;YNE;s{EW3TISJLNd+g>y*j%Lx~_3PVk;Yx{Q(k&RkAMz(<;A94n>#*xO-
zzcf1PEz265#Y*V2^509qMRreq2Rg>rk3hy!=}K3@30s$0w~x8UQpg(4lKtL-A*k-}
zZE!ek)h?6qRUy!enQ1#V(28v{49Q9u%1@FWez5Stg2rX^mf0}R2c9PRO^W>(cK+4s
ztv)D<`5LeRDN||oM<%%tZRIQ!70`I|=H^%o`+S<*LS;@uyI>JtYsuXVRqNg$K!akQ
zdxd*5?hPf|F&|;SMt_S@-bn;W@SlW=2iG?z_?ngUHWmQ$iA;L(Ap8}ehJ)J9@ojdb
zf-4RyFeSQs?VoF>UH3Z?+Ks5z+!HOuG`cYBs@yc%EX)$s4zXZQCd{|eCyi4dC}eQ&
z{$+3|6;Wj2d?DDbcRC_h-DsSr_mR6t%vD?nB4hOdSdW410=Jb><}JHFA)K>4`=_-o
zQ(7W27HA-Fn#nhxL{FmC?hlpTMUiPMg#S};bX+}CYP=W4X2YFl(fY*?SW*Sb_>E|y
zi(3ibw&A>O&m7t0oW2ygS@y|FnIBk0$ujJqn6~3HBb591bZC49B!&L?T(=LTG%#v5
z0moJ0St0bV(R=<Tr1^aYtRAE4Z^JR^w+YIYB!_m>Dtd1$Bra;UB)y51l}>0gRhJVT
zCf|-D=(~xn2`1xsWw7tiMZH;Jv7)qJpmN#$LCGBBOA&~D-Du#-Grq#`Cvv6~E2n%}
z-~^>D6nE|sXS`z#&;G7*qS@8Xb4+oeA&fh`EGyXQtAVUTav6?x^K&p+f<bTCLATbz
zZ9|jqj{bv|{xB%^R1e7erimL3f|Rgv>hA=i{VPwE_5RPp$nU>g2IG`Q>=eyNTE>^h
zfZv95&(MVh#~$&y%=xD_&Jc``>V|JE2j<((L`J1BuQ`8IDJnr!%@B_qGMCLxk`9Nw
zUiIZC{*plmtZ3reZH4AkQ(*KJ?TyJESm}sx7gK{1u_Vo~zxDM+aQqpOv%wzpC-B>Z
z8%E|zJ{QC_JHU$FH7AFHin5&Vx^~VF{*~|MYgDyY)3$2QDzuQ)8*^#zBB0}SnhKwf
zARF<$r1YkpODIO;SNy+cij0$7blrQ5p%-qdW#yq{O4<y3%t<~J)n$?sYa3j!hxPFG
zoz%mEiW*-aPb~$h*<|Lr@{g7`#DfBwg2_{Uq*z*?&=-E3j3WrL2^y~CU`!W*E<TNI
zsYy~@SW6Z?H*b19g&$|*9LA=ii8V>?r;<K`sT)b;ys9*qgt#3xxWgsmYh5>LGnK!k
z1^aEA2Gsi8Oc`<P;5q93zUcBJZp51Dd6=vBSH{O5SR$W)`R#i9aVB{~VKq|TWDomp
zTL6<LVEg|L(4T~_p{}FN#HyqVNO&}kRu}bVFWY?MX@gE9S?0#<(7U=r*B6VL3YwX2
zJ-=^K{cMa^Yv5N!J-9FM@h2v;$e+}7VJuF`Q&*AiJcvORy}1WP*llox3x{A`tb#DT
zBx%L0S9rnc`P-F2=WDme1DvTbN>hEuy?Y_)G!MH4v%z*`B<_j+UCtPATkYbpP%}@=
z=|LpJLAbR|79v~H=z*MhLUASm{;IuW&_;SjQ7hhpok#v-**|vey0u?rh|tWk-$2Z=
zi-=}xat72M-<m2r@WL@F<H+uu%cBRfvD;t?FS&b|8Cu_E$|}l#Ts0xrU{XaJpUjX|
z`%VcJ$91LF(#`Hn$L$+d<2?_>NZdu5ieA$eF*<g;ZVr2$7I~B5s4Y!|m7eVeV|WQ*
ztMB9Efw|}C)xOn$d%l)7h>Uz*B`1vV4|4rb;jyV^yZr;Hk1MV?sp)yKpsCVzGlNGu
z*{tEeNMXYN?|Z;bd4m@SHjke*l$s?LgD9-qn|}UF6sHi)Fy#xKF3ZDB^0ueANEx$>
zL8aH&tz+wfkqf+~K;F_C9$Za;=^OUv%YRxszd6KLWBxcG7{_Ze`nYC(b}4%EdmOdz
zO<(AqCR6mJY9NUftxi>r=a=0N_Qp+HLL&!nDsyO9X}^n(o=T=kUu-#;K&i~Ban!EX
z7PvhcvIYBCwqvJ1{n+nhh&Q*`VK#-+n@_$i_<LNO3l2r>MSm*tdPFw27JTR;$z0J*
zet;G$Z7tmViBV&X_)RJf75_Zm<KuvFzgiiua_<<S0}u-YHLY97J}hJ0IgVhsL976r
zlWemCAJqEHl&j6&;$7n36JR&WvYn&B%>#l7K)-da8WWm--lfvsmll&Iz}sbk3ngNw
zdq>P}V=0n<BZr9cw?^%{pEJu@Ss6O<&3rm!$)P>8G6*oksk3^ky36SOaOsR&ZUhk`
zTpv0#_V1ded2G8u6aMN{^tSJO$8QTCvmMFZ!WONun;75@jQ^;72{Pj{ovdI1XJG!{
z;*AbNHk^9*<LBlNz8W+%*$}y0*7yT|0I4#}T-WWGBH#$<T2N0{5ZSAB5UK7q$MLdO
z5BNhhbn>xB)n_E%j5nqc5)!)DA72MVIGfm2uSlH{4iV}Lba7L)t@`^`-S|zW4*MmE
zi!p*0Z8s>wU)hCK{9S0B6WeGmeM$*zmT)1e^cISY^~2z7Xw4_N?)8*MX|L0$;+IC_
zJayaWkFq~kWi;P?|LKURMd7InY5}piDF5f#zh@z-{>N3Wk*${UMb*^7v&YP*pAR11
z_&Miq(`38Z8RFic-$*=rhb_E`s+sO@N3v&wQ59KLlPU-uazCH^ek*Fdj8t&1@_E*#
zR=CvKtkrM<#4wdPEW<#1jvr4G7}WX{c_aU1fg6MA$76~>*_)!pEBA3yxGfAh&n|P^
zD}MRdo_hJDZLfOhU!C`SCWn-^zZ!2qA4||>p<n(<-Z56!#%tKdN&+F2(XO4^JIyVo
zOThiagG)l<&m*bsxEqu)_vC!-uulL=;h%R(IGujRm~QrRL21E<g3?N=^Bl(>YY$6V
zf=+fXqM|q7&f9#q^sQ-ftuA{9XCF)9tl#^q?)mHG)jaOl4x@+sWtloKq~-J>vieOC
zKaAUUgD{)~ctN8sYStqs3h0fKFg}(!wjS<nc;)}h;9(8jtdU=hl~!$Zg=}j2ZXk{k
zVDS)&NiC@ILAP~Rg;j_&t%%?lai6xu=MJm(7AQ?K=-j+(M&>{L*e<i3Wth;OxWQ*F
z0K&>BZ;GhvCa2dVK5wSHgL_DZx~eHg$$kWPw6j{fO@sd9U9+F%>|@1^3=XU52ZD*8
z2W*ag^X%z}yLY@7CobmgGKVwx1c%ZW3oBk8T0%g*`$6OBD>|NaPK&u1AL;F@iEl6Q
zyDfdH<Fy;lWepGQvpL@P`YYxOOdyU|?boH@i((PMMkr#H#A5aqm7sjqjJ9^*>E_#l
zE>>Dq+Id)ADK_;G2G>0h$zm88@n*AccDL9&nd_K$$1w)8k;fXAt^BIXN{q~zjlAqM
zn>8bCAQYRiP|}SySTWgd26u_E!a9bBMGc8sHx506nZQk<{|-|Q>k`df0g9M=U6T(R
zoJ5GsUa3McwZZRM)7sjX2(-E{P#MSM*1m(wKe`!Da>l!djiVNEnM6&9HqA9O@yr|D
z&Z)AxkNh=(M5lh7X#H|9Zz7s1K5vO*R0E+w?CO{tn%O1H8|jYruWpPCekD8<_m{Gn
zX~%?gpyUI`D!8MWh-?!=R#jmywoTGym`+%#*^V+sPg1LQgMzm6T=tKa$2Qy6ttUjO
zm}Wf~`!{-t^!RFJOd#b&cii{IfSbl{wH7_n03D0w5(HjTm%jN|fv}X3@wj=2Xy0`M
z<ZvQ2*n{NMV7&RC??an6@t=V}OsSkl8`X7LZb5B5qTBZ)dsVNC<D!!<g~Mu%gBlD1
z@pBAcpyu~DMMR+vk-2AU;SIFp>ARfOrMPadTjiBj0qtfIjWN06S0+Sz=J1+Xc8UP;
z&yP4bRO`1-#6pyn^;;KK+A%N}Ec0QN*oNy1F3m_;wyA<<KsVq$Ygk(cAhY%$T5WI*
zO?MEtAxbD67&ivMebOOn!GJV`h_q1yU0A^b9=mR-$G~i!HXVoq1%EO+;?m82$LxC?
z461eeRX2g)Rr0EZVvJ-8YM_f%1TQK~<TgFuOFdq1{#?b@Jhjh#+%+1fWby0eS1ndI
z>lVCHDU-}9N4M!mk}KRTIJ5LzSXM+H2#^+QrWdjjw7lZ+JaJ!m(J&SNc}dG!*y1F_
z9miPvq_m>{<GY^kb8z$LUeCP^+^anf)6m)%2r8{0XxYi1gnex5z5JgH`pIa+OKfU6
z%_`#E;U`z6obOSR1+!j*@cBV2^ZqfayZzoL2`o5L2o3K0*j83{+wLsVvZ{|{!8&sB
z9b7_qb;i|%a%x19wVWP4tAKGkL|$3Bqs52;w|2X3g`8$0V7R~e1(W3y)yt{QH3{pA
z8B1>p)P`4-BBcPN6l>oX_A;ONiPJx_!l2EuZzY!gI#u9!ix3l^8EvuH(`~R1N`Bxe
z_uo!ZAWNg7g9y^{Sd-KEyWH`&@8P%*t-V)s;U<SRPCAc2*%#?#l7TC2P30_c6i2rJ
zWHp;PL()Qs%UU#^UH4Bq%nb&%`>f<25hEk6CX92O|7GT<4C6P`HY`v5pV~H4A}_}o
zLIG@J$#JrA*<&LihwW$7YX+Eau`ee?dC897{UfU6Eu#H}V89!pT_Xj={5Xks`&_WS
znY|;KrWRY#8w&>=x`D&Igu&CmdT_U%aDk~{tmq-coJ{gaG5Jw?VNgX<vr7<NQMJ}l
zQ5NqOh$!w4KFBG@x!sH~R}PTJ)u1_Bw8>ji+uo|d8Q-xcrLATMUDT~<F03BW-gC|S
z1#SX~%{#1BxXekL20RMbe4-Y5$+67$-jvT*!!AChdN|oc8HSQ*E<TaCpZv6FORTV9
z_FXw{`*5pxZ6r`F6E0FMM(@p|_Bnr+@W<R4-dBGnkYR68mY`UlIQ;OnTI-KE3*I@+
z!pS_krAQR6R}mz6n0(*ku5vnxjrFf-%sZjKpL~@6ff3w^FC2IalJ?{LLwDD8QXOg_
z7X4&NQiV6(3hp1|GT0dNq>j%(ggQQ<n|mTbV=*I<?g8un{S6e)hMi%X?aRmVRSxbO
z3+8;3XJGH68sU*3ats9Bb-1(n^Yq^3+4Y(vHLfmqhYh*#GH-IU)3ch4Y;WriEP2&9
zp8|agkl6qn`$ZG$1)DEo)c7T><Uht2!-``;KcATEd2MWI@!%_C6;58yH2;Ji1bKr^
zl;iJ9X_sE}{hp}Tv?%-r7s1zGVZP}Xs_X&=EY$TL9i44|7BnzvzIP<ZaY#%M*6kC~
zl$!CfP``ukxXKVR)+^D0I&34kY-G4@Z%lnx>m{%B$;`i3@=%ES02ivYCQSkC9Degp
z-{2gaHnt>OD>8z1$zQz-9<6S+&Uj7KYmI!kT2BX(Mf5{XpF7$>s8;Ed-ljXb0x^P`
z<y*q+-H`41buBDN?HMVI6tDOMvurtKjC-W+I_sp``x8R@Dc&OPA1a^0)ZKZW1=JH0
z3+o4EHy$agRn?Wz7W}D>@O-;q2<1n|@j_f=TS>5(5h~UG_dB?=7ZcYV9X^fD1ZG=3
zwb(09aE|AKMjommAxRO~{ibGq=~2E$&(CwK{wWR{GTM<1Mvd3r*W|4;43mdhzNqqf
z+x9(C2NLoq_vUpkUU*7Dbs!Pg6(9b?p~@#c{}#9l8a&Wp8FF%GMt^RT*&=C!D>%77
z4+l9-AjhFMv}=3f_8W5Ak&XCmCwtXd-Z%ClBLI?Ju7v0b32@W@I<c_vY`;A@zka5~
zG@5?$QaLU*W}^hZtVtUhn#n+rn->D$AO1c29^6}Malh23aku!Q*H>7um>lICWvD6-
z#rGL&vRkvfke*fyQOo3A?YCXK>XZi(V?Q=p<ThY96P`bnSB@ki9F4dx9u`cezi1ni
zBt!qLj~k;>?#~rHYp#6~w(ES+>$Wy&u+KsfBSp;JL9nA(WF|qDPrSZ+!#lS?rKWG4
zmyhXHkm_%l`O7N&%zCONuQz4x3hAUCJb}@OX`*PP$FQ|Cm#0m`n!|lgb4sf#tGORf
z-iciT_1ykB5OTq@ByxRe6Mscf{+7@mb()(vLypRoOwyJ+QDW{5&6ogyWG%Hj6;@Ep
zC_wCnvc<l9GJA${YpLbga)OhC0tGqP=^Y$9t#gwMtp)%NC(uxz5wt{|fcAu$YVn6w
zE|_9Vrj{l2#}~kCE_mnKMP6QEW_-aig!?IC(0&k1=0Qwcu&T7}4~pG?3eV^vE&*ty
z7^!8cUQ^Tfkr|)Xoxb{~nv!zq@=wj+3<8Dpf3q*3*g0@y`A3M%t!{<y3SA^p{u}cB
zSZL3)({AHh`0Elgq87LU?#|zTq`!CXfTjzjXP^4I44hy3d)?oc(S-=HNT`bcOFGRE
zj!trIXvR7lJwDjLaL?O8z4}L34m5JXj2;r3go_xXD5Ui_QoqMRj&@TyjlIgSIY|lc
z8#u}@)lX}IZi6Na#%=8#1yCNb=&4gW_}13)%=5y-Z1c$(W7Z@bP$-#pI5b%sVNY_|
zhJ0zt(#s*RCqm=7(mPq@aEXxuuLr&rPz$34EGs7}@jD&*exLzu_?>*!N@Gi}GZkKY
zmSZPT0mxIAt(UIc31|OtaK<DXP4*B*gV`;9z*q*gl()ha)Ryp{HWuwn)ZazkKT@h>
z5z)2@kcM+yH$ag}zq(tPKqWZ3&s^#Anm0@7A@qy@PhSQB$JK}U*QjT+r?9N9UbsTy
z72W1xdS5EYYYJ5H?gisGltEidmb%W0L+I?d;w%*jlrJU#(eyz0H+CTNjljUawq*Sv
zJllH&40O7AEN`t4739LD0?JY<0=+Lf=J4efUMs%_dU6PqNRrC}Huru|fzplgJ8MJB
zT-!)c8q5A(PLWW^8VbrcO1<UgQg8+EwY7lN1z{LB=!dvYPmu?KciFMl|Nl-@edK?e
zFe!$5lY$x(nc!U@aqyv&0ORwq|GiYCP&@xJh6v3C0hrodaGJdT0fJ*k|F;U}YWE)+
zmN0-#l#;;tQp^fOgFvWUUd5gP;Ji`i1!!d8u`?9p%8X`NRQqVdRj}taAu6-fFX&>~
zT%wIq)s+&oLpDvG-M}%X({mlCUQq*GK`XD5uj`?d@}_G}<OQJ1*JUg?idj5UyeYC>
zS1O?K2|oPx?~m!5Sn%Mpc`uWwL7-s@5R8$Qf`9+S`V9PEX{7*xz6#h5D3l?#tu+5{
zg+>k$ti%M6A0>1z#gz;VcKWKGW8DGt4v#Zqsw%na|Mr6JrC`>^s6Zu9`SRoYHcXeQ
z&I@L|&j_pN|Kr>}3XP&GmQqA`E~*gbS2@O=bAT>n#DMdhCxadI6|<p=FTH8-3}@~2
z##MCeZVaC9`HsukKYr=?PU-nKiS#A9k5&iLbO>4sSR?TGXIne$PMy#b5@Y?&*=tEn
z*d?<cc?Wj`Pf`GU4F=k1C`ye`fawUL%#$5BM*`~vsHS6h2?>{4TH#PxFTQUFp}l#b
zj|y7=HxDDi(|FC`Oc1g(P6IxUGYX#4>WwWxK)%xFszB6GdWM42j4{^`pa-ddPAV%}
zKe}0bmO)s%9E!Tsau)`Qm7FFI4-7rtpl|6wXqbdU9w03ni^y6lIIC{j-M%-`ibuhB
zO@bT^WC>u{nxRI|Y9P#igqD}Eu2R!MWg3QKrm<jUTkBZ-P9zz+S=u?rTLjN0HvQ<0
z9(d_NL+~2O{dcT!U{R>$K$pkH^wAIY!uC`5FmD<BI(RFC+1Uc{0snzf?@HY<JKL62
zgMftvR4ZQ}IwftH3%(7gK+M|`W&M~gXKvubGi1^2U@{XvE_ZOs452E;Xu%_ehbh+m
zC5MO9g%WI)cTiXGJdK0N$sgMQCyStm0F4+<1Hb+FI2Ksxw`;gGyb#pg*WetMB|~v|
z7$H`d{CDQ$QP~jEW~PEpp~~9;k0s{wx7XWowR5w;0mH4(1C_&ga2!2%dJ<pl*LcxE
z0zG*XZh)=%M}*4@)j`fxC$4XSpm_Tj+r6gpDLFGMcNVJEpoUYe7CP8(juKqh?D&Kz
z<`?6mSS=J>I>DexqIsmo5Tcq9_Ko(obtk?=ywh|QWcxfRK*<kBx+rTD%`>etIBI5q
zlh4CR6GUm^vm2z~{kLj;utRKX`R?f-ULpm0hn1@<`3s~_;sVY_JFlIw0Cy0K<pmcd
z5cWkTOK$v#Q&3i$6A#j@8Ub0r6+Zh7VbBU>{bI!hx=+Lj;S+LPG=th}_QfIJJZgTw
z-F=0dsr%g6>=;a@U`|-Bi@GU&>vp>k9yW6FO&yw_)#3u^Z=0Pld!!azJa+?vnZr2T
zKYQ(<D!CGd8Ff4>o8|0bn?_F(y@T8%obLU|w_iHuOw=hVp8doc!qVlL2RXhO@xKCP
zoT!M1?B{<lKoKLVT3gKTop|FI7X?U?o>mu1O%TM%5ph&FG3sPBGAvSngV|r%)Wq@w
zSbxzTLo)reU92`kL}}asExp|L8FKfCE8>^xxJ42P=+=x%0GMt8*-<WVy*Q2_LN`^8
z`9mt=mnyAj1})^q>oo!YViF-F`kUeQ9hl>f87B9LS3cXSiOQM}rtXIbQzbiebJ+Ye
zOp!i|wz~OiTnB<;u&MV#v)d#6$1F1U(B2fBGQa9fjElpmtYp0V*6+hgIb0kzsHMZk
z_W!OG#{)nJv}!Q9>1>5Didgy{Npq!7KGY#z#{jn-oZ)eg=>AGRh+QcmNUYA|Psy4Y
zD`X8z>)hBnoLTK5K<PrG#;!I5aqhQZQ9?(7vA&-|qm^mx?6+#>2i;9`??sB;mCW#W
zi5%VdeIEuvTxrBvXHP2|&^2sV9{khov|L*{kn&_C@GHoP$GO7LV`mY&afu$F0^kZ#
zj#XAPD!HFVWn@dkR-WiwGS-WfPemC+UqAHC5}N1iHYIupjPHo*jXM2^Q$%_>&;BD@
zIsHwl`q=2_dudivx=#ySfEl7Cwi_JMk(6Qb$xUdz$voC@UVG%4QG88-o_#k5&voy-
z2y4_sNPv_a3T*F+NZ;>X9>W6;|1<jZxOh<-i21&dfw0f=-}GzH6~lvG*ye}ff0SB!
z$hq`S55YO%d_x&#6rH(vFF60^kFd2xO_QRrk8mg?BvHR=FDY%s=gNSEhgB|cE*LMc
zcUk2JWx(fT(d8d1(iJFi_Q^Up)DrQ{4w7jDv1+lAzx;W6h(;w1T#T2+&pdUzu(|zm
zdBa5J`M(<(Ti9^vS5|&bKZwpOK4BOm;ULTM`J^{TZ7pmI6rmt?b@r&eU+yDVt=;dK
z2?^V82ukbNEjUlL&?+ElTpk9-8A-^&n#(|TCG2_6rPc0l&bYO0kcxqzu6eusk>Vda
zlbh`iw@d4D^*}vs{At~m&eZ3>d~C<R#~o!SU*>szgpwbBGqhB$w%ZVs)(3gzWAle{
zUzvSC?UsyRg5tDpiSqcvJ`>yVN3Ks#cWncj`L)(2NYO1VjG6@BwI5cFV2}7$u(3D!
zK~6k8{G>_s*mAnv*S4EvLYr5EF_nm4)BpSA%N6e;2y*=i6voQoUk&!vrOXbePOY3T
z>;I5++9Uh5)r7B#YYB*sWEztA5dub^0|eIHVZ8mXD_9yNA+PTRFEi)L2R7P32@K?;
zS#m+uUjAj{v(?e5f$b;x71ggEuLO_$;PI)0AT&nyUE>~7PC`;|{*U+`hfljzLoA=l
z%}c~3dO;E7a`miQ4+SDZ2IG`1RM9pZCTE>)J`Ss<esE{dF(QPM3~lR1JuE2oomDly
z>3jVHWxpyuCq=8>;~UFfqA%2aYl+=`N8=CM<e9He#!M0X{&nmA{(sr8iA3CscaIe2
zme9>idoE@e9&h@&Pb0T$uX$ZRFN|}Nz4@AQ;_Ecv4$}Ckw@yZ!i1+hvtYkRfYxi*>
zgwP?fLcl?L-41IeqF&#Js5ka$Y;t2-mk2bw{Q|qWMz?I$xc*dw^CM9F6C=475Et)o
z*VmApXsEgV2tF+Y#a_54%p=h;OP=*=w_b6sH=-3(Fby75ATAUeXlQ7*zfUepO}<Xg
zHGgE+&GjdF#YeDHhwd(@vqpOyAt@*^RrQT>o;BnOop#g`9~hS9$gwsP8>A``z?jHQ
z)w8j$pKQ73gPXp(77Iv{EB2_fU=+*v)OWc#f2N}!YjHlUTd3l?7}!4lCYIMZAD^PT
zYbx+os=2tnWC#Zu_=$I$2|<iX)M>f_TPqUmET4FQ<6VRXHXcO*TqqB`^C<hfT-46u
zch@hrVwxx0&lDtu36Ad=jvl{+<a7enfLm8rmN_<<bREwOeRAvP;(<zt?>jUSf@l&+
zc4nwSRPm=zcNi92;cXnHOHA##GI3TyZUDnLxXY7gO?uEOD{Ce%YW~`^S`EB6;l_7e
zysGw5CdEiJQ6W@VhwUeZWeoXg0VvU2(E9<|`oOId@sOugXgUZIN$`hEx0RF=^MW-V
zM6&z0fTrLO9#uBd#Z{t8C>gYGL!u4wR}d;`l^r3hPt(TswO1WR;_f2410E%%vwweq
zBX|UXTgYqn&yvqDy!#JipZ{9;J@~HM+3C}KR4~$&Gxg%HF$!N8Z?A^m#n5~2{O$rg
zgl}5hk7TdUb&CfC$)fzP-e^YtVxaVeq5G;kQ;w9jQfLt2Ex*6yppM7yHMFx6rNY(?
z-6q0j6q=o}<PcgfObs?da$*8Hze6gLx;^*H8ajoE46F*f$-c^<K*iC>V|kQWl$vPo
zCRoy>tS@l&AM?yycS-(aHtTE7-ixaPZz_Ztg;|n;_%`24Lyy9C4?%F7@Z-q0w)4lk
z?sd?iv;`_eiG1$+Cn7#1tDRwb&{y~6SJADZ=8bX)ZC+zJ>^~~`-Gw)_xLTy5^5)iZ
zp`T(s`J+u$2r{soh6zz9D_eXD)}Zaz{JnpCHuQ)dB4hb$<U_c<6ZQy_c=FAVyr<3F
zISj*V&?LF(S^M6K{e|g5x0-k-9%#V*G%*`L5lNa6fM8eGY-l4T=>LxP>BPQiHDnD6
z0If&k19T9dLPzx}+nN($BZP0ICTvYIcZtOCCKNN@_}E@>25TvJ2{XxknVkwtd9~Q*
zCqF)U;VDe2RxJ2Jr#A)zzdT46y&Adbx&K#uh8Veiq=b@CA+tK+L%zLm8wji~SPPx|
zZk!&W?eh(kXb!vnt!*xtO>Z+bbd|8a<!t|2UMTGnVR`E^9|W1d#SYleAiVTJ4F*^j
z_ek~)u^?Gf(|oY-8SKd|C668B9<r8J)os80YBkWrD^#|*@{b?Yut^OyR32T_tsJpc
z%j+pE7Z5v(z3(LOm<WOsB%U#p;}8h12Q!#xSN9SGYf!`N_<0Fq9C<k=RX*a+q&&@6
zIFxn6W8DafQmVTLI!1fe0$GA&Jn!UXY~NY30abw_)^tuZ#Q*i`fd9?pNeVmQ-bF_@
zZZ(3%IZTLIYM;0RQ33VH%!=7DOnvn;$I2AWG|_!0-sTemGNLSQ|9(pX*S;7G%R3;<
z)0?LZz%Y3K@$;x|3I`1d6My+QQ#*5i9$YpPms})*Ac3%1^d2b(r!{D8z*mArq>6p=
zGHR-WP#Fc-m*0?#5}TJ|ai5Urq_yLLm{teo$*Y|3rsL(af49X%u8LOa3^-H;?t@xF
z=>cm5d|~IF;kNF?`#S*XSC^V$*x?B|gh~lW;C%MnBtbVj1<{$Y@Q%$8MNtGX{Oga_
z;fFe3Y(-!8Rd+a2My8iPWsiTQ*+kU7aO8bhx1m?kHB8q~b%7`XA$z!F;^CE#w*aMQ
zP?n|`0YU}TzG}S|bA30TWqKbzlZVD<bYo{<$`&+ULP}Hlas+>;$a(~Qy}qKa*q8o>
z3B$X^I%Xx3@irm)Qlw^x=lp}_5_~+ga2R=i!(nuWE{f%BJVDy$)f1>@7ezCR^zhM1
z+p~d)9c(UV_uw0D{#h_!d2YF5lZpdp?aGb*M(<EtfdFK8{r)1ai@M`yBb4$=fh)~>
zx@D6qB3huCEW3vWd`3qw&=GqpFUw4$M5sA#VE^y-4t0+0EHTa`o0PMq3WPpdQXG*4
zjOtId*Eylrz^n2GP)6PB+9T2I`0?>Y{f&V$FOWm8aSosb92uPvjN%|RFim;wPW-sK
zC|xBGOlNV3eE8mf(I6A{Yu~9B!~5q7=*gQt@->#gdTMCg-WYLj!bxLYy6&yAx8jZc
zbWV?@k>x#~wpMo6ucR%CpnG19n^O|sqq%Yiw4XehS4bOmK!WmD7|4-(`UN!$f|$G3
zV!GM4NS-K7ba}3P%}@SE|6e+r7>MPutZ4Ja%)KKhsj7lTg&+jG&lV;sV*xz?9hH&%
zny4Yk4kNaIp4~aGP_Prrr<S^H5cg-xu<AxH5^wWmhWK|4ekHrbI?oVUT?Wg+RURxU
zUkryBzNNtu{lK-IbGk0DYoS_uwiUA<G@;Eh_dDv!{^YX$puI=P12S1eAMcmGW;Imz
z*vDdm?<qq!>(|9v+(Cn3;@Y1PjVZ}>9@%{Y&?(9Vf{%TxpV6RYsWc$tkCvYTs$-5t
z5?HGoh5^|h<x3YgpDARc@BZb^-i-N)sgRw}9%%$^6~U9o*RC1uI;g$tS*j2ea~odt
z<F5>3>30JmaH-u^108WKz6+9@dDg>Vg#EZi;-SUF#8G0+oEzxy$|tPteNGbGV!;Co
zx>b0Bsx){N!cRy`q+2mtz^eES6u5jJtHCm6s-Cr<Ajyp8-`oryr5a>Y(}V?FanfGb
zMF@QFS47^XH7VQQ?`u~6!Q0Zm9Ei6k4YfxXE5`?o(7f+sh@@-I=dBbc7JdP8euviy
z!)lH3I33(Qd|7Mx{q=-s=vNoV09~CJ{cG~={yz%l#DZ|@An)P(o|}JmaCJh;hYSm@
zr|E1~e(n?r+3DfD52hM-1k=*56Q4JcUf?ANqa^X3-?_=Z2`UTVE;rr9_m6OV+Og{N
z4oZ#0UB>K@#mo+T8{<utt$tmgNq8M~$^+t(_pG@_)shbv7A9x|jE$l|Z^HP>%`DdE
z*=eG+*#f9DA{<LMEj_S8poe~jpzi=L@8?Fw`=>2puG-Bn5>rHI*m@21XC}I^AeIBy
zAStYaYI?~9qqm<*LEaso3a<M|yVLXpm#S>Gr)Q^kdS&9W3@2zQ#usxYT#I1eZ{gei
zwJ-gNaf;{fea%iUndG`C13B^af3}HkIeu{7&5Uv;e-$QAqsj%D-4^Z(ByUkU&lfdW
zqJ6$z-xiK&v9<Y~U&;iNLXIz;@j)}6qCxkyW&CWk-5juLZKV)W>vNt`$5df@7zRU%
z4(Zu3>i<?ULN#P)p$|b>+#WL?#J;9@P%5a``fsH71aU|cL<*ogp!%4lMGxN+kIbbQ
zmGoPznGpSa&5^uiG_9`6a={)lo_zr_N=2RJn>kw*%Qx&uet#+zgrNILFPfD(g3mk5
zwY#v@^L@wS;7hJTatK@^uAW^8Jtu+pQBN=!&RKED`&C>j41vkPfz^6VG&8<$ANluF
zgZFCHoc9+qqqb%U8o$0Y<5Zn$r?7v<4l-wP)*RQJ9Qp(6tHKDCltF06+nct>%;5vf
z8fP}C^`h|HQ}7|@2M|n|qYi4&hUdg{_O}h)M4jccawfE`n?l{!v#}pSL{n^XVZ@+O
zmNR?PM!DKBXw2@4w9i8an+xwT%rFRDB)x}|ARa@n63`&DK~wj>aC9XQM+o50or-M+
zX6znb9K3x@VvPb0r+09Stu$^rP$Z+esy1NkGTAvbK@@$B)`r`hyUQL3X>bl`7~`i8
zFA@~~hy%n}*cdv_&eYgOl78r6g-O{uD$bBLTgwrA*T-u|c=(;ahIuI(uPby=>LM!Z
zz#?AqSauc<We?*aqD2gr9YDs2-z32tWWdxWTQ4`0q?(=g(%XosK1q49#9+5V8lKM!
zr1C~wm_78U%7X@xc#G+{Lcrg7goP*joJYZ5H86g_RDglK7*XMrC%MO}C{<pS<J;)S
zyZD~C8%B38GlDi}42ET{BBJBNTGTL`;|LY)9c^(yjrGNqv4c)yVvcz|FObmEq;<S*
zX~yD&#|cQ>vHy+!HDU&s?fLYB{jMRskWg=$SuU&cEnRvbxi9@^6!bx8Vc`QHyI({0
ziXZ5;z6y_0WnZz!>x*W8u#Wi$&_K+Yop8M?^-@&DGH9?+*3o6afBR-H3pxXK2&Azu
z6E_R@X6(K9rgGBup;LHE!Cl0=CkiVDO^_hpxKlDzc%11F=^C2P4%|PfhM{%mj>xjg
zxp-JCH+Aive7!dzofb-j<p+~n@xRB8t|^@6umY0dcR5OO-z3}NM_cLhu*epRE`hqD
zyxG~UcTk=kUU49r?WQp^pWmZt-EXRLq%M#vT)Ivc^mELhiSvJBOU``uzz2&kGaO3O
zYr-nuajz8HXEI1b66cC1Wc0ggsa|%!CYC6R6@NN!R0p!5zu^Zx@;xX@N0)ZeYs77w
zivaj}WQ%Cbqa;+{Qa2~W1$-J9?@IoxQZ8{m;oI@W0Fp6ZBpHsZm(5TTeNu~`eUSOY
zDwpZ9mBP2@JWMt3nFjxb7SvgvUZ!6SfKcZ<)K^DYTW@1$L4||05<llBWONla4ooc=
zA+fse*#6S+x*yfR<R1=AnCDRbmAZCg3Atyjw)hzdP<(82KN#T8@YD+TPH+X&HkuQ%
ze8KgXt~0eT;0ef1hxj|ZZT@0s{lNTB_CnMa@j(pGSpf`2{chs(Dq%rz`O`$HOM#I$
z2B)61dj2k}d+<^Xp!?dDh%An-UDLj>;4v`i^R|m_Y9!~!Rs+R<lU#$X4Rcki#n_3{
z>2?I#`BA3FENs}^j0}4Pj~GI0)3Kyuc>@3rWPy<;tcCtuPuIh>qwDnWTR%J0$4k5+
z*TuXPB1{T79}$YFy9efqv<XEmH;+I>*=$epoLc}=OwdmD2@_oo0t3=uV~gTWg`fv1
z;(K;Dj>HHe4)vj^Cqeo#vIHE8_>EI1Y&x161U!P?c_*lMz-g3!M|%vQV<rC^5<)(a
zh{iS7a<62>PAGEHmB2}}hrS$S)(VVq6X@U>&Kncp*pbl_7VPRq$Uzt(c&c)zkaZi5
z=EL)nWqs*t{LaFEw)VLvAS(p-Vxxq~F)v^e+QSdAxtIFozUI^M+oYz9gPs{Y(l*Ut
z^x%PB;q@SLlZi{Lw|r%$gq8bEDa6}jknD50L)Hn6#FO>)8q8Sx%}cZQtlSge<?SQ$
zF5B87eL}_(fR%a}EM-zCz-9aWF>b;z8!-U8|3Jd}^n<~qe+sE7{$L}YZkapICRg}v
zj<q8oC*mZ%WdFBNP1p(%jrO90M8a=tLl5>JRJ1q)h=nsyd)<#lmQsjWSvk=mI^?9W
zyzqd(DvSoq`{mp~1;5)bd~w;}oEzVjH=Sd;WnxK!SIl9q2}yh&-c-AJi=>P3kZ=86
z&L~1vba3)oaqZ(1Lyuj%$aiP+Nc62-!8cwXuk0=~=q6y=#@R5&X6Igw!;_mHBDsX0
zA0<D{gC66L-XEmshEYnO)8F_0*FbaDqyd!X+^VZTfgUj;+82Z3y;(T3t!yP)*2?>m
zgO#3mTpMmRfZCK*2MdQO*KEv19^plfYC;R@zg+G!$z^>O7s~^Sc%IZJoq1?9uNjUt
z#Pc>%6k7Jfo#bs{+2z1^X2cNvk|teiAcP+=%+Bw=c+!R>S4d_4Z|aH~z8?p~74>MN
zPUQpsK-@)<da;4`y#~ogI$X8n3R+-Kr1&3bO7G3<GY&#k6onThxdI`hj_<bmrQ%?i
zlor5BrC}`ANlUo*wOKLiv(J%49HA_-lGq?9u(E<iyW=AgTb)#R%u^Y5-+c2Ln)E?R
z`dR2ev>WHvdpv9wPG3dGy$iDYYV>iCa{j-zhF+K$y8?b3n9RtUe$l4W&_jNNN-3lp
zfhK+UMtpB+|A|%vbOvFE`)?^*bds2x22|L<sZSF@l0=)s>Tk@87^i7!tOme*(N$HC
zyie<H`L_IUoKkBi0?m8>y5H%ap}yqXT|M{UBepA-Iad%_#9e(Xj!9Yzt@1LigdAV-
zJyJdeW2Xe@-;gmF?Nv`lB<6uabJ5%WCXaY;Mlx1<H1A8(1W8!R7ZMk74P+IefyJfQ
z+9@~ptZ)LzrvlJmp8}ZDZx~{%BWMAQl*-8KOYq}jZn3l5#fuD$1`z4TUKb5@TTJHV
z78e*dn1s*HvZKMWxpmH%%KnafiW7MsO@yXmIy(I)SKWQWPp}BX`~cL#tfG5{g|K4{
zZq$L{MCdq?Pp(fslW$2vufm$Sl;_sHCMB9Dlf_)py=mO(gKBl^yO2v6w&>YQVH;u1
z{K^(8=v?A`Cp*2ML~nZct3mn0qSgW#ZnQBwD0&#@idf4d#^bv4XjEX)@mM5}>aJOu
ze?-||5}42kICp9We4N48Ud!%@XZ+M+ZG|hOr|5}TWn*?wq(X!JN=O`4;KIPy;6#$3
zKNR%PpYrn&8*>TJ%mrRLQIxWD9qJg4rb+r?^NP)3S<aVCq_zcK^_($<k+;n9Wt$&C
z#Am$U`pxoLpS4|*FXS`tqVH$kh^%808G|rD*hob%YvU9X=x^A6Y&~JYdmVJFwBUdF
zfis1lK|~V%D>proH%1)p+eYoN2eLZ3U`;uG5<lkA{wg<r^}$ovhj#NyZ?<fa2hM1J
ztDB=?_;ivMibCZoNb<;uP2Mqk;DK;_4NKi_nMGxLr)pUg)-^u8(NsHMNzyoB$6Nk4
zELG`R4tBY$+Im_bfjvx;jT4SmZIK0uH+*oR0k)#1haQNY|F8nL$+eU^gla#K9MYcn
z17U*NwCQa7rGZAD5^uIBIq(Yw&Klb5&x5R-kv$K=rE!~vNIMK>8af^i3VFgKRJR6W
ziO?)uI2==%x@)G3;qH$v>~-K6G?YU)TF3~tAIEg>aBKbH$^#`Qwf}~Gy-kQNjVXy}
zqNufPm=c{zaW=-zeht14g318Z?PXBHLN47>I%oAkNZBG8-e<FA=j79y!yILelG4u*
zuYO(Ov4Ul7T7bc6zl$n;)Q|>|h&qZEfqsPWBj^e$wz1q+GHp_u3k5&=@Zz;oF2fHE
zxa(^~T?`ii(((tqftX9!@9;`+Y0%V~9!^0yy%Vm0xLm}$fdsz_!D!s&LCnh-(<8xn
zbnouwoiFPq(XO53L4clVpiBrGjk#}Jl!e<I?!}_DGz!!>4P=TVOb@>GVYYmB{D$4k
zO4@LqdK*|N=yec#;d$%@Y0EwpRae-?ZH4aAi^SxY4Ss>5(E|J?5`-qapJD`iE?LyO
z7MFcPlJ>*Co1rd#w1@87D$}<mh2wuE0W{qpuYrb@Mhb4LFEJNm1%IM7h(#{6KsW=W
zt`|h#p2|t!;PxQCYf082Gn<Ya8XSu;3Ah_EP|fpP^hIoH1z{4^De8jFB;}^4ate(1
z+GL}LxYi8UA+7v4CeCtv-t2uxYU(i2vgf$F3}If_FtO%CYV#|~DbJmRJq$<7DPMc|
z*hO->U!#YzHJJ}u|IG=-JHQ>KZxiysQXzBh{SCgygyWR0*Cr+SVTN`wRsR=F0kZyb
zQ}&!qJWHRb<M@F^6aZ<<X7qBky2cyZ91q|)BRJlOj1^;sf&b3Bqe_5vMzGP?SZ821
zh-KE5WzMhC4n`F2_`yUJMB@r>>fuA|SYz3vVKCZ=84Zz92QNn)48YRP=m2{FY!Z<&
z+csqo*(8=JbG7)&co`m1@UQ?7DM@>KMy#DE!z|kx46%n=kf9I^HZVg4l4`y@*PtJO
zIPI)m)C!;qKqY|P0_?J3c8FNp1ojTGq`bkhwwIqZr%5{~Q8Z!!Aab}dOK$7w^*0X`
zpq~i(Sw@nKL9zjoEg;#5NrFKySV#uMEe1Ue5DOqdfCPN`-v9su!~jTaXH)e8_`deo
zR(yG#e4_v@0P2hwzd&k?kwyTu0@NDVH3HPyf?6?BBes7}v)f;}ee(S^+V33^weSA}
X)1C$M`J{G100000NkvXXu0mjf8T-ry

literal 0
HcmV?d00001

diff --git a/web/src/app/api/[...path]/route.ts b/web/src/app/api/[...path]/route.ts
index 6ca13aba146..8c3d4522be4 100644
--- a/web/src/app/api/[...path]/route.ts
+++ b/web/src/app/api/[...path]/route.ts
@@ -75,9 +75,13 @@ async function handleRequest(request: NextRequest, path: string[]) {
       backendUrl.searchParams.append(key, value);
     });
 
+    // Create a new headers object, omitting the 'connection' header
+    const headers = new Headers(request.headers);
+    headers.delete('connection');
+
     const response = await fetch(backendUrl, {
       method: request.method,
-      headers: request.headers,
+      headers: headers,
       body: request.body,
       signal: request.signal,
       // @ts-ignore
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 8a7179a0640..30424644c07 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -75,6 +75,7 @@ import slackIcon from "../../../public/Slack.png";
 import s3Icon from "../../../public/S3.png";
 import r2Icon from "../../../public/r2.png";
 import salesforceIcon from "../../../public/Salesforce.png";
+import freshdeskIcon from "../../../public/Freshdesk.png";
 
 import sharepointIcon from "../../../public/Sharepoint.png";
 import teamsIcon from "../../../public/Teams.png";
@@ -1301,6 +1302,13 @@ export const AsanaIcon = ({
   className = defaultTailwindCSS,
 }: IconProps) => <LogoIcon size={size} className={className} src={asanaIcon} />;
 
+export const FreshdeskIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => (
+  <LogoIcon size={size} className={className} src={freshdeskIcon} />
+);
+
 /* 
 EE Icons
 */
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index d722fcf9848..dd06a3436a1 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -922,6 +922,12 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
     ],
     advanced_values: [],
   },
+  freshdesk: {
+    description: "Configure Freshdesk connector",
+    values: [],
+    advanced_values: [],
+  },
+  
 };
 export function createConnectorInitialValues(
   connector: ConfigurableSources
@@ -1180,6 +1186,11 @@ export interface AsanaConfig {
   asana_team_id?: string;
 }
 
+export interface FreshdeskConfig {
+  requested_objects?: string[];
+}
+
+
 export interface MediaWikiConfig extends MediaWikiBaseConfig {
   hostname: string;
 }
diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts
index d7bcef0adaf..de6b6094a40 100644
--- a/web/src/lib/connectors/credentials.ts
+++ b/web/src/lib/connectors/credentials.ts
@@ -186,6 +186,12 @@ export interface AxeroCredentialJson {
   axero_api_token: string;
 }
 
+export interface FreshdeskCredentialJson {
+  freshdesk_domain: string;
+  freshdesk_password: string;
+  freshdesk_api_key: string;
+}
+
 export interface MediaWikiCredentialJson {}
 export interface WikipediaCredentialJson extends MediaWikiCredentialJson {}
 
@@ -289,6 +295,11 @@ export const credentialTemplates: Record<ValidSources, any> = {
     access_key_id: "",
     secret_access_key: "",
   } as OCICredentialJson,
+  freshdesk: {
+    freshdesk_domain: "",
+    freshdesk_password: "",
+    freshdesk_api_key: "",
+  } as FreshdeskCredentialJson,
   xenforo: null,
   google_sites: null,
   file: null,
@@ -435,6 +446,11 @@ export const credentialDisplayNames: Record<string, string> = {
   // Axero
   base_url: "Axero Base URL",
   axero_api_token: "Axero API Token",
+
+  // Freshdesk
+  freshdesk_domain: "Freshdesk Domain",
+  freshdesk_password: "Freshdesk Password",
+  freshdesk_api_key: "Freshdesk API Key",
 };
 export function getDisplayNameForCredentialKey(key: string): string {
   return credentialDisplayNames[key] || key;
diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts
index 18bc3336ae7..3c1cebe008a 100644
--- a/web/src/lib/sources.ts
+++ b/web/src/lib/sources.ts
@@ -37,6 +37,7 @@ import {
   GoogleStorageIcon,
   ColorSlackIcon,
   XenforoIcon,
+  FreshdeskIcon,
 } from "@/components/icons/icons";
 import { ValidSources } from "./types";
 import {
@@ -289,6 +290,12 @@ const SOURCE_METADATA_MAP: SourceMap = {
     displayName: "Ingestion",
     category: SourceCategory.Other,
   },
+  freshdesk: {
+    icon: FreshdeskIcon,
+    displayName: "Freshdesk",
+    category: SourceCategory.CustomerSupport,
+    docs: "https://docs.danswer.dev/connectors/freshdesk",
+  },
   // currently used for the Internet Search tool docs, which is why
   // a globe is used
   not_applicable: {
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index d4f43a024d6..d92b6a0d139 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -263,6 +263,7 @@ const validSources = [
   "oci_storage",
   "not_applicable",
   "ingestion_api",
+  "freshdesk",
 ] as const;
 
 export type ValidSources = (typeof validSources)[number];

From 143da5bc0d6a49c4e58a904818e31cb38604a00d Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 23 Oct 2024 10:26:54 -0700
Subject: [PATCH 183/376] add copying for unrecognized languages (#2883)

* add copying for unrecognized languages

* k
---
 web/src/app/chat/lib.tsx               |  1 -
 web/src/app/chat/message/CodeBlock.tsx |  1 +
 web/src/app/chat/message/codeUtils.ts  | 24 +++++++++++++++++++++++-
 3 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx
index 3fbb9397abc..38fdac037a6 100644
--- a/web/src/app/chat/lib.tsx
+++ b/web/src/app/chat/lib.tsx
@@ -348,7 +348,6 @@ export function getCitedDocumentsFromMessage(message: Message) {
 }
 
 export function groupSessionsByDateRange(chatSessions: ChatSession[]) {
-  console.log(chatSessions);
   const today = new Date();
   today.setHours(0, 0, 0, 0); // Set to start of today for accurate comparison
 
diff --git a/web/src/app/chat/message/CodeBlock.tsx b/web/src/app/chat/message/CodeBlock.tsx
index c1f2f99397c..5ab6b73b56e 100644
--- a/web/src/app/chat/message/CodeBlock.tsx
+++ b/web/src/app/chat/message/CodeBlock.tsx
@@ -116,6 +116,7 @@ export const CodeBlock = memo(function CodeBlock({
           {codeText && <CopyButton />}
         </div>
       )}
+
       <CodeContent />
     </div>
   );
diff --git a/web/src/app/chat/message/codeUtils.ts b/web/src/app/chat/message/codeUtils.ts
index d2aad299044..a9cd13944f0 100644
--- a/web/src/app/chat/message/codeUtils.ts
+++ b/web/src/app/chat/message/codeUtils.ts
@@ -1,3 +1,5 @@
+import React from "react";
+
 export function extractCodeText(
   node: any,
   content: string,
@@ -32,7 +34,27 @@ export function extractCodeText(
     codeText = formattedCodeLines.join("\n").trim();
   } else {
     // Fallback if position offsets are not available
-    codeText = children?.toString() || null;
+    const extractTextFromReactNode = (node: React.ReactNode): string => {
+      if (typeof node === "string") return node;
+      if (typeof node === "number") return String(node);
+      if (!node) return "";
+
+      if (React.isValidElement(node)) {
+        const children = node.props.children;
+        if (Array.isArray(children)) {
+          return children.map(extractTextFromReactNode).join("");
+        }
+        return extractTextFromReactNode(children);
+      }
+
+      if (Array.isArray(node)) {
+        return node.map(extractTextFromReactNode).join("");
+      }
+
+      return "";
+    };
+
+    codeText = extractTextFromReactNode(children);
   }
 
   return codeText || "";

From 85b56e39c9352599b8592359c0921ac7a4d1654f Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Wed, 23 Oct 2024 14:01:03 -0400
Subject: [PATCH 184/376] Fix Freshdesk connector date parsing for UTC
 timestamps

---
 backend/danswer/connectors/freshdesk/connector.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index be2b1e0e3ea..9173ba34c9a 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -13,10 +13,7 @@
 
 
 class FreshdeskConnector(PollConnector):
-    def __init__(self, api_key: str, domain: str, password: str, batch_size: int = INDEX_BATCH_SIZE) -> None:
-        self.api_key = api_key
-        self.domain = domain
-        self.password = password
+    def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
 
     def ticket_link(self, tid: int) -> str:
@@ -63,6 +60,8 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
             for ticket in tickets:
                 # Convert the "created_at", "updated_at", and "due_by" values to ISO 8601 strings
                 for date_field in ["created_at", "updated_at", "due_by"]:
+                    if ticket[date_field].endswith('Z'):
+                        ticket[date_field] = ticket[date_field][:-1] + '+00:00'
                     ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")
 
                 # Convert all other values to strings
@@ -113,4 +112,4 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
                 yield doc_batch
 
     def poll_source(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
-        yield from self._process_tickets(start, end)
\ No newline at end of file
+        yield from self._process_tickets(start, end)

From 7abbfa37bb8a0831b71b7917af461b5b1ca0714f Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Wed, 23 Oct 2024 12:57:00 -0700
Subject: [PATCH 185/376] Tiny confluence fix (#2885)

* Tiny confluence fix

* Update utils.py

---------

Co-authored-by: pablodanswer <pablo@danswer.ai>
---
 backend/danswer/connectors/confluence/utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/connectors/confluence/utils.py b/backend/danswer/connectors/confluence/utils.py
index 029e35e6538..beb0465be60 100644
--- a/backend/danswer/connectors/confluence/utils.py
+++ b/backend/danswer/connectors/confluence/utils.py
@@ -37,7 +37,7 @@ def get_user_email_from_username__server(
 
 
 _USER_NOT_FOUND = "Unknown Confluence User"
-_USER_ID_TO_DISPLAY_NAME_CACHE: dict[str, str] = {}
+_USER_ID_TO_DISPLAY_NAME_CACHE: dict[str, str | None] = {}
 
 
 def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str:
@@ -67,7 +67,7 @@ def _get_user(confluence_client: OnyxConfluence, user_id: str) -> str:
 
         _USER_ID_TO_DISPLAY_NAME_CACHE[user_id] = found_display_name
 
-    return _USER_ID_TO_DISPLAY_NAME_CACHE.get(user_id, _USER_NOT_FOUND)
+    return _USER_ID_TO_DISPLAY_NAME_CACHE.get(user_id) or _USER_NOT_FOUND
 
 
 def extract_text_from_confluence_html(

From 786a46cbd02178945a2ca17ba81e77a56cdec987 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 23 Oct 2024 12:59:14 -0700
Subject: [PATCH 186/376] sticky credential description (#2886)

---
 .../app/admin/connectors/[connector]/AddConnectorPage.tsx | 4 +---
 .../components/credentials/actions/ModifyCredential.tsx   | 8 ++++----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index cc8173082da..c2e903e2776 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { FetchError, errorHandlingFetcher } from "@/lib/fetcher";
+import { errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR, { mutate } from "swr";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 
@@ -40,8 +40,6 @@ import {
   useGoogleDriveCredentials,
 } from "./pages/utils/hooks";
 import { Formik } from "formik";
-import { AccessTypeForm } from "@/components/admin/connectors/AccessTypeForm";
-import { AccessTypeGroupSelector } from "@/components/admin/connectors/AccessTypeGroupSelector";
 import NavigationRow from "./NavigationRow";
 import { useRouter } from "next/navigation";
 export interface AdvancedConfig {
diff --git a/web/src/components/credentials/actions/ModifyCredential.tsx b/web/src/components/credentials/actions/ModifyCredential.tsx
index 8dd801e45af..01433f33443 100644
--- a/web/src/components/credentials/actions/ModifyCredential.tsx
+++ b/web/src/components/credentials/actions/ModifyCredential.tsx
@@ -1,4 +1,4 @@
-import React, { useState, useEffect } from "react";
+import React, { useState } from "react";
 import { Modal } from "@/components/Modal";
 import { Button, Text, Badge } from "@tremor/react";
 import { ValidSources } from "@/lib/types";
@@ -54,9 +54,9 @@ const CredentialSelectionTable = ({
   };
 
   return (
-    <div className="w-full overflow-auto">
+    <div className="w-full max-h-[50vh] overflow-auto">
       <table className="w-full text-sm border-collapse">
-        <thead>
+        <thead className="sticky top-0 w-full">
           <tr className="bg-gray-100">
             <th className="p-2 text-left font-medium text-gray-600"></th>
             <th className="p-2 text-left font-medium text-gray-600">ID</th>
@@ -70,7 +70,7 @@ const CredentialSelectionTable = ({
         </thead>
 
         {allCredentials.length > 0 && (
-          <tbody>
+          <tbody className="w-full">
             {allCredentials.map((credential, ind) => {
               const selected = currentCredentialId
                 ? credential.id == (selectedCredentialId || currentCredentialId)

From 8b72264535826aa3366872f1e1b32970bb708389 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 23 Oct 2024 13:20:20 -0700
Subject: [PATCH 187/376] Gating Notifications (#2868)

* functional notifications

* typing

* minor

* ports

* nit

* verify functionality

* pretty
---
 backend/danswer/configs/constants.py          |  1 +
 backend/danswer/db/notification.py            |  5 +-
 backend/danswer/server/settings/api.py        | 34 ++++++++----
 backend/ee/danswer/server/tenants/api.py      |  8 ++-
 backend/ee/danswer/server/tenants/models.py   |  2 +
 web/src/app/admin/settings/interfaces.ts      |  1 +
 web/src/app/auth/logout/route.ts              |  4 +-
 web/src/app/auth/signup/page.tsx              |  4 +-
 web/src/app/layout.tsx                        | 14 -----
 web/src/components/admin/Layout.tsx           |  1 -
 .../components/header/AnnouncementBanner.tsx  | 53 ++++++++++---------
 web/src/lib/constants.ts                      |  2 +-
 12 files changed, 71 insertions(+), 58 deletions(-)

diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 3c9a7cffcc2..2c86c7f0547 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -136,6 +136,7 @@ class DocumentSource(str, Enum):
 class NotificationType(str, Enum):
     REINDEX = "reindex"
     PERSONA_SHARED = "persona_shared"
+    TRIAL_ENDS_TWO_DAYS = "two_day_trial_ending"  # 2 days left in trial
 
 
 class BlobType(str, Enum):
diff --git a/backend/danswer/db/notification.py b/backend/danswer/db/notification.py
index bd58add14a0..a6cdf989177 100644
--- a/backend/danswer/db/notification.py
+++ b/backend/danswer/db/notification.py
@@ -4,6 +4,7 @@
 from sqlalchemy.orm import Session
 from sqlalchemy.sql import func
 
+from danswer.auth.schemas import UserRole
 from danswer.configs.constants import NotificationType
 from danswer.db.models import Notification
 from danswer.db.models import User
@@ -54,7 +55,9 @@ def get_notification_by_id(
     notif = db_session.get(Notification, notification_id)
     if not notif:
         raise ValueError(f"No notification found with id {notification_id}")
-    if notif.user_id != user_id:
+    if notif.user_id != user_id and not (
+        notif.user_id is None and user is not None and user.role == UserRole.ADMIN
+    ):
         raise PermissionError(
             f"User {user_id} is not authorized to access notification {notification_id}"
         )
diff --git a/backend/danswer/server/settings/api.py b/backend/danswer/server/settings/api.py
index 35ab0b12c6d..7ff4de55c2a 100644
--- a/backend/danswer/server/settings/api.py
+++ b/backend/danswer/server/settings/api.py
@@ -53,7 +53,7 @@ def fetch_settings(
     """Settings and notifications are stuffed into this single endpoint to reduce number of
     Postgres calls"""
     general_settings = load_settings()
-    user_notifications = get_reindex_notification(user, db_session)
+    settings_notifications = get_settings_notifications(user, db_session)
 
     try:
         kv_store = get_kv_store()
@@ -63,20 +63,29 @@ def fetch_settings(
 
     return UserSettings(
         **general_settings.model_dump(),
-        notifications=user_notifications,
+        notifications=settings_notifications,
         needs_reindexing=needs_reindexing,
     )
 
 
-def get_reindex_notification(
+def get_settings_notifications(
     user: User | None, db_session: Session
 ) -> list[Notification]:
-    """Get notifications for the user, currently the logic is very specific to the reindexing flag"""
+    """Get notifications for settings page, including product gating and reindex notifications"""
+    # Check for product gating notification
+    product_notif = get_notifications(
+        user=None,
+        notif_type=NotificationType.TRIAL_ENDS_TWO_DAYS,
+        db_session=db_session,
+    )
+    notifications = [Notification.from_model(product_notif[0])] if product_notif else []
+
+    # Only show reindex notifications to admins
     is_admin = is_user_admin(user)
     if not is_admin:
-        # Reindexing flag should only be shown to admins, basic users can't trigger it anyway
-        return []
+        return notifications
 
+    # Check if reindexing is needed
     kv_store = get_kv_store()
     try:
         needs_index = cast(bool, kv_store.load(KV_REINDEX_KEY))
@@ -84,12 +93,12 @@ def get_reindex_notification(
             dismiss_all_notifications(
                 notif_type=NotificationType.REINDEX, db_session=db_session
             )
-            return []
+            return notifications
     except KvKeyNotFoundError:
         # If something goes wrong and the flag is gone, better to not start a reindexing
         # it's a heavyweight long running job and maybe this flag is cleaned up later
         logger.warning("Could not find reindex flag")
-        return []
+        return notifications
 
     try:
         # Need a transaction in order to prevent under-counting current notifications
@@ -107,7 +116,9 @@ def get_reindex_notification(
             )
             db_session.flush()
             db_session.commit()
-            return [Notification.from_model(notif)]
+
+            notifications.append(Notification.from_model(notif))
+            return notifications
 
         if len(reindex_notifs) > 1:
             logger.error("User has multiple reindex notifications")
@@ -118,8 +129,9 @@ def get_reindex_notification(
         )
 
         db_session.commit()
-        return [Notification.from_model(reindex_notif)]
+        notifications.append(Notification.from_model(reindex_notif))
+        return notifications
     except SQLAlchemyError:
         logger.exception("Error while processing notifications")
         db_session.rollback()
-        return []
+        return notifications
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index 0438772486a..9ee598a2e26 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -8,6 +8,7 @@
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.db.engine import get_session_with_tenant
+from danswer.db.notification import create_notification
 from danswer.server.settings.store import load_settings
 from danswer.server.settings.store import store_settings
 from danswer.setup import setup_danswer
@@ -87,12 +88,17 @@ def gate_product(
     1) User has ended free trial without adding payment method
     2) User's card has declined
     """
-    token = current_tenant_id.set(current_tenant_id.get())
+    tenant_id = product_gating_request.tenant_id
+    token = current_tenant_id.set(tenant_id)
 
     settings = load_settings()
     settings.product_gating = product_gating_request.product_gating
     store_settings(settings)
 
+    if product_gating_request.notification:
+        with get_session_with_tenant(tenant_id) as db_session:
+            create_notification(None, product_gating_request.notification, db_session)
+
     if token is not None:
         current_tenant_id.reset(token)
 
diff --git a/backend/ee/danswer/server/tenants/models.py b/backend/ee/danswer/server/tenants/models.py
index 32642ecfcda..30f656c0824 100644
--- a/backend/ee/danswer/server/tenants/models.py
+++ b/backend/ee/danswer/server/tenants/models.py
@@ -1,5 +1,6 @@
 from pydantic import BaseModel
 
+from danswer.configs.constants import NotificationType
 from danswer.server.settings.models import GatingType
 
 
@@ -15,6 +16,7 @@ class CreateTenantRequest(BaseModel):
 class ProductGatingRequest(BaseModel):
     tenant_id: str
     product_gating: GatingType
+    notification: NotificationType | None = None
 
 
 class BillingInformation(BaseModel):
diff --git a/web/src/app/admin/settings/interfaces.ts b/web/src/app/admin/settings/interfaces.ts
index 2c18a2c8262..38959fc8cd2 100644
--- a/web/src/app/admin/settings/interfaces.ts
+++ b/web/src/app/admin/settings/interfaces.ts
@@ -18,6 +18,7 @@ export interface Settings {
 export enum NotificationType {
   PERSONA_SHARED = "persona_shared",
   REINDEX_NEEDED = "reindex_needed",
+  TRIAL_ENDS_TWO_DAYS = "two_day_trial_ending",
 }
 
 export interface Notification {
diff --git a/web/src/app/auth/logout/route.ts b/web/src/app/auth/logout/route.ts
index e3bae04bb22..cd731810ca3 100644
--- a/web/src/app/auth/logout/route.ts
+++ b/web/src/app/auth/logout/route.ts
@@ -1,4 +1,4 @@
-import { CLOUD_ENABLED } from "@/lib/constants";
+import { NEXT_PUBLIC_CLOUD_ENABLED } from "@/lib/constants";
 import { getAuthTypeMetadataSS, logoutSS } from "@/lib/userSS";
 import { NextRequest } from "next/server";
 
@@ -13,7 +13,7 @@ export const POST = async (request: NextRequest) => {
   }
 
   // Delete cookies only if cloud is enabled (jwt auth)
-  if (CLOUD_ENABLED) {
+  if (NEXT_PUBLIC_CLOUD_ENABLED) {
     const cookiesToDelete = ["fastapiusersauth", "tenant_details"];
     const cookieOptions = {
       path: "/",
diff --git a/web/src/app/auth/signup/page.tsx b/web/src/app/auth/signup/page.tsx
index f44b53247ec..5a90535df59 100644
--- a/web/src/app/auth/signup/page.tsx
+++ b/web/src/app/auth/signup/page.tsx
@@ -8,10 +8,8 @@ import {
 } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 import { EmailPasswordForm } from "../login/EmailPasswordForm";
-import { Card, Title, Text } from "@tremor/react";
+import { Text } from "@tremor/react";
 import Link from "next/link";
-import { Logo } from "@/components/Logo";
-import { CLOUD_ENABLED } from "@/lib/constants";
 import { SignInButton } from "../login/SignInButton";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index 41611266f61..dbf86e3bb34 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -174,20 +174,6 @@ export default async function RootLayout({
             process.env.THEME_IS_DARK?.toLowerCase() === "true" ? "dark" : ""
           }`}
         >
-          {productGating === GatingType.PARTIAL && (
-            <div className="fixed top-0 left-0 right-0 z-50 bg-warning-100 text-warning-900 p-2 text-center">
-              <p className="text-sm font-medium">
-                Your account is pending payment!{" "}
-                <a
-                  href="/admin/cloud-settings"
-                  className="font-bold underline hover:text-warning-700 transition-colors"
-                >
-                  Update your billing information
-                </a>{" "}
-                or access will be suspended soon.
-              </p>
-            </div>
-          )}
           <UserProvider>
             <ProviderContextProvider>
               <SettingsProvider settings={combinedSettings}>
diff --git a/web/src/components/admin/Layout.tsx b/web/src/components/admin/Layout.tsx
index 97fc6343285..2b1cef8cc2c 100644
--- a/web/src/components/admin/Layout.tsx
+++ b/web/src/components/admin/Layout.tsx
@@ -27,7 +27,6 @@ export async function Layout({ children }: { children: React.ReactNode }) {
 
   const authTypeMetadata = results[0] as AuthTypeMetadata | null;
   const user = results[1] as User | null;
-  console.log("authTypeMetadata", authTypeMetadata);
   const authDisabled = authTypeMetadata?.authType === "disabled";
   const requiresVerification = authTypeMetadata?.requiresVerification;
 
diff --git a/web/src/components/header/AnnouncementBanner.tsx b/web/src/components/header/AnnouncementBanner.tsx
index 19b461a1e48..1312e321bdb 100644
--- a/web/src/components/header/AnnouncementBanner.tsx
+++ b/web/src/components/header/AnnouncementBanner.tsx
@@ -32,7 +32,7 @@ export function AnnouncementBanner() {
   const handleDismiss = async (notificationId: number) => {
     try {
       const response = await fetch(
-        `/api/settings/notifications/${notificationId}/dismiss`,
+        `/api/notifications/${notificationId}/dismiss`,
         {
           method: "POST",
         }
@@ -61,12 +61,12 @@ export function AnnouncementBanner() {
       {localNotifications
         .filter((notification) => !notification.dismissed)
         .map((notification) => {
-          if (notification.notif_type == "reindex") {
-            return (
-              <div
-                key={notification.id}
-                className="absolute top-0 left-1/2 transform -translate-x-1/2 bg-blue-600 rounded-sm text-white px-4 pr-8 py-3 mx-auto"
-              >
+          return (
+            <div
+              key={notification.id}
+              className="absolute top-0 left-1/2 transform -translate-x-1/2 bg-blue-600 rounded-sm text-white px-4 pr-8 py-3 mx-auto"
+            >
+              {notification.notif_type == "reindex" ? (
                 <p className="text-center">
                   Your index is out of date - we strongly recommend updating
                   your search settings.{" "}
@@ -77,24 +77,29 @@ export function AnnouncementBanner() {
                     Update here
                   </Link>
                 </p>
-                <button
-                  onClick={() => handleDismiss(notification.id)}
-                  className="absolute top-0 right-0 mt-2 mr-2"
-                  aria-label="Dismiss"
-                >
-                  <CustomTooltip
-                    showTick
-                    citation
-                    delay={100}
-                    content="Dismiss"
+              ) : notification.notif_type == "two_day_trial_ending" ? (
+                <p className="text-center">
+                  Your trial is ending soon - submit your billing information to
+                  continue using Danswer.{" "}
+                  <Link
+                    href="/admin/cloud-settings"
+                    className="ml-2 underline cursor-pointer"
                   >
-                    <XIcon className="h-5 w-5" />
-                  </CustomTooltip>
-                </button>
-              </div>
-            );
-          }
-          return null;
+                    Update here
+                  </Link>
+                </p>
+              ) : null}
+              <button
+                onClick={() => handleDismiss(notification.id)}
+                className="absolute top-0 right-0 mt-2 mr-2"
+                aria-label="Dismiss"
+              >
+                <CustomTooltip showTick citation delay={100} content="Dismiss">
+                  <XIcon className="h-5 w-5" />
+                </CustomTooltip>
+              </button>
+            </div>
+          );
         })}
     </>
   );
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index 806a1d447a6..de9303c012b 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -62,7 +62,7 @@ export const CUSTOM_ANALYTICS_ENABLED = process.env.CUSTOM_ANALYTICS_SECRET_KEY
 export const DISABLE_LLM_DOC_RELEVANCE =
   process.env.DISABLE_LLM_DOC_RELEVANCE?.toLowerCase() === "true";
 
-export const CLOUD_ENABLED =
+export const NEXT_PUBLIC_CLOUD_ENABLED =
   process.env.NEXT_PUBLIC_CLOUD_ENABLED?.toLowerCase() === "true";
 
 export const REGISTRATION_URL =

From 3eb67baf5b1c2f9f4b183b9bc28a226900e1d438 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 23 Oct 2024 13:25:52 -0700
Subject: [PATCH 188/376] Bugfix/indexing UI (#2879)

* fresh indexing feature branch

* cherry pick test

* Revert "cherry pick test"

This reverts commit 2a624220687affdda3de347e30f2011136f64bda.

* set multitenant so that vespa fields match when indexing

* cleanup pass

* mypy

* pass through env var to control celery indexing concurrency

* comments on task kickoff and some logging improvements

* disentangle configuration for different workers and beats.

* use get_session_with_tenant

* comment out all of update.py

* rename to RedisConnectorIndexingFenceData

* first check num_indexing_workers

* refactor RedisConnectorIndexingFenceData

* comment out on_worker_process_init

* missed a file

* scope db sessions to short lengths

* update launch.json template

* fix types

* keep index button disabled until indexing is truly finished

* change priority order of tooltips

* should be using the logger from app_base

* if we run out of retries, just mark the doc as modified so it gets synced later

* tighten up the logging ... we know these are ID's

* add logging
---
 .../danswer/background/celery/apps/primary.py |  5 +---
 .../danswer/background/celery/celery_redis.py | 15 +++++-----
 .../background/celery/tasks/shared/tasks.py   | 28 +++++++++++++++----
 backend/danswer/db/document.py                | 14 ++++++++++
 backend/danswer/server/documents/cc_pair.py   | 14 ++++++++--
 backend/danswer/server/documents/models.py    |  3 ++
 backend/danswer/utils/logger.py               |  5 ++--
 .../connector/[ccPairId]/ReIndexButton.tsx    | 10 +++++--
 .../app/admin/connector/[ccPairId]/page.tsx   |  2 ++
 .../app/admin/connector/[ccPairId]/types.ts   |  1 +
 10 files changed, 73 insertions(+), 24 deletions(-)

diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index c99607f4bc3..58e464f3768 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -11,9 +11,9 @@
 from celery.signals import worker_init
 from celery.signals import worker_ready
 from celery.signals import worker_shutdown
-from celery.utils.log import get_task_logger
 
 import danswer.background.celery.apps.app_base as app_base
+from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.background.celery.celery_redis import RedisConnectorIndexing
@@ -31,9 +31,6 @@
 
 logger = setup_logger()
 
-# use this within celery tasks to get celery task specific logging
-task_logger = get_task_logger(__name__)
-
 celery_app = Celery(__name__)
 celery_app.config_from_object("danswer.background.celery.configs.primary")
 
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index f1a5697e246..1ea5e3b1765 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -465,14 +465,8 @@ def generate_tasks(
 
         return len(async_results)
 
-    def is_pruning(self, db_session: Session, redis_client: Redis) -> bool:
+    def is_pruning(self, redis_client: Redis) -> bool:
         """A single example of a helper method being refactored into the redis helper"""
-        cc_pair = get_connector_credential_pair_from_id(
-            cc_pair_id=int(self._id), db_session=db_session
-        )
-        if not cc_pair:
-            raise ValueError(f"cc_pair_id {self._id} does not exist.")
-
         if redis_client.exists(self.fence_key):
             return True
 
@@ -538,6 +532,13 @@ def generate_tasks(
     ) -> int | None:
         return None
 
+    def is_indexing(self, redis_client: Redis) -> bool:
+        """A single example of a helper method being refactored into the redis helper"""
+        if redis_client.exists(self.fence_key):
+            return True
+
+        return False
+
 
 def celery_get_queue_length(queue: str, r: Redis) -> int:
     """This is a redis specific way to get the length of a celery queue.
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 474a749e786..7ce43454aa3 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -11,6 +11,7 @@
 from danswer.db.document import delete_documents_complete__no_commit
 from danswer.db.document import get_document
 from danswer.db.document import get_document_connector_count
+from danswer.db.document import mark_document_as_modified
 from danswer.db.document import mark_document_as_synced
 from danswer.db.document_set import fetch_document_sets_for_document
 from danswer.db.engine import get_session_with_tenant
@@ -19,6 +20,8 @@
 from danswer.document_index.interfaces import VespaDocumentFields
 from danswer.server.documents.models import ConnectorCredentialPairIdentifier
 
+DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES = 3
+
 
 class RedisConnectorIndexingFenceData(BaseModel):
     index_attempt_id: int | None
@@ -32,7 +35,7 @@ class RedisConnectorIndexingFenceData(BaseModel):
     bind=True,
     soft_time_limit=45,
     time_limit=60,
-    max_retries=3,
+    max_retries=DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES,
 )
 def document_by_cc_pair_cleanup_task(
     self: Task,
@@ -56,7 +59,7 @@ def document_by_cc_pair_cleanup_task(
     connector / credential pair from the access list
     (6) delete all relevant entries from postgres
     """
-    task_logger.info(f"document_id={document_id}")
+    task_logger.info(f"tenant_id={tenant_id} document_id={document_id}")
 
     try:
         with get_session_with_tenant(tenant_id) as db_session:
@@ -122,6 +125,8 @@ def document_by_cc_pair_cleanup_task(
             else:
                 pass
 
+            db_session.commit()
+
             task_logger.info(
                 f"tenant_id={tenant_id} "
                 f"document_id={document_id} "
@@ -129,16 +134,27 @@ def document_by_cc_pair_cleanup_task(
                 f"refcount={count} "
                 f"chunks={chunks_affected}"
             )
-            db_session.commit()
     except SoftTimeLimitExceeded:
         task_logger.info(
             f"SoftTimeLimitExceeded exception. tenant_id={tenant_id} doc_id={document_id}"
         )
+        return False
     except Exception as e:
         task_logger.exception("Unexpected exception")
 
-        # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
-        countdown = 2 ** (self.request.retries + 4)
-        self.retry(exc=e, countdown=countdown)
+        if self.request.retries < DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES:
+            # Still retrying. Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
+            countdown = 2 ** (self.request.retries + 4)
+            self.retry(exc=e, countdown=countdown)
+        else:
+            # This is the last attempt! mark the document as dirty in the db so that it
+            # eventually gets fixed out of band via stale document reconciliation
+            task_logger.info(
+                f"Max retries reached. Marking doc as dirty for reconciliation: "
+                f"tenant_id={tenant_id} document_id={document_id}"
+            )
+            with get_session_with_tenant(tenant_id):
+                mark_document_as_modified(document_id, db_session)
+        return False
 
     return True
diff --git a/backend/danswer/db/document.py b/backend/danswer/db/document.py
index 8aee28aef05..2e142a2c0b5 100644
--- a/backend/danswer/db/document.py
+++ b/backend/danswer/db/document.py
@@ -375,6 +375,20 @@ def update_docs_last_modified__no_commit(
         doc.last_modified = now
 
 
+def mark_document_as_modified(
+    document_id: str,
+    db_session: Session,
+) -> None:
+    stmt = select(DbDocument).where(DbDocument.id == document_id)
+    doc = db_session.scalar(stmt)
+    if doc is None:
+        raise ValueError(f"No document with ID: {document_id}")
+
+    # update last_synced
+    doc.last_modified = datetime.now(timezone.utc)
+    db_session.commit()
+
+
 def mark_document_as_synced(document_id: str, db_session: Session) -> None:
     stmt = select(DbDocument).where(DbDocument.id == document_id)
     doc = db_session.scalar(stmt)
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index db35807ad54..92a94a63878 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -11,6 +11,7 @@
 
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
 from danswer.background.celery.tasks.pruning.tasks import (
@@ -34,6 +35,7 @@
 from danswer.db.index_attempt import get_latest_index_attempt_for_cc_pair_id
 from danswer.db.index_attempt import get_paginated_index_attempts_for_cc_pair_id
 from danswer.db.models import User
+from danswer.db.search_settings import get_current_search_settings
 from danswer.db.tasks import check_task_is_live_and_not_timed_out
 from danswer.db.tasks import get_latest_task
 from danswer.redis.redis_pool import get_redis_client
@@ -93,6 +95,8 @@ def get_cc_pair_full_info(
     user: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
 ) -> CCPairFullInfo:
+    r = get_redis_client()
+
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id, db_session, user, get_editable=False
     )
@@ -122,11 +126,16 @@ def get_cc_pair_full_info(
 
     latest_attempt = get_latest_index_attempt_for_cc_pair_id(
         db_session=db_session,
-        connector_credential_pair_id=cc_pair.id,
+        connector_credential_pair_id=cc_pair_id,
         secondary_index=False,
         only_finished=False,
     )
 
+    search_settings = get_current_search_settings(db_session)
+    rci = RedisConnectorIndexing(
+        cc_pair_id=cc_pair_id, search_settings_id=search_settings.id
+    )
+
     return CCPairFullInfo.from_models(
         cc_pair_model=cc_pair,
         number_of_index_attempts=count_index_attempts_for_connector(
@@ -141,6 +150,7 @@ def get_cc_pair_full_info(
         ),
         num_docs_indexed=documents_indexed,
         is_editable_for_current_user=is_editable_for_current_user,
+        indexing=rci.is_indexing(r),
     )
 
 
@@ -250,7 +260,7 @@ def prune_cc_pair(
 
     r = get_redis_client()
     rcp = RedisConnectorPruning(cc_pair_id)
-    if rcp.is_pruning(db_session, r):
+    if rcp.is_pruning(r):
         raise HTTPException(
             status_code=HTTPStatus.CONFLICT,
             detail="Pruning task already in progress.",
diff --git a/backend/danswer/server/documents/models.py b/backend/danswer/server/documents/models.py
index 780d8a3f28e..fcbc0a76a12 100644
--- a/backend/danswer/server/documents/models.py
+++ b/backend/danswer/server/documents/models.py
@@ -222,6 +222,7 @@ class CCPairFullInfo(BaseModel):
     access_type: AccessType
     is_editable_for_current_user: bool
     deletion_failure_message: str | None
+    indexing: bool
 
     @classmethod
     def from_models(
@@ -232,6 +233,7 @@ def from_models(
         last_index_attempt: IndexAttempt | None,
         num_docs_indexed: int,  # not ideal, but this must be computed separately
         is_editable_for_current_user: bool,
+        indexing: bool,
     ) -> "CCPairFullInfo":
         # figure out if we need to artificially deflate the number of docs indexed.
         # This is required since the total number of docs indexed by a CC Pair is
@@ -265,6 +267,7 @@ def from_models(
             access_type=cc_pair_model.access_type,
             is_editable_for_current_user=is_editable_for_current_user,
             deletion_failure_message=cc_pair_model.deletion_failure_message,
+            indexing=indexing,
         )
 
 
diff --git a/backend/danswer/utils/logger.py b/backend/danswer/utils/logger.py
index 065d6282cbf..2aadbe379d6 100644
--- a/backend/danswer/utils/logger.py
+++ b/backend/danswer/utils/logger.py
@@ -61,10 +61,10 @@ def process(
         cc_pair_id = IndexAttemptSingleton.get_connector_credential_pair_id()
 
         if attempt_id is not None:
-            msg = f"[Attempt ID: {attempt_id}] {msg}"
+            msg = f"[Attempt: {attempt_id}] {msg}"
 
         if cc_pair_id is not None:
-            msg = f"[CC Pair ID: {cc_pair_id}] {msg}"
+            msg = f"[CC Pair: {cc_pair_id}] {msg}"
 
         # For Slack Bot, logs the channel relevant to the request
         channel_id = self.extra.get(SLACK_CHANNEL_ID) if self.extra else None
@@ -185,6 +185,7 @@ def setup_logger(
 
 
 def print_loggers() -> None:
+    """Print information about all loggers. Use to debug logging issues."""
     root_logger = logging.getLogger()
     loggers: list[logging.Logger | logging.PlaceHolder] = [root_logger]
     loggers.extend(logging.Logger.manager.loggerDict.values())
diff --git a/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx b/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx
index dced8811a3f..75bb2eca95d 100644
--- a/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx
@@ -94,12 +94,14 @@ export function ReIndexButton({
   connectorId,
   credentialId,
   isDisabled,
+  isIndexing,
   isDeleting,
 }: {
   ccPairId: number;
   connectorId: number;
   credentialId: number;
   isDisabled: boolean;
+  isIndexing: boolean;
   isDeleting: boolean;
 }) {
   const { popup, setPopup } = usePopup();
@@ -128,9 +130,11 @@ export function ReIndexButton({
         tooltip={
           isDeleting
             ? "Cannot index while connector is deleting"
-            : isDisabled
-              ? "Connector must be re-enabled before indexing"
-              : undefined
+            : isIndexing
+              ? "Indexing is already in progress"
+              : isDisabled
+                ? "Connector must be re-enabled before indexing"
+                : undefined
         }
       >
         Index
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index 9cdf7c83ec2..e2576cc9b81 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -188,8 +188,10 @@ function Main({ ccPairId }: { ccPairId: number }) {
                 connectorId={ccPair.connector.id}
                 credentialId={ccPair.credential.id}
                 isDisabled={
+                  ccPair.indexing ||
                   ccPair.status === ConnectorCredentialPairStatus.PAUSED
                 }
+                isIndexing={ccPair.indexing}
                 isDeleting={isDeleting}
               />
             )}
diff --git a/web/src/app/admin/connector/[ccPairId]/types.ts b/web/src/app/admin/connector/[ccPairId]/types.ts
index 55bbe955730..5e9cec428c1 100644
--- a/web/src/app/admin/connector/[ccPairId]/types.ts
+++ b/web/src/app/admin/connector/[ccPairId]/types.ts
@@ -25,6 +25,7 @@ export interface CCPairFullInfo {
   is_public: boolean;
   is_editable_for_current_user: boolean;
   deletion_failure_message: string | null;
+  indexing: boolean;
 }
 
 export interface PaginatedIndexAttempts {

From 14e75bbd24b067ff2b3138af4f6c7937201b34e2 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 23 Oct 2024 16:12:17 -0700
Subject: [PATCH 189/376] add default schema config (#2888)

* add default schema config

* resolve circular import

* k
---
 backend/alembic/env.py                             |  9 +++++++--
 backend/danswer/auth/users.py                      | 11 ++++++++---
 backend/danswer/configs/constants.py               |  1 -
 backend/danswer/connectors/file/connector.py       |  2 +-
 backend/danswer/danswerbot/slack/listener.py       |  5 ++++-
 backend/danswer/db/engine.py                       | 14 +++++++-------
 backend/danswer/key_value_store/store.py           |  3 ++-
 .../danswer/server/middleware/tenant_tracking.py   |  2 +-
 backend/ee/danswer/server/tenants/provisioning.py  |  7 ++++---
 .../scripts/query_time_check/seed_dummy_docs.py    |  3 ++-
 backend/shared_configs/configs.py                  |  7 ++++++-
 deployment/docker_compose/docker-compose.dev.yml   |  2 ++
 deployment/kubernetes/env-configmap.yaml           |  1 +
 13 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index b4b0ecb4665..7ccd04cf16a 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -14,6 +14,7 @@
 from danswer.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore
 from danswer.db.engine import get_all_tenant_ids
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 # Alembic Config object
 config = context.config
@@ -57,11 +58,15 @@ def get_schema_options() -> tuple[str, bool, bool]:
             if "=" in pair:
                 key, value = pair.split("=", 1)
                 x_args[key.strip()] = value.strip()
-    schema_name = x_args.get("schema", "public")
+    schema_name = x_args.get("schema", POSTGRES_DEFAULT_SCHEMA)
     create_schema = x_args.get("create_schema", "true").lower() == "true"
     upgrade_all_tenants = x_args.get("upgrade_all_tenants", "false").lower() == "true"
 
-    if MULTI_TENANT and schema_name == "public" and not upgrade_all_tenants:
+    if (
+        MULTI_TENANT
+        and schema_name == POSTGRES_DEFAULT_SCHEMA
+        and not upgrade_all_tenants
+    ):
         raise ValueError(
             "Cannot run default migrations in public schema when multi-tenancy is enabled. "
             "Please specify a tenant-specific schema."
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 4565073b6af..51cac314fdb 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -94,6 +94,7 @@
 from danswer.utils.telemetry import RecordType
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import current_tenant_id
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
 
@@ -187,7 +188,7 @@ def verify_email_domain(email: str) -> None:
 
 def get_tenant_id_for_email(email: str) -> str:
     if not MULTI_TENANT:
-        return "public"
+        return POSTGRES_DEFAULT_SCHEMA
     # Implement logic to get tenant_id from the mapping table
     with Session(get_sqlalchemy_engine()) as db_session:
         result = db_session.execute(
@@ -235,7 +236,9 @@ async def create(
     ) -> User:
         try:
             tenant_id = (
-                get_tenant_id_for_email(user_create.email) if MULTI_TENANT else "public"
+                get_tenant_id_for_email(user_create.email)
+                if MULTI_TENANT
+                else POSTGRES_DEFAULT_SCHEMA
             )
         except exceptions.UserNotExists:
             raise HTTPException(status_code=401, detail="User not found")
@@ -327,7 +330,9 @@ async def oauth_callback(
         # Get tenant_id from mapping table
         try:
             tenant_id = (
-                get_tenant_id_for_email(account_email) if MULTI_TENANT else "public"
+                get_tenant_id_for_email(account_email)
+                if MULTI_TENANT
+                else POSTGRES_DEFAULT_SCHEMA
             )
         except exceptions.UserNotExists:
             raise HTTPException(status_code=401, detail="User not found")
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 2c86c7f0547..6a3385b9fa8 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -46,7 +46,6 @@
 POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME = "celery_worker_indexing_child"
 POSTGRES_PERMISSIONS_APP_NAME = "permissions"
 POSTGRES_UNKNOWN_APP_NAME = "unknown"
-POSTGRES_DEFAULT_SCHEMA = "public"
 
 # API Keys
 DANSWER_API_KEY_PREFIX = "API_KEY__"
diff --git a/backend/danswer/connectors/file/connector.py b/backend/danswer/connectors/file/connector.py
index 9992159eb35..eb79cce579e 100644
--- a/backend/danswer/connectors/file/connector.py
+++ b/backend/danswer/connectors/file/connector.py
@@ -10,7 +10,6 @@
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
-from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import LoadConnector
@@ -29,6 +28,7 @@
 from danswer.file_store.file_store import get_default_file_store
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import current_tenant_id
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
 
diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index e3b2d213e83..b05c3a5ce55 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -60,6 +60,7 @@
 from shared_configs.configs import current_tenant_id
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import SLACK_CHANNEL_ID
 
 logger = setup_logger()
@@ -510,7 +511,9 @@ def _initialize_socket_client(socket_client: TenantSocketModeClient) -> None:
             for tenant_id in tenant_ids:
                 with get_session_with_tenant(tenant_id) as db_session:
                     try:
-                        token = current_tenant_id.set(tenant_id or "public")
+                        token = current_tenant_id.set(
+                            tenant_id or POSTGRES_DEFAULT_SCHEMA
+                        )
                         latest_slack_bot_tokens = fetch_tokens()
                         current_tenant_id.reset(token)
 
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 625c36435cd..7bf813b44f8 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -36,11 +36,11 @@
 from danswer.configs.app_configs import POSTGRES_PORT
 from danswer.configs.app_configs import POSTGRES_USER
 from danswer.configs.app_configs import SECRET_JWT_KEY
-from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
 from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import current_tenant_id
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
 
@@ -192,13 +192,13 @@ def get_app_name(cls) -> str:
 def get_all_tenant_ids() -> list[str] | list[None]:
     if not MULTI_TENANT:
         return [None]
-    with get_session_with_tenant(tenant_id="public") as session:
+    with get_session_with_tenant(tenant_id=POSTGRES_DEFAULT_SCHEMA) as session:
         result = session.execute(
             text(
-                """
-            SELECT schema_name
-            FROM information_schema.schemata
-            WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
+                f"""
+                SELECT schema_name
+                FROM information_schema.schemata
+                WHERE schema_name NOT IN ('pg_catalog', 'information_schema', '{POSTGRES_DEFAULT_SCHEMA}')"""
             )
         )
         tenant_ids = [row[0] for row in result]
@@ -365,7 +365,7 @@ def get_session_generator_with_tenant() -> Generator[Session, None, None]:
 def get_session() -> Generator[Session, None, None]:
     """Generate a database session with the appropriate tenant schema set."""
     tenant_id = current_tenant_id.get()
-    if tenant_id == "public" and MULTI_TENANT:
+    if tenant_id == POSTGRES_DEFAULT_SCHEMA and MULTI_TENANT:
         raise HTTPException(status_code=401, detail="User must authenticate")
 
     engine = get_sqlalchemy_engine()
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index 98f3d7ec1cb..b461ca22feb 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -17,6 +17,7 @@
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import current_tenant_id
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
 
@@ -35,7 +36,7 @@ def get_session(self) -> Iterator[Session]:
         with Session(engine, expire_on_commit=False) as session:
             if MULTI_TENANT:
                 tenant_id = current_tenant_id.get()
-                if tenant_id == "public":
+                if tenant_id == POSTGRES_DEFAULT_SCHEMA:
                     raise HTTPException(
                         status_code=401, detail="User must authenticate"
                     )
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index f564a4fc683..63b0f82be8f 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -10,9 +10,9 @@
 
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import SECRET_JWT_KEY
-from danswer.configs.constants import POSTGRES_DEFAULT_SCHEMA
 from danswer.db.engine import is_valid_schema_name
 from shared_configs.configs import current_tenant_id
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 
 def add_tenant_id_middleware(app: FastAPI, logger: logging.LoggerAdapter) -> None:
diff --git a/backend/ee/danswer/server/tenants/provisioning.py b/backend/ee/danswer/server/tenants/provisioning.py
index 9ec7b8061aa..311698e6a3d 100644
--- a/backend/ee/danswer/server/tenants/provisioning.py
+++ b/backend/ee/danswer/server/tenants/provisioning.py
@@ -12,6 +12,7 @@
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.models import UserTenantMapping
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
 
@@ -71,7 +72,7 @@ def ensure_schema_exists(tenant_id: str) -> bool:
 # For now, we're implementing a primitive mapping between users and tenants.
 # This function is only used to determine a user's relationship to a tenant upon creation (implying ownership).
 def user_owns_a_tenant(email: str) -> bool:
-    with get_session_with_tenant("public") as db_session:
+    with get_session_with_tenant(POSTGRES_DEFAULT_SCHEMA) as db_session:
         result = (
             db_session.query(UserTenantMapping)
             .filter(UserTenantMapping.email == email)
@@ -81,7 +82,7 @@ def user_owns_a_tenant(email: str) -> bool:
 
 
 def add_users_to_tenant(emails: list[str], tenant_id: str) -> None:
-    with get_session_with_tenant("public") as db_session:
+    with get_session_with_tenant(POSTGRES_DEFAULT_SCHEMA) as db_session:
         try:
             for email in emails:
                 db_session.add(UserTenantMapping(email=email, tenant_id=tenant_id))
@@ -91,7 +92,7 @@ def add_users_to_tenant(emails: list[str], tenant_id: str) -> None:
 
 
 def remove_users_from_tenant(emails: list[str], tenant_id: str) -> None:
-    with get_session_with_tenant("public") as db_session:
+    with get_session_with_tenant(POSTGRES_DEFAULT_SCHEMA) as db_session:
         try:
             mappings_to_delete = (
                 db_session.query(UserTenantMapping)
diff --git a/backend/scripts/query_time_check/seed_dummy_docs.py b/backend/scripts/query_time_check/seed_dummy_docs.py
index 70cb2a4a6a8..e7aa65fba76 100644
--- a/backend/scripts/query_time_check/seed_dummy_docs.py
+++ b/backend/scripts/query_time_check/seed_dummy_docs.py
@@ -21,6 +21,7 @@
 from danswer.indexing.models import DocMetadataAwareIndexChunk
 from danswer.indexing.models import IndexChunk
 from danswer.utils.timing import log_function_time
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.model_server_models import Embedding
 
 
@@ -94,7 +95,7 @@ def generate_dummy_chunk(
         ),
         document_sets={document_set for document_set in document_set_names},
         boost=random.randint(-1, 1),
-        tenant_id="public",
+        tenant_id=POSTGRES_DEFAULT_SCHEMA,
     )
 
 
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index f10855f103f..77139125f6e 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -128,7 +128,12 @@ def validate_cors_origin(origin: str) -> None:
     # If the environment variable is empty, allow all origins
     CORS_ALLOWED_ORIGIN = ["*"]
 
-current_tenant_id = contextvars.ContextVar("current_tenant_id", default="public")
+
+POSTGRES_DEFAULT_SCHEMA = os.environ.get("POSTGRES_DEFAULT_SCHEMA") or "public"
+
+current_tenant_id = contextvars.ContextVar(
+    "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
+)
 
 
 SUPPORTED_EMBEDDING_MODELS = [
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index d22bde5b465..7b31689c8f3 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -63,6 +63,7 @@ services:
       - QA_PROMPT_OVERRIDE=${QA_PROMPT_OVERRIDE:-}
       # Other services
       - POSTGRES_HOST=relational_db
+      - POSTGRES_DEFAULT_SCHEMA=${POSTGRES_DEFAULT_SCHEMA:-}
       - VESPA_HOST=index
       - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-} # For frontend redirect auth purpose
@@ -147,6 +148,7 @@ services:
       - POSTGRES_USER=${POSTGRES_USER:-}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-}
       - POSTGRES_DB=${POSTGRES_DB:-}
+      - POSTGRES_DEFAULT_SCHEMA=${POSTGRES_DEFAULT_SCHEMA:-}
       - VESPA_HOST=index
       - REDIS_HOST=cache
       - WEB_DOMAIN=${WEB_DOMAIN:-} # For frontend redirect auth purpose for OAuth2 connectors
diff --git a/deployment/kubernetes/env-configmap.yaml b/deployment/kubernetes/env-configmap.yaml
index 1d4bf1cffd7..e1eefaeca90 100644
--- a/deployment/kubernetes/env-configmap.yaml
+++ b/deployment/kubernetes/env-configmap.yaml
@@ -32,6 +32,7 @@ data:
   QA_PROMPT_OVERRIDE: ""
   # Other Services
   POSTGRES_HOST: "relational-db-service"
+  POSTGRES_DEFAULT_SCHEMA: ""
   VESPA_HOST: "document-index-service"
   REDIS_HOST: "redis-service"
   # Internet Search Tool

From b9fb657d811a20d009a48aaef53da613d3b7cd84 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Wed, 23 Oct 2024 17:49:04 -0700
Subject: [PATCH 190/376] Temporary fix for empty Google App credentials
 (#2892)

* Temporary fix for empty Google App credentials

* added it to credential creation
---
 backend/danswer/db/credentials.py              | 18 ++++++++++++++++++
 backend/danswer/server/documents/connector.py  | 10 ++++++++++
 backend/danswer/server/documents/credential.py |  8 ++++++++
 3 files changed, 36 insertions(+)

diff --git a/backend/danswer/db/credentials.py b/backend/danswer/db/credentials.py
index abab904cc48..b4ecfc888fe 100644
--- a/backend/danswer/db/credentials.py
+++ b/backend/danswer/db/credentials.py
@@ -406,6 +406,24 @@ def create_initial_public_credential(db_session: Session) -> None:
     db_session.commit()
 
 
+def cleanup_gmail_credentials(db_session: Session) -> None:
+    gmail_credentials = fetch_credentials_by_source(
+        db_session=db_session, user=None, document_source=DocumentSource.GMAIL
+    )
+    for credential in gmail_credentials:
+        db_session.delete(credential)
+    db_session.commit()
+
+
+def cleanup_google_drive_credentials(db_session: Session) -> None:
+    google_drive_credentials = fetch_credentials_by_source(
+        db_session=db_session, user=None, document_source=DocumentSource.GOOGLE_DRIVE
+    )
+    for credential in google_drive_credentials:
+        db_session.delete(credential)
+    db_session.commit()
+
+
 def delete_gmail_service_account_credentials(
     user: User | None, db_session: Session
 ) -> None:
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index 54d11e867bd..a6ce87ad8a6 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -60,6 +60,8 @@
 from danswer.db.connector_credential_pair import get_cc_pair_groups_for_ids
 from danswer.db.connector_credential_pair import get_connector_credential_pair
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
+from danswer.db.credentials import cleanup_gmail_credentials
+from danswer.db.credentials import cleanup_google_drive_credentials
 from danswer.db.credentials import create_credential
 from danswer.db.credentials import delete_gmail_service_account_credentials
 from danswer.db.credentials import delete_google_drive_service_account_credentials
@@ -143,9 +145,11 @@ def upsert_google_app_gmail_credentials(
 @router.delete("/admin/connector/gmail/app-credential")
 def delete_google_app_gmail_credentials(
     _: User = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
         delete_google_app_gmail_cred()
+        cleanup_gmail_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -181,9 +185,11 @@ def upsert_google_app_credentials(
 @router.delete("/admin/connector/google-drive/app-credential")
 def delete_google_app_credentials(
     _: User = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
         delete_google_app_cred()
+        cleanup_google_drive_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -221,9 +227,11 @@ def upsert_google_service_gmail_account_key(
 @router.delete("/admin/connector/gmail/service-account-key")
 def delete_google_service_gmail_account_key(
     _: User = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
         delete_gmail_service_account_key()
+        cleanup_gmail_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -261,9 +269,11 @@ def upsert_google_service_account_key(
 @router.delete("/admin/connector/google-drive/service-account-key")
 def delete_google_service_account_key(
     _: User = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
         delete_service_account_key()
+        cleanup_google_drive_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
diff --git a/backend/danswer/server/documents/credential.py b/backend/danswer/server/documents/credential.py
index 3d965481bf5..2c6e41bf968 100644
--- a/backend/danswer/server/documents/credential.py
+++ b/backend/danswer/server/documents/credential.py
@@ -8,6 +8,8 @@
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
 from danswer.db.credentials import alter_credential
+from danswer.db.credentials import cleanup_gmail_credentials
+from danswer.db.credentials import cleanup_google_drive_credentials
 from danswer.db.credentials import create_credential
 from danswer.db.credentials import CREDENTIAL_PERMISSIONS_TO_IGNORE
 from danswer.db.credentials import delete_credential
@@ -138,6 +140,12 @@ def create_credential_from_model(
             object_is_public=credential_info.curator_public,
         )
 
+    # Temporary fix for empty Google App credentials
+    if credential_info.source == DocumentSource.GMAIL:
+        cleanup_gmail_credentials(db_session=db_session)
+    if credential_info.source == DocumentSource.GOOGLE_DRIVE:
+        cleanup_google_drive_credentials(db_session=db_session)
+
     credential = create_credential(credential_info, user, db_session)
     return ObjectCreationIdResponse(
         id=credential.id,

From 0545fb44434a9500a976e2a7962188568a354b27 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 23 Oct 2024 19:12:25 -0700
Subject: [PATCH 191/376] Multitenant redis update (#2889)

* add multi tenancy to redis

* rename context var

* k

* args -> kwargs

* minor update to kv interface

* robustify
---
 backend/danswer/auth/users.py                 |  10 +-
 .../background/celery/apps/app_base.py        |  77 +++++--
 .../danswer/background/celery/apps/beat.py    |   2 +-
 .../danswer/background/celery/apps/primary.py | 192 ++++++++++--------
 .../danswer/background/celery/celery_utils.py |  16 +-
 .../celery/tasks/connector_deletion/tasks.py  |   4 +-
 .../background/celery/tasks/indexing/tasks.py |   6 +-
 .../background/celery/tasks/pruning/tasks.py  |   6 +-
 .../background/celery/tasks/vespa/tasks.py    |   9 +-
 backend/danswer/connectors/file/connector.py  |   6 +-
 backend/danswer/danswerbot/slack/listener.py  |  13 +-
 backend/danswer/db/credentials.py             |   1 -
 backend/danswer/db/engine.py                  |  54 +++--
 backend/danswer/key_value_store/store.py      |  16 +-
 backend/danswer/redis/redis_pool.py           | 105 +++++++++-
 .../search/preprocessing/preprocessing.py     |   4 +-
 backend/danswer/server/documents/cc_pair.py   |  16 +-
 backend/danswer/server/documents/connector.py |   9 +-
 backend/danswer/server/manage/users.py        |  14 +-
 .../server/query_and_chat/token_limit.py      |   4 +-
 .../ee/danswer/background/celery/apps/beat.py |   2 +-
 .../danswer/background/celery/apps/primary.py |  22 +-
 .../server/middleware/tenant_tracking.py      |   4 +-
 backend/ee/danswer/server/tenants/api.py      |  16 +-
 backend/ee/danswer/server/tenants/billing.py  |   2 -
 backend/shared_configs/configs.py             |   2 +-
 26 files changed, 408 insertions(+), 204 deletions(-)

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 51cac314fdb..f4f069a5bac 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -93,7 +93,7 @@
 from danswer.utils.telemetry import optional_telemetry
 from danswer.utils.telemetry import RecordType
 from danswer.utils.variable_functionality import fetch_versioned_implementation
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
@@ -249,7 +249,7 @@ async def create(
             )
 
         async with get_async_session_with_tenant(tenant_id) as db_session:
-            token = current_tenant_id.set(tenant_id)
+            token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
             verify_email_is_invited(user_create.email)
             verify_email_domain(user_create.email)
@@ -288,7 +288,7 @@ async def create(
                 else:
                     raise exceptions.UserAlreadyExists()
 
-            current_tenant_id.reset(token)
+            CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
             return user
 
     async def on_after_login(
@@ -342,7 +342,7 @@ async def oauth_callback(
 
         token = None
         async with get_async_session_with_tenant(tenant_id) as db_session:
-            token = current_tenant_id.set(tenant_id)
+            token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
             verify_email_in_whitelist(account_email, tenant_id)
             verify_email_domain(account_email)
@@ -432,7 +432,7 @@ async def oauth_callback(
                 user.oidc_expiry = None  # type: ignore
 
             if token:
-                current_tenant_id.reset(token)
+                CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
             return user
 
diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
index 2a52abde5d1..05c0eabbbd9 100644
--- a/backend/danswer/background/celery/apps/app_base.py
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -19,6 +19,7 @@
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
 from danswer.configs.constants import DanswerRedisLocks
+from danswer.db.engine import get_all_tenant_ids
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import ColoredFormatter
 from danswer.utils.logger import PlainFormatter
@@ -56,7 +57,7 @@ def on_task_postrun(
     task_id: str | None = None,
     task: Task | None = None,
     args: tuple | None = None,
-    kwargs: dict | None = None,
+    kwargs: dict[str, Any] | None = None,
     retval: Any | None = None,
     state: str | None = None,
     **kwds: Any,
@@ -83,7 +84,19 @@ def on_task_postrun(
     if not task_id:
         return
 
-    r = get_redis_client()
+    # Get tenant_id directly from kwargs- each celery task has a tenant_id kwarg
+    if not kwargs:
+        logger.error(f"Task {task.name} (ID: {task_id}) is missing kwargs")
+        tenant_id = None
+    else:
+        tenant_id = kwargs.get("tenant_id")
+
+    task_logger.debug(
+        f"Task {task.name} (ID: {task_id}) completed with state: {state} "
+        f"{f'for tenant_id={tenant_id}' if tenant_id else ''}"
+    )
+
+    r = get_redis_client(tenant_id=tenant_id)
 
     if task_id.startswith(RedisConnectorCredentialPair.PREFIX):
         r.srem(RedisConnectorCredentialPair.get_taskset_key(), task_id)
@@ -124,7 +137,7 @@ def on_celeryd_init(sender: Any = None, conf: Any = None, **kwargs: Any) -> None
 
 
 def wait_for_redis(sender: Any, **kwargs: Any) -> None:
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=None)
 
     WAIT_INTERVAL = 5
     WAIT_LIMIT = 60
@@ -157,26 +170,44 @@ def wait_for_redis(sender: Any, **kwargs: Any) -> None:
 
 
 def on_secondary_worker_init(sender: Any, **kwargs: Any) -> None:
-    r = get_redis_client()
-
     WAIT_INTERVAL = 5
     WAIT_LIMIT = 60
 
     logger.info("Running as a secondary celery worker.")
-    logger.info("Waiting for primary worker to be ready...")
+    logger.info("Waiting for all tenant primary workers to be ready...")
     time_start = time.monotonic()
+
     while True:
-        if r.exists(DanswerRedisLocks.PRIMARY_WORKER):
+        tenant_ids = get_all_tenant_ids()
+        # Check if we have a primary worker lock for each tenant
+        all_tenants_ready = all(
+            get_redis_client(tenant_id=tenant_id).exists(
+                DanswerRedisLocks.PRIMARY_WORKER
+            )
+            for tenant_id in tenant_ids
+        )
+
+        if all_tenants_ready:
             break
 
-        time.monotonic()
         time_elapsed = time.monotonic() - time_start
+        ready_tenants = sum(
+            1
+            for tenant_id in tenant_ids
+            if get_redis_client(tenant_id=tenant_id).exists(
+                DanswerRedisLocks.PRIMARY_WORKER
+            )
+        )
+
         logger.info(
-            f"Primary worker is not ready yet. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
+            f"Not all tenant primary workers are ready yet. "
+            f"Ready tenants: {ready_tenants}/{len(tenant_ids)} "
+            f"elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
         )
+
         if time_elapsed > WAIT_LIMIT:
             msg = (
-                f"Primary worker was not ready within the timeout. "
+                f"Not all tenant primary workers were ready within the timeout "
                 f"({WAIT_LIMIT} seconds). Exiting..."
             )
             logger.error(msg)
@@ -184,7 +215,7 @@ def on_secondary_worker_init(sender: Any, **kwargs: Any) -> None:
 
         time.sleep(WAIT_INTERVAL)
 
-    logger.info("Wait for primary worker completed successfully. Continuing...")
+    logger.info("All tenant primary workers are ready. Continuing...")
     return
 
 
@@ -196,14 +227,26 @@ def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
     if not celery_is_worker_primary(sender):
         return
 
-    if not sender.primary_worker_lock:
+    if not hasattr(sender, "primary_worker_locks"):
         return
 
-    logger.info("Releasing primary worker lock.")
-    lock = sender.primary_worker_lock
-    if lock.owned():
-        lock.release()
-        sender.primary_worker_lock = None
+    for tenant_id, lock in sender.primary_worker_locks.items():
+        try:
+            if lock and lock.owned():
+                logger.debug(f"Attempting to release lock for tenant {tenant_id}")
+                try:
+                    lock.release()
+                    logger.debug(f"Successfully released lock for tenant {tenant_id}")
+                except Exception as e:
+                    logger.error(
+                        f"Failed to release lock for tenant {tenant_id}. Error: {str(e)}"
+                    )
+                finally:
+                    sender.primary_worker_locks[tenant_id] = None
+        except Exception as e:
+            logger.error(
+                f"Error checking lock status for tenant {tenant_id}. Error: {str(e)}"
+            )
 
 
 def on_setup_logging(
diff --git a/backend/danswer/background/celery/apps/beat.py b/backend/danswer/background/celery/apps/beat.py
index 47be61e36be..8ddc17efc52 100644
--- a/backend/danswer/background/celery/apps/beat.py
+++ b/backend/danswer/background/celery/apps/beat.py
@@ -88,7 +88,7 @@ def on_setup_logging(
             "task": task["task"],
             "schedule": task["schedule"],
             "options": task["options"],
-            "args": (tenant_id,),  # Must pass tenant_id as an argument
+            "kwargs": {"tenant_id": tenant_id},  # Must pass tenant_id as an argument
         }
 
 # Include any existing beat schedules
diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 58e464f3768..d86f0d60fe4 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -1,7 +1,6 @@
 import multiprocessing
 from typing import Any
 
-import redis
 from celery import bootsteps  # type: ignore
 from celery import Celery
 from celery import signals
@@ -24,6 +23,7 @@
 from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
+from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import SqlEngine
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
@@ -80,81 +80,83 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
 
     # This is singleton work that should be done on startup exactly once
     # by the primary worker
-    r = get_redis_client()
-
-    # For the moment, we're assuming that we are the only primary worker
-    # that should be running.
-    # TODO: maybe check for or clean up another zombie primary worker if we detect it
-    r.delete(DanswerRedisLocks.PRIMARY_WORKER)
-
-    # this process wide lock is taken to help other workers start up in order.
-    # it is planned to use this lock to enforce singleton behavior on the primary
-    # worker, since the primary worker does redis cleanup on startup, but this isn't
-    # implemented yet.
-    lock = r.lock(
-        DanswerRedisLocks.PRIMARY_WORKER,
-        timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
-    )
+    tenant_ids = get_all_tenant_ids()
+    for tenant_id in tenant_ids:
+        r = get_redis_client(tenant_id=tenant_id)
+
+        # For the moment, we're assuming that we are the only primary worker
+        # that should be running.
+        # TODO: maybe check for or clean up another zombie primary worker if we detect it
+        r.delete(DanswerRedisLocks.PRIMARY_WORKER)
+
+        # this process wide lock is taken to help other workers start up in order.
+        # it is planned to use this lock to enforce singleton behavior on the primary
+        # worker, since the primary worker does redis cleanup on startup, but this isn't
+        # implemented yet.
+        lock = r.lock(
+            DanswerRedisLocks.PRIMARY_WORKER,
+            timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
+        )
 
-    logger.info("Primary worker lock: Acquire starting.")
-    acquired = lock.acquire(blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2)
-    if acquired:
-        logger.info("Primary worker lock: Acquire succeeded.")
-    else:
-        logger.error("Primary worker lock: Acquire failed!")
-        raise WorkerShutdown("Primary worker lock could not be acquired!")
+        logger.info("Primary worker lock: Acquire starting.")
+        acquired = lock.acquire(blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2)
+        if acquired:
+            logger.info("Primary worker lock: Acquire succeeded.")
+        else:
+            logger.error("Primary worker lock: Acquire failed!")
+            raise WorkerShutdown("Primary worker lock could not be acquired!")
 
-    sender.primary_worker_lock = lock
+        sender.primary_worker_locks[tenant_id] = lock
 
-    # As currently designed, when this worker starts as "primary", we reinitialize redis
-    # to a clean state (for our purposes, anyway)
-    r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
-    r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+        # As currently designed, when this worker starts as "primary", we reinitialize redis
+        # to a clean state (for our purposes, anyway)
+        r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
+        r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
 
-    r.delete(RedisConnectorCredentialPair.get_taskset_key())
-    r.delete(RedisConnectorCredentialPair.get_fence_key())
+        r.delete(RedisConnectorCredentialPair.get_taskset_key())
+        r.delete(RedisConnectorCredentialPair.get_fence_key())
 
-    for key in r.scan_iter(RedisDocumentSet.TASKSET_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisDocumentSet.TASKSET_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisUserGroup.TASKSET_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisUserGroup.TASKSET_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorDeletion.TASKSET_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorDeletion.TASKSET_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorPruning.TASKSET_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorPruning.TASKSET_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_COMPLETE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorPruning.GENERATOR_COMPLETE_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorPruning.GENERATOR_PROGRESS_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorPruning.GENERATOR_PROGRESS_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorIndexing.TASKSET_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorIndexing.TASKSET_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_COMPLETE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_COMPLETE_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_PROGRESS_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_PROGRESS_PREFIX + "*"):
+            r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
+            r.delete(key)
 
 
 # @worker_process_init.connect
@@ -217,42 +219,58 @@ def start(self, worker: Any) -> None:
 
     def run_periodic_task(self, worker: Any) -> None:
         try:
-            if not worker.primary_worker_lock:
+            if not celery_is_worker_primary(worker):
                 return
 
-            if not hasattr(worker, "primary_worker_lock"):
+            if not hasattr(worker, "primary_worker_locks"):
                 return
 
-            r = get_redis_client()
-
-            lock: redis.lock.Lock = worker.primary_worker_lock
-
-            if lock.owned():
-                task_logger.debug("Reacquiring primary worker lock.")
-                lock.reacquire()
-            else:
-                task_logger.warning(
-                    "Full acquisition of primary worker lock. "
-                    "Reasons could be computer sleep or a clock change."
-                )
-                lock = r.lock(
-                    DanswerRedisLocks.PRIMARY_WORKER,
-                    timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
-                )
-
-                task_logger.info("Primary worker lock: Acquire starting.")
-                acquired = lock.acquire(
-                    blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2
-                )
-                if acquired:
-                    task_logger.info("Primary worker lock: Acquire succeeded.")
+            # Retrieve all tenant IDs
+            tenant_ids = get_all_tenant_ids()
+
+            for tenant_id in tenant_ids:
+                lock = worker.primary_worker_locks.get(tenant_id)
+                if not lock:
+                    continue  # Skip if no lock for this tenant
+
+                r = get_redis_client(tenant_id=tenant_id)
+
+                if lock.owned():
+                    task_logger.debug(
+                        f"Reacquiring primary worker lock for tenant {tenant_id}."
+                    )
+                    lock.reacquire()
                 else:
-                    task_logger.error("Primary worker lock: Acquire failed!")
-                    raise TimeoutError("Primary worker lock could not be acquired!")
+                    task_logger.warning(
+                        f"Full acquisition of primary worker lock for tenant {tenant_id}. "
+                        "Reasons could be worker restart or lock expiration."
+                    )
+                    lock = r.lock(
+                        DanswerRedisLocks.PRIMARY_WORKER,
+                        timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
+                    )
+
+                    task_logger.info(
+                        f"Primary worker lock for tenant {tenant_id}: Acquire starting."
+                    )
+                    acquired = lock.acquire(
+                        blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2
+                    )
+                    if acquired:
+                        task_logger.info(
+                            f"Primary worker lock for tenant {tenant_id}: Acquire succeeded."
+                        )
+                        worker.primary_worker_locks[tenant_id] = lock
+                    else:
+                        task_logger.error(
+                            f"Primary worker lock for tenant {tenant_id}: Acquire failed!"
+                        )
+                        raise TimeoutError(
+                            f"Primary worker lock for tenant {tenant_id} could not be acquired!"
+                        )
 
-                worker.primary_worker_lock = lock
         except Exception:
-            task_logger.exception("HubPeriodicTask.run_periodic_task exceptioned.")
+            task_logger.exception("Periodic task failed.")
 
     def stop(self, worker: Any) -> None:
         # Cancel the scheduled task when the worker stops
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 794f89232c5..b1e9c2113e2 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -27,7 +27,10 @@
 
 
 def _get_deletion_status(
-    connector_id: int, credential_id: int, db_session: Session
+    connector_id: int,
+    credential_id: int,
+    db_session: Session,
+    tenant_id: str | None = None,
 ) -> TaskQueueState | None:
     """We no longer store TaskQueueState in the DB for a deletion attempt.
     This function populates TaskQueueState by just checking redis.
@@ -40,7 +43,7 @@ def _get_deletion_status(
 
     rcd = RedisConnectorDeletion(cc_pair.id)
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
     if not r.exists(rcd.fence_key):
         return None
 
@@ -50,9 +53,14 @@ def _get_deletion_status(
 
 
 def get_deletion_attempt_snapshot(
-    connector_id: int, credential_id: int, db_session: Session
+    connector_id: int,
+    credential_id: int,
+    db_session: Session,
+    tenant_id: str | None = None,
 ) -> DeletionAttemptSnapshot | None:
-    deletion_task = _get_deletion_status(connector_id, credential_id, db_session)
+    deletion_task = _get_deletion_status(
+        connector_id, credential_id, db_session, tenant_id
+    )
     if not deletion_task:
         return None
 
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index b3c2eea30b0..f6a59d03e3a 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -24,8 +24,8 @@
     trail=False,
     bind=True,
 )
-def check_for_connector_deletion_task(self: Task, tenant_id: str | None) -> None:
-    r = get_redis_client()
+def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> None:
+    r = get_redis_client(tenant_id=tenant_id)
 
     lock_beat = r.lock(
         DanswerRedisLocks.CHECK_CONNECTOR_DELETION_BEAT_LOCK,
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index ed08787d53e..bdd55f77f32 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -55,10 +55,10 @@
     soft_time_limit=300,
     bind=True,
 )
-def check_for_indexing(self: Task, tenant_id: str | None) -> int | None:
+def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
     tasks_created = 0
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     lock_beat = r.lock(
         DanswerRedisLocks.CHECK_INDEXING_BEAT_LOCK,
@@ -398,7 +398,7 @@ def connector_indexing_task(
     attempt = None
     n_final_progress = 0
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
 
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index 698c2937299..9f290d6f23a 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -41,8 +41,8 @@
     soft_time_limit=JOB_TIMEOUT,
     bind=True,
 )
-def check_for_pruning(self: Task, tenant_id: str | None) -> None:
-    r = get_redis_client()
+def check_for_pruning(self: Task, *, tenant_id: str | None) -> None:
+    r = get_redis_client(tenant_id=tenant_id)
 
     lock_beat = r.lock(
         DanswerRedisLocks.CHECK_PRUNE_BEAT_LOCK,
@@ -222,7 +222,7 @@ def connector_pruning_generator_task(
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
     from the most recently pulled document ID list"""
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     rcp = RedisConnectorPruning(cc_pair_id)
 
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 53e26be6954..812074b91e9 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -60,6 +60,7 @@
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import VespaDocumentFields
 from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from danswer.utils.variable_functionality import (
     fetch_versioned_implementation_with_fallback,
@@ -67,6 +68,8 @@
 from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import noop_fallback
 
+logger = setup_logger()
+
 
 # celery auto associates tasks created inside another task,
 # which bloats the result metadata considerably. trail=False prevents this.
@@ -76,11 +79,11 @@
     trail=False,
     bind=True,
 )
-def check_for_vespa_sync_task(self: Task, tenant_id: str | None) -> None:
+def check_for_vespa_sync_task(self: Task, *, tenant_id: str | None) -> None:
     """Runs periodically to check if any document needs syncing.
     Generates sets of tasks for Celery if syncing is needed."""
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     lock_beat = r.lock(
         DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK,
@@ -680,7 +683,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
 
     Returns True if the task actually did work, False
     """
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     lock_beat: redis.lock.Lock = r.lock(
         DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK,
diff --git a/backend/danswer/connectors/file/connector.py b/backend/danswer/connectors/file/connector.py
index eb79cce579e..d07a224478e 100644
--- a/backend/danswer/connectors/file/connector.py
+++ b/backend/danswer/connectors/file/connector.py
@@ -27,7 +27,7 @@
 from danswer.file_processing.extract_file_text import read_text_file
 from danswer.file_store.file_store import get_default_file_store
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
@@ -175,7 +175,7 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
 
     def load_from_state(self) -> GenerateDocumentsOutput:
         documents: list[Document] = []
-        token = current_tenant_id.set(self.tenant_id)
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(self.tenant_id)
 
         with get_session_with_tenant(self.tenant_id) as db_session:
             for file_path in self.file_locations:
@@ -199,7 +199,7 @@ def load_from_state(self) -> GenerateDocumentsOutput:
             if documents:
                 yield documents
 
-        current_tenant_id.reset(token)
+        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 if __name__ == "__main__":
diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index b05c3a5ce55..a40dbe9a9b9 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -57,10 +57,9 @@
 from danswer.server.manage.models import SlackBotTokens
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
-from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import SLACK_CHANNEL_ID
 
 logger = setup_logger()
@@ -364,7 +363,7 @@ def process_message(
     # Set the current tenant ID at the beginning for all DB calls within this thread
     if client.tenant_id:
         logger.info(f"Setting tenant ID to {client.tenant_id}")
-        token = current_tenant_id.set(client.tenant_id)
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(client.tenant_id)
     try:
         with get_session_with_tenant(client.tenant_id) as db_session:
             slack_bot_config = get_slack_bot_config_for_channel(
@@ -413,7 +412,7 @@ def process_message(
                     apologize_for_fail(details, client)
     finally:
         if client.tenant_id:
-            current_tenant_id.reset(token)
+            CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 def acknowledge_message(req: SocketModeRequest, client: TenantSocketModeClient) -> None:
@@ -511,11 +510,9 @@ def _initialize_socket_client(socket_client: TenantSocketModeClient) -> None:
             for tenant_id in tenant_ids:
                 with get_session_with_tenant(tenant_id) as db_session:
                     try:
-                        token = current_tenant_id.set(
-                            tenant_id or POSTGRES_DEFAULT_SCHEMA
-                        )
+                        token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id or "public")
                         latest_slack_bot_tokens = fetch_tokens()
-                        current_tenant_id.reset(token)
+                        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
                         if (
                             tenant_id not in slack_bot_tokens
diff --git a/backend/danswer/db/credentials.py b/backend/danswer/db/credentials.py
index b4ecfc888fe..5da5099f1e3 100644
--- a/backend/danswer/db/credentials.py
+++ b/backend/danswer/db/credentials.py
@@ -242,7 +242,6 @@ def create_credential(
     )
     db_session.add(credential)
     db_session.flush()  # This ensures the credential gets an ID
-
     _relate_credential_to_user_groups__no_commit(
         db_session=db_session,
         credential_id=credential.id,
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 7bf813b44f8..c03a47d7f44 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -39,7 +39,7 @@
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
 from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
@@ -260,12 +260,12 @@ def get_current_tenant_id(request: Request) -> str:
     """Dependency that extracts the tenant ID from the JWT token in the request and sets the context variable."""
     if not MULTI_TENANT:
         tenant_id = POSTGRES_DEFAULT_SCHEMA
-        current_tenant_id.set(tenant_id)
+        CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
         return tenant_id
 
     token = request.cookies.get("tenant_details")
     if not token:
-        current_value = current_tenant_id.get()
+        current_value = CURRENT_TENANT_ID_CONTEXTVAR.get()
         # If no token is present, use the default schema or handle accordingly
         return current_value
 
@@ -273,14 +273,14 @@ def get_current_tenant_id(request: Request) -> str:
         payload = jwt.decode(token, SECRET_JWT_KEY, algorithms=["HS256"])
         tenant_id = payload.get("tenant_id")
         if not tenant_id:
-            return current_tenant_id.get()
+            return CURRENT_TENANT_ID_CONTEXTVAR.get()
         if not is_valid_schema_name(tenant_id):
             raise HTTPException(status_code=400, detail="Invalid tenant ID format")
-        current_tenant_id.set(tenant_id)
+        CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
         return tenant_id
     except jwt.InvalidTokenError:
-        return current_tenant_id.get()
+        return CURRENT_TENANT_ID_CONTEXTVAR.get()
     except Exception as e:
         logger.error(f"Unexpected error in get_current_tenant_id: {str(e)}")
         raise HTTPException(status_code=500, detail="Internal server error")
@@ -291,7 +291,7 @@ async def get_async_session_with_tenant(
     tenant_id: str | None = None,
 ) -> AsyncGenerator[AsyncSession, None]:
     if tenant_id is None:
-        tenant_id = current_tenant_id.get()
+        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
 
     if not is_valid_schema_name(tenant_id):
         logger.error(f"Invalid tenant ID: {tenant_id}")
@@ -319,30 +319,32 @@ async def get_async_session_with_tenant(
 def get_session_with_tenant(
     tenant_id: str | None = None,
 ) -> Generator[Session, None, None]:
-    """Generate a database session with the appropriate tenant schema set."""
+    """Generate a database session bound to a connection with the appropriate tenant schema set."""
     engine = get_sqlalchemy_engine()
+
     if tenant_id is None:
-        tenant_id = current_tenant_id.get()
+        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+    else:
+        CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
+
+    event.listen(engine, "checkout", set_search_path_on_checkout)
 
     if not is_valid_schema_name(tenant_id):
         raise HTTPException(status_code=400, detail="Invalid tenant ID")
 
-    # Establish a raw connection without starting a transaction
+    # Establish a raw connection
     with engine.connect() as connection:
-        # Access the raw DBAPI connection
+        # Access the raw DBAPI connection and set the search_path
         dbapi_connection = connection.connection
 
-        # Execute SET search_path outside of any transaction
+        # Set the search_path outside of any transaction
         cursor = dbapi_connection.cursor()
         try:
-            cursor.execute(f'SET search_path TO "{tenant_id}"')
-            # Optionally verify the search_path was set correctly
-            cursor.execute("SHOW search_path")
-            cursor.fetchone()
+            cursor.execute(f'SET search_path = "{tenant_id}"')
         finally:
             cursor.close()
 
-        # Proceed to create a session using the connection
+        # Bind the session to the connection
         with Session(bind=connection, expire_on_commit=False) as session:
             try:
                 yield session
@@ -356,15 +358,27 @@ def get_session_with_tenant(
                         cursor.close()
 
 
+def set_search_path_on_checkout(
+    dbapi_conn: Any, connection_record: Any, connection_proxy: Any
+) -> None:
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+    if tenant_id and is_valid_schema_name(tenant_id):
+        with dbapi_conn.cursor() as cursor:
+            cursor.execute(f'SET search_path TO "{tenant_id}"')
+            logger.debug(
+                f"Set search_path to {tenant_id} for connection {connection_record}"
+            )
+
+
 def get_session_generator_with_tenant() -> Generator[Session, None, None]:
-    tenant_id = current_tenant_id.get()
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     with get_session_with_tenant(tenant_id) as session:
         yield session
 
 
 def get_session() -> Generator[Session, None, None]:
     """Generate a database session with the appropriate tenant schema set."""
-    tenant_id = current_tenant_id.get()
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     if tenant_id == POSTGRES_DEFAULT_SCHEMA and MULTI_TENANT:
         raise HTTPException(status_code=401, detail="User must authenticate")
 
@@ -381,7 +395,7 @@ def get_session() -> Generator[Session, None, None]:
 
 async def get_async_session() -> AsyncGenerator[AsyncSession, None]:
     """Generate an async database session with the appropriate tenant schema set."""
-    tenant_id = current_tenant_id.get()
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     engine = get_sqlalchemy_async_engine()
     async with AsyncSession(engine, expire_on_commit=False) as async_session:
         if MULTI_TENANT:
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index b461ca22feb..d0a17b26565 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -4,6 +4,7 @@
 from typing import cast
 
 from fastapi import HTTPException
+from redis.client import Redis
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
@@ -16,7 +17,7 @@
 from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
@@ -27,15 +28,22 @@
 
 
 class PgRedisKVStore(KeyValueStore):
-    def __init__(self) -> None:
-        self.redis_client = get_redis_client()
+    def __init__(
+        self, redis_client: Redis | None = None, tenant_id: str | None = None
+    ) -> None:
+        # If no redis_client is provided, fall back to the context var
+        if redis_client is not None:
+            self.redis_client = redis_client
+        else:
+            tenant_id = tenant_id or CURRENT_TENANT_ID_CONTEXTVAR.get()
+            self.redis_client = get_redis_client(tenant_id=tenant_id)
 
     @contextmanager
     def get_session(self) -> Iterator[Session]:
         engine = get_sqlalchemy_engine()
         with Session(engine, expire_on_commit=False) as session:
             if MULTI_TENANT:
-                tenant_id = current_tenant_id.get()
+                tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
                 if tenant_id == POSTGRES_DEFAULT_SCHEMA:
                     raise HTTPException(
                         status_code=401, detail="User must authenticate"
diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py
index fd08b9157bd..3f2ec03d77f 100644
--- a/backend/danswer/redis/redis_pool.py
+++ b/backend/danswer/redis/redis_pool.py
@@ -1,4 +1,7 @@
+import functools
 import threading
+from collections.abc import Callable
+from typing import Any
 from typing import Optional
 
 import redis
@@ -14,6 +17,98 @@
 from danswer.configs.app_configs import REDIS_SSL_CA_CERTS
 from danswer.configs.app_configs import REDIS_SSL_CERT_REQS
 from danswer.configs.constants import REDIS_SOCKET_KEEPALIVE_OPTIONS
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+class TenantRedis(redis.Redis):
+    def __init__(self, tenant_id: str, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.tenant_id: str = tenant_id
+
+    def _prefixed(self, key: str | bytes | memoryview) -> str | bytes | memoryview:
+        prefix: str = f"{self.tenant_id}:"
+        if isinstance(key, str):
+            if key.startswith(prefix):
+                return key
+            else:
+                return prefix + key
+        elif isinstance(key, bytes):
+            prefix_bytes = prefix.encode()
+            if key.startswith(prefix_bytes):
+                return key
+            else:
+                return prefix_bytes + key
+        elif isinstance(key, memoryview):
+            key_bytes = key.tobytes()
+            prefix_bytes = prefix.encode()
+            if key_bytes.startswith(prefix_bytes):
+                return key
+            else:
+                return memoryview(prefix_bytes + key_bytes)
+        else:
+            raise TypeError(f"Unsupported key type: {type(key)}")
+
+    def _prefix_method(self, method: Callable) -> Callable:
+        @functools.wraps(method)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            if "name" in kwargs:
+                kwargs["name"] = self._prefixed(kwargs["name"])
+            elif len(args) > 0:
+                args = (self._prefixed(args[0]),) + args[1:]
+            return method(*args, **kwargs)
+
+        return wrapper
+
+    def _prefix_scan_iter(self, method: Callable) -> Callable:
+        @functools.wraps(method)
+        def wrapper(*args: Any, **kwargs: Any) -> Any:
+            # Prefix the match pattern if provided
+            if "match" in kwargs:
+                kwargs["match"] = self._prefixed(kwargs["match"])
+            elif len(args) > 0:
+                args = (self._prefixed(args[0]),) + args[1:]
+
+            # Get the iterator
+            iterator = method(*args, **kwargs)
+
+            # Remove prefix from returned keys
+            prefix = f"{self.tenant_id}:".encode()
+            prefix_len = len(prefix)
+
+            for key in iterator:
+                if isinstance(key, bytes) and key.startswith(prefix):
+                    yield key[prefix_len:]
+                else:
+                    yield key
+
+        return wrapper
+
+    def __getattribute__(self, item: str) -> Any:
+        original_attr = super().__getattribute__(item)
+        methods_to_wrap = [
+            "lock",
+            "unlock",
+            "get",
+            "set",
+            "delete",
+            "exists",
+            "incrby",
+            "hset",
+            "hget",
+            "getset",
+            "owned",
+            "reacquire",
+            "create_lock",
+            "startswith",
+        ]  # Regular methods that need simple prefixing
+
+        if item == "scan_iter":
+            return self._prefix_scan_iter(original_attr)
+        elif item in methods_to_wrap and callable(original_attr):
+            return self._prefix_method(original_attr)
+        return original_attr
 
 
 class RedisPool:
@@ -32,8 +127,10 @@ def __new__(cls) -> "RedisPool":
     def _init_pool(self) -> None:
         self._pool = RedisPool.create_pool(ssl=REDIS_SSL)
 
-    def get_client(self) -> Redis:
-        return redis.Redis(connection_pool=self._pool)
+    def get_client(self, tenant_id: str | None) -> Redis:
+        if tenant_id is None:
+            tenant_id = "public"
+        return TenantRedis(tenant_id, connection_pool=self._pool)
 
     @staticmethod
     def create_pool(
@@ -84,8 +181,8 @@ def create_pool(
 redis_pool = RedisPool()
 
 
-def get_redis_client() -> Redis:
-    return redis_pool.get_client()
+def get_redis_client(*, tenant_id: str | None) -> Redis:
+    return redis_pool.get_client(tenant_id)
 
 
 # # Usage example
diff --git a/backend/danswer/search/preprocessing/preprocessing.py b/backend/danswer/search/preprocessing/preprocessing.py
index aa3124617e5..10832ae7cbf 100644
--- a/backend/danswer/search/preprocessing/preprocessing.py
+++ b/backend/danswer/search/preprocessing/preprocessing.py
@@ -10,7 +10,7 @@
 from danswer.configs.chat_configs import HYBRID_ALPHA_KEYWORD
 from danswer.configs.chat_configs import NUM_POSTPROCESSED_RESULTS
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
-from danswer.db.engine import current_tenant_id
+from danswer.db.engine import CURRENT_TENANT_ID_CONTEXTVAR
 from danswer.db.models import User
 from danswer.db.search_settings import get_current_search_settings
 from danswer.llm.interfaces import LLM
@@ -162,7 +162,7 @@ def retrieval_preprocessing(
         time_cutoff=time_filter or predicted_time_cutoff,
         tags=preset_filters.tags,  # Tags are never auto-extracted
         access_control_list=user_acl_filters,
-        tenant_id=current_tenant_id.get() if MULTI_TENANT else None,
+        tenant_id=CURRENT_TENANT_ID_CONTEXTVAR.get() if MULTI_TENANT else None,
     )
 
     llm_evaluation_type = LLMEvaluationType.BASIC
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index 92a94a63878..ddc084498ba 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -25,7 +25,8 @@
     update_connector_credential_pair_from_id,
 )
 from danswer.db.document import get_document_counts_for_cc_pairs
-from danswer.db.engine import current_tenant_id
+from danswer.db.engine import CURRENT_TENANT_ID_CONTEXTVAR
+from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import AccessType
 from danswer.db.enums import ConnectorCredentialPairStatus
@@ -94,8 +95,9 @@ def get_cc_pair_full_info(
     cc_pair_id: int,
     user: User | None = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
+    tenant_id: str | None = Depends(get_current_tenant_id),
 ) -> CCPairFullInfo:
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id, db_session, user, get_editable=False
@@ -147,6 +149,7 @@ def get_cc_pair_full_info(
             connector_id=cc_pair.connector_id,
             credential_id=cc_pair.credential_id,
             db_session=db_session,
+            tenant_id=tenant_id,
         ),
         num_docs_indexed=documents_indexed,
         is_editable_for_current_user=is_editable_for_current_user,
@@ -243,6 +246,7 @@ def prune_cc_pair(
     cc_pair_id: int,
     user: User = Depends(current_curator_or_admin_user),
     db_session: Session = Depends(get_session),
+    tenant_id: str | None = Depends(get_current_tenant_id),
 ) -> StatusResponse[list[int]]:
     """Triggers pruning on a particular cc_pair immediately"""
 
@@ -258,7 +262,7 @@ def prune_cc_pair(
             detail="Connection not found for current user's permissions",
         )
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
     rcp = RedisConnectorPruning(cc_pair_id)
     if rcp.is_pruning(r):
         raise HTTPException(
@@ -273,7 +277,7 @@ def prune_cc_pair(
         f"{cc_pair.connector.name} connector."
     )
     tasks_created = try_creating_prune_generator_task(
-        primary_app, cc_pair, db_session, r, current_tenant_id.get()
+        primary_app, cc_pair, db_session, r, CURRENT_TENANT_ID_CONTEXTVAR.get()
     )
     if not tasks_created:
         raise HTTPException(
@@ -359,7 +363,9 @@ def sync_cc_pair(
 
     logger.info(f"Syncing the {cc_pair.connector.name} connector.")
     sync_external_doc_permissions_task.apply_async(
-        kwargs=dict(cc_pair_id=cc_pair_id, tenant_id=current_tenant_id.get()),
+        kwargs=dict(
+            cc_pair_id=cc_pair_id, tenant_id=CURRENT_TENANT_ID_CONTEXTVAR.get()
+        ),
     )
 
     return StatusResponse(
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index a6ce87ad8a6..1ba0ab13e2c 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -493,10 +493,11 @@ def get_connector_indexing_status(
     get_editable: bool = Query(
         False, description="If true, return editable document sets"
     ),
+    tenant_id: str | None = Depends(get_current_tenant_id),
 ) -> list[ConnectorIndexingStatus]:
     indexing_statuses: list[ConnectorIndexingStatus] = []
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     # NOTE: If the connector is deleting behind the scenes,
     # accessing cc_pairs can be inconsistent and members like
@@ -617,6 +618,7 @@ def get_connector_indexing_status(
                     connector_id=connector.id,
                     credential_id=credential.id,
                     db_session=db_session,
+                    tenant_id=tenant_id,
                 ),
                 is_deletable=check_deletion_attempt_is_allowed(
                     connector_credential_pair=cc_pair,
@@ -694,15 +696,18 @@ def create_connector_with_mock_credential(
         connector_response = create_connector(
             db_session=db_session, connector_data=connector_data
         )
+
         mock_credential = CredentialBase(
             credential_json={}, admin_public=True, source=connector_data.source
         )
         credential = create_credential(
             mock_credential, user=user, db_session=db_session
         )
+
         access_type = (
             AccessType.PUBLIC if connector_data.is_public else AccessType.PRIVATE
         )
+
         response = add_credential_to_connector(
             db_session=db_session,
             user=user,
@@ -786,7 +791,7 @@ def connector_run_once(
     """Used to trigger indexing on a set of cc_pairs associated with a
     single connector."""
 
-    r = get_redis_client()
+    r = get_redis_client(tenant_id=tenant_id)
 
     connector_id = run_info.connector_id
     specified_credential_ids = run_info.credential_ids
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index ae2ab8c6e8c..cd2ffe08422 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -38,7 +38,7 @@
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
 from danswer.db.auth import get_total_users
-from danswer.db.engine import current_tenant_id
+from danswer.db.engine import CURRENT_TENANT_ID_CONTEXTVAR
 from danswer.db.engine import get_session
 from danswer.db.models import AccessToken
 from danswer.db.models import DocumentSet__User
@@ -188,7 +188,7 @@ def bulk_invite_users(
             status_code=400, detail="Auth is disabled, cannot invite users"
         )
 
-    tenant_id = current_tenant_id.get()
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
 
     normalized_emails = []
     try:
@@ -222,7 +222,9 @@ def bulk_invite_users(
         return number_of_invited_users
     try:
         logger.info("Registering tenant users")
-        register_tenant_users(current_tenant_id.get(), get_total_users(db_session))
+        register_tenant_users(
+            CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users(db_session)
+        )
         if ENABLE_EMAIL_INVITES:
             try:
                 for email in all_emails:
@@ -250,13 +252,15 @@ def remove_invited_user(
     user_emails = get_invited_users()
     remaining_users = [user for user in user_emails if user != user_email.user_email]
 
-    tenant_id = current_tenant_id.get()
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
     remove_users_from_tenant([user_email.user_email], tenant_id)
     number_of_invited_users = write_invited_users(remaining_users)
 
     try:
         if MULTI_TENANT:
-            register_tenant_users(current_tenant_id.get(), get_total_users(db_session))
+            register_tenant_users(
+                CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users(db_session)
+            )
     except Exception:
         logger.error(
             "Request to update number of seats taken in control plane failed. "
diff --git a/backend/danswer/server/query_and_chat/token_limit.py b/backend/danswer/server/query_and_chat/token_limit.py
index 6221eae3346..ec94e2ece4d 100644
--- a/backend/danswer/server/query_and_chat/token_limit.py
+++ b/backend/danswer/server/query_and_chat/token_limit.py
@@ -21,7 +21,7 @@
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from ee.danswer.db.token_limit import fetch_all_global_token_rate_limits
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 logger = setup_logger()
@@ -41,7 +41,7 @@ def check_token_rate_limits(
     versioned_rate_limit_strategy = fetch_versioned_implementation(
         "danswer.server.query_and_chat.token_limit", "_check_token_rate_limits"
     )
-    return versioned_rate_limit_strategy(user, current_tenant_id.get())
+    return versioned_rate_limit_strategy(user, CURRENT_TENANT_ID_CONTEXTVAR.get())
 
 
 def _check_token_rate_limits(_: User | None, tenant_id: str | None) -> None:
diff --git a/backend/ee/danswer/background/celery/apps/beat.py b/backend/ee/danswer/background/celery/apps/beat.py
index 20325e77df6..bee219e2471 100644
--- a/backend/ee/danswer/background/celery/apps/beat.py
+++ b/backend/ee/danswer/background/celery/apps/beat.py
@@ -41,7 +41,7 @@
         beat_schedule[task_name] = {
             "task": task["task"],
             "schedule": task["schedule"],
-            "args": (tenant_id,),  # Must pass tenant_id as an argument
+            "kwargs": {"tenant_id": tenant_id},  # Must pass tenant_id as an argument
         }
 
 # Include any existing beat schedules
diff --git a/backend/ee/danswer/background/celery/apps/primary.py b/backend/ee/danswer/background/celery/apps/primary.py
index 97c5b0221ca..be27d22868e 100644
--- a/backend/ee/danswer/background/celery/apps/primary.py
+++ b/backend/ee/danswer/background/celery/apps/primary.py
@@ -29,7 +29,7 @@
     run_external_group_permission_sync,
 )
 from ee.danswer.server.reporting.usage_export_generation import create_new_usage_report
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
@@ -39,7 +39,9 @@
 
 @build_celery_task_wrapper(name_sync_external_doc_permissions_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
-def sync_external_doc_permissions_task(cc_pair_id: int, tenant_id: str | None) -> None:
+def sync_external_doc_permissions_task(
+    cc_pair_id: int, *, tenant_id: str | None
+) -> None:
     with get_session_with_tenant(tenant_id) as db_session:
         run_external_doc_permission_sync(db_session=db_session, cc_pair_id=cc_pair_id)
 
@@ -47,7 +49,7 @@ def sync_external_doc_permissions_task(cc_pair_id: int, tenant_id: str | None) -
 @build_celery_task_wrapper(name_sync_external_group_permissions_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
 def sync_external_group_permissions_task(
-    cc_pair_id: int, tenant_id: str | None
+    cc_pair_id: int, *, tenant_id: str | None
 ) -> None:
     with get_session_with_tenant(tenant_id) as db_session:
         run_external_group_permission_sync(db_session=db_session, cc_pair_id=cc_pair_id)
@@ -56,7 +58,7 @@ def sync_external_group_permissions_task(
 @build_celery_task_wrapper(name_chat_ttl_task)
 @celery_app.task(soft_time_limit=JOB_TIMEOUT)
 def perform_ttl_management_task(
-    retention_limit_days: int, tenant_id: str | None
+    retention_limit_days: int, *, tenant_id: str | None
 ) -> None:
     with get_session_with_tenant(tenant_id) as db_session:
         delete_chat_sessions_older_than(retention_limit_days, db_session)
@@ -69,7 +71,7 @@ def perform_ttl_management_task(
     name="check_sync_external_doc_permissions_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_sync_external_doc_permissions_task(tenant_id: str | None) -> None:
+def check_sync_external_doc_permissions_task(*, tenant_id: str | None) -> None:
     """Runs periodically to sync external permissions"""
     with get_session_with_tenant(tenant_id) as db_session:
         cc_pairs = get_all_auto_sync_cc_pairs(db_session)
@@ -86,7 +88,7 @@ def check_sync_external_doc_permissions_task(tenant_id: str | None) -> None:
     name="check_sync_external_group_permissions_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_sync_external_group_permissions_task(tenant_id: str | None) -> None:
+def check_sync_external_group_permissions_task(*, tenant_id: str | None) -> None:
     """Runs periodically to sync external group permissions"""
     with get_session_with_tenant(tenant_id) as db_session:
         cc_pairs = get_all_auto_sync_cc_pairs(db_session)
@@ -103,12 +105,12 @@ def check_sync_external_group_permissions_task(tenant_id: str | None) -> None:
     name="check_ttl_management_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def check_ttl_management_task(tenant_id: str | None) -> None:
+def check_ttl_management_task(*, tenant_id: str | None) -> None:
     """Runs periodically to check if any ttl tasks should be run and adds them
     to the queue"""
     token = None
     if MULTI_TENANT and tenant_id is not None:
-        token = current_tenant_id.set(tenant_id)
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
     settings = load_settings()
     retention_limit_days = settings.maximum_chat_retention_days
@@ -120,14 +122,14 @@ def check_ttl_management_task(tenant_id: str | None) -> None:
                 ),
             )
     if token is not None:
-        current_tenant_id.reset(token)
+        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 @celery_app.task(
     name="autogenerate_usage_report_task",
     soft_time_limit=JOB_TIMEOUT,
 )
-def autogenerate_usage_report_task(tenant_id: str | None) -> None:
+def autogenerate_usage_report_task(*, tenant_id: str | None) -> None:
     """This generates usage report under the /admin/generate-usage/report endpoint"""
     with get_session_with_tenant(tenant_id) as db_session:
         create_new_usage_report(
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index 63b0f82be8f..eba02d0428b 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -11,7 +11,7 @@
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import SECRET_JWT_KEY
 from danswer.db.engine import is_valid_schema_name
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 
@@ -49,7 +49,7 @@ async def set_tenant_id(
                 else:
                     tenant_id = POSTGRES_DEFAULT_SCHEMA
 
-            current_tenant_id.set(tenant_id)
+            CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
             logger.info(f"Middleware set current_tenant_id to: {tenant_id}")
 
             response = await call_next(request)
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index 9ee598a2e26..342554c1c43 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -24,7 +24,7 @@
 from ee.danswer.server.tenants.provisioning import ensure_schema_exists
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
 from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
-from shared_configs.configs import current_tenant_id
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 stripe.api_key = STRIPE_SECRET_KEY
@@ -55,7 +55,7 @@ def create_tenant(
         else:
             logger.info(f"Schema already exists for tenant {tenant_id}")
 
-        token = current_tenant_id.set(tenant_id)
+        token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
         run_alembic_migrations(tenant_id)
 
         with get_session_with_tenant(tenant_id) as db_session:
@@ -74,7 +74,7 @@ def create_tenant(
         )
     finally:
         if token is not None:
-            current_tenant_id.reset(token)
+            CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 @router.post("/product-gating")
@@ -89,7 +89,7 @@ def gate_product(
     2) User's card has declined
     """
     tenant_id = product_gating_request.tenant_id
-    token = current_tenant_id.set(tenant_id)
+    token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
     settings = load_settings()
     settings.product_gating = product_gating_request.product_gating
@@ -100,7 +100,7 @@ def gate_product(
             create_notification(None, product_gating_request.notification, db_session)
 
     if token is not None:
-        current_tenant_id.reset(token)
+        CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
 
 
 @router.get("/billing-information", response_model=BillingInformation)
@@ -108,14 +108,16 @@ async def billing_information(
     _: User = Depends(current_admin_user),
 ) -> BillingInformation:
     logger.info("Fetching billing information")
-    return BillingInformation(**fetch_billing_information(current_tenant_id.get()))
+    return BillingInformation(
+        **fetch_billing_information(CURRENT_TENANT_ID_CONTEXTVAR.get())
+    )
 
 
 @router.post("/create-customer-portal-session")
 async def create_customer_portal_session(_: User = Depends(current_admin_user)) -> dict:
     try:
         # Fetch tenant_id and current tenant's information
-        tenant_id = current_tenant_id.get()
+        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
         stripe_info = fetch_tenant_stripe_information(tenant_id)
         stripe_customer_id = stripe_info.get("stripe_customer_id")
         if not stripe_customer_id:
diff --git a/backend/ee/danswer/server/tenants/billing.py b/backend/ee/danswer/server/tenants/billing.py
index 5dcd96713de..681ac835e5f 100644
--- a/backend/ee/danswer/server/tenants/billing.py
+++ b/backend/ee/danswer/server/tenants/billing.py
@@ -8,7 +8,6 @@
 from ee.danswer.configs.app_configs import STRIPE_PRICE_ID
 from ee.danswer.configs.app_configs import STRIPE_SECRET_KEY
 from ee.danswer.server.tenants.access import generate_data_plane_token
-from shared_configs.configs import current_tenant_id
 
 stripe.api_key = STRIPE_SECRET_KEY
 
@@ -50,7 +49,6 @@ def register_tenant_users(tenant_id: str, number_of_users: int) -> stripe.Subscr
     if not STRIPE_PRICE_ID:
         raise Exception("STRIPE_PRICE_ID is not set")
 
-    tenant_id = current_tenant_id.get()
     response = fetch_tenant_stripe_information(tenant_id)
     stripe_subscription_id = cast(str, response.get("stripe_subscription_id"))
 
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index 77139125f6e..5c24aebe749 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -131,7 +131,7 @@ def validate_cors_origin(origin: str) -> None:
 
 POSTGRES_DEFAULT_SCHEMA = os.environ.get("POSTGRES_DEFAULT_SCHEMA") or "public"
 
-current_tenant_id = contextvars.ContextVar(
+CURRENT_TENANT_ID_CONTEXTVAR = contextvars.ContextVar(
     "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
 )
 

From 87b597509154654fb4d8f07cac4351edfc11e8e9 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Thu, 24 Oct 2024 11:38:29 -0400
Subject: [PATCH 192/376] Remove unnecessary log & Add LoadConnector

---
 backend/danswer/connectors/freshdesk/connector.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 9173ba34c9a..8d82f59a23c 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -5,14 +5,14 @@
 from bs4 import BeautifulSoup  # Add this import for HTML parsing
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
-from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector
+from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector, LoadConnector
 from danswer.connectors.models import ConnectorMissingCredentialError, Document, Section
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
 
 
-class FreshdeskConnector(PollConnector):
+class FreshdeskConnector(PollConnector, LoadConnector):
     def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
 
@@ -37,7 +37,6 @@ def strip_html_tags(self, html: str) -> str:
         return soup.get_text()
 
     def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
-        logger.info("Loading credentials")
         self.api_key = credentials.get("freshdesk_api_key")
         self.domain = credentials.get("freshdesk_domain")
         self.password = credentials.get("freshdesk_password")

From cc1e1c178b4c0f1202ce1f82d1c3e8197e4f87f2 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Thu, 24 Oct 2024 11:49:11 -0400
Subject: [PATCH 193/376] Replace html processing library with danswer util

---
 backend/danswer/connectors/freshdesk/connector.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 8d82f59a23c..6148df2f2aa 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -2,7 +2,7 @@
 import json
 from datetime import datetime, timezone
 from typing import Any, List, Optional
-from bs4 import BeautifulSoup  # Add this import for HTML parsing
+from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector, LoadConnector
@@ -33,8 +33,7 @@ def build_doc_sections_from_ticket(self, ticket: dict) -> List[Section]:
         ]
 
     def strip_html_tags(self, html: str) -> str:
-        soup = BeautifulSoup(html, 'html.parser')
-        return soup.get_text()
+        return parse_html_page_basic(html)
 
     def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
         self.api_key = credentials.get("freshdesk_api_key")
@@ -43,7 +42,6 @@ def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, An
         return None
 
     def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
-        logger.info("Processing tickets")
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
 

From 4ad35d76b00bd33174d148bcb781eb32c1d13393 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Thu, 24 Oct 2024 12:25:29 -0400
Subject: [PATCH 194/376] Make ticket fetching a seperate function from
 processing

---
 .../danswer/connectors/freshdesk/connector.py | 124 +++++++++---------
 1 file changed, 65 insertions(+), 59 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 6148df2f2aa..51cc74ec854 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -1,6 +1,6 @@
 import requests
 import json
-from datetime import datetime, timezone
+from datetime import datetime
 from typing import Any, List, Optional
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
@@ -40,73 +40,79 @@ def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, An
         self.domain = credentials.get("freshdesk_domain")
         self.password = credentials.get("freshdesk_password")
         return None
-
-    def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+    
+    def _fetch_tickets(self) -> List[dict]:
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
-
+        
         freshdesk_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets?include=description"
         response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
         response.raise_for_status()  # raises exception when not a 2xx response
-
+        
         if response.status_code!= 204:
             tickets = json.loads(response.content)
             logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API")
-            doc_batch: List[Document] = []
-
-            for ticket in tickets:
-                # Convert the "created_at", "updated_at", and "due_by" values to ISO 8601 strings
-                for date_field in ["created_at", "updated_at", "due_by"]:
-                    if ticket[date_field].endswith('Z'):
-                        ticket[date_field] = ticket[date_field][:-1] + '+00:00'
-                    ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")
-
-                # Convert all other values to strings
-                ticket = {
-                    key: str(value) if not isinstance(value, str) else value
-                    for key, value in ticket.items()
-                }
-
-                # Checking for overdue tickets
-                today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-                ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
-
-                # Mapping the status field values
-                status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
-                ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
-
-                # Stripping HTML tags from the description field
-                ticket["description"] = self.strip_html_tags(ticket["description"])
-
-                # Remove extra white spaces from the description field
-                ticket["description"] = " ".join(ticket["description"].split())
-
-                # Use list comprehension for building sections
-                sections = self.build_doc_sections_from_ticket(ticket)
-
-                created_at = datetime.fromisoformat(ticket["created_at"])
-                today = datetime.now()
-                if (today - created_at).days / 30.4375 <= 2:
-                    doc = Document(
-                        id=ticket["id"],
-                        sections=sections,
-                        source=DocumentSource.FRESHDESK,
-                        semantic_identifier=ticket["subject"],
-                        metadata={
-                            key: value
-                            for key, value in ticket.items()
-                            if isinstance(value, str) and key not in ["description", "description_text"]
-                        },
-                    )
-
-                    doc_batch.append(doc)
-
-                    if len(doc_batch) >= self.batch_size:
-                        yield doc_batch
-                        doc_batch = []
-
-            if doc_batch:
+            return tickets
+        else:
+            return []
+
+    def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+        tickets = self._fetch_tickets()
+        doc_batch: List[Document] = []
+
+        for ticket in tickets:
+            #convert to iso format
+            for date_field in ["created_at", "updated_at", "due_by"]:
+                if ticket[date_field].endswith('Z'):
+                    ticket[date_field] = ticket[date_field][:-1] + '+00:00'
+                ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")
+
+            #convert all other values to strings
+            ticket = {
+                key: str(value) if not isinstance(value, str) else value
+                for key, value in ticket.items()
+            }
+
+            # Checking for overdue tickets
+            today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
+
+            # Mapping the status field values
+            status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
+            ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
+
+            # Stripping HTML tags from the description field
+            ticket["description"] = self.strip_html_tags(ticket["description"])
+
+            # Remove extra white spaces from the description field
+            ticket["description"] = " ".join(ticket["description"].split())
+
+            # Use list comprehension for building sections
+            sections = self.build_doc_sections_from_ticket(ticket)
+
+            created_at = datetime.fromisoformat(ticket["created_at"])
+            today = datetime.now()
+            if (today - created_at).days / 30.4375 <= 2:
+                doc = Document(
+                    id=ticket["id"],
+                    sections=sections,
+                    source=DocumentSource.FRESHDESK,
+                    semantic_identifier=ticket["subject"],
+                    metadata={
+                        key: value
+                        for key, value in ticket.items()
+                        if isinstance(value, str) and key not in ["description", "description_text"]
+                    },
+                )
+
+                doc_batch.append(doc)
+
+            if len(doc_batch) >= self.batch_size:
                 yield doc_batch
+                doc_batch = []
+
+        if doc_batch:
+            yield doc_batch
 
     def poll_source(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
         yield from self._process_tickets(start, end)

From 245adc4d3d222cc5f1c9c28608c5db534c93b93f Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Thu, 24 Oct 2024 12:42:08 -0400
Subject: [PATCH 195/376] Remove 2 month time check & Add time range to fetch
 and process

---
 backend/danswer/connectors/freshdesk/connector.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 51cc74ec854..9275ee26d04 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -41,11 +41,15 @@ def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, An
         self.password = credentials.get("freshdesk_password")
         return None
     
-    def _fetch_tickets(self) -> List[dict]:
+    def _fetch_tickets(self, start: datetime, end: datetime) -> List[dict]:
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
         
-        freshdesk_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets?include=description"
+        #convert start and end time to the format required by Freshdesk API
+        start_time = start.strftime("%Y-%m-%dT%H:%M:%SZ")
+        end_time = end.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+        freshdesk_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets?include=description&updated_since={start_time}&updated_before={end_time}"
         response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
         response.raise_for_status()  # raises exception when not a 2xx response
         
@@ -57,7 +61,7 @@ def _fetch_tickets(self) -> List[dict]:
             return []
 
     def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
-        tickets = self._fetch_tickets()
+        tickets = self._fetch_tickets(start, end)
         doc_batch: List[Document] = []
 
         for ticket in tickets:
@@ -91,8 +95,7 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
             sections = self.build_doc_sections_from_ticket(ticket)
 
             created_at = datetime.fromisoformat(ticket["created_at"])
-            today = datetime.now()
-            if (today - created_at).days / 30.4375 <= 2:
+            if start <= created_at <= end:
                 doc = Document(
                     id=ticket["id"],
                     sections=sections,

From 1b6b1347222b79e218560cfa892a6ca443a2d4fa Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 24 Oct 2024 10:29:36 -0700
Subject: [PATCH 196/376] Clearer azure models (#2898)

* clear up llm

* remove logs
---
 backend/danswer/llm/factory.py              | 4 +++-
 backend/danswer/server/manage/llm/api.py    | 2 ++
 backend/danswer/server/manage/llm/models.py | 1 +
 backend/requirements/model_server.txt       | 4 ++--
 4 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/llm/factory.py b/backend/danswer/llm/factory.py
index f930c3d3358..eedf7ccc763 100644
--- a/backend/danswer/llm/factory.py
+++ b/backend/danswer/llm/factory.py
@@ -51,6 +51,7 @@ def _create_llm(model: str) -> LLM:
         return get_llm(
             provider=llm_provider.provider,
             model=model,
+            deployment_name=llm_provider.deployment_name,
             api_key=llm_provider.api_key,
             api_base=llm_provider.api_base,
             api_version=llm_provider.api_version,
@@ -104,7 +105,7 @@ def _create_llm(model: str) -> LLM:
 def get_llm(
     provider: str,
     model: str,
-    deployment_name: str | None = None,
+    deployment_name: str | None,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -116,6 +117,7 @@ def get_llm(
     return DefaultMultiLLM(
         model_provider=provider,
         model_name=model,
+        deployment_name=deployment_name,
         api_key=api_key,
         api_base=api_base,
         api_version=api_version,
diff --git a/backend/danswer/server/manage/llm/api.py b/backend/danswer/server/manage/llm/api.py
index 06501d6834c..9cac96236b0 100644
--- a/backend/danswer/server/manage/llm/api.py
+++ b/backend/danswer/server/manage/llm/api.py
@@ -54,6 +54,7 @@ def test_llm_configuration(
         api_base=test_llm_request.api_base,
         api_version=test_llm_request.api_version,
         custom_config=test_llm_request.custom_config,
+        deployment_name=test_llm_request.deployment_name,
     )
 
     functions_with_args: list[tuple[Callable, tuple]] = [(test_llm, (llm,))]
@@ -70,6 +71,7 @@ def test_llm_configuration(
             api_base=test_llm_request.api_base,
             api_version=test_llm_request.api_version,
             custom_config=test_llm_request.custom_config,
+            deployment_name=test_llm_request.deployment_name,
         )
         functions_with_args.append((test_llm, (fast_llm,)))
 
diff --git a/backend/danswer/server/manage/llm/models.py b/backend/danswer/server/manage/llm/models.py
index 2e3b3844807..9b371099c57 100644
--- a/backend/danswer/server/manage/llm/models.py
+++ b/backend/danswer/server/manage/llm/models.py
@@ -21,6 +21,7 @@ class TestLLMRequest(BaseModel):
     # model level
     default_model_name: str
     fast_default_model_name: str | None = None
+    deployment_name: str | None = None
 
 
 class LLMProviderDescriptor(BaseModel):
diff --git a/backend/requirements/model_server.txt b/backend/requirements/model_server.txt
index 2ea6df66d8b..6160555f7b1 100644
--- a/backend/requirements/model_server.txt
+++ b/backend/requirements/model_server.txt
@@ -1,5 +1,5 @@
-cohere==5.6.1
 einops==0.8.0
+cohere==5.6.1
 fastapi==0.109.2
 google-cloud-aiplatform==1.58.0
 numpy==1.26.4
@@ -13,4 +13,4 @@ transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
 litellm==1.49.5
-sentry-sdk[fastapi,celery,starlette]==2.14.0
+sentry-sdk[fastapi,celery,starlette]==2.14.0
\ No newline at end of file

From 2b9a751b96f7c730890c8dda6714ddcbd4402b45 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 12:50:09 -0700
Subject: [PATCH 197/376] working chat feedback dump script (with api addition)
 (#2891)

* working chat feedback dump script (with api addition)

* mypy fix

* comment out pydantic models (but leave for reference)

* small code review tweaks

* bump to clear vercel issue?
---
 .../ee/danswer/server/query_history/api.py    |  33 +++
 backend/scripts/chat_feedback_dump.py         | 239 ++++++++++++++++++
 2 files changed, 272 insertions(+)
 create mode 100644 backend/scripts/chat_feedback_dump.py

diff --git a/backend/ee/danswer/server/query_history/api.py b/backend/ee/danswer/server/query_history/api.py
index 1411f973e36..f50b9cc5230 100644
--- a/backend/ee/danswer/server/query_history/api.py
+++ b/backend/ee/danswer/server/query_history/api.py
@@ -20,10 +20,13 @@
 from danswer.configs.constants import QAFeedbackType
 from danswer.configs.constants import SessionType
 from danswer.db.chat import get_chat_session_by_id
+from danswer.db.chat import get_chat_sessions_by_user
 from danswer.db.engine import get_session
 from danswer.db.models import ChatMessage
 from danswer.db.models import ChatSession
 from danswer.db.models import User
+from danswer.server.query_and_chat.models import ChatSessionDetails
+from danswer.server.query_and_chat.models import ChatSessionsResponse
 from ee.danswer.db.query_history import fetch_chat_sessions_eagerly_by_time
 
 router = APIRouter()
@@ -330,6 +333,36 @@ def snapshot_from_chat_session(
     )
 
 
+@router.get("/admin/chat-sessions")
+def get_user_chat_sessions(
+    user_id: UUID,
+    _: User | None = Depends(current_admin_user),
+    db_session: Session = Depends(get_session),
+) -> ChatSessionsResponse:
+    try:
+        chat_sessions = get_chat_sessions_by_user(
+            user_id=user_id, deleted=False, db_session=db_session, limit=0
+        )
+
+    except ValueError:
+        raise ValueError("Chat session does not exist or has been deleted")
+
+    return ChatSessionsResponse(
+        sessions=[
+            ChatSessionDetails(
+                id=chat.id,
+                name=chat.description,
+                persona_id=chat.persona_id,
+                time_created=chat.time_created.isoformat(),
+                shared_status=chat.shared_status,
+                folder_id=chat.folder_id,
+                current_alternate_model=chat.current_alternate_model,
+            )
+            for chat in chat_sessions
+        ]
+    )
+
+
 @router.get("/admin/chat-session-history")
 def get_chat_session_history(
     feedback_type: QAFeedbackType | None = None,
diff --git a/backend/scripts/chat_feedback_dump.py b/backend/scripts/chat_feedback_dump.py
new file mode 100644
index 00000000000..f0d6d3cbb37
--- /dev/null
+++ b/backend/scripts/chat_feedback_dump.py
@@ -0,0 +1,239 @@
+# This file is used to demonstrate how to use the backend APIs directly
+# to query out feedback for all messages
+import argparse
+import logging
+from logging import getLogger
+from typing import Any
+from uuid import UUID
+
+import requests
+
+from danswer.server.manage.models import AllUsersResponse
+from danswer.server.query_and_chat.models import ChatSessionsResponse
+from ee.danswer.server.query_history.api import ChatSessionSnapshot
+
+# Configure the logger
+logging.basicConfig(
+    level=logging.INFO,  # Set the log level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",  # Log format
+    handlers=[logging.StreamHandler()],  # Output logs to console
+)
+
+logger = getLogger(__name__)
+
+# uncomment the following pydantic models if you need the script to be independent
+# from pydantic import BaseModel
+# from datetime import datetime
+# from enum import Enum
+
+# class UserRole(str, Enum):
+#     """
+#     User roles
+#     - Basic can't perform any admin actions
+#     - Admin can perform all admin actions
+#     - Curator can perform admin actions for
+#         groups they are curators of
+#     - Global Curator can perform admin actions
+#         for all groups they are a member of
+#     """
+
+#     BASIC = "basic"
+#     ADMIN = "admin"
+#     CURATOR = "curator"
+#     GLOBAL_CURATOR = "global_curator"
+
+
+# class UserStatus(str, Enum):
+#     LIVE = "live"
+#     INVITED = "invited"
+#     DEACTIVATED = "deactivated"
+
+
+# class FullUserSnapshot(BaseModel):
+#     id: UUID
+#     email: str
+#     role: UserRole
+#     status: UserStatus
+
+
+# class InvitedUserSnapshot(BaseModel):
+#     email: str
+
+
+# class AllUsersResponse(BaseModel):
+#     accepted: list[FullUserSnapshot]
+#     invited: list[InvitedUserSnapshot]
+#     accepted_pages: int
+#     invited_pages: int
+
+
+# class ChatSessionSharedStatus(str, Enum):
+#     PUBLIC = "public"
+#     PRIVATE = "private"
+
+
+# class ChatSessionDetails(BaseModel):
+#     id: UUID
+#     name: str
+#     persona_id: int | None = None
+#     time_created: str
+#     shared_status: ChatSessionSharedStatus
+#     folder_id: int | None = None
+#     current_alternate_model: str | None = None
+
+
+# class ChatSessionsResponse(BaseModel):
+#     sessions: list[ChatSessionDetails]
+
+
+# class SessionType(str, Enum):
+#     CHAT = "Chat"
+#     SEARCH = "Search"
+#     SLACK = "Slack"
+
+
+# class AbridgedSearchDoc(BaseModel):
+#     """A subset of the info present in `SearchDoc`"""
+
+#     document_id: str
+#     semantic_identifier: str
+#     link: str | None
+
+
+# class QAFeedbackType(str, Enum):
+#     LIKE = "like"  # User likes the answer, used for metrics
+#     DISLIKE = "dislike"  # User dislikes the answer, used for metrics
+
+
+# class MessageType(str, Enum):
+#     # Using OpenAI standards, Langchain equivalent shown in comment
+#     # System message is always constructed on the fly, not saved
+#     SYSTEM = "system"  # SystemMessage
+#     USER = "user"  # HumanMessage
+#     ASSISTANT = "assistant"  # AIMessage
+
+
+# class MessageSnapshot(BaseModel):
+#     message: str
+#     message_type: MessageType
+#     documents: list[AbridgedSearchDoc]
+#     feedback_type: QAFeedbackType | None
+#     feedback_text: str | None
+#     time_created: datetime
+
+
+# class ChatSessionSnapshot(BaseModel):
+#     id: UUID
+#     user_email: str
+#     name: str | None
+#     messages: list[MessageSnapshot]
+#     persona_name: str | None
+#     time_created: datetime
+#     flow_type: SessionType
+
+
+def create_new_chat_session(danswer_url: str, api_key: str | None) -> int:
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else None
+    session_endpoint = danswer_url + "/api/chat/create-chat-session"
+
+    response = requests.get(session_endpoint, headers=headers)
+    response.raise_for_status()
+
+    new_session_id = response.json()["chat_session_id"]
+    return new_session_id
+
+
+def manage_users(danswer_url: str, headers: dict[str, str] | None) -> AllUsersResponse:
+    endpoint = danswer_url + "/manage/users"
+
+    response = requests.get(
+        endpoint,
+        headers=headers,
+    )
+    response.raise_for_status()
+
+    all_users = AllUsersResponse(**response.json())
+    return all_users
+
+
+def get_chat_sessions(
+    danswer_url: str, headers: dict[str, str] | None, user_id: UUID
+) -> ChatSessionsResponse:
+    endpoint = danswer_url + "/admin/chat-sessions"
+
+    params: dict[str, Any] = {"user_id": user_id}
+    response = requests.get(
+        endpoint,
+        params=params,
+        headers=headers,
+    )
+    response.raise_for_status()
+
+    sessions = ChatSessionsResponse(**response.json())
+    return sessions
+
+
+def get_session_history(
+    danswer_url: str, headers: dict[str, str] | None, session_id: UUID
+) -> ChatSessionSnapshot:
+    endpoint = danswer_url + f"/admin/chat-session-history/{session_id}"
+
+    response = requests.get(
+        endpoint,
+        headers=headers,
+    )
+    response.raise_for_status()
+
+    sessions = ChatSessionSnapshot(**response.json())
+    return sessions
+
+
+def process_all_chat_feedback(danswer_url: str, api_key: str | None) -> None:
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else None
+
+    all_users = manage_users(danswer_url, headers)
+    if not all_users:
+        raise RuntimeError("manage_users returned None")
+
+    logger.info(f"Accepted users: {len(all_users.accepted)}")
+
+    user_ids: list[UUID] = [user.id for user in all_users.accepted]
+
+    for user_id in user_ids:
+        r_sessions = get_chat_sessions(danswer_url, headers, user_id)
+        logger.info(f"user={user_id} num_sessions={len(r_sessions.sessions)}")
+        for session in r_sessions.sessions:
+            try:
+                s = get_session_history(danswer_url, headers, session.id)
+            except requests.exceptions.HTTPError:
+                logger.exception("get_session_history failed.")
+
+            for m in s.messages:
+                logger.info(
+                    f"user={user_id} "
+                    f"session={session.id} "
+                    f"message={m.message} "
+                    f"feedback_type={m.feedback_type} "
+                    f"feedback_text={m.feedback_text}"
+                )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Sample API Usage - Chat Feedback")
+    parser.add_argument(
+        "--url",
+        type=str,
+        default="http://localhost:8080",
+        help="Danswer URL, should point to Danswer nginx.",
+    )
+
+    # Not needed if Auth is disabled?
+    # Or for Danswer MIT Edition API key must be replaced with session cookie
+    parser.add_argument(
+        "--api-key",
+        type=str,
+        help="Danswer Admin Level API key",
+    )
+
+    args = parser.parse_args()
+    process_all_chat_feedback(danswer_url=args.url, api_key=args.api_key)

From 32b595dfe1210218de40d3083a239927da261220 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 13:31:39 -0700
Subject: [PATCH 198/376] update stale workflow

---
 .github/workflows/nightly-close-stale-issues.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-close-stale-issues.yml b/.github/workflows/nightly-close-stale-issues.yml
index 393d3ec950f..a7d296e0a92 100644
--- a/.github/workflows/nightly-close-stale-issues.yml
+++ b/.github/workflows/nightly-close-stale-issues.yml
@@ -1,8 +1,13 @@
-name: 'Close stale issues and PRs'
+name: 'Nightly - Close stale issues and PRs'
 on:
   schedule:
     - cron: '0 11 * * *' # Runs every day at 3 AM PST / 4 AM PDT / 11 AM UTC
 
+permissions:
+  # contents: write # only for delete-branch option
+  issues: write
+  pull-requests: write
+  
 jobs:
   stale:
     runs-on: ubuntu-latest

From 705b8255808eef66cc614008412489267c4a4b5a Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 14:21:38 -0700
Subject: [PATCH 199/376] fix typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 087b0df531f..1ff4338c530 100644
--- a/README.md
+++ b/README.md
@@ -68,7 +68,7 @@ We also have built-in support for deployment on Kubernetes. Files for that can b
 
 ## 🚧 Roadmap
 * Chat/Prompt sharing with specific teammates and user groups.
-* Multi-Model model support, chat with images, video etc.
+* Multimodal model support, chat with images, video etc.
 * Choosing between LLMs and parameters during chat session.
 * Tool calling and agent configurations options.
 * Organizational understanding and ability to locate and suggest experts from your team.

From da979e574531e1e9c943001087ddb5156039d434 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 24 Oct 2024 14:27:34 -0700
Subject: [PATCH 200/376] More intuitive search settings interfaces (#2899)

* clearer search settings interfaces

* nits
---
 .../admin/embeddings/RerankingFormPage.tsx    |  7 ++--
 web/src/app/admin/embeddings/interfaces.ts    |  5 ---
 .../pages/AdvancedEmbeddingFormPage.tsx       | 12 ++++---
 .../embeddings/pages/EmbeddingFormPage.tsx    | 32 ++++++++++++-------
 web/src/app/chat/ChatPage.tsx                 |  1 -
 .../embedding/CustomEmbeddingModelForm.tsx    |  9 ------
 web/src/components/embedding/interfaces.tsx   | 31 ------------------
 7 files changed, 32 insertions(+), 65 deletions(-)

diff --git a/web/src/app/admin/embeddings/RerankingFormPage.tsx b/web/src/app/admin/embeddings/RerankingFormPage.tsx
index 1e0aae06594..67f5fc1614b 100644
--- a/web/src/app/admin/embeddings/RerankingFormPage.tsx
+++ b/web/src/app/admin/embeddings/RerankingFormPage.tsx
@@ -90,9 +90,10 @@ const RerankingDetailsForm = forwardRef<
 
           return (
             <div className="p-2 rounded-lg max-w-4xl mx-auto">
-              <h2 className="text-2xl font-bold mb-4 text-text-800">
-                Post-processing
-              </h2>
+              <p className="mb-4">
+                Select from cloud, self-hosted models, or use no reranking
+                model.
+              </p>
               <div className="text-sm mr-auto mb-6 divide-x-2 flex">
                 {originalRerankingDetails.rerank_model_name && (
                   <button
diff --git a/web/src/app/admin/embeddings/interfaces.ts b/web/src/app/admin/embeddings/interfaces.ts
index 2fc328eab86..3f7549fa741 100644
--- a/web/src/app/admin/embeddings/interfaces.ts
+++ b/web/src/app/admin/embeddings/interfaces.ts
@@ -21,11 +21,6 @@ export enum RerankerProvider {
 }
 
 export interface AdvancedSearchConfiguration {
-  model_name: string;
-  model_dim: number;
-  normalize: boolean;
-  query_prefix: string;
-  passage_prefix: string;
   index_name: string | null;
   multipass_indexing: boolean;
   multilingual_expansion: string[];
diff --git a/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
index b708675116b..bea80322e1a 100644
--- a/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/AdvancedEmbeddingFormPage.tsx
@@ -4,7 +4,11 @@ import * as Yup from "yup";
 import { TrashIcon } from "@/components/icons/icons";
 import { FaPlus } from "react-icons/fa";
 import { AdvancedSearchConfiguration } from "../interfaces";
-import { BooleanFormField } from "@/components/admin/connectors/Field";
+import {
+  BooleanFormField,
+  Label,
+  SubLabel,
+} from "@/components/admin/connectors/Field";
 import NumberInput from "../../connectors/[connector]/pages/ConnectorInput/NumberInput";
 
 interface AdvancedEmbeddingFormPageProps {
@@ -21,9 +25,6 @@ const AdvancedEmbeddingFormPage = forwardRef<
 >(({ updateAdvancedEmbeddingDetails, advancedEmbeddingDetails }, ref) => {
   return (
     <div className="py-4 rounded-lg max-w-4xl px-4 mx-auto">
-      <h2 className="text-2xl font-bold mb-4 text-text-800">
-        Advanced Configuration
-      </h2>
       <Formik
         innerRef={ref}
         initialValues={advancedEmbeddingDetails}
@@ -52,6 +53,9 @@ const AdvancedEmbeddingFormPage = forwardRef<
             <FieldArray name="multilingual_expansion">
               {({ push, remove }) => (
                 <div className="w-full">
+                  <Label>Multi-lingual Expansion</Label>
+
+                  <SubLabel>Add additional languages to the search.</SubLabel>
                   {values.multilingual_expansion.map(
                     (_: any, index: number) => (
                       <div key={index} className="w-full flex mb-4">
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index 90f1b70fa04..4947e8759d5 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -33,11 +33,6 @@ export default function EmbeddingForm() {
 
   const [advancedEmbeddingDetails, setAdvancedEmbeddingDetails] =
     useState<AdvancedSearchConfiguration>({
-      model_name: "",
-      model_dim: 0,
-      normalize: false,
-      query_prefix: "",
-      passage_prefix: "",
       index_name: "",
       multipass_indexing: true,
       multilingual_expansion: [],
@@ -109,11 +104,6 @@ export default function EmbeddingForm() {
   useEffect(() => {
     if (searchSettings) {
       setAdvancedEmbeddingDetails({
-        model_name: searchSettings.model_name,
-        model_dim: searchSettings.model_dim,
-        normalize: searchSettings.normalize,
-        query_prefix: searchSettings.query_prefix,
-        passage_prefix: searchSettings.passage_prefix,
         index_name: searchSettings.index_name,
         multipass_indexing: searchSettings.multipass_indexing,
         multilingual_expansion: searchSettings.multilingual_expansion,
@@ -263,8 +253,8 @@ export default function EmbeddingForm() {
       // This is a cloud model
       newModel = {
         ...selectedProvider,
-        ...rerankingDetails,
         ...advancedEmbeddingDetails,
+        ...rerankingDetails,
         provider_type:
           (selectedProvider.provider_type
             ?.toLowerCase()
@@ -274,8 +264,8 @@ export default function EmbeddingForm() {
       // This is a locally hosted model
       newModel = {
         ...selectedProvider,
-        ...rerankingDetails,
         ...advancedEmbeddingDetails,
+        ...rerankingDetails,
         provider_type: null,
       };
     }
@@ -388,6 +378,16 @@ export default function EmbeddingForm() {
 
         {formStep == 1 && (
           <>
+            <h2 className="text-2xl font-bold mb-4 text-text-800">
+              Select a Reranking Model
+            </h2>
+            <Text className="mb-4">
+              Updating the reranking model does not require re-indexing
+              documents. The reranker helps improve search quality by reordering
+              results after the initial embedding search. Changes will take
+              effect immediately for all new searches.
+            </Text>
+
             <Card>
               <RerankingDetailsForm
                 setModelTab={setModelTab}
@@ -429,6 +429,14 @@ export default function EmbeddingForm() {
         )}
         {formStep == 2 && (
           <>
+            <h2 className="text-2xl font-bold mb-4 text-text-800">
+              Advanced Search Configuration
+            </h2>
+            <Text className="mb-4">
+              Configure advanced embedding and search settings. Changes will
+              require re-indexing documents.
+            </Text>
+
             <Card>
               <AdvancedEmbeddingFormPage
                 advancedEmbeddingDetails={advancedEmbeddingDetails}
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index b67ec473026..2ce657b2caf 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -809,7 +809,6 @@ export function ChatPage({
 
       // Check if all messages are currently rendered
       if (currentVisibleRange.end < messageHistory.length) {
-        console.log("Updating visible range");
         // Update visible range to include the last messages
         updateCurrentVisibleRange({
           start: Math.max(
diff --git a/web/src/components/embedding/CustomEmbeddingModelForm.tsx b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
index 8dcd34f0ba6..7ba29531863 100644
--- a/web/src/components/embedding/CustomEmbeddingModelForm.tsx
+++ b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
@@ -33,10 +33,6 @@ export function CustomEmbeddingModelForm({
             api_url: provider.api_url,
             description: "",
             index_name: "",
-            pricePerMillion: 0,
-            mtebScore: 0,
-            maxContext: 4096,
-            max_tokens: 1024,
           }
         }
         validationSchema={Yup.object().shape({
@@ -51,13 +47,8 @@ export function CustomEmbeddingModelForm({
           api_url: Yup.string().required("API base URL is required"),
           description: Yup.string(),
           index_name: Yup.string().nullable(),
-          pricePerMillion: Yup.number(),
-          mtebScore: Yup.number(),
-          maxContext: Yup.number(),
-          max_tokens: Yup.number(),
         })}
         onSubmit={async (values) => {
-          console.log(values);
           setShowTentativeModel(values as CloudEmbeddingModel);
         }}
       >
diff --git a/web/src/components/embedding/interfaces.tsx b/web/src/components/embedding/interfaces.tsx
index ff80ffd6ace..87a16959864 100644
--- a/web/src/components/embedding/interfaces.tsx
+++ b/web/src/components/embedding/interfaces.tsx
@@ -58,17 +58,10 @@ export interface EmbeddingModelDescriptor {
 
 export interface CloudEmbeddingModel extends EmbeddingModelDescriptor {
   pricePerMillion: number;
-  enabled?: boolean;
-  mtebScore: number;
-  maxContext: number;
 }
 
 export interface HostedEmbeddingModel extends EmbeddingModelDescriptor {
   link?: string;
-  model_dim: number;
-  normalize: boolean;
-  query_prefix: string;
-  passage_prefix: string;
   isDefault?: boolean;
 }
 
@@ -197,9 +190,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Cohere's English embedding model. Good performance for English-language tasks.",
         pricePerMillion: 0.1,
-        mtebScore: 64.5,
-        maxContext: 512,
-        enabled: false,
         model_dim: 1024,
         normalize: false,
         query_prefix: "",
@@ -214,9 +204,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Cohere's lightweight English embedding model. Faster and more efficient for simpler tasks.",
         pricePerMillion: 0.1,
-        mtebScore: 62,
-        maxContext: 512,
-        enabled: false,
         model_dim: 384,
         normalize: false,
         query_prefix: "",
@@ -247,9 +234,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         normalize: false,
         query_prefix: "",
         passage_prefix: "",
-        mtebScore: 64.6,
-        maxContext: 8191,
-        enabled: false,
         index_name: "",
         api_key: null,
         api_url: null,
@@ -264,9 +248,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "OpenAI's newer, more efficient embedding model. Good balance of performance and cost.",
         pricePerMillion: 0.02,
-        enabled: false,
-        mtebScore: 62.3,
-        maxContext: 8191,
         index_name: "",
         api_key: null,
         api_url: null,
@@ -290,9 +271,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         model_name: "text-embedding-004",
         description: "Google's most recent text embedding model.",
         pricePerMillion: 0.025,
-        mtebScore: 66.31,
-        maxContext: 2048,
-        enabled: false,
         model_dim: 768,
         normalize: false,
         query_prefix: "",
@@ -306,9 +284,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         model_name: "textembedding-gecko@003",
         description: "Google's Gecko embedding model. Powerful and efficient.",
         pricePerMillion: 0.025,
-        mtebScore: 66.31,
-        maxContext: 2048,
-        enabled: false,
         model_dim: 768,
         normalize: false,
         query_prefix: "",
@@ -335,9 +310,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Voyage's large embedding model. High performance with instruction fine-tuning.",
         pricePerMillion: 0.12,
-        mtebScore: 68.28,
-        maxContext: 4000,
-        enabled: false,
         model_dim: 1024,
         normalize: false,
         query_prefix: "",
@@ -352,9 +324,6 @@ export const AVAILABLE_CLOUD_PROVIDERS: CloudEmbeddingProvider[] = [
         description:
           "Voyage's lightweight embedding model. Good balance of performance and efficiency.",
         pricePerMillion: 0.12,
-        mtebScore: 67.13,
-        maxContext: 16000,
-        enabled: false,
         model_dim: 1024,
         normalize: false,
         query_prefix: "",

From 33eabf1b253ac2af8243aabd66fc8f354e767984 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 24 Oct 2024 14:27:55 -0700
Subject: [PATCH 201/376] Add global assistants context (#2900)

* add global assistants context

* nit

* minor cleanup

* minor clarity

* nit
---
 backend/danswer/db/persona.py                 |  1 -
 .../app/admin/assistants/AssistantEditor.tsx  |  3 +
 web/src/app/admin/assistants/PersonaTable.tsx | 56 +++++++--------
 web/src/app/admin/assistants/page.tsx         | 28 +-------
 web/src/app/assistants/gallery/page.tsx       | 25 +++----
 web/src/app/assistants/mine/page.tsx          | 11 +--
 web/src/app/chat/page.tsx                     | 39 ++++-------
 web/src/app/layout.tsx                        | 21 ++++--
 web/src/app/search/page.tsx                   | 43 +++++-------
 web/src/components/context/AppProvider.tsx    | 39 +++++++++++
 .../components/context/AssistantsContext.tsx  | 65 ++++++++++++++++-
 web/src/lib/chat/fetchAssistantdata.ts        | 69 +++++++++++++++++++
 web/src/lib/chat/fetchChatData.ts             | 52 ++------------
 13 files changed, 265 insertions(+), 187 deletions(-)
 create mode 100644 web/src/components/context/AppProvider.tsx
 create mode 100644 web/src/lib/chat/fetchAssistantdata.ts

diff --git a/backend/danswer/db/persona.py b/backend/danswer/db/persona.py
index 36d2d25c402..f23b3f6d071 100644
--- a/backend/danswer/db/persona.py
+++ b/backend/danswer/db/persona.py
@@ -328,7 +328,6 @@ def update_all_personas_display_priority(
 
     for persona in personas:
         persona.display_priority = display_priority_map[persona.id]
-
     db_session.commit()
 
 
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 7ca1d087641..2eec9632c66 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -61,6 +61,7 @@ import {
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import { buildImgUrl } from "@/app/chat/files/images/utils";
 import { LlmList } from "@/components/llm/LLMList";
+import { useAssistants } from "@/components/context/AssistantsContext";
 
 function findSearchTool(tools: ToolSnapshot[]) {
   return tools.find((tool) => tool.in_code_tool_id === "SearchTool");
@@ -105,6 +106,7 @@ export function AssistantEditor({
   shouldAddAssistantToUserPreferences?: boolean;
   admin?: boolean;
 }) {
+  const { refreshAssistants } = useAssistants();
   const router = useRouter();
 
   const { popup, setPopup } = usePopup();
@@ -433,6 +435,7 @@ export function AssistantEditor({
                 });
               }
             }
+            await refreshAssistants();
             router.push(
               redirectType === SuccessfulPersonaUpdateRedirectType.ADMIN
                 ? `/admin/assistants?u=${Date.now()}`
diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index 15775c7cdfb..6adf22a525e 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -5,7 +5,7 @@ import { Persona } from "./interfaces";
 import { useRouter } from "next/navigation";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { useState, useMemo } from "react";
+import { useState, useMemo, useEffect } from "react";
 import { UniqueIdentifier } from "@dnd-kit/core";
 import { DraggableTable } from "@/components/table/DraggableTable";
 import {
@@ -16,6 +16,7 @@ import {
 import { FiEdit2 } from "react-icons/fi";
 import { TrashIcon } from "@/components/icons/icons";
 import { useUser } from "@/components/user/UserProvider";
+import { useAssistants } from "@/components/context/AssistantsContext";
 
 function PersonaTypeDisplay({ persona }: { persona: Persona }) {
   if (persona.builtin_persona) {
@@ -37,43 +38,36 @@ function PersonaTypeDisplay({ persona }: { persona: Persona }) {
   return <Text>Personal {persona.owner && <>({persona.owner.email})</>}</Text>;
 }
 
-export function PersonasTable({
-  allPersonas,
-  editablePersonas,
-}: {
-  allPersonas: Persona[];
-  editablePersonas: Persona[];
-}) {
+export function PersonasTable() {
   const router = useRouter();
   const { popup, setPopup } = usePopup();
-
-  const { isLoadingUser, isAdmin } = useUser();
+  const { refreshUser, isLoadingUser, isAdmin } = useUser();
+  const {
+    allAssistants: assistants,
+    refreshAssistants,
+    editablePersonas,
+  } = useAssistants();
 
   const editablePersonaIds = useMemo(() => {
     return new Set(editablePersonas.map((p) => p.id.toString()));
   }, [editablePersonas]);
 
-  const sortedPersonas = useMemo(() => {
+  const [finalPersonas, setFinalPersonas] = useState<Persona[]>([]);
+
+  useEffect(() => {
     const editable = editablePersonas.sort(personaComparator);
-    const nonEditable = allPersonas
+    const nonEditable = assistants
       .filter((p) => !editablePersonaIds.has(p.id.toString()))
       .sort(personaComparator);
-    return [...editable, ...nonEditable];
-  }, [allPersonas, editablePersonas]);
-
-  const [finalPersonas, setFinalPersonas] = useState<string[]>(
-    sortedPersonas.map((persona) => persona.id.toString())
-  );
-  const finalPersonaValues = finalPersonas
-    .filter((id) => new Set(allPersonas.map((p) => p.id.toString())).has(id))
-    .map((id) => {
-      return sortedPersonas.find(
-        (persona) => persona.id.toString() === id
-      ) as Persona;
-    });
+    setFinalPersonas([...editable, ...nonEditable]);
+  }, [editablePersonas, assistants, editablePersonaIds]);
 
   const updatePersonaOrder = async (orderedPersonaIds: UniqueIdentifier[]) => {
-    setFinalPersonas(orderedPersonaIds.map((id) => id.toString()));
+    const reorderedAssistants = orderedPersonaIds.map(
+      (id) => assistants.find((assistant) => assistant.id.toString() === id)!
+    );
+
+    setFinalPersonas(reorderedAssistants);
 
     const displayPriorityMap = new Map<UniqueIdentifier, number>();
     orderedPersonaIds.forEach((personaId, ind) => {
@@ -89,13 +83,19 @@ export function PersonasTable({
         display_priority_map: Object.fromEntries(displayPriorityMap),
       }),
     });
+
     if (!response.ok) {
       setPopup({
         type: "error",
         message: `Failed to update persona order - ${await response.text()}`,
       });
+      setFinalPersonas(assistants);
       router.refresh();
+      return;
     }
+
+    await refreshAssistants();
+    await refreshUser();
   };
 
   if (isLoadingUser) {
@@ -115,8 +115,8 @@ export function PersonasTable({
       <DraggableTable
         headers={["Name", "Description", "Type", "Is Visible", "Delete"]}
         isAdmin={isAdmin}
-        rows={finalPersonaValues.map((persona) => {
-          const isEditable = editablePersonaIds.has(persona.id.toString());
+        rows={finalPersonas.map((persona) => {
+          const isEditable = editablePersonas.includes(persona);
           return {
             id: persona.id.toString(),
             cells: [
diff --git a/web/src/app/admin/assistants/page.tsx b/web/src/app/admin/assistants/page.tsx
index 7f3922ac4a7..6c13bbb66dd 100644
--- a/web/src/app/admin/assistants/page.tsx
+++ b/web/src/app/admin/assistants/page.tsx
@@ -2,33 +2,10 @@ import { PersonasTable } from "./PersonaTable";
 import { FiPlusSquare } from "react-icons/fi";
 import Link from "next/link";
 import { Divider, Text, Title } from "@tremor/react";
-import { fetchSS } from "@/lib/utilsSS";
-import { ErrorCallout } from "@/components/ErrorCallout";
-import { Persona } from "./interfaces";
 import { AssistantsIcon } from "@/components/icons/icons";
 import { AdminPageTitle } from "@/components/admin/Title";
 
 export default async function Page() {
-  const allPersonaResponse = await fetchSS("/admin/persona");
-  const editablePersonaResponse = await fetchSS(
-    "/admin/persona?get_editable=true"
-  );
-
-  if (!allPersonaResponse.ok || !editablePersonaResponse.ok) {
-    return (
-      <ErrorCallout
-        errorTitle="Something went wrong :("
-        errorMsg={`Failed to fetch personas - ${
-          (await allPersonaResponse.text()) ||
-          (await editablePersonaResponse.text())
-        }`}
-      />
-    );
-  }
-
-  const allPersonas = (await allPersonaResponse.json()) as Persona[];
-  const editablePersonas = (await editablePersonaResponse.json()) as Persona[];
-
   return (
     <div className="mx-auto container">
       <AdminPageTitle icon={<AssistantsIcon size={32} />} title="Assistants" />
@@ -64,10 +41,7 @@ export default async function Page() {
         <Divider />
 
         <Title>Existing Assistants</Title>
-        <PersonasTable
-          allPersonas={allPersonas}
-          editablePersonas={editablePersonas}
-        />
+        <PersonasTable />
       </div>
     </div>
   );
diff --git a/web/src/app/assistants/gallery/page.tsx b/web/src/app/assistants/gallery/page.tsx
index 24b36cd2bcc..1ae928c598f 100644
--- a/web/src/app/assistants/gallery/page.tsx
+++ b/web/src/app/assistants/gallery/page.tsx
@@ -22,33 +22,24 @@ export default async function GalleryPage({
   const {
     user,
     chatSessions,
-    assistants,
     folders,
     openedFolders,
     shouldShowWelcomeModal,
     toggleSidebar,
-    hasAnyConnectors,
-    hasImageCompatibleModel,
   } = data;
 
   return (
     <>
-      <AssistantsProvider
-        initialAssistants={assistants}
-        hasAnyConnectors={hasAnyConnectors}
-        hasImageCompatibleModel={hasImageCompatibleModel}
-      >
-        {shouldShowWelcomeModal && <WelcomeModal user={user} />}
+      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
 
-        <InstantSSRAutoRefresh />
+      <InstantSSRAutoRefresh />
 
-        <WrappedAssistantsGallery
-          initiallyToggled={toggleSidebar}
-          chatSessions={chatSessions}
-          folders={folders}
-          openedFolders={openedFolders}
-        />
-      </AssistantsProvider>
+      <WrappedAssistantsGallery
+        initiallyToggled={toggleSidebar}
+        chatSessions={chatSessions}
+        folders={folders}
+        openedFolders={openedFolders}
+      />
     </>
   );
 }
diff --git a/web/src/app/assistants/mine/page.tsx b/web/src/app/assistants/mine/page.tsx
index a2655593e1b..d14de250bae 100644
--- a/web/src/app/assistants/mine/page.tsx
+++ b/web/src/app/assistants/mine/page.tsx
@@ -23,20 +23,13 @@ export default async function GalleryPage({
     user,
     chatSessions,
     folders,
-    assistants,
     openedFolders,
     shouldShowWelcomeModal,
     toggleSidebar,
-    hasAnyConnectors,
-    hasImageCompatibleModel,
   } = data;
 
   return (
-    <AssistantsProvider
-      initialAssistants={assistants}
-      hasAnyConnectors={hasAnyConnectors}
-      hasImageCompatibleModel={hasImageCompatibleModel}
-    >
+    <>
       {shouldShowWelcomeModal && <WelcomeModal user={user} />}
 
       <InstantSSRAutoRefresh />
@@ -46,6 +39,6 @@ export default async function GalleryPage({
         folders={folders}
         openedFolders={openedFolders}
       />
-    </AssistantsProvider>
+    </>
   );
 }
diff --git a/web/src/app/chat/page.tsx b/web/src/app/chat/page.tsx
index 3c872390c1b..4f91699b48c 100644
--- a/web/src/app/chat/page.tsx
+++ b/web/src/app/chat/page.tsx
@@ -32,38 +32,29 @@ export default async function Page({
     openedFolders,
     defaultAssistantId,
     shouldShowWelcomeModal,
-    assistants,
     userInputPrompts,
-    hasAnyConnectors,
-    hasImageCompatibleModel,
   } = data;
 
   return (
     <>
       <InstantSSRAutoRefresh />
       {shouldShowWelcomeModal && <WelcomeModal user={user} />}
-      <AssistantsProvider
-        initialAssistants={assistants}
-        hasAnyConnectors={hasAnyConnectors}
-        hasImageCompatibleModel={hasImageCompatibleModel}
+      <ChatProvider
+        value={{
+          chatSessions,
+          availableSources,
+          availableDocumentSets: documentSets,
+          availableTags: tags,
+          llmProviders,
+          folders,
+          openedFolders,
+          userInputPrompts,
+          shouldShowWelcomeModal,
+          defaultAssistantId,
+        }}
       >
-        <ChatProvider
-          value={{
-            chatSessions,
-            availableSources,
-            availableDocumentSets: documentSets,
-            availableTags: tags,
-            llmProviders,
-            folders,
-            openedFolders,
-            userInputPrompts,
-            shouldShowWelcomeModal,
-            defaultAssistantId,
-          }}
-        >
-          <WrappedChat initiallyToggled={toggleSidebar} />
-        </ChatProvider>
-      </AssistantsProvider>
+        <WrappedChat initiallyToggled={toggleSidebar} />
+      </ChatProvider>
     </>
   );
 }
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index dbf86e3bb34..e6339ade9fb 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -19,6 +19,8 @@ import { HeaderTitle } from "@/components/header/HeaderTitle";
 import { Logo } from "@/components/Logo";
 import { UserProvider } from "@/components/user/UserProvider";
 import { ProviderContextProvider } from "@/components/chat_search/ProviderContext";
+import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
+import { AppProvider } from "@/components/context/AppProvider";
 
 const inter = Inter({
   subsets: ["latin"],
@@ -55,6 +57,10 @@ export default async function RootLayout({
 }) {
   const combinedSettings = await fetchSettingsSS();
 
+  const data = await fetchAssistantData();
+
+  const { assistants, hasAnyConnectors, hasImageCompatibleModel } = data;
+
   const productGating =
     combinedSettings?.settings.product_gating ?? GatingType.NONE;
 
@@ -174,13 +180,14 @@ export default async function RootLayout({
             process.env.THEME_IS_DARK?.toLowerCase() === "true" ? "dark" : ""
           }`}
         >
-          <UserProvider>
-            <ProviderContextProvider>
-              <SettingsProvider settings={combinedSettings}>
-                {children}
-              </SettingsProvider>
-            </ProviderContextProvider>
-          </UserProvider>
+          <AppProvider
+            settings={combinedSettings}
+            assistants={assistants}
+            hasAnyConnectors={hasAnyConnectors}
+            hasImageCompatibleModel={hasImageCompatibleModel}
+          >
+            {children}
+          </AppProvider>
         </div>
       </body>
     </html>
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index d515a1fb8df..cfbc72e564b 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -201,32 +201,25 @@ export default async function Home({
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 
       Only used in the EE version of the app. */}
       <ChatPopup />
-      <AssistantsProvider
-        initialAssistants={assistants}
-        hasAnyConnectors={hasAnyConnectors}
-        hasImageCompatibleModel={false}
+      <SearchProvider
+        value={{
+          querySessions,
+          ccPairs,
+          documentSets,
+          assistants,
+          tags,
+          agenticSearchEnabled,
+          disabledAgentic: DISABLE_LLM_DOC_RELEVANCE,
+          initiallyToggled: toggleSidebar,
+          shouldShowWelcomeModal,
+          shouldDisplayNoSources: shouldDisplayNoSourcesModal,
+        }}
       >
-        <SearchProvider
-          value={{
-            querySessions,
-            ccPairs,
-            documentSets,
-            assistants,
-            tags,
-            agenticSearchEnabled,
-            disabledAgentic: DISABLE_LLM_DOC_RELEVANCE,
-            initiallyToggled: toggleSidebar,
-            shouldShowWelcomeModal,
-            shouldDisplayNoSources: shouldDisplayNoSourcesModal,
-          }}
-        >
-          <WrappedSearch
-            initiallyToggled={toggleSidebar}
-            searchTypeDefault={searchTypeDefault}
-          />
-        </SearchProvider>
-      </AssistantsProvider>
-      s
+        <WrappedSearch
+          initiallyToggled={toggleSidebar}
+          searchTypeDefault={searchTypeDefault}
+        />
+      </SearchProvider>
     </>
   );
 }
diff --git a/web/src/components/context/AppProvider.tsx b/web/src/components/context/AppProvider.tsx
new file mode 100644
index 00000000000..9ce4d864fbf
--- /dev/null
+++ b/web/src/components/context/AppProvider.tsx
@@ -0,0 +1,39 @@
+"use server";
+import { CombinedSettings } from "@/app/admin/settings/interfaces";
+import { UserProvider } from "../user/UserProvider";
+import { ProviderContextProvider } from "../chat_search/ProviderContext";
+import { SettingsProvider } from "../settings/SettingsProvider";
+import { AssistantsProvider } from "./AssistantsContext";
+import { Persona } from "@/app/admin/assistants/interfaces";
+
+interface AppProviderProps {
+  children: React.ReactNode;
+  settings: CombinedSettings;
+  assistants: Persona[];
+  hasAnyConnectors: boolean;
+  hasImageCompatibleModel: boolean;
+}
+
+export const AppProvider = ({
+  children,
+  settings,
+  assistants,
+  hasAnyConnectors,
+  hasImageCompatibleModel,
+}: AppProviderProps) => {
+  return (
+    <UserProvider>
+      <ProviderContextProvider>
+        <SettingsProvider settings={settings}>
+          <AssistantsProvider
+            initialAssistants={assistants}
+            hasAnyConnectors={hasAnyConnectors}
+            hasImageCompatibleModel={hasImageCompatibleModel}
+          >
+            {children}
+          </AssistantsProvider>
+        </SettingsProvider>
+      </ProviderContextProvider>
+    </UserProvider>
+  );
+};
diff --git a/web/src/components/context/AssistantsContext.tsx b/web/src/components/context/AssistantsContext.tsx
index 30503a29c67..821429e1090 100644
--- a/web/src/components/context/AssistantsContext.tsx
+++ b/web/src/components/context/AssistantsContext.tsx
@@ -1,5 +1,11 @@
 "use client";
-import React, { createContext, useState, useContext, useMemo } from "react";
+import React, {
+  createContext,
+  useState,
+  useContext,
+  useMemo,
+  useEffect,
+} from "react";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import {
   classifyAssistants,
@@ -15,6 +21,10 @@ interface AssistantsContextProps {
   finalAssistants: Persona[];
   ownedButHiddenAssistants: Persona[];
   refreshAssistants: () => Promise<void>;
+
+  // Admin only
+  editablePersonas: Persona[];
+  allAssistants: Persona[];
 }
 
 const AssistantsContext = createContext<AssistantsContextProps | undefined>(
@@ -35,7 +45,54 @@ export const AssistantsProvider: React.FC<{
   const [assistants, setAssistants] = useState<Persona[]>(
     initialAssistants || []
   );
-  const { user } = useUser();
+  const { user, isLoadingUser, isAdmin } = useUser();
+  const [editablePersonas, setEditablePersonas] = useState<Persona[]>([]);
+
+  useEffect(() => {
+    const fetchEditablePersonas = async () => {
+      if (!isAdmin) {
+        return;
+      }
+
+      try {
+        const response = await fetch("/api/admin/persona?get_editable=true");
+        if (!response.ok) {
+          console.error("Failed to fetch editable personas");
+          return;
+        }
+        const personas = await response.json();
+        setEditablePersonas(personas);
+      } catch (error) {
+        console.error("Error fetching editable personas:", error);
+      }
+    };
+
+    fetchEditablePersonas();
+  }, [isAdmin]);
+
+  const [allAssistants, setAllAssistants] = useState<Persona[]>([]);
+
+  useEffect(() => {
+    const fetchAllAssistants = async () => {
+      if (!isAdmin) {
+        return;
+      }
+
+      try {
+        const response = await fetch("/api/admin/persona");
+        if (!response.ok) {
+          console.error("Failed to fetch all personas");
+          return;
+        }
+        const personas = await response.json();
+        setAllAssistants(personas);
+      } catch (error) {
+        console.error("Error fetching all personas:", error);
+      }
+    };
+
+    fetchAllAssistants();
+  }, [isAdmin]);
 
   const refreshAssistants = async () => {
     try {
@@ -92,7 +149,7 @@ export const AssistantsProvider: React.FC<{
       finalAssistants,
       ownedButHiddenAssistants,
     };
-  }, [user, assistants]);
+  }, [user, assistants, isLoadingUser]);
 
   return (
     <AssistantsContext.Provider
@@ -103,6 +160,8 @@ export const AssistantsProvider: React.FC<{
         finalAssistants,
         ownedButHiddenAssistants,
         refreshAssistants,
+        editablePersonas,
+        allAssistants,
       }}
     >
       {children}
diff --git a/web/src/lib/chat/fetchAssistantdata.ts b/web/src/lib/chat/fetchAssistantdata.ts
new file mode 100644
index 00000000000..08910320a83
--- /dev/null
+++ b/web/src/lib/chat/fetchAssistantdata.ts
@@ -0,0 +1,69 @@
+import { fetchSS } from "@/lib/utilsSS";
+import { CCPairBasicInfo } from "@/lib/types";
+import { Persona } from "@/app/admin/assistants/interfaces";
+import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
+import { personaComparator } from "@/app/admin/assistants/lib";
+import { fetchAssistantsSS } from "../assistants/fetchAssistantsSS";
+import { checkLLMSupportsImageInput } from "../llm/utils";
+
+interface AssistantData {
+  assistants: Persona[];
+  hasAnyConnectors: boolean;
+  hasImageCompatibleModel: boolean;
+}
+export async function fetchAssistantData(): Promise<AssistantData> {
+  const [assistants, assistantsFetchError] = await fetchAssistantsSS();
+  const ccPairsResponse = await fetchSS("/manage/indexing-status");
+
+  let ccPairs: CCPairBasicInfo[] = [];
+  if (ccPairsResponse?.ok) {
+    ccPairs = await ccPairsResponse.json();
+  } else {
+    console.log(`Failed to fetch connectors - ${ccPairsResponse?.status}`);
+  }
+
+  const hasAnyConnectors = ccPairs.length > 0;
+
+  // if no connectors are setup, only show personas that are pure
+  // passthrough and don't do any retrieval
+  let filteredAssistants = assistants;
+  if (assistantsFetchError) {
+    console.log(`Failed to fetch assistants - ${assistantsFetchError}`);
+  }
+
+  // remove those marked as hidden by an admin
+  filteredAssistants = filteredAssistants.filter(
+    (assistant) => assistant.is_visible
+  );
+
+  if (!hasAnyConnectors) {
+    filteredAssistants = filteredAssistants.filter(
+      (assistant) => assistant.num_chunks === 0
+    );
+  }
+
+  // sort them in priority order
+  filteredAssistants.sort(personaComparator);
+
+  const llmProviders = await fetchLLMProvidersSS();
+  const hasImageCompatibleModel = llmProviders.some(
+    (provider) =>
+      provider.provider === "openai" ||
+      provider.model_names.some((model) => checkLLMSupportsImageInput(model))
+  );
+
+  if (!hasImageCompatibleModel) {
+    filteredAssistants = filteredAssistants.filter(
+      (assistant) =>
+        !assistant.tools.some(
+          (tool) => tool.in_code_tool_id === "ImageGenerationTool"
+        )
+    );
+  }
+
+  return {
+    assistants: filteredAssistants,
+    hasAnyConnectors,
+    hasImageCompatibleModel,
+  };
+}
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index 8c2f0fb5a3d..32838aeac6f 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -37,7 +37,6 @@ interface FetchChatDataResult {
   ccPairs: CCPairBasicInfo[];
   availableSources: ValidSources[];
   documentSets: DocumentSet[];
-  assistants: Persona[];
   tags: Tag[];
   llmProviders: LLMProviderDescriptor[];
   folders: Folder[];
@@ -47,8 +46,6 @@ interface FetchChatDataResult {
   finalDocumentSidebarInitialWidth?: number;
   shouldShowWelcomeModal: boolean;
   userInputPrompts: InputPrompt[];
-  hasAnyConnectors: boolean;
-  hasImageCompatibleModel: boolean;
 }
 
 export async function fetchChatData(searchParams: {
@@ -59,7 +56,6 @@ export async function fetchChatData(searchParams: {
     getCurrentUserSS(),
     fetchSS("/manage/indexing-status"),
     fetchSS("/manage/document-set"),
-    fetchAssistantsSS(),
     fetchSS("/chat/get-user-chat-sessions"),
     fetchSS("/query/valid-tags"),
     fetchLLMProvidersSS(),
@@ -76,7 +72,7 @@ export async function fetchChatData(searchParams: {
     | LLMProviderDescriptor[]
     | [Persona[], string | null]
     | null
-  )[] = [null, null, null, null, null, null, null, null, null, null];
+  )[] = [null, null, null, null, null, null, null, null, null];
   try {
     results = await Promise.all(tasks);
   } catch (e) {
@@ -87,17 +83,13 @@ export async function fetchChatData(searchParams: {
   const user = results[1] as User | null;
   const ccPairsResponse = results[2] as Response | null;
   const documentSetsResponse = results[3] as Response | null;
-  const [rawAssistantsList, assistantsFetchError] = results[4] as [
-    Persona[],
-    string | null,
-  ];
 
-  const chatSessionsResponse = results[5] as Response | null;
+  const chatSessionsResponse = results[4] as Response | null;
 
-  const tagsResponse = results[6] as Response | null;
-  const llmProviders = (results[7] || []) as LLMProviderDescriptor[];
-  const foldersResponse = results[8] as Response | null;
-  const userInputPromptsResponse = results[9] as Response | null;
+  const tagsResponse = results[5] as Response | null;
+  const llmProviders = (results[6] || []) as LLMProviderDescriptor[];
+  const foldersResponse = results[7] as Response | null;
+  const userInputPromptsResponse = results[8] as Response | null;
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
@@ -161,17 +153,6 @@ export async function fetchChatData(searchParams: {
     );
   }
 
-  let assistants = rawAssistantsList;
-  if (assistantsFetchError) {
-    console.log(`Failed to fetch assistants - ${assistantsFetchError}`);
-  }
-  // remove those marked as hidden by an admin
-
-  assistants = assistants.filter((assistant) => assistant.is_visible);
-
-  // sort them in priority order
-  assistants.sort(personaComparator);
-
   let tags: Tag[] = [];
   if (tagsResponse?.ok) {
     tags = (await tagsResponse.json()).tags;
@@ -206,24 +187,6 @@ export async function fetchChatData(searchParams: {
 
   // if no connectors are setup, only show personas that are pure
   // passthrough and don't do any retrieval
-  if (!hasAnyConnectors) {
-    assistants = assistants.filter((assistant) => assistant.num_chunks === 0);
-  }
-
-  const hasImageCompatibleModel = llmProviders.some(
-    (provider) =>
-      provider.provider === "openai" ||
-      provider.model_names.some((model) => checkLLMSupportsImageInput(model))
-  );
-
-  if (!hasImageCompatibleModel) {
-    assistants = assistants.filter(
-      (assistant) =>
-        !assistant.tools.some(
-          (tool) => tool.in_code_tool_id === "ImageGenerationTool"
-        )
-    );
-  }
 
   let folders: Folder[] = [];
   if (foldersResponse?.ok) {
@@ -243,7 +206,6 @@ export async function fetchChatData(searchParams: {
     ccPairs,
     availableSources,
     documentSets,
-    assistants,
     tags,
     llmProviders,
     folders,
@@ -253,7 +215,5 @@ export async function fetchChatData(searchParams: {
     toggleSidebar,
     shouldShowWelcomeModal,
     userInputPrompts,
-    hasAnyConnectors,
-    hasImageCompatibleModel,
   };
 }

From 94b4dc1656379698fa0436b2cea34360538230e1 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 14:49:18 -0700
Subject: [PATCH 202/376] can't add to primary_worker_locks if it doesn't exist
 (#2903)

* can't add to primary_worker_locks if it doesn't exist

* move init
---
 backend/danswer/background/celery/apps/primary.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index d86f0d60fe4..23e25fa92b8 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -78,6 +78,8 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
 
     logger.info("Running as the primary celery worker.")
 
+    sender.primary_worker_locks = {}
+
     # This is singleton work that should be done on startup exactly once
     # by the primary worker
     tenant_ids = get_all_tenant_ids()
@@ -106,6 +108,7 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
             logger.error("Primary worker lock: Acquire failed!")
             raise WorkerShutdown("Primary worker lock could not be acquired!")
 
+        # tacking on our own user data to the sender
         sender.primary_worker_locks[tenant_id] = lock
 
         # As currently designed, when this worker starts as "primary", we reinitialize redis

From 9f504171095d00628c7523be4750fd6bc32542ff Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 15:44:20 -0700
Subject: [PATCH 203/376] try hiding celery task spam (#2905)

* try hiding celery task spam

* mypy fix
---
 backend/danswer/background/celery/apps/app_base.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
index 05c0eabbbd9..b74cba1dfe6 100644
--- a/backend/danswer/background/celery/apps/app_base.py
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -5,9 +5,11 @@
 
 import sentry_sdk
 from celery import Task
+from celery.app import trace
 from celery.exceptions import WorkerShutdown
 from celery.states import READY_STATES
 from celery.utils.log import get_task_logger
+from celery.worker import strategy  # type: ignore
 from sentry_sdk.integrations.celery import CeleryIntegration
 
 from danswer.background.celery.apps.task_formatters import CeleryTaskColoredFormatter
@@ -297,3 +299,11 @@ def on_setup_logging(
 
     task_logger.setLevel(loglevel)
     task_logger.propagate = False
+
+    # hide celery task received spam
+    # e.g. "Task check_for_pruning[a1e96171-0ba8-4e00-887b-9fbf7442eab3] received"
+    strategy.logger.setLevel(logging.WARNING)
+
+    # hide celery task succeeded/failed spam
+    # e.g. "Task check_for_pruning[a1e96171-0ba8-4e00-887b-9fbf7442eab3] succeeded in 0.03137450001668185s: None"
+    trace.logger.setLevel(logging.WARNING)

From b49a9ab171204c25e5c472b5a18dd911e20fc73a Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 16:45:48 -0700
Subject: [PATCH 204/376] Seeding (#2902)

* checkpoint

* k

* k

* k

* fixed slack api calls

* missed one

---------

Co-authored-by: hagen-danswer <hagen@danswer.ai>
---
 backend/danswer/access/models.py              |     9 +
 backend/danswer/configs/constants.py          |     1 +
 backend/danswer/connectors/axero/connector.py |     2 +-
 .../danswer/connectors/clickup/connector.py   |     2 +-
 .../danswer/connectors/discourse/connector.py |     2 +-
 .../connectors/document360/connector.py       |     2 +-
 .../connectors/google_drive/connector.py      |     2 +-
 backend/danswer/connectors/slack/utils.py     |     2 +-
 .../danswer/db/connector_credential_pair.py   |     5 +-
 backend/danswer/db/credentials.py             |     7 +-
 backend/danswer/db/index_attempt.py           |    27 +
 backend/danswer/indexing/embedder.py          |     8 +
 backend/danswer/indexing/indexing_pipeline.py |     2 +
 backend/danswer/key_value_store/factory.py    |     3 +-
 backend/danswer/key_value_store/interface.py  |     2 +
 backend/danswer/seeding/__init__.py           |     0
 backend/danswer/seeding/initial_docs.json     | 10824 ++++++++++++++++
 backend/danswer/seeding/load_docs.py          |   212 +
 backend/danswer/setup.py                      |    14 +-
 .../retry_wrapper.py                          |    18 +-
 .../google_drive/doc_sync.py                  |     2 +-
 .../google_drive/group_sync.py                |     2 +-
 backend/scripts/document_seeding_prep.py      |   240 +
 .../slack/slack_api_utils.py                  |    34 +-
 24 files changed, 11372 insertions(+), 50 deletions(-)
 create mode 100644 backend/danswer/seeding/__init__.py
 create mode 100644 backend/danswer/seeding/initial_docs.json
 create mode 100644 backend/danswer/seeding/load_docs.py
 rename backend/danswer/{connectors/cross_connector_utils => utils}/retry_wrapper.py (79%)
 create mode 100644 backend/scripts/document_seeding_prep.py

diff --git a/backend/danswer/access/models.py b/backend/danswer/access/models.py
index af5a021ca97..46b9c0efa93 100644
--- a/backend/danswer/access/models.py
+++ b/backend/danswer/access/models.py
@@ -70,3 +70,12 @@ def build(
             user_groups=set(user_groups),
             is_public=is_public,
         )
+
+
+default_public_access = DocumentAccess(
+    external_user_emails=set(),
+    external_user_group_ids=set(),
+    user_emails=set(),
+    user_groups=set(),
+    is_public=True,
+)
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index 6a3385b9fa8..caa903af71d 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -70,6 +70,7 @@
 KV_INSTANCE_DOMAIN_KEY = "instance_domain"
 KV_ENTERPRISE_SETTINGS_KEY = "danswer_enterprise_settings"
 KV_CUSTOM_ANALYTICS_SCRIPT_KEY = "__custom_analytics_script__"
+KV_DOCUMENTS_SEEDED_KEY = "documents_seeded"
 
 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT = 60
 CELERY_PRIMARY_WORKER_LOCK_TIMEOUT = 120
diff --git a/backend/danswer/connectors/axero/connector.py b/backend/danswer/connectors/axero/connector.py
index a4d5162b6ce..000151209de 100644
--- a/backend/danswer/connectors/axero/connector.py
+++ b/backend/danswer/connectors/axero/connector.py
@@ -15,7 +15,6 @@
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
 )
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
@@ -24,6 +23,7 @@
 from danswer.connectors.models import Section
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 
 
 logger = setup_logger()
diff --git a/backend/danswer/connectors/clickup/connector.py b/backend/danswer/connectors/clickup/connector.py
index 78d572af413..6ef60763baf 100644
--- a/backend/danswer/connectors/clickup/connector.py
+++ b/backend/danswer/connectors/clickup/connector.py
@@ -10,7 +10,6 @@
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
 )
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
@@ -19,6 +18,7 @@
 from danswer.connectors.models import ConnectorMissingCredentialError
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
+from danswer.utils.retry_wrapper import retry_builder
 
 
 CLICKUP_API_BASE_URL = "https://api.clickup.com/api/v2"
diff --git a/backend/danswer/connectors/discourse/connector.py b/backend/danswer/connectors/discourse/connector.py
index d74aad0f276..d1b6395a189 100644
--- a/backend/danswer/connectors/discourse/connector.py
+++ b/backend/danswer/connectors/discourse/connector.py
@@ -14,7 +14,6 @@
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
 )
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
@@ -24,6 +23,7 @@
 from danswer.connectors.models import Section
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 
 logger = setup_logger()
 
diff --git a/backend/danswer/connectors/document360/connector.py b/backend/danswer/connectors/document360/connector.py
index 6a9f4ba6a56..7ccf3c92e62 100644
--- a/backend/danswer/connectors/document360/connector.py
+++ b/backend/danswer/connectors/document360/connector.py
@@ -11,7 +11,6 @@
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
 )
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.document360.utils import flatten_child_categories
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import LoadConnector
@@ -22,6 +21,7 @@
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
 from danswer.file_processing.html_utils import parse_html_page_basic
+from danswer.utils.retry_wrapper import retry_builder
 
 # Limitations and Potential Improvements
 # 1. The "Categories themselves contain potentially relevant information" but they're not pulled in
diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index 9f8c6fbfda8..c48df5bb741 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -19,7 +19,6 @@
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import IGNORE_FOR_QA
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.google_drive.connector_auth import get_google_drive_creds
 from danswer.connectors.google_drive.constants import (
     DB_CREDENTIALS_DICT_DELEGATED_USER_KEY,
@@ -40,6 +39,7 @@
 from danswer.file_processing.unstructured import unstructured_to_text
 from danswer.utils.batching import batch_generator
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 
 logger = setup_logger()
 
diff --git a/backend/danswer/connectors/slack/utils.py b/backend/danswer/connectors/slack/utils.py
index 20e859a6d44..78bc42a0926 100644
--- a/backend/danswer/connectors/slack/utils.py
+++ b/backend/danswer/connectors/slack/utils.py
@@ -10,9 +10,9 @@
 from slack_sdk.errors import SlackApiError
 from slack_sdk.web import SlackResponse
 
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.models import BasicExpertInfo
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 
 logger = setup_logger()
 
diff --git a/backend/danswer/db/connector_credential_pair.py b/backend/danswer/db/connector_credential_pair.py
index b3e1de7647a..b333dd78603 100644
--- a/backend/danswer/db/connector_credential_pair.py
+++ b/backend/danswer/db/connector_credential_pair.py
@@ -341,6 +341,8 @@ def add_credential_to_connector(
     access_type: AccessType,
     groups: list[int] | None,
     auto_sync_options: dict | None = None,
+    initial_status: ConnectorCredentialPairStatus = ConnectorCredentialPairStatus.ACTIVE,
+    last_successful_index_time: datetime | None = None,
 ) -> StatusResponse:
     connector = fetch_connector_by_id(connector_id, db_session)
     credential = fetch_credential_by_id(credential_id, user, db_session)
@@ -384,9 +386,10 @@ def add_credential_to_connector(
         connector_id=connector_id,
         credential_id=credential_id,
         name=cc_pair_name,
-        status=ConnectorCredentialPairStatus.ACTIVE,
+        status=initial_status,
         access_type=access_type,
         auto_sync_options=auto_sync_options,
+        last_successful_index_time=last_successful_index_time,
     )
     db_session.add(association)
     db_session.flush()  # make sure the association has an id
diff --git a/backend/danswer/db/credentials.py b/backend/danswer/db/credentials.py
index 5da5099f1e3..80ebe1b1538 100644
--- a/backend/danswer/db/credentials.py
+++ b/backend/danswer/db/credentials.py
@@ -40,6 +40,8 @@
     DocumentSource.MEDIAWIKI,
 }
 
+PUBLIC_CREDENTIAL_ID = 0
+
 
 def _add_user_filters(
     stmt: Select,
@@ -384,12 +386,11 @@ def delete_credential(
 
 
 def create_initial_public_credential(db_session: Session) -> None:
-    public_cred_id = 0
     error_msg = (
         "DB is not in a valid initial state."
         "There must exist an empty public credential for data connectors that do not require additional Auth."
     )
-    first_credential = fetch_credential_by_id(public_cred_id, None, db_session)
+    first_credential = fetch_credential_by_id(PUBLIC_CREDENTIAL_ID, None, db_session)
 
     if first_credential is not None:
         if first_credential.credential_json != {} or first_credential.user is not None:
@@ -397,7 +398,7 @@ def create_initial_public_credential(db_session: Session) -> None:
         return
 
     credential = Credential(
-        id=public_cred_id,
+        id=PUBLIC_CREDENTIAL_ID,
         credential_json={},
         user_id=None,
     )
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index fbbbef1bbfc..45a07387949 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -1,5 +1,6 @@
 from collections.abc import Sequence
 from datetime import datetime
+from datetime import timedelta
 from datetime import timezone
 
 from sqlalchemy import and_
@@ -66,6 +67,32 @@ def create_index_attempt(
     return new_attempt.id
 
 
+def mock_successful_index_attempt(
+    connector_credential_pair_id: int,
+    search_settings_id: int,
+    docs_indexed: int,
+    db_session: Session,
+) -> int:
+    """Should not be used in any user triggered flows"""
+    db_time = func.now()
+    new_attempt = IndexAttempt(
+        connector_credential_pair_id=connector_credential_pair_id,
+        search_settings_id=search_settings_id,
+        from_beginning=True,
+        status=IndexingStatus.SUCCESS,
+        total_docs_indexed=docs_indexed,
+        new_docs_indexed=docs_indexed,
+        # Need this to be some convincing random looking value and it can't be 0
+        # or the indexing rate would calculate out to infinity
+        time_started=db_time - timedelta(seconds=1.92),
+        time_updated=db_time,
+    )
+    db_session.add(new_attempt)
+    db_session.commit()
+
+    return new_attempt.id
+
+
 def get_in_progress_index_attempts(
     connector_id: int | None,
     db_session: Session,
diff --git a/backend/danswer/indexing/embedder.py b/backend/danswer/indexing/embedder.py
index 3e9383cb828..1c11a01b390 100644
--- a/backend/danswer/indexing/embedder.py
+++ b/backend/danswer/indexing/embedder.py
@@ -103,6 +103,9 @@ def embed_chunks(
         self,
         chunks: list[DocAwareChunk],
     ) -> list[IndexChunk]:
+        """Adds embeddings to the chunks, the title and metadata suffixes are added to the chunk as well
+        if they exist. If there is no space for it, it would have been thrown out at the chunking step.
+        """
         # All chunks at this point must have some non-empty content
         flat_chunk_texts: list[str] = []
         large_chunks_present = False
@@ -121,6 +124,11 @@ def embed_chunks(
             flat_chunk_texts.append(chunk_text)
 
             if chunk.mini_chunk_texts:
+                if chunk.large_chunk_reference_ids:
+                    # A large chunk does not contain mini chunks, if it matches the large chunk
+                    # with a high score, then mini chunks would not be used anyway
+                    # otherwise it should match the normal chunk
+                    raise RuntimeError("Large chunk contains mini chunks")
                 flat_chunk_texts.extend(chunk.mini_chunk_texts)
 
         embeddings = self.embedding_model.encode(
diff --git a/backend/danswer/indexing/indexing_pipeline.py b/backend/danswer/indexing/indexing_pipeline.py
index d40bd341fdf..507956ff40f 100644
--- a/backend/danswer/indexing/indexing_pipeline.py
+++ b/backend/danswer/indexing/indexing_pipeline.py
@@ -195,6 +195,8 @@ def index_doc_batch_prepare(
     db_session: Session,
     ignore_time_skip: bool = False,
 ) -> DocumentBatchPrepareContext | None:
+    """This sets up the documents in the relational DB (source of truth) for permissions, metadata, etc.
+    This preceeds indexing it into the actual document index."""
     documents = []
     for document in document_batch:
         empty_contents = not any(section.text.strip() for section in document.sections)
diff --git a/backend/danswer/key_value_store/factory.py b/backend/danswer/key_value_store/factory.py
index 7b52fdabef8..142e9031b77 100644
--- a/backend/danswer/key_value_store/factory.py
+++ b/backend/danswer/key_value_store/factory.py
@@ -3,5 +3,6 @@
 
 
 def get_kv_store() -> KeyValueStore:
-    # this is the only one supported currently
+    # In the Multi Tenant case, the tenant context is picked up automatically, it does not need to be passed in
+    # It's read from the global thread level variable
     return PgRedisKVStore()
diff --git a/backend/danswer/key_value_store/interface.py b/backend/danswer/key_value_store/interface.py
index 190c53189dd..aa815171f06 100644
--- a/backend/danswer/key_value_store/interface.py
+++ b/backend/danswer/key_value_store/interface.py
@@ -14,6 +14,8 @@ class KvKeyNotFoundError(Exception):
 
 
 class KeyValueStore:
+    # In the Multi Tenant case, the tenant context is picked up automatically, it does not need to be passed in
+    # It's read from the global thread level variable
     @abc.abstractmethod
     def store(self, key: str, val: JSON_ro, encrypt: bool = False) -> None:
         raise NotImplementedError
diff --git a/backend/danswer/seeding/__init__.py b/backend/danswer/seeding/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/backend/danswer/seeding/initial_docs.json b/backend/danswer/seeding/initial_docs.json
new file mode 100644
index 00000000000..cded36721b3
--- /dev/null
+++ b/backend/danswer/seeding/initial_docs.json
@@ -0,0 +1,10824 @@
+[
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/overview",
+        "title": "Use Cases Overview",
+        "content": "How to leverage Danswer in your organization\n\nDanswer Overview\nDanswer is the AI Assistant connected to your organization's docs, apps, and people. Danswer makes Generative AI more versatile for work by enabling new types of questions like \"What is the most common feature request we've heard from customers this month\". Whereas other AI systems have no context of your team and are generally unhelpful with work related questions, Danswer makes it possible to ask these questions in natural language and get back answers in seconds.\n\nDanswer can connect to +30 different tools and the use cases are not limited to the ones in the following pages. The highlighted use cases are for inspiration and come from feedback gathered from our users and customers.\n\n\nCommon Getting Started Questions:\n\nWhy are these docs connected in my Danswer deployment?\nAnswer: This is just an example of how connectors work in Danswer. You can connect up your own team's knowledge and you will be able to ask questions unique to your organization. Danswer will keep all of the knowledge up to date and in sync with your connected applications.\n\nIs my data being sent anywhere when I connect it up to Danswer?\nAnswer: No! Danswer is built with data security as our highest priority. We open sourced it so our users can know exactly what is going on with their data. By default all of the document processing happens within Danswer. The only time it is sent outward is for the GenAI call to generate answers.\n\nWhere is the feature for auto sync-ing document level access permissions from all connected sources?\nAnswer: This falls under the Enterprise Edition set of Danswer features built on top of the MIT/community edition. If you are on Danswer Cloud, you have access to them by default. If you're running it yourself, reach out to the Danswer team to receive access.",
+        "title_embedding": [
+            0.013585364446043968,
+            0.06531507521867752,
+            -0.0001728831703076139,
+            -0.003940831869840622,
+            0.044078364968299866,
+            -0.006206007208675146,
+            -0.008377128280699253,
+            -0.0193742997944355,
+            -0.018904175609350204,
+            0.00868070125579834,
+            -0.005770757794380188,
+            0.018564216792583466,
+            0.030414527282118797,
+            0.0327068567276001,
+            -0.0336286760866642,
+            -0.0517829954624176,
+            0.0029869426507502794,
+            -3.836356700048782e-05,
+            -0.006240167189389467,
+            0.011168955825269222,
+            -0.04732134938240051,
+            -0.05257624015212059,
+            0.017937077209353447,
+            -0.029843101277947426,
+            0.03417196497321129,
+            0.008637758903205395,
+            -0.016427641734480858,
+            0.017053034156560898,
+            -0.02532368339598179,
+            -0.016002299264073372,
+            0.04696495085954666,
+            0.03518024459481239,
+            0.02884317748248577,
+            -0.06098122522234917,
+            -0.024405447766184807,
+            -0.07693222165107727,
+            0.026796545833349228,
+            -0.01345108263194561,
+            0.030546706169843674,
+            0.00011662459291983396,
+            0.0362892709672451,
+            0.020864704623818398,
+            -0.0030571012757718563,
+            -0.014566082507371902,
+            0.056138113141059875,
+            -0.01727251335978508,
+            0.05477291718125343,
+            0.019774138927459717,
+            0.01646329089999199,
+            -0.020768629387021065,
+            -0.025477997958660126,
+            0.010012250393629074,
+            0.0037975965533405542,
+            -0.076152004301548,
+            0.032632406800985336,
+            -0.00799622479826212,
+            0.029365766793489456,
+            0.02749769017100334,
+            0.030807621777057648,
+            -0.031911835074424744,
+            0.029850834980607033,
+            0.05788583680987358,
+            -0.07022606581449509,
+            0.057270754128694534,
+            -0.012120618484914303,
+            -0.0351836234331131,
+            0.002640453167259693,
+            0.01869964227080345,
+            0.010610891506075859,
+            -0.06439802795648575,
+            0.06629050523042679,
+            0.00746738538146019,
+            0.01690092496573925,
+            -0.025001635774970055,
+            -0.04047262296080589,
+            -0.04058482125401497,
+            0.01863245666027069,
+            -0.021404679864645004,
+            -0.006766777019947767,
+            0.05071299150586128,
+            0.02962121181190014,
+            -0.06122489646077156,
+            0.019276190549135208,
+            -0.03599821403622627,
+            0.07253828644752502,
+            -0.001938252942636609,
+            -0.00785142183303833,
+            -0.015793368220329285,
+            -0.06096868962049484,
+            -0.022668933495879173,
+            0.014094856567680836,
+            0.03107546642422676,
+            0.030937792733311653,
+            0.04295564815402031,
+            -0.06191089749336243,
+            0.010305442847311497,
+            0.006790813058614731,
+            -0.05027518793940544,
+            0.026334501802921295,
+            -0.03490821272134781,
+            -0.03132909536361694,
+            0.00332127814181149,
+            0.03326006978750229,
+            0.05009974539279938,
+            0.05102463811635971,
+            0.080863356590271,
+            0.008220085874199867,
+            0.015490916557610035,
+            -0.029478086158633232,
+            0.004051747731864452,
+            -0.05233829841017723,
+            0.032338161021471024,
+            0.016430383548140526,
+            0.033909399062395096,
+            -0.0069341897033154964,
+            -0.01092524453997612,
+            0.08201614767313004,
+            -0.061916135251522064,
+            -0.0202189888805151,
+            0.06966497749090195,
+            0.01732165738940239,
+            0.020277557894587517,
+            0.005021766293793917,
+            0.03159264847636223,
+            0.027503052726387978,
+            -0.03912171721458435,
+            -0.03356969729065895,
+            0.018767613917589188,
+            0.02705945260822773,
+            -0.06412986665964127,
+            0.01829575188457966,
+            0.030256258323788643,
+            0.0074773263186216354,
+            -0.059738945215940475,
+            0.042067185044288635,
+            0.05707620456814766,
+            -0.02195868454873562,
+            -0.018797017633914948,
+            0.07043126970529556,
+            -0.007300470490008593,
+            0.04988619685173035,
+            -0.01761087030172348,
+            0.024358391761779785,
+            0.00520830973982811,
+            0.007853846065700054,
+            -0.040487151592969894,
+            0.013271456584334373,
+            0.01356235146522522,
+            -0.07054886221885681,
+            0.046908896416425705,
+            0.0032781255431473255,
+            0.022826792672276497,
+            0.016794828698039055,
+            -0.0015572791453450918,
+            -0.03220144659280777,
+            -0.05249498412013054,
+            -0.023642878979444504,
+            -0.0013240000698715448,
+            0.020749850198626518,
+            -0.032788924872875214,
+            0.01523237768560648,
+            0.03563430905342102,
+            -0.011741658672690392,
+            0.08171892166137695,
+            -0.04966546595096588,
+            -0.02209051325917244,
+            0.005958004854619503,
+            0.02892436273396015,
+            0.03561494126915932,
+            0.02638504095375538,
+            0.041660238057374954,
+            -0.05757247656583786,
+            0.027456382289528847,
+            -0.011119958013296127,
+            0.05332427844405174,
+            0.03419065102934837,
+            0.09803897142410278,
+            -0.0104225380346179,
+            0.06645305454730988,
+            0.02061033807694912,
+            -0.0188175980001688,
+            -0.03409148380160332,
+            -0.010915222577750683,
+            0.016926083713769913,
+            -0.01010509766638279,
+            -0.031197063624858856,
+            0.064297154545784,
+            -0.047280170023441315,
+            -0.022006161510944366,
+            0.04089798405766487,
+            0.0013760487781837583,
+            0.0012917317217215896,
+            -0.010142299346625805,
+            -0.05629380792379379,
+            -0.058489665389060974,
+            -0.06434599310159683,
+            0.04390622675418854,
+            0.03466123342514038,
+            -0.002495409222319722,
+            -0.01867988333106041,
+            -0.012142776511609554,
+            0.025940915569663048,
+            -0.05517507344484329,
+            0.026919366791844368,
+            -0.05310383439064026,
+            0.0020175466779619455,
+            0.0407392643392086,
+            -0.0055900681763887405,
+            0.028038354590535164,
+            0.10158932954072952,
+            0.056325044482946396,
+            0.016724230721592903,
+            0.005659179296344519,
+            0.04764577001333237,
+            -0.03514963388442993,
+            0.03311126306653023,
+            -0.05855907127261162,
+            -0.007677929475903511,
+            -0.0368916280567646,
+            0.02390834502875805,
+            0.021506410092115402,
+            -0.022855432704091072,
+            0.02669590339064598,
+            0.03190927952528,
+            0.026299884542822838,
+            0.04545223340392113,
+            -0.04817903786897659,
+            0.08401483297348022,
+            -0.0017600113060325384,
+            -0.0026402128860354424,
+            -0.06519021838903427,
+            -0.08366627246141434,
+            0.025473223999142647,
+            -0.03265143185853958,
+            -0.026078224182128906,
+            -0.014162144623696804,
+            -0.024846363812685013,
+            0.042588867247104645,
+            -0.00620845053344965,
+            0.0341552197933197,
+            -0.005032224114984274,
+            0.039284951984882355,
+            0.02678983099758625,
+            -0.02592509239912033,
+            -0.0334317646920681,
+            -0.017748532816767693,
+            -0.03175748512148857,
+            -0.03699929639697075,
+            -0.0009614137816242874,
+            0.029971860349178314,
+            0.03400350362062454,
+            0.03034038282930851,
+            0.005932188127189875,
+            0.05225752666592598,
+            -0.032566532492637634,
+            -0.04808121547102928,
+            -0.023166747763752937,
+            0.02398361638188362,
+            -0.03062198869884014,
+            -0.046609822660684586,
+            0.019089654088020325,
+            0.0017758660251274705,
+            0.015751969069242477,
+            -0.029143214225769043,
+            0.0024112602695822716,
+            -0.02520643174648285,
+            0.023578567430377007,
+            -0.023457514122128487,
+            0.013982303440570831,
+            -0.016386305913329124,
+            -0.002706830855458975,
+            -0.016093457117676735,
+            -0.037887830287218094,
+            0.06287679821252823,
+            0.00989477802067995,
+            -0.026199528947472572,
+            0.0037161086220294237,
+            -0.027242952957749367,
+            -0.03319230675697327,
+            -0.026061702519655228,
+            0.015717124566435814,
+            -0.055130068212747574,
+            -0.02499731443822384,
+            -0.014253406785428524,
+            0.046770382672548294,
+            0.008143531158566475,
+            0.005510109476745129,
+            -0.02712511457502842,
+            -0.03787349909543991,
+            0.013756319880485535,
+            0.00579818757250905,
+            0.008403831161558628,
+            0.029755474999547005,
+            -0.0032762265764176846,
+            0.0044027529656887054,
+            0.03601987659931183,
+            0.0909135565161705,
+            -0.007387327961623669,
+            -0.005328672006726265,
+            -0.03983118385076523,
+            -0.045495130121707916,
+            0.022088482975959778,
+            -0.04040846228599548,
+            -0.0028130451682955027,
+            0.03781481459736824,
+            0.03704448416829109,
+            0.03319826349616051,
+            0.0018484846223145723,
+            0.0547247938811779,
+            0.019755663350224495,
+            -0.07568438351154327,
+            0.05122718587517738,
+            -0.02555399388074875,
+            0.06782808154821396,
+            -0.0756291076540947,
+            0.05646894872188568,
+            0.06156547740101814,
+            -0.0010290262289345264,
+            0.02769431658089161,
+            0.003549074986949563,
+            0.02812255173921585,
+            -0.016937678679823875,
+            0.018674779683351517,
+            -0.037636883556842804,
+            -0.002664038445800543,
+            0.023414231836795807,
+            0.040655869990587234,
+            0.027929119765758514,
+            0.03510269895195961,
+            -0.012380925938487053,
+            0.024845607578754425,
+            0.027425218373537064,
+            -0.05437726899981499,
+            0.015804018825292587,
+            0.05077793449163437,
+            -0.0003959169262088835,
+            0.016312288120388985,
+            -0.007089096121490002,
+            -0.018367605283856392,
+            0.02974492497742176,
+            0.08662278950214386,
+            -0.021586472168564796,
+            -0.01729869470000267,
+            -0.04846135899424553,
+            -0.03031736984848976,
+            0.002749247709289193,
+            0.02350122295320034,
+            -0.0211945790797472,
+            0.03907554969191551,
+            -0.023193899542093277,
+            -0.017260679975152016,
+            -0.03159818798303604,
+            -0.03952740877866745,
+            0.010126064531505108,
+            -0.04888703301548958,
+            0.06297406554222107,
+            0.03254289552569389,
+            0.004272142890840769,
+            -0.03231256827712059,
+            -0.04512784630060196,
+            0.0043722353875637054,
+            -0.02994321472942829,
+            0.05848870426416397,
+            0.003534652292728424,
+            0.007630845997482538,
+            0.017482444643974304,
+            0.04071490466594696,
+            0.008809284307062626,
+            -0.03566472604870796,
+            -0.029327288269996643,
+            -0.017241651192307472,
+            -0.012668757699429989,
+            0.05879855155944824,
+            0.05893324315547943,
+            0.09900359064340591,
+            0.028096094727516174,
+            -0.036374326795339584,
+            0.06244330108165741,
+            -0.03114512376487255,
+            -0.028666621074080467,
+            0.06343588978052139,
+            0.025132114067673683,
+            -0.01625697687268257,
+            0.019650116562843323,
+            -0.049646493047475815,
+            -0.03520796075463295,
+            0.03757908195257187,
+            0.002519423607736826,
+            0.03556838259100914,
+            -0.017592694610357285,
+            0.0010467531392350793,
+            -0.06738362461328506,
+            -0.025265797972679138,
+            0.008135112002491951,
+            -0.01762012392282486,
+            -0.024728305637836456,
+            -0.03567385673522949,
+            0.018016908317804337,
+            0.06866948306560516,
+            0.03130311518907547,
+            -0.0297296904027462,
+            -0.006176969967782497,
+            0.04329727590084076,
+            0.044129278510808945,
+            -0.020673662424087524,
+            0.06023940071463585,
+            -0.004932863637804985,
+            -0.050380971282720566,
+            -0.034760732203722,
+            0.00199303706176579,
+            0.05686243996024132,
+            -0.0148441381752491,
+            -0.012425840832293034,
+            -0.011634211987257004,
+            0.022722775116562843,
+            -0.008717222139239311,
+            0.020749682560563087,
+            -0.0277851615101099,
+            0.0007777228020131588,
+            0.013342801481485367,
+            0.03622204810380936,
+            -0.023042850196361542,
+            -0.026700101792812347,
+            -0.034892451018095016,
+            -0.028433026745915413,
+            0.06670085340738297,
+            0.013445812277495861,
+            0.03833755850791931,
+            0.01010140310972929,
+            -0.03759188950061798,
+            -0.05855119228363037,
+            0.00781426765024662,
+            -0.04906706139445305,
+            0.03342912718653679,
+            -0.05243462324142456,
+            0.040698058903217316,
+            0.06868159025907516,
+            0.022752607241272926,
+            -0.005430352408438921,
+            -0.006812892388552427,
+            -0.04902511462569237,
+            -0.006551826372742653,
+            -0.03979682922363281,
+            -0.013956423848867416,
+            -0.06136368587613106,
+            0.0740000531077385,
+            0.05603933334350586,
+            0.02190348319709301,
+            -0.043786339461803436,
+            -0.0392116904258728,
+            -0.01866808719933033,
+            0.01707339473068714,
+            -0.026303859427571297,
+            -0.01817542500793934,
+            0.03552285581827164,
+            0.0276781152933836,
+            0.05265122279524803,
+            -0.03358357027173042,
+            -0.020007848739624023,
+            0.04865119233727455,
+            0.02959197200834751,
+            -0.0032693049870431423,
+            0.02495887503027916,
+            0.03446371853351593,
+            -0.011217310093343258,
+            -0.09030335396528244,
+            0.014422472566366196,
+            -0.008989378809928894,
+            -0.011282369494438171,
+            0.049398381263017654,
+            -0.01687331311404705,
+            0.025424139574170113,
+            0.024985041469335556,
+            -0.009084195829927921,
+            0.004050575662404299,
+            0.0007717382395640016,
+            -0.03172731399536133,
+            -0.017505444586277008,
+            -0.014687484130263329,
+            0.03803866356611252,
+            0.016156280413269997,
+            -0.010017951019108295,
+            -0.026353944092988968,
+            0.019050614908337593,
+            -0.03580506518483162,
+            0.02924525737762451,
+            0.02443450316786766,
+            -0.01770329475402832,
+            0.04581848904490471,
+            -0.01908605918288231,
+            0.012714254669845104,
+            0.08363562822341919,
+            0.037286512553691864,
+            -0.003420531051233411,
+            -0.06909038126468658,
+            -0.0591881163418293,
+            -0.007654525339603424,
+            0.053144630044698715,
+            0.03045589104294777,
+            -0.04600578546524048,
+            0.026682959869503975,
+            -0.0019753179512917995,
+            -0.017073772847652435,
+            -0.012071357108652592,
+            0.028171954676508904,
+            0.00024773634504526854,
+            -0.03256797045469284,
+            -0.09742321819067001,
+            0.040483273565769196,
+            -0.025031624361872673,
+            0.03650636970996857,
+            0.0011886897264048457,
+            0.016929153352975845,
+            0.054483890533447266,
+            0.03752107173204422,
+            0.019491281360387802,
+            0.006253591738641262,
+            0.02451430633664131,
+            -0.05976274237036705,
+            0.060739971697330475,
+            -0.04400366172194481,
+            0.028709039092063904,
+            -0.02141660451889038,
+            0.08152823895215988,
+            -0.00450171809643507,
+            -0.03484562411904335,
+            -0.00046958858729340136,
+            -0.017397938296198845,
+            0.07823023945093155,
+            -0.011110293678939342,
+            0.004524719901382923,
+            0.03619854897260666,
+            -0.02478216402232647,
+            -0.011563056148588657,
+            -0.012815544381737709,
+            -0.03503820300102234,
+            -0.04771020635962486,
+            -0.030619489029049873,
+            0.0669066309928894,
+            0.030025487765669823,
+            -0.011697783134877682,
+            -0.006708705797791481,
+            -0.0061534675769507885,
+            0.0365905724465847,
+            -0.006860053166747093,
+            -0.07040797173976898,
+            -0.057646144181489944,
+            0.04284966364502907,
+            -0.01533683855086565,
+            -0.06859996914863586,
+            0.009425769560039043,
+            -9.838528785621747e-05,
+            0.010796179063618183,
+            -0.06541100144386292,
+            0.01059884112328291,
+            -0.028843343257904053,
+            0.029019653797149658,
+            -0.005446962546557188,
+            -0.0120149040594697,
+            -0.0471968911588192,
+            0.008648250252008438,
+            0.021961573511362076,
+            -0.010606558993458748,
+            0.0008718566386960447,
+            -0.014988702721893787,
+            -0.11522816866636276,
+            -0.023671753704547882,
+            -0.004968483000993729,
+            0.0307041984051466,
+            -0.0020613274537026882,
+            -0.03271760419011116,
+            -0.04547363147139549,
+            -0.00812614057213068,
+            -0.013890305534005165,
+            0.048099175095558167,
+            -0.015408700332045555,
+            0.06658884882926941,
+            0.012633614242076874,
+            -0.05337975174188614,
+            0.0033035692758858204,
+            0.03610198199748993,
+            -0.0405871607363224,
+            0.008806376717984676,
+            -0.017653945833444595,
+            -0.05865860357880592,
+            0.03825455904006958,
+            -0.00478429114446044,
+            -0.04127506911754608,
+            0.01231306791305542,
+            0.0008735111332498491,
+            0.02923770435154438,
+            0.005922738928347826,
+            -0.01829770766198635,
+            -0.00685579888522625,
+            -0.03903493285179138,
+            0.009158597327768803,
+            -0.03491708263754845,
+            0.04114120453596115,
+            -0.0014327293029055,
+            -0.019274454563856125,
+            0.02704671025276184,
+            0.01738886535167694,
+            -0.02327372133731842,
+            -0.03135831654071808,
+            -0.01305293757468462,
+            0.04163745418190956,
+            0.01710107922554016,
+            0.06454417109489441,
+            0.020267069339752197,
+            -0.08408207446336746,
+            -0.010505065321922302,
+            -0.0073319086804986,
+            0.004039655905216932,
+            -0.01633611135184765,
+            -0.02889716438949108,
+            -0.0806351900100708,
+            -0.023603465408086777,
+            -0.06304290890693665,
+            0.007231221999973059,
+            -0.038828227669000626,
+            0.014790577813982964,
+            -0.03915632143616676,
+            0.05616161227226257,
+            0.00311578088440001,
+            -0.02434428222477436,
+            0.006431886460632086,
+            -0.06326194852590561,
+            -0.0166602972894907,
+            0.03630464896559715,
+            0.01622965931892395,
+            0.026233987882733345,
+            0.06605540215969086,
+            -0.05635184794664383,
+            -0.08930846303701401,
+            -0.05207853391766548,
+            0.027004040777683258,
+            -0.031913693994283676,
+            -0.009139630943536758,
+            -0.028410566970705986,
+            0.06700566411018372,
+            0.0423152893781662,
+            -0.010422738268971443,
+            -0.04085265100002289,
+            0.029382970184087753,
+            0.052883222699165344,
+            0.02239867113530636,
+            -0.0012815282680094242,
+            0.014223611913621426,
+            0.02597920596599579,
+            -0.015063034370541573,
+            0.0828455239534378,
+            0.03366050869226456,
+            -0.022025907412171364,
+            -0.0019613192416727543,
+            -0.02539178729057312,
+            0.0399317741394043,
+            -0.006493750028312206,
+            -0.0013236093800514936,
+            -0.02036309242248535,
+            -0.0065197269432246685,
+            -0.030695058405399323,
+            -0.03585388883948326,
+            -0.045742426067590714,
+            0.015121972188353539,
+            0.08081705123186111,
+            0.007906812243163586,
+            0.059827424585819244,
+            -0.04740464314818382,
+            -0.0743480697274208,
+            -0.025416050106287003,
+            -0.05693193897604942,
+            -0.001481675892136991,
+            -0.017177585512399673,
+            -0.03311903402209282,
+            0.022755322977900505,
+            -0.003895542584359646,
+            0.02692737802863121,
+            -0.0032731543760746717,
+            -0.0031116430182009935,
+            -0.030539495870471,
+            -0.006427450571209192,
+            -0.0015021534636616707,
+            0.0017666849307715893,
+            -0.03059082292020321,
+            0.0005832729511894286,
+            -0.05637278035283089,
+            0.04087543487548828,
+            0.00220437441021204,
+            0.0021564762573689222,
+            -0.0314127579331398,
+            -0.025518659502267838,
+            -0.07314060628414154,
+            -0.014426291920244694,
+            -0.08714891970157623,
+            -0.02331671305000782,
+            0.013582085259258747,
+            -0.0025384915061295033,
+            -0.01540769450366497,
+            -0.0110056446865201,
+            0.04654880613088608,
+            0.010653696954250336,
+            0.0018328200094401836,
+            0.007387213874608278,
+            0.07984212785959244,
+            -0.02893732860684395,
+            -0.04140201583504677,
+            -0.07618758082389832,
+            -0.00793982483446598,
+            -0.0377434641122818,
+            0.032935332506895065,
+            -0.013259266503155231,
+            0.02015708014369011,
+            0.09388656169176102,
+            0.0017843206878751516,
+            0.034253206104040146,
+            -0.017240997403860092,
+            0.009084933437407017,
+            -0.048595622181892395,
+            -0.03737767040729523,
+            -0.04036621376872063,
+            -0.009442481212317944,
+            0.01705838553607464,
+            -0.03709835931658745,
+            0.004579882137477398,
+            -0.02558705396950245,
+            -0.010287507437169552,
+            -0.00969093106687069,
+            0.012930587865412235,
+            -0.0026530276518315077,
+            -0.055302973836660385,
+            -0.0007084248936735094,
+            0.0027114865370094776,
+            -0.022337302565574646,
+            0.049817051738500595,
+            -0.015339787118136883,
+            -0.01575980708003044,
+            -0.0211472287774086,
+            -0.041779227554798126,
+            -0.00043109135003760457,
+            -0.002638365374878049,
+            0.0003785403096117079,
+            0.04666115716099739,
+            -0.031109463423490524,
+            0.03951709344983101,
+            0.007409846410155296,
+            0.032062821090221405,
+            -0.019406728446483612,
+            -0.03020879067480564,
+            0.06802312284708023,
+            0.018488138914108276,
+            -0.053909264504909515,
+            -0.007893281057476997,
+            0.02533031813800335,
+            0.03132852911949158,
+            -0.053483180701732635,
+            0.025661734864115715,
+            0.002397680189460516,
+            0.04062856733798981,
+            0.02830035611987114,
+            -0.00479720626026392,
+            -0.008729430846869946,
+            -0.0034766148310154676,
+            0.03173350542783737,
+            0.0071125393733382225,
+            -0.03040342777967453,
+            -0.006032709032297134,
+            -0.06926627457141876,
+            -0.0381772480905056
+        ],
+        "content_embedding": [
+            -0.02845170348882675,
+            0.020628532394766808,
+            0.003312832210212946,
+            -0.029765071347355843,
+            0.016959644854068756,
+            -0.004110109526664019,
+            0.054954126477241516,
+            -0.03696386516094208,
+            -0.06003747880458832,
+            -0.016737867146730423,
+            -0.04143841937184334,
+            0.010524315759539604,
+            0.01846286654472351,
+            0.012900668196380138,
+            -0.01821877434849739,
+            -0.022333195433020592,
+            0.016231827437877655,
+            -0.00692401546984911,
+            -0.009705417789518833,
+            0.0043431734666228294,
+            -0.01035444438457489,
+            -0.03510449081659317,
+            -0.01101984828710556,
+            -0.029713021591305733,
+            0.08172306418418884,
+            -0.008759400807321072,
+            -0.040999725461006165,
+            0.04106973484158516,
+            -0.05768377706408501,
+            -0.008512589149177074,
+            0.05944962799549103,
+            -0.012553821317851543,
+            0.013645646162331104,
+            -0.02660560794174671,
+            -0.057905036956071854,
+            -0.054687026888132095,
+            0.003909541759639978,
+            -0.04956740885972977,
+            -0.042125821113586426,
+            0.06187684088945389,
+            0.06303229182958603,
+            -0.012631679885089397,
+            -0.004673871211707592,
+            -0.02207319252192974,
+            0.052802763879299164,
+            0.014762785285711288,
+            0.04115021601319313,
+            -0.006632254458963871,
+            0.03773806244134903,
+            -0.03468457981944084,
+            -0.014101233333349228,
+            0.013350501656532288,
+            -0.024982236325740814,
+            -0.009867328219115734,
+            -0.007960042916238308,
+            0.005127797368913889,
+            0.002303300891071558,
+            -0.004433336202055216,
+            0.03658096119761467,
+            -0.04504770040512085,
+            0.027889715507626534,
+            0.05441499873995781,
+            -0.04908447712659836,
+            0.041611816734075546,
+            -0.00782090611755848,
+            -0.05460766702890396,
+            0.0005653056432493031,
+            0.0009949197992682457,
+            0.013009139336645603,
+            0.004702548962086439,
+            -0.0066951001062989235,
+            -0.009612455032765865,
+            0.027976926416158676,
+            0.013144126161932945,
+            -0.009398404508829117,
+            -0.009249510243535042,
+            0.02228953316807747,
+            -0.05003415420651436,
+            -0.03484565392136574,
+            0.039622433483600616,
+            0.03127755597233772,
+            -0.07711455225944519,
+            0.026068583130836487,
+            -0.03025561012327671,
+            0.03434577211737633,
+            0.02756066806614399,
+            -0.016127552837133408,
+            0.0031622813548892736,
+            -0.011191527359187603,
+            0.10279087722301483,
+            0.07508235424757004,
+            0.014161021448671818,
+            0.04303860291838646,
+            0.02421264722943306,
+            -0.060081642121076584,
+            0.08023173362016678,
+            -0.016117870807647705,
+            -0.040795255452394485,
+            -0.006737033370882273,
+            -0.02539793588221073,
+            -0.005812298972159624,
+            0.027351481840014458,
+            0.02652551420032978,
+            0.034308984875679016,
+            0.07952407002449036,
+            0.012120738625526428,
+            -0.002102786907926202,
+            0.02581837773323059,
+            0.0036945617757737637,
+            0.03335866704583168,
+            -0.05533025786280632,
+            -0.029806576669216156,
+            0.014511525630950928,
+            0.028494026511907578,
+            -0.028353745117783546,
+            -0.0015628034016117454,
+            0.0542825423181057,
+            -0.06842301040887833,
+            0.013071774505078793,
+            0.035904042422771454,
+            -0.060427047312259674,
+            -0.010712354443967342,
+            -0.010741145350039005,
+            0.00589279318228364,
+            0.03916572034358978,
+            0.0011838098289445043,
+            -0.04358551278710365,
+            -0.02426866628229618,
+            -0.02629699930548668,
+            -0.016508616507053375,
+            0.0038987575098872185,
+            0.00010461249621585011,
+            -0.06473322957754135,
+            0.027538873255252838,
+            0.03787471354007721,
+            0.024383891373872757,
+            -0.04171127453446388,
+            -0.03238093852996826,
+            0.007360804360359907,
+            -0.014501902274787426,
+            0.014242740347981453,
+            -0.0012311796890571713,
+            -0.013716178946197033,
+            -0.009915472939610481,
+            0.026615049690008163,
+            -0.07398053258657455,
+            0.0030485496390610933,
+            0.025813661515712738,
+            -0.022065768018364906,
+            0.0349227599799633,
+            0.0045135351829230785,
+            -0.053763143718242645,
+            -0.013968654908239841,
+            0.016600387170910835,
+            0.029198968783020973,
+            -0.03825172409415245,
+            -0.03900526836514473,
+            0.02822844497859478,
+            0.052716661244630814,
+            -0.00427692336961627,
+            0.029389938339591026,
+            0.01127107534557581,
+            -0.02288925088942051,
+            0.06506737321615219,
+            -0.011876849457621574,
+            -0.009232635609805584,
+            0.059180255979299545,
+            0.060491811484098434,
+            0.04768436402082443,
+            0.04782063513994217,
+            -0.007591789122670889,
+            -0.012142209336161613,
+            -0.00854392908513546,
+            -0.03645598515868187,
+            0.02366817742586136,
+            0.028424806892871857,
+            0.03254731744527817,
+            -0.0650848001241684,
+            0.05803924798965454,
+            -0.006124107167124748,
+            0.007514724973589182,
+            -0.06995245814323425,
+            0.03610721975564957,
+            -0.025534681975841522,
+            -0.047099191695451736,
+            0.0024543125182390213,
+            0.013705895282328129,
+            -0.08660408854484558,
+            0.013458521105349064,
+            -0.05938595533370972,
+            0.025314588099718094,
+            -0.06279927492141724,
+            -0.008528811857104301,
+            -0.04051665961742401,
+            -0.02572588622570038,
+            -0.05028638243675232,
+            0.029650729149580002,
+            0.03656933456659317,
+            0.027842504903674126,
+            -0.017784448340535164,
+            -0.06566111743450165,
+            -0.016097936779260635,
+            -0.07754653692245483,
+            0.02611452341079712,
+            -0.012319186702370644,
+            0.03830364719033241,
+            0.05927351489663124,
+            -0.0005797847989015281,
+            0.05858585610985756,
+            0.013468705117702484,
+            0.08553440123796463,
+            0.010187739506363869,
+            -0.023877883329987526,
+            0.027608737349510193,
+            -0.04135579988360405,
+            -0.004526825156062841,
+            0.01695535145699978,
+            -0.043227668851614,
+            -0.03456792235374451,
+            0.06477289646863937,
+            0.031624119728803635,
+            -0.04087601602077484,
+            0.0010430653346702456,
+            0.017958510667085648,
+            0.009248117916285992,
+            0.010219916701316833,
+            -0.05485055223107338,
+            -0.01347501389682293,
+            -0.015884561464190483,
+            -0.008806952275335789,
+            -0.04478437826037407,
+            -0.09141774475574493,
+            0.07184188067913055,
+            0.02080371417105198,
+            0.03414024040102959,
+            0.02681431546807289,
+            -0.02171824313700199,
+            0.023230157792568207,
+            0.0034705817233771086,
+            0.023832201957702637,
+            0.04260754957795143,
+            -0.023710861802101135,
+            0.017519451677799225,
+            -0.023114347830414772,
+            -0.07241662591695786,
+            0.043135177344083786,
+            -0.03519831597805023,
+            0.01728164590895176,
+            -0.007306656800210476,
+            0.029966725036501884,
+            0.005133960861712694,
+            0.010730396956205368,
+            0.014178331010043621,
+            0.02290872484445572,
+            0.04147600382566452,
+            -0.0711970180273056,
+            0.011968120001256466,
+            0.012014097534120083,
+            -0.00941413827240467,
+            -0.048221614211797714,
+            0.02721494808793068,
+            -0.036967791616916656,
+            0.03305060788989067,
+            -0.023104682564735413,
+            -0.0021078407298773527,
+            0.01056760549545288,
+            0.003825176041573286,
+            -0.02744617499411106,
+            -0.011484067887067795,
+            -0.019249368458986282,
+            0.012087506242096424,
+            0.016815317794680595,
+            0.008888418786227703,
+            0.09483875334262848,
+            0.005403030198067427,
+            -0.006538084242492914,
+            -0.00812787376344204,
+            0.010251098312437534,
+            0.025141935795545578,
+            -0.016502706333994865,
+            -0.07583127170801163,
+            -0.059361476451158524,
+            -0.03975491225719452,
+            0.005571363028138876,
+            0.025980276986956596,
+            0.06575164198875427,
+            -0.022391004487872124,
+            -0.0014668750809505582,
+            0.0309857539832592,
+            -0.01333113107830286,
+            0.024967554956674576,
+            0.008801382035017014,
+            0.004801678936928511,
+            0.011097696609795094,
+            -0.02484068274497986,
+            0.07475821673870087,
+            0.06004296988248825,
+            0.008063849061727524,
+            -0.007297527976334095,
+            -0.0225421991199255,
+            -0.020057078450918198,
+            0.04824424162507057,
+            0.019114485010504723,
+            -0.024972988292574883,
+            -0.013590608723461628,
+            -0.026848217472434044,
+            0.013158710673451424,
+            0.03205424174666405,
+            0.06152794137597084,
+            0.06059333309531212,
+            -0.03528637811541557,
+            0.03574252501130104,
+            0.011881774291396141,
+            -0.014821416698396206,
+            -0.03766554221510887,
+            0.02960871160030365,
+            0.043620847165584564,
+            -0.0008511713240295649,
+            -0.012452763505280018,
+            -0.008131926879286766,
+            0.02682739496231079,
+            -0.027776895090937614,
+            -0.017724614590406418,
+            -0.074460469186306,
+            0.007388352416455746,
+            0.052085623145103455,
+            0.005944994743913412,
+            0.05980602651834488,
+            0.004958854056894779,
+            0.004315464291721582,
+            0.009470906108617783,
+            0.08363069593906403,
+            -0.06266297399997711,
+            -0.02252691425383091,
+            -0.0047216094098985195,
+            0.01363289449363947,
+            0.00803599413484335,
+            0.017915869131684303,
+            0.01683986745774746,
+            0.0054694474674761295,
+            0.00690553430467844,
+            -0.033289551734924316,
+            -0.041059620678424835,
+            -0.05957230553030968,
+            -0.07026804983615875,
+            0.0026692922692745924,
+            0.01784949004650116,
+            -0.0003522790502756834,
+            0.044131647795438766,
+            0.05823194235563278,
+            -0.01914701797068119,
+            0.012501074001193047,
+            -0.0076722633093595505,
+            -0.040374498814344406,
+            0.010002685710787773,
+            -0.006841403432190418,
+            0.024179449304938316,
+            0.01219668984413147,
+            -0.005650076549500227,
+            0.010313056409358978,
+            0.0951242670416832,
+            -0.012245064601302147,
+            0.02261069230735302,
+            -0.02354615181684494,
+            0.04120791703462601,
+            0.03377315774559975,
+            0.03468254953622818,
+            0.033613745123147964,
+            0.03305840864777565,
+            -0.033862534910440445,
+            0.034367989748716354,
+            -0.022536078467965126,
+            0.04874858632683754,
+            0.0415329709649086,
+            0.06666682660579681,
+            0.0036932802759110928,
+            -0.09809356927871704,
+            -0.0017088145250454545,
+            0.0024299651850014925,
+            -0.018637090921401978,
+            0.06801039725542068,
+            0.02409985661506653,
+            0.009735392406582832,
+            -0.02749275043606758,
+            0.030437270179390907,
+            -0.00898370798677206,
+            -0.020128484815359116,
+            -0.009687880054116249,
+            0.01668565906584263,
+            -0.04497738555073738,
+            -0.019772959873080254,
+            -0.022632960230112076,
+            -0.02253716252744198,
+            0.010271976701915264,
+            -0.00696501974016428,
+            0.030849630013108253,
+            -0.04239751771092415,
+            0.03944450616836548,
+            -0.013430316932499409,
+            0.032022625207901,
+            -0.05952562019228935,
+            -0.043423160910606384,
+            -0.0024594010319560766,
+            -0.0009159342152997851,
+            -0.01097820233553648,
+            0.02963317185640335,
+            -0.02188674546778202,
+            -0.048581305891275406,
+            0.03341398760676384,
+            -0.011065436527132988,
+            0.047042280435562134,
+            0.04042183235287666,
+            -0.02600206807255745,
+            -0.05695529654622078,
+            0.05499875172972679,
+            -0.03984459489583969,
+            0.0649806335568428,
+            0.02108422853052616,
+            -0.07841797918081284,
+            -0.02946053445339203,
+            -0.01747039519250393,
+            -0.013214156962931156,
+            0.008581981062889099,
+            -0.0022455912549048662,
+            -0.022153383120894432,
+            0.011744811199605465,
+            0.017487658187747,
+            0.008825760334730148,
+            0.025580866262316704,
+            0.0082536730915308,
+            0.03269948437809944,
+            0.012900054454803467,
+            0.04077104851603508,
+            0.0378829762339592,
+            -0.06819288432598114,
+            0.02784581482410431,
+            0.06434649229049683,
+            0.03351795673370361,
+            0.011211752891540527,
+            0.01894824393093586,
+            0.004370532464236021,
+            -0.014345336705446243,
+            0.006097803357988596,
+            -0.08543102443218231,
+            -0.02757532149553299,
+            0.06072119623422623,
+            0.10378460586071014,
+            0.009428516030311584,
+            -0.04370144382119179,
+            -0.01193047035485506,
+            0.04444314166903496,
+            0.011696353554725647,
+            -0.030421355739235878,
+            -0.014331959187984467,
+            0.04900198057293892,
+            0.027842359617352486,
+            -0.009193948470056057,
+            -0.06911113113164902,
+            -0.011863719671964645,
+            0.035546496510505676,
+            0.022603457793593407,
+            0.017083479091525078,
+            0.015593679621815681,
+            0.021234845742583275,
+            0.004202473908662796,
+            -0.0537634901702404,
+            0.04333661124110222,
+            -0.025838447734713554,
+            -0.024251429364085197,
+            0.03204401955008507,
+            -0.002707387087866664,
+            0.03231189027428627,
+            -0.0030726506374776363,
+            0.024067431688308716,
+            -0.03183548524975777,
+            0.007890576496720314,
+            0.054388418793678284,
+            -0.017090266570448875,
+            0.01572081632912159,
+            0.04539618268609047,
+            0.01960766315460205,
+            0.009962057694792747,
+            -0.06782109290361404,
+            0.03449762612581253,
+            -0.004095849581062794,
+            0.011451792903244495,
+            0.01291556004434824,
+            0.009039049968123436,
+            -0.023024702444672585,
+            -0.019358525052666664,
+            0.004383507184684277,
+            0.01303566712886095,
+            0.008749599568545818,
+            -0.0019463954959064722,
+            -0.05059671401977539,
+            -0.03929319232702255,
+            -0.038611579686403275,
+            -0.0004830050456803292,
+            0.03260226547718048,
+            -0.0484665147960186,
+            0.04194365814328194,
+            0.028087828308343887,
+            0.0015577803133055568,
+            -0.0020338960457593203,
+            0.00992380827665329,
+            0.029156159609556198,
+            0.013400931842625141,
+            -0.08322624862194061,
+            0.03542347997426987,
+            0.029148025438189507,
+            0.03978026285767555,
+            0.012040375731885433,
+            -0.015034311451017857,
+            0.016610056161880493,
+            0.030412640422582626,
+            -0.051336318254470825,
+            0.057814277708530426,
+            -0.009117085486650467,
+            -0.055816203355789185,
+            0.05705805867910385,
+            -0.013596060685813427,
+            0.07485361397266388,
+            0.04783453419804573,
+            0.03237048164010048,
+            -0.1252431571483612,
+            -0.024511752650141716,
+            0.009948192164301872,
+            -0.001071324571967125,
+            0.03724309056997299,
+            0.0011626302730292082,
+            -0.024294020608067513,
+            0.09550263732671738,
+            0.001125041046179831,
+            -0.032950934022665024,
+            0.03814041614532471,
+            -0.015817970037460327,
+            -0.042719099670648575,
+            0.02066672407090664,
+            0.03320891037583351,
+            0.04236403852701187,
+            0.026531731709837914,
+            -0.026540761813521385,
+            0.018515214323997498,
+            -0.0034993020817637444,
+            -0.019887158647179604,
+            -0.03968263417482376,
+            -0.048776015639305115,
+            0.058169975876808167,
+            -0.033802736550569534,
+            -0.06506575644016266,
+            0.06649087369441986,
+            -0.0008670967072248459,
+            0.03118874505162239,
+            -0.04512554034590721,
+            0.017889900133013725,
+            -0.0688585713505745,
+            -0.005668118130415678,
+            -0.04627913981676102,
+            -0.005911638960242271,
+            -0.004173378925770521,
+            0.030776705592870712,
+            -0.003502912586554885,
+            0.06485340744256973,
+            -0.002432354027405381,
+            0.016381509602069855,
+            -0.11775369197130203,
+            -0.03817284479737282,
+            -0.0002618363650981337,
+            0.020635321736335754,
+            -0.00496688112616539,
+            -0.025879040360450745,
+            -0.03508525341749191,
+            0.03343107923865318,
+            0.0065256268717348576,
+            0.03733794391155243,
+            -0.05374712869524956,
+            0.06329557299613953,
+            -0.012232488952577114,
+            -0.07295915484428406,
+            -0.03410065174102783,
+            -0.036963820457458496,
+            -0.040072306990623474,
+            0.08864720910787582,
+            -0.025265397503972054,
+            0.006373850163072348,
+            0.03589979186654091,
+            -0.04953531548380852,
+            -0.014813058078289032,
+            0.03135038912296295,
+            -0.045539740473032,
+            -0.013331865891814232,
+            -0.0077253468334674835,
+            0.02402244322001934,
+            -0.01973932422697544,
+            -0.07107116281986237,
+            0.029835300520062447,
+            0.027613440528512,
+            0.027596186846494675,
+            0.0323333777487278,
+            -0.059915486723184586,
+            -0.03399650380015373,
+            -0.011191067285835743,
+            0.04654889553785324,
+            0.02089870348572731,
+            -0.02999742142856121,
+            0.03440370410680771,
+            0.007507022004574537,
+            0.040571920573711395,
+            -0.03278252109885216,
+            -0.06050700321793556,
+            0.021276405081152916,
+            -0.016155162826180458,
+            0.00010897620086325333,
+            -0.0010203487472608685,
+            -0.03746471554040909,
+            -0.025609171018004417,
+            0.005009123589843512,
+            -0.08643782883882523,
+            0.031217029318213463,
+            -0.0032753287814557552,
+            -0.025097588077187538,
+            -0.03903978690505028,
+            0.04100614786148071,
+            -0.015351627953350544,
+            -0.027274904772639275,
+            0.03702486678957939,
+            -0.08083852380514145,
+            0.005576241761445999,
+            -0.006957546342164278,
+            0.09723483771085739,
+            0.018242431804537773,
+            0.05415903404355049,
+            -0.06101224571466446,
+            -0.025462020188570023,
+            -0.05338318645954132,
+            -0.014500913210213184,
+            0.017682785168290138,
+            -0.017082780599594116,
+            0.028426188975572586,
+            0.039222750812768936,
+            0.02545572631061077,
+            -0.03942421078681946,
+            -0.06022893264889717,
+            0.04819706454873085,
+            0.06749513745307922,
+            0.01035595778375864,
+            0.011470649391412735,
+            -0.0024080125149339437,
+            -0.033233992755413055,
+            0.028164777904748917,
+            0.0553852841258049,
+            0.06263226270675659,
+            -0.016413886100053787,
+            0.021145500242710114,
+            0.01179521344602108,
+            0.033495236188173294,
+            -0.013982322998344898,
+            0.015615278854966164,
+            -0.04715987294912338,
+            0.029921408742666245,
+            0.00394752062857151,
+            -0.028457310050725937,
+            0.018736237660050392,
+            -0.02897241711616516,
+            -0.006964333355426788,
+            0.001459103194065392,
+            0.020680002868175507,
+            -0.045486945658922195,
+            -0.0186879001557827,
+            0.006018372252583504,
+            -0.005646763369441032,
+            0.031949128955602646,
+            0.015718143433332443,
+            -0.0680442824959755,
+            -0.040316201746463776,
+            0.027103520929813385,
+            0.007122257724404335,
+            -0.02554631046950817,
+            -0.015084164217114449,
+            -0.05808757618069649,
+            -0.01925673894584179,
+            -0.030870718881487846,
+            0.04750153049826622,
+            -0.05464727431535721,
+            -0.03634507954120636,
+            -0.022956492379307747,
+            -0.001869303290732205,
+            0.009947973303496838,
+            -0.0452248640358448,
+            -0.05064425989985466,
+            0.0033088859636336565,
+            0.032645225524902344,
+            0.011128626763820648,
+            -0.07038814574480057,
+            -0.00398398470133543,
+            -0.029936205595731735,
+            -0.037302739918231964,
+            -0.026515178382396698,
+            -0.005808456335216761,
+            0.011362303048372269,
+            0.013931548222899437,
+            0.053531426936388016,
+            -0.017702942714095116,
+            0.04049023985862732,
+            0.0490780733525753,
+            -0.0034894803538918495,
+            0.0072046807035803795,
+            0.05128946155309677,
+            -0.02248883992433548,
+            -0.016904350370168686,
+            -0.007111264392733574,
+            0.013148962520062923,
+            0.04168686643242836,
+            -0.011360356584191322,
+            -0.01462612859904766,
+            0.036358367651700974,
+            -0.000562329194508493,
+            -0.037175074219703674,
+            0.002623690525069833,
+            -0.04284300655126572,
+            0.045784976333379745,
+            0.017441436648368835,
+            -0.09851012378931046,
+            0.06095545366406441,
+            -0.0374034121632576,
+            -0.02720695361495018,
+            -0.007703973911702633,
+            0.03689894452691078,
+            0.008105777204036713,
+            0.019800135865807533,
+            -0.02071499079465866,
+            -0.022336198017001152,
+            -0.009109229780733585,
+            0.02298821695148945,
+            -0.04302436485886574,
+            -0.023912018164992332,
+            0.007846477441489697,
+            -0.04115860536694527,
+            -0.052512455731630325,
+            -0.0262643713504076,
+            0.00893806479871273,
+            -0.032936349511146545,
+            -0.015261095948517323,
+            0.005558508913964033,
+            -0.008528356440365314,
+            -0.023201653733849525,
+            -0.056550297886133194,
+            0.025247847661376,
+            0.04831540212035179,
+            -0.019267458468675613,
+            -0.03474835306406021,
+            -0.001712511875666678,
+            -0.04063638299703598,
+            -0.01707102544605732,
+            -0.01384702231734991,
+            -0.0023981977719813585,
+            0.03153757378458977,
+            0.030312344431877136,
+            0.040820326656103134,
+            -0.013783353380858898,
+            0.012157448567450047,
+            -0.015558870509266853,
+            -0.056085314601659775,
+            0.03875841945409775,
+            0.021351536735892296,
+            -0.021598603576421738,
+            -0.01058109663426876,
+            0.001237297197803855
+        ],
+        "chunk_ind": 0
+    },
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/enterprise_search",
+        "title": "Enterprise Search",
+        "content": "Value of Enterprise Search with Danswer\n\nWhat is Enterprise Search and why is it Important?\nAn Enterprise Search system gives team members a single place to access all of the disparate knowledge of an organization. Critical information is saved across a host of channels like call transcripts with prospects, engineering design docs, IT runbooks, customer support email exchanges, project management tickets, and more. As fast moving teams scale up, information gets spread out and more disorganized.\n\nSince it quickly becomes infeasible to check across every source, decisions get made on incomplete information, employee satisfaction decreases, and the most valuable members of your team are tied up with constant distractions as junior teammates are unable to unblock themselves. Danswer solves this problem by letting anyone on the team access all of the knowledge across your organization in a permissioned and secure way. Users can ask questions in natural language and get back answers and documents across all of the connected sources instantly.\n\nWhat's the real cost?\nA typical knowledge worker spends over 2 hours a week on search, but more than that, the cost of incomplete or incorrect information can be extremely high. Customer support/success that isn't able to find the reference to similar cases could cause hours or even days of delay leading to lower customer satisfaction or in the worst case - churn. An account exec not realizing that a prospect had previously mentioned a specific need could lead to lost deals. An engineer not realizing a similar feature had previously been built could result in weeks of wasted development time and tech debt with duplicate implementation. With a lack of knowledge, your whole organization is navigating in the dark - inefficient and mistake prone.",
+        "title_embedding": [
+            -0.011060578748583794,
+            0.05994883179664612,
+            0.008845113217830658,
+            0.011364905163645744,
+            0.03147757425904274,
+            0.04506697878241539,
+            -0.025942707434296608,
+            -0.011002028360962868,
+            -0.03507396578788757,
+            -0.01727098599076271,
+            -0.016820134595036507,
+            0.04671240597963333,
+            0.023456331342458725,
+            -0.005752791650593281,
+            -0.011421029455959797,
+            -0.04169125109910965,
+            0.0652366429567337,
+            -0.011136278510093689,
+            -0.013501451350748539,
+            -0.006273901090025902,
+            0.0012236927868798375,
+            -0.030771249905228615,
+            0.010098040103912354,
+            -0.02360220067203045,
+            0.006734110414981842,
+            0.001379420980811119,
+            -0.0047225081361830235,
+            -0.013901330530643463,
+            0.014645840041339397,
+            -0.02156134508550167,
+            -0.026707857847213745,
+            -0.00676271365955472,
+            0.056067030876874924,
+            -0.0455806590616703,
+            -0.02762053906917572,
+            -0.0965149849653244,
+            0.04567359760403633,
+            0.053895801305770874,
+            0.029093541204929352,
+            0.0199823547154665,
+            0.047089505940675735,
+            0.026028119027614594,
+            -0.0034626282285898924,
+            -0.021002190187573433,
+            0.04217635095119476,
+            -0.015198000706732273,
+            0.047393105924129486,
+            0.008588545024394989,
+            0.07058116048574448,
+            -0.09135723114013672,
+            -0.009591161273419857,
+            0.014971816912293434,
+            -8.222273208957631e-07,
+            -0.05534408614039421,
+            -0.029965048655867577,
+            -0.028381407260894775,
+            0.025547217577695847,
+            0.038583844900131226,
+            0.01984122209250927,
+            -0.02435469813644886,
+            0.04955849424004555,
+            0.0128632802516222,
+            -0.022573867812752724,
+            0.025284791365265846,
+            0.04496009647846222,
+            0.0004200333496555686,
+            -0.0015001222491264343,
+            0.02609623409807682,
+            0.023677939549088478,
+            -0.05961468443274498,
+            0.06799754500389099,
+            -0.014409428462386131,
+            -0.010374268516898155,
+            0.019634529948234558,
+            -0.03720257058739662,
+            -0.04654879495501518,
+            0.0056068566627800465,
+            -0.021244503557682037,
+            -0.03198164328932762,
+            0.05707596614956856,
+            0.043927326798439026,
+            -0.033356692641973495,
+            0.015723733231425285,
+            -0.027493512257933617,
+            0.04525380581617355,
+            0.025808652862906456,
+            -0.007083983160555363,
+            -0.011038954369723797,
+            -0.004390218295156956,
+            -0.006583297159522772,
+            0.003319315379485488,
+            0.061810243874788284,
+            0.05887124314904213,
+            0.07722297310829163,
+            -0.06601747870445251,
+            0.036486171185970306,
+            0.05119618400931358,
+            0.00945530366152525,
+            -0.03005358576774597,
+            -0.047870855778455734,
+            -0.0359003059566021,
+            -0.005699747242033482,
+            0.053807053714990616,
+            -0.001554036163724959,
+            0.060548700392246246,
+            0.05476829782128334,
+            0.00426551653072238,
+            -0.005215689539909363,
+            -0.0352163203060627,
+            -0.03284529969096184,
+            -0.03373449295759201,
+            0.021254340186715126,
+            -0.010481598787009716,
+            0.02651101164519787,
+            -0.00481840968132019,
+            0.045160870999097824,
+            0.09704204648733139,
+            -0.03473421558737755,
+            0.015584945678710938,
+            0.06579536944627762,
+            0.0651017278432846,
+            0.007380738854408264,
+            0.00624364148825407,
+            0.07893780618906021,
+            -0.019876087084412575,
+            0.006619091611355543,
+            -0.030776498839259148,
+            -0.016426965594291687,
+            0.014603338204324245,
+            -0.03326896205544472,
+            0.003784433240070939,
+            0.025205042213201523,
+            0.03047170303761959,
+            -0.03364298865199089,
+            0.005974944215267897,
+            0.015994269400835037,
+            -0.10205432027578354,
+            -0.026497431099414825,
+            0.07166463136672974,
+            -0.007370935752987862,
+            0.034912627190351486,
+            -0.004656887147575617,
+            0.03219066560268402,
+            -0.02239271067082882,
+            0.012679396197199821,
+            -0.07867992669343948,
+            -0.026638884097337723,
+            -0.008346030488610268,
+            -0.027243634685873985,
+            0.012043806724250317,
+            0.024078860878944397,
+            0.0006219774950295687,
+            0.0032065671402961016,
+            -0.008564895950257778,
+            -0.03275461867451668,
+            -0.05791788548231125,
+            -0.044815272092819214,
+            0.006680188700556755,
+            0.04072298854589462,
+            0.016144724562764168,
+            -0.0008583687013015151,
+            0.03699830546975136,
+            -0.04675738513469696,
+            0.06570404022932053,
+            -0.011776894330978394,
+            -0.011386243626475334,
+            -0.0003363603027537465,
+            0.039485324174165726,
+            0.014156803488731384,
+            0.018139634281396866,
+            -0.014710970222949982,
+            -0.052651647478342056,
+            0.02912742830812931,
+            0.024101730436086655,
+            0.0413704477250576,
+            0.03631320223212242,
+            0.046750932931900024,
+            -0.017062805593013763,
+            0.017990263178944588,
+            -0.03954370319843292,
+            -0.006972718983888626,
+            -0.03784368559718132,
+            0.02746269851922989,
+            0.04107878357172012,
+            -0.005694024730473757,
+            -0.03896583244204521,
+            0.026265999302268028,
+            -0.035318680107593536,
+            -0.018394622951745987,
+            0.013594037853181362,
+            0.0381510891020298,
+            -0.010223937220871449,
+            0.032417282462120056,
+            0.01610656827688217,
+            -0.013205642811954021,
+            -0.03757423907518387,
+            0.03799910470843315,
+            -0.039449408650398254,
+            -0.011290505528450012,
+            -0.016824476420879364,
+            0.007128347177058458,
+            0.030213234946131706,
+            -0.09385695308446884,
+            0.014417118392884731,
+            -0.021249795332551003,
+            -0.021371016278862953,
+            0.031582340598106384,
+            -0.015021033585071564,
+            0.03207740932703018,
+            0.04465494304895401,
+            0.051139406859874725,
+            -0.004539252258837223,
+            -0.004026447422802448,
+            0.036198731511831284,
+            0.002513982355594635,
+            -0.022555746138095856,
+            -0.023142442107200623,
+            0.026506206020712852,
+            -0.0208470169454813,
+            0.01958189532160759,
+            0.02606782503426075,
+            -0.050900157541036606,
+            0.001175468903966248,
+            0.0026185859460383654,
+            0.01644700951874256,
+            0.047048378735780716,
+            -0.006155692040920258,
+            0.013264120556414127,
+            -0.004277337808161974,
+            0.022337032482028008,
+            -0.030710609629750252,
+            -0.06784506887197495,
+            0.010662010870873928,
+            -0.020733945071697235,
+            -0.01206474844366312,
+            -0.0005046974983997643,
+            -0.004159707576036453,
+            0.028128545731306076,
+            -0.011551725678145885,
+            0.057953692972660065,
+            0.028500419110059738,
+            0.02070418931543827,
+            0.029373178258538246,
+            -0.053878165781497955,
+            -0.03885475918650627,
+            -0.011427262797951698,
+            -0.040592946112155914,
+            0.019192807376384735,
+            -0.013966009952127934,
+            0.002324307570233941,
+            0.027266129851341248,
+            0.02721570059657097,
+            -0.013851913623511791,
+            0.06292124837636948,
+            -0.019983768463134766,
+            -0.06498263776302338,
+            -0.014787066727876663,
+            0.07545251399278641,
+            0.009921795688569546,
+            -0.02266773208975792,
+            -0.0174646507948637,
+            -0.0037801002617925406,
+            0.037214070558547974,
+            -0.033669255673885345,
+            -0.0033054312225431204,
+            -0.004362864885479212,
+            -0.010861773043870926,
+            -0.041649043560028076,
+            0.02711806818842888,
+            -0.001099557732231915,
+            0.0007163260015659034,
+            0.01317980233579874,
+            -0.011158796027302742,
+            0.03966476768255234,
+            0.023275790736079216,
+            -0.011645027436316013,
+            0.0030634249560534954,
+            -0.01243121363222599,
+            0.01271719578653574,
+            0.003938829991966486,
+            -0.00769989937543869,
+            -0.039121564477682114,
+            0.0005735178128816187,
+            0.02157283015549183,
+            0.005828005261719227,
+            0.03934130072593689,
+            0.015216797590255737,
+            0.017237801104784012,
+            -0.037648268043994904,
+            -0.007132838945835829,
+            -0.018956882879137993,
+            -0.0597093440592289,
+            0.058341480791568756,
+            -0.0008284997311420739,
+            0.02095271646976471,
+            0.043099164962768555,
+            0.09887702018022537,
+            -0.01221393421292305,
+            -0.02239784225821495,
+            0.016775032505393028,
+            0.013331425376236439,
+            -0.004451168701052666,
+            -0.02870352193713188,
+            -0.020854242146015167,
+            0.05349724739789963,
+            0.03315908834338188,
+            0.018541062250733376,
+            -0.03136591613292694,
+            0.03549784794449806,
+            -0.0076525891199707985,
+            -0.06454484909772873,
+            0.049847088754177094,
+            0.012184737250208855,
+            0.03575005754828453,
+            -0.050804175436496735,
+            0.09406977146863937,
+            0.05103312432765961,
+            -0.0036910600028932095,
+            0.10705005377531052,
+            0.011394658125936985,
+            -0.014218435622751713,
+            -0.042272791266441345,
+            0.018426422029733658,
+            -0.08213183283805847,
+            -0.010240674018859863,
+            0.051353540271520615,
+            0.016103247180581093,
+            0.04293083772063255,
+            -0.00462630670517683,
+            0.001971749123185873,
+            -0.05101824551820755,
+            -0.017815101891756058,
+            -0.0788436159491539,
+            -0.019784294068813324,
+            0.006863154470920563,
+            0.04096531495451927,
+            0.016416994854807854,
+            0.018884461373090744,
+            -0.03645262494683266,
+            -0.02363709919154644,
+            0.08447448164224625,
+            0.027652855962514877,
+            -0.005039512179791927,
+            -0.05533800646662712,
+            0.006148343440145254,
+            -0.03248206898570061,
+            -0.015117023140192032,
+            -0.056908346712589264,
+            0.057090409100055695,
+            0.02987913228571415,
+            -0.0642392635345459,
+            -0.01212853193283081,
+            -0.04195745661854744,
+            -0.008033841848373413,
+            -0.05249612778425217,
+            0.05965931713581085,
+            0.08591161668300629,
+            -0.012983623892068863,
+            0.002055486897006631,
+            -0.002928174799308181,
+            -0.023014886304736137,
+            -0.05307631567120552,
+            0.0325687900185585,
+            -0.008586175739765167,
+            -0.005393583793193102,
+            0.009566529653966427,
+            0.06500132381916046,
+            -0.02100509963929653,
+            -0.018470296636223793,
+            0.001247459789738059,
+            0.007388024125248194,
+            0.012469757348299026,
+            0.08475572615861893,
+            0.06918514519929886,
+            0.054265547543764114,
+            0.03292711451649666,
+            -0.08437038213014603,
+            0.07744771242141724,
+            -0.0004291488730814308,
+            -0.020394261926412582,
+            0.039096955209970474,
+            0.015851527452468872,
+            -0.009922537952661514,
+            0.02087295800447464,
+            -0.019477976486086845,
+            -0.06510577350854874,
+            0.008559669367969036,
+            0.015032066963613033,
+            -0.022979427129030228,
+            -0.017166415229439735,
+            -0.014456263743340969,
+            -0.034205030649900436,
+            -0.04903494939208031,
+            0.073653943836689,
+            -0.041798241436481476,
+            0.0035302129108458757,
+            0.031043095514178276,
+            0.038764648139476776,
+            0.03582717105746269,
+            -0.003121789079159498,
+            0.03909862041473389,
+            -0.03283870965242386,
+            0.06343409419059753,
+            0.085169717669487,
+            0.0037416887935250998,
+            0.043896209448575974,
+            -0.02215113304555416,
+            -0.04062772914767265,
+            -0.029482074081897736,
+            0.0013964198296889663,
+            0.04621904715895653,
+            0.030072476714849472,
+            -0.023583346977829933,
+            -0.016047311946749687,
+            -0.04016166180372238,
+            -0.026690224185585976,
+            0.034725841134786606,
+            -0.08011004328727722,
+            -0.023635270074009895,
+            -0.01675681211054325,
+            0.02217511460185051,
+            -0.018720457330346107,
+            0.0413116030395031,
+            -0.0045730252750217915,
+            -0.08402986079454422,
+            0.03641941770911217,
+            0.028000695630908012,
+            0.042173273861408234,
+            0.024761751294136047,
+            -0.051845893263816833,
+            -0.07877497375011444,
+            -0.020710380747914314,
+            -0.035789184272289276,
+            0.04824375733733177,
+            -0.04493764415383339,
+            -0.0014088008319959044,
+            0.09272980690002441,
+            -0.030772028490900993,
+            0.027623610571026802,
+            -0.0008853759500198066,
+            -0.015347420237958431,
+            -0.0006863650633022189,
+            0.02924676053225994,
+            0.03864092007279396,
+            -0.043402496725320816,
+            0.11410719156265259,
+            0.01606914773583412,
+            0.03158045932650566,
+            -0.049648500978946686,
+            -0.026801105588674545,
+            0.013934214599430561,
+            -0.04582132399082184,
+            -0.02133217453956604,
+            0.013296819292008877,
+            0.030687933787703514,
+            0.0014671665849164128,
+            0.005454834550619125,
+            -0.024595070630311966,
+            0.036868833005428314,
+            -0.003586424048990011,
+            -0.007300499361008406,
+            0.00619609747081995,
+            0.004614396020770073,
+            0.06406176835298538,
+            0.010256785899400711,
+            -0.050202082842588425,
+            -0.013155301101505756,
+            -0.04005127400159836,
+            -0.027943719178438187,
+            0.05738724395632744,
+            -0.002920332597568631,
+            -0.00731270294636488,
+            0.04419538751244545,
+            0.024069754406809807,
+            0.012176074087619781,
+            0.004615467507392168,
+            -0.04112132638692856,
+            -0.04844773933291435,
+            -0.012684458866715431,
+            0.0071298484690487385,
+            -0.010914848186075687,
+            -0.03592529892921448,
+            -0.05016973987221718,
+            -0.011797907762229443,
+            -0.043843258172273636,
+            -0.03715396672487259,
+            0.016528192907571793,
+            0.024301515892148018,
+            0.01335576456040144,
+            0.021006477996706963,
+            -0.021391959860920906,
+            0.05299517139792442,
+            0.0070807491429150105,
+            -0.08096124231815338,
+            -0.07334060221910477,
+            -0.034530941396951675,
+            -0.04421507194638252,
+            0.010524646379053593,
+            0.009575314819812775,
+            -0.031711090356111526,
+            0.023479584604501724,
+            -0.04212309420108795,
+            0.016264619305729866,
+            0.03907531499862671,
+            -0.0011187525233253837,
+            -0.03998023644089699,
+            -0.027464834973216057,
+            -0.07113838940858841,
+            -0.028915319591760635,
+            -0.01282753050327301,
+            -0.0033073138911277056,
+            0.026715606451034546,
+            -0.002769897459074855,
+            0.020033732056617737,
+            0.014502385631203651,
+            -0.017903830856084824,
+            0.06932531297206879,
+            0.0432068407535553,
+            0.01685408502817154,
+            0.04834728315472603,
+            -0.009553197771310806,
+            0.019799189642071724,
+            0.01173039898276329,
+            0.04158413037657738,
+            -0.018829666078090668,
+            -0.008410722948610783,
+            0.008009687066078186,
+            0.034592460840940475,
+            0.07790639251470566,
+            -0.022050900384783745,
+            0.04081638529896736,
+            0.046872470527887344,
+            0.0010260086273774505,
+            -0.05322079360485077,
+            0.009096509777009487,
+            -0.06831686198711395,
+            -0.01390997413545847,
+            -0.020475609228014946,
+            0.017393099144101143,
+            -0.007532020565122366,
+            -0.06435851007699966,
+            -0.014785194769501686,
+            0.02654031664133072,
+            0.004753720946609974,
+            0.026440177112817764,
+            -0.028890414163470268,
+            -0.011440729722380638,
+            0.003554105758666992,
+            -0.0022926912643015385,
+            -0.02393224649131298,
+            0.03711748123168945,
+            -0.06023703143000603,
+            -0.008778683841228485,
+            -0.05984162166714668,
+            -0.024247022345662117,
+            -0.036919932812452316,
+            0.05249374359846115,
+            0.03022468276321888,
+            -0.011348876170814037,
+            0.0008303995127789676,
+            0.001597013440914452,
+            -0.015491127036511898,
+            -0.035073015838861465,
+            -0.024477796629071236,
+            -0.030328145250678062,
+            -0.09301470220088959,
+            -0.046440113335847855,
+            0.036719027906656265,
+            -0.021899227052927017,
+            0.04666316881775856,
+            -0.07481305301189423,
+            -0.04928148165345192,
+            -0.01480096485465765,
+            0.0014140848070383072,
+            0.016779841855168343,
+            -0.04318199306726456,
+            0.011910341680049896,
+            -0.04019855335354805,
+            -0.027363713830709457,
+            0.006433602888137102,
+            0.023732252418994904,
+            -0.013081788085401058,
+            0.02489032782614231,
+            0.005415213759988546,
+            -0.058724161237478256,
+            0.032487478107213974,
+            -0.014332194812595844,
+            -0.020952431485056877,
+            0.055405858904123306,
+            -0.02239573374390602,
+            0.016315918415784836,
+            0.04710645601153374,
+            0.006866136100143194,
+            -0.019589263945817947,
+            -0.046199049800634384,
+            0.04977096989750862,
+            -0.03211359679698944,
+            0.06759121268987656,
+            -0.007805021945387125,
+            0.009877636097371578,
+            -0.003194598713889718,
+            -0.0014034705236554146,
+            0.024012917652726173,
+            0.0007609894964843988,
+            0.04028927534818649,
+            0.047299597412347794,
+            0.04644732549786568,
+            0.06253348290920258,
+            -0.03101237863302231,
+            -0.04797065258026123,
+            -0.02459110878407955,
+            -0.06663094460964203,
+            -0.012946722097694874,
+            -0.046321313828229904,
+            -0.03617801144719124,
+            -0.06608668714761734,
+            0.01371682621538639,
+            -0.040183935314416885,
+            0.027353622019290924,
+            -0.013125114142894745,
+            0.020482128486037254,
+            -0.10186963528394699,
+            0.03741387277841568,
+            -0.048566944897174835,
+            0.0017904090927913785,
+            0.0444694422185421,
+            -0.02355058304965496,
+            -0.04245513305068016,
+            0.01599632203578949,
+            -0.00974870752543211,
+            -0.02246273122727871,
+            0.011107604950666428,
+            -0.006354854442179203,
+            -0.08260829746723175,
+            -0.054969724267721176,
+            -0.038703542202711105,
+            -0.02590899169445038,
+            -0.012424441985785961,
+            0.033952418714761734,
+            0.032632969319820404,
+            0.03585505858063698,
+            -0.027734532952308655,
+            -0.05185376852750778,
+            0.005663866177201271,
+            0.01415393128991127,
+            0.007472912315279245,
+            -0.0325092077255249,
+            -0.0008526426972821355,
+            0.05909401550889015,
+            -0.006496420595794916,
+            0.06674317270517349,
+            0.06033811718225479,
+            -0.04705937206745148,
+            0.01221691444516182,
+            -0.005195186473429203,
+            0.017006050795316696,
+            0.015768419951200485,
+            -0.02346021682024002,
+            -0.04318040981888771,
+            -0.00965888798236847,
+            -0.012831253930926323,
+            -0.023086808621883392,
+            -0.043478451669216156,
+            0.02215973101556301,
+            0.01018955372273922,
+            -0.0029477940406650305,
+            -0.026364397257566452,
+            -0.04219489544630051,
+            -0.0690244510769844,
+            0.0017003740649670362,
+            -0.03498053178191185,
+            -0.01891854591667652,
+            -0.020380523055791855,
+            -0.07183944433927536,
+            0.01474913302809,
+            0.012818068265914917,
+            0.02298390306532383,
+            0.006645163521170616,
+            -0.014497633092105389,
+            -0.05751577392220497,
+            -0.01127719134092331,
+            0.014469895511865616,
+            0.039319343864917755,
+            -0.002891098614782095,
+            0.0038161359261721373,
+            -0.0176107045263052,
+            -0.02695712260901928,
+            0.023520348593592644,
+            0.053624920547008514,
+            -0.0472102165222168,
+            -0.021724319085478783,
+            -0.04204733297228813,
+            0.004941252060234547,
+            -0.07744265347719193,
+            -0.028974706307053566,
+            -6.1493665270973e-05,
+            -0.020630594342947006,
+            -0.014794640243053436,
+            -0.045572925359010696,
+            0.03233763575553894,
+            0.00969443004578352,
+            0.03665856271982193,
+            0.027483846992254257,
+            0.074271060526371,
+            -0.07454165071249008,
+            -0.034101732075214386,
+            -0.07216823101043701,
+            -0.001424514572136104,
+            -0.0025912360288202763,
+            -0.002444307319819927,
+            -0.012540637515485287,
+            0.009027975611388683,
+            0.06855443120002747,
+            -0.0013480151537805796,
+            0.027303414419293404,
+            -0.019723499193787575,
+            0.033644214272499084,
+            -0.04313155263662338,
+            -0.016152892261743546,
+            -0.020085612311959267,
+            0.029526935890316963,
+            0.0004591972683556378,
+            -0.013712934218347073,
+            0.015895912423729897,
+            -0.046559300273656845,
+            -0.00015638815239071846,
+            0.0015497541753575206,
+            -0.0015048328787088394,
+            0.06692329794168472,
+            0.0013934546150267124,
+            0.008921030908823013,
+            -0.010347972624003887,
+            -0.039798807352781296,
+            0.06892028450965881,
+            0.021145053207874298,
+            0.007431029342114925,
+            -0.05281573906540871,
+            0.015844792127609253,
+            0.014578046277165413,
+            -0.0020482230465859175,
+            0.03509555384516716,
+            -0.021227506920695305,
+            -0.03619229048490524,
+            0.004116897005587816,
+            0.02835669554769993,
+            -0.0028248224407434464,
+            0.00836214143782854,
+            -0.004688165616244078,
+            0.04566347226500511,
+            -0.0352579727768898,
+            -0.007859165780246258,
+            -0.003958444111049175,
+            0.023938892409205437,
+            0.04262895882129669,
+            -0.02836589328944683,
+            0.0456448458135128,
+            -0.062015753239393234,
+            0.03518408164381981,
+            0.06333593279123306,
+            -0.0155468275770545,
+            0.013991734012961388,
+            0.02207978442311287,
+            0.0032898876816034317,
+            0.05948015674948692,
+            0.010670959949493408,
+            -0.00624996330589056,
+            -0.04401599243283272,
+            -0.0022705462761223316
+        ],
+        "content_embedding": [
+            -0.02403288148343563,
+            0.08599621057510376,
+            -0.003520619124174118,
+            -0.002186001278460026,
+            -0.002845448674634099,
+            0.005484029185026884,
+            0.016410797834396362,
+            -0.02119613252580166,
+            -0.04811510443687439,
+            -0.013274849392473698,
+            -0.043441254645586014,
+            0.009376521222293377,
+            0.019551504403352737,
+            0.0036566888447850943,
+            0.004842979833483696,
+            -0.006052782759070396,
+            0.025645455345511436,
+            -0.012315846979618073,
+            -0.027703408151865005,
+            0.02152254432439804,
+            0.011010917834937572,
+            -0.012258552014827728,
+            -0.01729186624288559,
+            -0.02833859808743,
+            0.04027653858065605,
+            -0.009032614529132843,
+            -0.017036406323313713,
+            0.003077560570091009,
+            -0.04315951466560364,
+            0.029963837936520576,
+            0.01716647669672966,
+            -0.02696092799305916,
+            -0.006828296463936567,
+            -0.026644738391041756,
+            -0.0889354720711708,
+            -0.05207673832774162,
+            0.05015599727630615,
+            -0.019386274740099907,
+            -0.03684607893228531,
+            0.046758152544498444,
+            0.05076799914240837,
+            0.007221075240522623,
+            0.016260067000985146,
+            -0.03850802034139633,
+            0.054756514728069305,
+            -0.012776038609445095,
+            0.036689598113298416,
+            -0.02616005390882492,
+            0.033269986510276794,
+            -0.05991198495030403,
+            -0.00043452056706883013,
+            -0.004230191465467215,
+            -0.008319171145558357,
+            0.0068639665842056274,
+            -0.016133679077029228,
+            0.005670355167239904,
+            -0.005839305464178324,
+            0.027315128594636917,
+            0.04275438189506531,
+            -0.05024448782205582,
+            0.05199997127056122,
+            0.05441230162978172,
+            -0.049353599548339844,
+            -0.008365850895643234,
+            0.0066192797385156155,
+            -0.055218830704689026,
+            0.01654958724975586,
+            0.013456150889396667,
+            -0.01148252934217453,
+            -0.014086110517382622,
+            0.024422504007816315,
+            -0.001102397684007883,
+            -0.022180721163749695,
+            0.022023402154445648,
+            -0.03140854462981224,
+            0.0038229606579989195,
+            0.01081792451441288,
+            -0.006220541428774595,
+            -0.02462770976126194,
+            0.051595576107501984,
+            0.04433179274201393,
+            -0.05173564329743385,
+            0.00422016903758049,
+            0.01004322525113821,
+            0.01985878124833107,
+            0.028202056884765625,
+            -0.0016253730282187462,
+            0.01617475040256977,
+            0.010118816047906876,
+            0.0603351853787899,
+            0.0140571603551507,
+            0.0029350141994655132,
+            0.04240429773926735,
+            0.06991686671972275,
+            -0.10119865089654922,
+            0.08219177275896072,
+            0.02891121245920658,
+            0.00130809610709548,
+            -0.016763439401984215,
+            -0.00509023480117321,
+            -0.0035567383747547865,
+            0.02000737562775612,
+            -0.002168829319998622,
+            0.01889166608452797,
+            0.04547121003270149,
+            0.04019659012556076,
+            0.024593960493803024,
+            0.017190879210829735,
+            0.007603269536048174,
+            0.007314966060221195,
+            -0.06791973859071732,
+            -0.036731328815221786,
+            0.007499238010495901,
+            0.02694091759622097,
+            -0.02129160799086094,
+            0.021507520228624344,
+            0.09310256689786911,
+            -0.03434507176280022,
+            0.006634920369833708,
+            0.03453971818089485,
+            0.0155464056879282,
+            0.003056582296267152,
+            -0.004192651715129614,
+            0.03274714946746826,
+            0.04909229651093483,
+            0.025430802255868912,
+            -0.01268640998750925,
+            -0.047261349856853485,
+            0.0018452038057148457,
+            -0.0002589405339676887,
+            -0.07050265371799469,
+            0.004126655403524637,
+            -0.07842830568552017,
+            0.00013916153693571687,
+            0.047662656754255295,
+            0.01362426858395338,
+            -0.07088430225849152,
+            -0.026547620072960854,
+            0.010091855190694332,
+            -0.015962867066264153,
+            0.03528159111738205,
+            0.011798265390098095,
+            0.020107097923755646,
+            -0.013524978421628475,
+            0.016901858150959015,
+            -0.08753035962581635,
+            -0.062227677553892136,
+            -0.028078285977244377,
+            -0.03297634422779083,
+            0.008013743907213211,
+            0.018041228875517845,
+            -0.022115394473075867,
+            0.00595641927793622,
+            0.019160043448209763,
+            0.008510938845574856,
+            -0.0474565215408802,
+            -0.038813307881355286,
+            -0.016643475741147995,
+            0.06800767034292221,
+            0.041471801698207855,
+            0.03696686029434204,
+            0.03421548008918762,
+            -0.03440884128212929,
+            0.06769654899835587,
+            -0.01683412306010723,
+            0.028133966028690338,
+            0.018801912665367126,
+            0.015075244940817356,
+            0.012032945640385151,
+            0.03569433093070984,
+            -0.021484674885869026,
+            -0.01213730126619339,
+            0.023888660594820976,
+            -0.03447817265987396,
+            0.03329891338944435,
+            -0.007350335828959942,
+            0.06528840214014053,
+            -0.03317185863852501,
+            0.024836916476488113,
+            -0.061747901141643524,
+            -0.01068184245377779,
+            -0.021780723705887794,
+            0.06678029894828796,
+            0.03825325518846512,
+            -0.02603997103869915,
+            0.0200904980301857,
+            0.025599440559744835,
+            -0.05657019838690758,
+            0.028341008350253105,
+            -0.0439138226211071,
+            0.05886855348944664,
+            -0.049358345568180084,
+            0.014541592448949814,
+            0.005707047879695892,
+            0.008378228172659874,
+            -0.030232897028326988,
+            0.06261618435382843,
+            -0.013355602510273457,
+            -0.036993358284235,
+            -0.028917213901877403,
+            -0.0680958554148674,
+            -0.027451951056718826,
+            -0.07567653805017471,
+            0.014718701131641865,
+            -0.009075576439499855,
+            0.013478180393576622,
+            0.03208685666322708,
+            0.031021032482385635,
+            0.016195151954889297,
+            0.008199494332075119,
+            0.08357387781143188,
+            -0.01200099941343069,
+            0.022620532661676407,
+            0.05445336923003197,
+            -0.018056273460388184,
+            -0.04719870164990425,
+            0.04062207415699959,
+            0.0009855915559455752,
+            -0.0462096631526947,
+            0.05879806727170944,
+            0.03913828358054161,
+            -0.05726383253931999,
+            0.02152623049914837,
+            0.002137464936822653,
+            0.01444965973496437,
+            -0.019534891471266747,
+            -0.0375588983297348,
+            -0.03905639797449112,
+            -0.027474306523799896,
+            -0.001400938956066966,
+            -0.033295221626758575,
+            -0.06691068410873413,
+            0.0620984211564064,
+            0.020130982622504234,
+            0.027853885665535927,
+            0.028353361412882805,
+            0.007000260055065155,
+            -0.015244328416883945,
+            0.028457706794142723,
+            0.05079026147723198,
+            0.0265045203268528,
+            -0.008008715696632862,
+            0.011166643351316452,
+            -0.02545643411576748,
+            -0.09122578054666519,
+            -0.000896137673407793,
+            0.0055070980452001095,
+            0.023860882967710495,
+            -0.056958671659231186,
+            0.002000730484724045,
+            0.000531299039721489,
+            0.01964678056538105,
+            0.02459172159433365,
+            0.010496687144041061,
+            0.032775767147541046,
+            -0.040455516427755356,
+            -0.01898832432925701,
+            0.048115238547325134,
+            0.008294769562780857,
+            -0.02248159423470497,
+            -0.0020450311712920666,
+            -0.02413240633904934,
+            0.0423247255384922,
+            -0.02917350083589554,
+            -0.0197658222168684,
+            0.009233975782990456,
+            -0.02438087947666645,
+            -0.057745061814785004,
+            0.020396480336785316,
+            -0.028454614803195,
+            -0.007276479620486498,
+            -0.0060751899145543575,
+            0.016126802191138268,
+            0.07733260095119476,
+            0.0055052717216312885,
+            -0.0241200253367424,
+            -0.009856182150542736,
+            -0.01288821641355753,
+            0.021394196897745132,
+            -0.0027394252829253674,
+            -0.057746946811676025,
+            -0.055244673043489456,
+            -0.03518827632069588,
+            0.020108383148908615,
+            -0.037429675459861755,
+            0.06402620673179626,
+            0.014570947736501694,
+            0.0011715830769389868,
+            0.04670550301671028,
+            -0.03730842098593712,
+            -0.002726265462115407,
+            -0.03393309563398361,
+            0.03357642516493797,
+            0.006151925306767225,
+            0.027046309784054756,
+            0.06079886853694916,
+            0.08915705978870392,
+            -0.040912795811891556,
+            -0.009531376883387566,
+            -0.008656186051666737,
+            -0.010746185667812824,
+            0.011325616389513016,
+            0.00910742674022913,
+            -0.00870103295892477,
+            -0.02257593534886837,
+            -0.008474824018776417,
+            -0.01126043125987053,
+            -0.006183316465467215,
+            0.03318650647997856,
+            -0.005288233514875174,
+            -0.031032271683216095,
+            0.02630523219704628,
+            0.02767125330865383,
+            -0.01024201512336731,
+            -0.02395681105554104,
+            0.07798302173614502,
+            0.06453987956047058,
+            -0.005852920934557915,
+            0.08618523925542831,
+            -0.009387078694999218,
+            0.007869970984756947,
+            -0.03182069584727287,
+            -0.022106602787971497,
+            -0.0868132933974266,
+            0.028115050867199898,
+            0.07332660257816315,
+            -0.0037628302816301584,
+            0.03760993853211403,
+            -0.027132470160722733,
+            0.030093027278780937,
+            -0.037918947637081146,
+            0.039932165294885635,
+            -0.07345228642225266,
+            -0.046965666115283966,
+            -0.0013359235599637032,
+            0.00791996717453003,
+            0.03006441704928875,
+            0.04222951829433441,
+            -0.0141807422041893,
+            -0.021912341937422752,
+            -0.0065930006094276905,
+            -0.0038735137786716223,
+            -0.038659993559122086,
+            -0.057126715779304504,
+            0.006521300878375769,
+            -0.030727874487638474,
+            -0.022539950907230377,
+            -0.06316803395748138,
+            0.06865260004997253,
+            0.031939368695020676,
+            -0.055947039276361465,
+            0.0066061182878911495,
+            -0.014607742428779602,
+            -0.02204318344593048,
+            -0.05172397196292877,
+            0.02495967596769333,
+            0.07759078592061996,
+            0.0027070387732237577,
+            0.008220532909035683,
+            0.02342107705771923,
+            0.03180982172489166,
+            -0.03099866956472397,
+            0.03512701019644737,
+            -0.03168865293264389,
+            0.012847676873207092,
+            0.06514899432659149,
+            0.08987598121166229,
+            0.0024377063382416964,
+            0.02394464798271656,
+            -0.041963983327150345,
+            -0.004438851028680801,
+            0.015682004392147064,
+            0.0410960391163826,
+            0.05460710451006889,
+            0.057952240109443665,
+            0.020986247807741165,
+            -0.08822161704301834,
+            0.01074486318975687,
+            0.014192330650985241,
+            -0.025726256892085075,
+            0.0719577744603157,
+            0.0021957557182759047,
+            0.022048326209187508,
+            -0.04020603001117706,
+            0.0014428661670535803,
+            -0.0357256680727005,
+            -0.030243121087551117,
+            -0.0376482829451561,
+            -0.020463477820158005,
+            -0.022432789206504822,
+            -0.03096373938024044,
+            -0.01816924288868904,
+            -0.05358648672699928,
+            0.07382772862911224,
+            -0.014173741452395916,
+            0.0201816875487566,
+            0.006632740143686533,
+            0.0025384302716702223,
+            0.04055432602763176,
+            0.0069578299298882484,
+            -0.019879184663295746,
+            -0.059168167412281036,
+            0.028969064354896545,
+            0.05784929171204567,
+            -0.002147398190572858,
+            0.043272342532873154,
+            -0.004542165901511908,
+            -0.0482858382165432,
+            -0.004174860659986734,
+            0.020621255040168762,
+            0.04293094202876091,
+            0.0292718093842268,
+            -0.00980047881603241,
+            -0.021710650995373726,
+            -0.02639775723218918,
+            -0.019148416817188263,
+            0.09012293070554733,
+            -0.045379895716905594,
+            -0.023026002570986748,
+            -0.030370570719242096,
+            0.008383749052882195,
+            0.014925302006304264,
+            -0.0011008139699697495,
+            0.006763918325304985,
+            -0.025421440601348877,
+            -0.004525069613009691,
+            0.03806034475564957,
+            0.005547006148844957,
+            0.05000557750463486,
+            -0.005488873925060034,
+            0.021936720237135887,
+            0.020678924396634102,
+            -0.004737663082778454,
+            0.040749210864305496,
+            -0.0533074289560318,
+            0.025417080149054527,
+            0.08257681876420975,
+            -0.005508026573807001,
+            -0.009805315174162388,
+            0.07595512270927429,
+            -0.0018210643902420998,
+            -0.029579052701592445,
+            0.009883117862045765,
+            -0.015399829484522343,
+            -0.017134232446551323,
+            0.03538937494158745,
+            0.0827752947807312,
+            0.012051745317876339,
+            -0.07159247249364853,
+            -0.02079680748283863,
+            0.03473742678761482,
+            0.018268825486302376,
+            0.023407628759741783,
+            -0.036390434950590134,
+            0.07932467013597488,
+            0.004754354711622,
+            -0.012676632963120937,
+            -0.06851805001497269,
+            0.02255256660282612,
+            0.03780437260866165,
+            0.04691546410322189,
+            0.018480120226740837,
+            0.0005508657777681947,
+            0.05573705583810806,
+            -0.009221675805747509,
+            -0.06587770581245422,
+            0.015470701269805431,
+            -0.012271493673324585,
+            -0.025784730911254883,
+            0.022757982835173607,
+            -0.01213389914482832,
+            0.017422374337911606,
+            0.012241406366229057,
+            0.04379018396139145,
+            0.01124424859881401,
+            0.002584748901426792,
+            0.02793707512319088,
+            -0.04307323694229126,
+            0.03207562863826752,
+            0.05286982282996178,
+            0.01086041983217001,
+            0.009665313176810741,
+            -0.054988693445920944,
+            0.01324005052447319,
+            -0.04261464625597,
+            -0.02707112766802311,
+            -0.002658748533576727,
+            0.03499991446733475,
+            -0.005491453222930431,
+            0.006562606431543827,
+            0.018722862005233765,
+            0.07151596993207932,
+            -0.003824777202680707,
+            -0.04148973524570465,
+            -0.06528852880001068,
+            -0.018773429095745087,
+            -0.023220594972372055,
+            0.021337825804948807,
+            0.003552130190655589,
+            -0.07254927605390549,
+            0.030997687950730324,
+            0.009675328619778156,
+            -0.007739027962088585,
+            -0.001004970632493496,
+            -0.0009698161156848073,
+            -0.03183043375611305,
+            -0.003764253342524171,
+            -0.06521959602832794,
+            0.0077109793201088905,
+            0.008421109057962894,
+            0.02024395577609539,
+            0.06566902250051498,
+            0.011374534107744694,
+            0.040655992925167084,
+            0.0274888314306736,
+            -0.0748000368475914,
+            0.06930309534072876,
+            0.014980202540755272,
+            -0.03328235074877739,
+            0.07670122385025024,
+            -0.013236696831882,
+            0.09516690671443939,
+            0.0004450292617548257,
+            0.01539886835962534,
+            -0.11376772075891495,
+            -0.0004633136559277773,
+            -0.023844275623559952,
+            0.023186970502138138,
+            0.0542912632226944,
+            0.006978484336286783,
+            0.03704620897769928,
+            0.0761408805847168,
+            0.0018389171455055475,
+            -0.02292831614613533,
+            0.035566531121730804,
+            -0.06125196814537048,
+            -0.01740599237382412,
+            -0.03189321979880333,
+            -0.023606419563293457,
+            0.0002929234178736806,
+            -0.032161861658096313,
+            -0.02417462132871151,
+            0.007371667306870222,
+            0.01384897343814373,
+            0.0011207011993974447,
+            -0.054523780941963196,
+            -0.03664090484380722,
+            0.012376014143228531,
+            0.005946264136582613,
+            -0.05214802548289299,
+            0.06363234668970108,
+            -0.01850913278758526,
+            0.03264418616890907,
+            -0.08298838883638382,
+            0.028580913320183754,
+            -0.06874261051416397,
+            0.04560680687427521,
+            -0.01221420057117939,
+            -0.015291322953999043,
+            0.011163976043462753,
+            -0.01707146316766739,
+            -0.021233493462204933,
+            0.0009499920415692031,
+            -0.011884773150086403,
+            0.031535957008600235,
+            -0.07693900167942047,
+            -0.030928723514080048,
+            0.02938068099319935,
+            0.013103127479553223,
+            0.009228850714862347,
+            -0.04399878904223442,
+            -0.038614701479673386,
+            0.021263988688588142,
+            0.0270336102694273,
+            -0.0022124540992081165,
+            -0.032499391585588455,
+            0.029354240745306015,
+            -0.028516946360468864,
+            -0.03277367725968361,
+            -0.04755333065986633,
+            -0.03938357159495354,
+            -0.029368583112955093,
+            0.06943269073963165,
+            0.017946777865290642,
+            -0.01990826241672039,
+            0.014896579086780548,
+            -0.06675421446561813,
+            -0.04962918534874916,
+            0.10290152579545975,
+            -0.05442032590508461,
+            0.0268304031342268,
+            0.01750801131129265,
+            0.0006768505554646254,
+            -0.007724875118583441,
+            -0.05064627528190613,
+            0.03560181334614754,
+            0.005476392339915037,
+            0.008490868844091892,
+            -0.005553610157221556,
+            -0.04698188602924347,
+            -0.025146158412098885,
+            0.0026807712856680155,
+            0.0254969522356987,
+            0.005350390914827585,
+            0.004036207217723131,
+            0.02843003161251545,
+            0.008211316540837288,
+            0.03748054802417755,
+            -0.05300099402666092,
+            -0.012365839444100857,
+            -0.0130928261205554,
+            -0.03939966484904289,
+            -0.026050617918372154,
+            -0.04415596276521683,
+            -0.03128521516919136,
+            -0.0388399139046669,
+            0.05186399444937706,
+            -0.049164481461048126,
+            0.043122462928295135,
+            -0.0315178819000721,
+            0.012280933558940887,
+            -0.0792573019862175,
+            0.05075725167989731,
+            -0.04304235801100731,
+            0.018651138991117477,
+            0.03076835162937641,
+            -0.060538437217473984,
+            -0.023055853322148323,
+            0.01177286822348833,
+            0.058492839336395264,
+            0.025716299191117287,
+            0.009599392302334309,
+            -0.012054546736180782,
+            -0.027742642909288406,
+            -0.05367058888077736,
+            -0.026801493018865585,
+            -0.014112668111920357,
+            -0.006383270025253296,
+            1.2056754712830298e-05,
+            0.039540693163871765,
+            0.02213987335562706,
+            -0.08540242910385132,
+            -0.04058465361595154,
+            0.008699232712388039,
+            0.031218260526657104,
+            0.0021884969901293516,
+            0.011582552455365658,
+            0.025049764662981033,
+            0.04276714473962784,
+            0.009781924076378345,
+            0.05123818293213844,
+            0.07441077381372452,
+            -0.029336893931031227,
+            0.02714505046606064,
+            0.041163086891174316,
+            -0.006217346992343664,
+            0.025060802698135376,
+            0.023126818239688873,
+            -0.07503696531057358,
+            -0.0020585027523338795,
+            0.005981603171676397,
+            -0.027166299521923065,
+            -0.020568007603287697,
+            0.005853605456650257,
+            -0.006091856863349676,
+            -0.033637579530477524,
+            -0.039759427309036255,
+            -0.06260950118303299,
+            -0.024897020310163498,
+            0.02462431788444519,
+            0.01859314925968647,
+            0.010398009791970253,
+            -0.00020126033632550389,
+            -0.06035298481583595,
+            -0.019108809530735016,
+            0.042335279285907745,
+            -0.03559218347072601,
+            -0.02529655024409294,
+            -0.02809930220246315,
+            -0.05607590824365616,
+            -0.026691321283578873,
+            -0.026792382821440697,
+            0.04120280221104622,
+            -0.015540994703769684,
+            -0.005803580395877361,
+            0.020203134045004845,
+            -0.05952906608581543,
+            -0.004206392448395491,
+            -0.011308858171105385,
+            -0.037488050758838654,
+            0.007830106653273106,
+            -0.009608179330825806,
+            0.00015318443183787167,
+            -0.0684049054980278,
+            0.0018899703864008188,
+            -0.023107590153813362,
+            -0.015158215537667274,
+            -0.030714333057403564,
+            0.025599345564842224,
+            0.018543586134910583,
+            -0.0075812251307070255,
+            0.04323196783661842,
+            -0.005424505099654198,
+            0.06189188361167908,
+            -0.01650432124733925,
+            0.0035911088343709707,
+            0.01841658726334572,
+            0.012203766033053398,
+            -0.015994010493159294,
+            -0.0018007376929745078,
+            0.011197488754987717,
+            -0.01184547133743763,
+            0.06119342893362045,
+            -0.04449119791388512,
+            -0.010956074111163616,
+            0.015267443843185902,
+            0.03397256135940552,
+            -0.022375188767910004,
+            -0.010562969371676445,
+            -0.030176721513271332,
+            0.0649082288146019,
+            -0.026252834126353264,
+            -0.043264783918857574,
+            0.020383840426802635,
+            -0.014332938008010387,
+            -0.019906938076019287,
+            0.002487052930518985,
+            0.016441592946648598,
+            0.05937374755740166,
+            0.029459767043590546,
+            -0.03393784165382385,
+            -0.016614725813269615,
+            -0.03633803129196167,
+            0.04786395654082298,
+            -0.014543719589710236,
+            0.0030611655674874783,
+            -0.03296193480491638,
+            0.024570109322667122,
+            -0.08628548681735992,
+            -0.008082202635705471,
+            0.05895440652966499,
+            -0.05567137897014618,
+            -0.05882163718342781,
+            -0.005672273691743612,
+            -0.022155780345201492,
+            -0.03165644034743309,
+            -0.04472680389881134,
+            0.025338545441627502,
+            0.053867027163505554,
+            -0.020717058330774307,
+            -0.019026240333914757,
+            -0.03037080727517605,
+            -0.005734192673116922,
+            -0.014109884388744831,
+            -0.005240253172814846,
+            0.056925658136606216,
+            0.006881027482450008,
+            0.006321505177766085,
+            0.025533199310302734,
+            0.0066923401318490505,
+            0.014867548830807209,
+            0.01877731829881668,
+            -0.03573253005743027,
+            0.05504361167550087,
+            0.044875118881464005,
+            0.008996511809527874,
+            -0.020861415192484856,
+            0.0196152962744236
+        ],
+        "chunk_ind": 0
+    },
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/enterprise_search",
+        "title": "Enterprise Search",
+        "content": "More than Search\nWhen analyzing the entire corpus of knowledge within your company is as easy as asking a question in a search bar, your entire team can stay informed and up to date. Danswer also makes it trivial to identify where knowledge is well documented and where it is lacking. Team members who are centers of knowledge can begin to effectively document their expertise since it is no longer being thrown into a black hole. All of this allows the organization to achieve higher efficiency and drive business outcomes.\n\nWith Generative AI, the entire user experience has evolved as well. For example, instead of just finding similar cases for your customer support team to reference, Danswer breaks down the issue and explains it so that even the most junior members can understand it. This in turn lets them give the most holistic and technically accurate response possible to your customers. On the other end, even the super stars of your sales team will not be able to review 10 hours of transcripts before hopping on that critical call, but Danswer can easily parse through it in mere seconds and give crucial context to help your team close.",
+        "title_embedding": [
+            -0.011060578748583794,
+            0.05994883179664612,
+            0.008845113217830658,
+            0.011364905163645744,
+            0.03147757425904274,
+            0.04506697878241539,
+            -0.025942707434296608,
+            -0.011002028360962868,
+            -0.03507396578788757,
+            -0.01727098599076271,
+            -0.016820134595036507,
+            0.04671240597963333,
+            0.023456331342458725,
+            -0.005752791650593281,
+            -0.011421029455959797,
+            -0.04169125109910965,
+            0.0652366429567337,
+            -0.011136278510093689,
+            -0.013501451350748539,
+            -0.006273901090025902,
+            0.0012236927868798375,
+            -0.030771249905228615,
+            0.010098040103912354,
+            -0.02360220067203045,
+            0.006734110414981842,
+            0.001379420980811119,
+            -0.0047225081361830235,
+            -0.013901330530643463,
+            0.014645840041339397,
+            -0.02156134508550167,
+            -0.026707857847213745,
+            -0.00676271365955472,
+            0.056067030876874924,
+            -0.0455806590616703,
+            -0.02762053906917572,
+            -0.0965149849653244,
+            0.04567359760403633,
+            0.053895801305770874,
+            0.029093541204929352,
+            0.0199823547154665,
+            0.047089505940675735,
+            0.026028119027614594,
+            -0.0034626282285898924,
+            -0.021002190187573433,
+            0.04217635095119476,
+            -0.015198000706732273,
+            0.047393105924129486,
+            0.008588545024394989,
+            0.07058116048574448,
+            -0.09135723114013672,
+            -0.009591161273419857,
+            0.014971816912293434,
+            -8.222273208957631e-07,
+            -0.05534408614039421,
+            -0.029965048655867577,
+            -0.028381407260894775,
+            0.025547217577695847,
+            0.038583844900131226,
+            0.01984122209250927,
+            -0.02435469813644886,
+            0.04955849424004555,
+            0.0128632802516222,
+            -0.022573867812752724,
+            0.025284791365265846,
+            0.04496009647846222,
+            0.0004200333496555686,
+            -0.0015001222491264343,
+            0.02609623409807682,
+            0.023677939549088478,
+            -0.05961468443274498,
+            0.06799754500389099,
+            -0.014409428462386131,
+            -0.010374268516898155,
+            0.019634529948234558,
+            -0.03720257058739662,
+            -0.04654879495501518,
+            0.0056068566627800465,
+            -0.021244503557682037,
+            -0.03198164328932762,
+            0.05707596614956856,
+            0.043927326798439026,
+            -0.033356692641973495,
+            0.015723733231425285,
+            -0.027493512257933617,
+            0.04525380581617355,
+            0.025808652862906456,
+            -0.007083983160555363,
+            -0.011038954369723797,
+            -0.004390218295156956,
+            -0.006583297159522772,
+            0.003319315379485488,
+            0.061810243874788284,
+            0.05887124314904213,
+            0.07722297310829163,
+            -0.06601747870445251,
+            0.036486171185970306,
+            0.05119618400931358,
+            0.00945530366152525,
+            -0.03005358576774597,
+            -0.047870855778455734,
+            -0.0359003059566021,
+            -0.005699747242033482,
+            0.053807053714990616,
+            -0.001554036163724959,
+            0.060548700392246246,
+            0.05476829782128334,
+            0.00426551653072238,
+            -0.005215689539909363,
+            -0.0352163203060627,
+            -0.03284529969096184,
+            -0.03373449295759201,
+            0.021254340186715126,
+            -0.010481598787009716,
+            0.02651101164519787,
+            -0.00481840968132019,
+            0.045160870999097824,
+            0.09704204648733139,
+            -0.03473421558737755,
+            0.015584945678710938,
+            0.06579536944627762,
+            0.0651017278432846,
+            0.007380738854408264,
+            0.00624364148825407,
+            0.07893780618906021,
+            -0.019876087084412575,
+            0.006619091611355543,
+            -0.030776498839259148,
+            -0.016426965594291687,
+            0.014603338204324245,
+            -0.03326896205544472,
+            0.003784433240070939,
+            0.025205042213201523,
+            0.03047170303761959,
+            -0.03364298865199089,
+            0.005974944215267897,
+            0.015994269400835037,
+            -0.10205432027578354,
+            -0.026497431099414825,
+            0.07166463136672974,
+            -0.007370935752987862,
+            0.034912627190351486,
+            -0.004656887147575617,
+            0.03219066560268402,
+            -0.02239271067082882,
+            0.012679396197199821,
+            -0.07867992669343948,
+            -0.026638884097337723,
+            -0.008346030488610268,
+            -0.027243634685873985,
+            0.012043806724250317,
+            0.024078860878944397,
+            0.0006219774950295687,
+            0.0032065671402961016,
+            -0.008564895950257778,
+            -0.03275461867451668,
+            -0.05791788548231125,
+            -0.044815272092819214,
+            0.006680188700556755,
+            0.04072298854589462,
+            0.016144724562764168,
+            -0.0008583687013015151,
+            0.03699830546975136,
+            -0.04675738513469696,
+            0.06570404022932053,
+            -0.011776894330978394,
+            -0.011386243626475334,
+            -0.0003363603027537465,
+            0.039485324174165726,
+            0.014156803488731384,
+            0.018139634281396866,
+            -0.014710970222949982,
+            -0.052651647478342056,
+            0.02912742830812931,
+            0.024101730436086655,
+            0.0413704477250576,
+            0.03631320223212242,
+            0.046750932931900024,
+            -0.017062805593013763,
+            0.017990263178944588,
+            -0.03954370319843292,
+            -0.006972718983888626,
+            -0.03784368559718132,
+            0.02746269851922989,
+            0.04107878357172012,
+            -0.005694024730473757,
+            -0.03896583244204521,
+            0.026265999302268028,
+            -0.035318680107593536,
+            -0.018394622951745987,
+            0.013594037853181362,
+            0.0381510891020298,
+            -0.010223937220871449,
+            0.032417282462120056,
+            0.01610656827688217,
+            -0.013205642811954021,
+            -0.03757423907518387,
+            0.03799910470843315,
+            -0.039449408650398254,
+            -0.011290505528450012,
+            -0.016824476420879364,
+            0.007128347177058458,
+            0.030213234946131706,
+            -0.09385695308446884,
+            0.014417118392884731,
+            -0.021249795332551003,
+            -0.021371016278862953,
+            0.031582340598106384,
+            -0.015021033585071564,
+            0.03207740932703018,
+            0.04465494304895401,
+            0.051139406859874725,
+            -0.004539252258837223,
+            -0.004026447422802448,
+            0.036198731511831284,
+            0.002513982355594635,
+            -0.022555746138095856,
+            -0.023142442107200623,
+            0.026506206020712852,
+            -0.0208470169454813,
+            0.01958189532160759,
+            0.02606782503426075,
+            -0.050900157541036606,
+            0.001175468903966248,
+            0.0026185859460383654,
+            0.01644700951874256,
+            0.047048378735780716,
+            -0.006155692040920258,
+            0.013264120556414127,
+            -0.004277337808161974,
+            0.022337032482028008,
+            -0.030710609629750252,
+            -0.06784506887197495,
+            0.010662010870873928,
+            -0.020733945071697235,
+            -0.01206474844366312,
+            -0.0005046974983997643,
+            -0.004159707576036453,
+            0.028128545731306076,
+            -0.011551725678145885,
+            0.057953692972660065,
+            0.028500419110059738,
+            0.02070418931543827,
+            0.029373178258538246,
+            -0.053878165781497955,
+            -0.03885475918650627,
+            -0.011427262797951698,
+            -0.040592946112155914,
+            0.019192807376384735,
+            -0.013966009952127934,
+            0.002324307570233941,
+            0.027266129851341248,
+            0.02721570059657097,
+            -0.013851913623511791,
+            0.06292124837636948,
+            -0.019983768463134766,
+            -0.06498263776302338,
+            -0.014787066727876663,
+            0.07545251399278641,
+            0.009921795688569546,
+            -0.02266773208975792,
+            -0.0174646507948637,
+            -0.0037801002617925406,
+            0.037214070558547974,
+            -0.033669255673885345,
+            -0.0033054312225431204,
+            -0.004362864885479212,
+            -0.010861773043870926,
+            -0.041649043560028076,
+            0.02711806818842888,
+            -0.001099557732231915,
+            0.0007163260015659034,
+            0.01317980233579874,
+            -0.011158796027302742,
+            0.03966476768255234,
+            0.023275790736079216,
+            -0.011645027436316013,
+            0.0030634249560534954,
+            -0.01243121363222599,
+            0.01271719578653574,
+            0.003938829991966486,
+            -0.00769989937543869,
+            -0.039121564477682114,
+            0.0005735178128816187,
+            0.02157283015549183,
+            0.005828005261719227,
+            0.03934130072593689,
+            0.015216797590255737,
+            0.017237801104784012,
+            -0.037648268043994904,
+            -0.007132838945835829,
+            -0.018956882879137993,
+            -0.0597093440592289,
+            0.058341480791568756,
+            -0.0008284997311420739,
+            0.02095271646976471,
+            0.043099164962768555,
+            0.09887702018022537,
+            -0.01221393421292305,
+            -0.02239784225821495,
+            0.016775032505393028,
+            0.013331425376236439,
+            -0.004451168701052666,
+            -0.02870352193713188,
+            -0.020854242146015167,
+            0.05349724739789963,
+            0.03315908834338188,
+            0.018541062250733376,
+            -0.03136591613292694,
+            0.03549784794449806,
+            -0.0076525891199707985,
+            -0.06454484909772873,
+            0.049847088754177094,
+            0.012184737250208855,
+            0.03575005754828453,
+            -0.050804175436496735,
+            0.09406977146863937,
+            0.05103312432765961,
+            -0.0036910600028932095,
+            0.10705005377531052,
+            0.011394658125936985,
+            -0.014218435622751713,
+            -0.042272791266441345,
+            0.018426422029733658,
+            -0.08213183283805847,
+            -0.010240674018859863,
+            0.051353540271520615,
+            0.016103247180581093,
+            0.04293083772063255,
+            -0.00462630670517683,
+            0.001971749123185873,
+            -0.05101824551820755,
+            -0.017815101891756058,
+            -0.0788436159491539,
+            -0.019784294068813324,
+            0.006863154470920563,
+            0.04096531495451927,
+            0.016416994854807854,
+            0.018884461373090744,
+            -0.03645262494683266,
+            -0.02363709919154644,
+            0.08447448164224625,
+            0.027652855962514877,
+            -0.005039512179791927,
+            -0.05533800646662712,
+            0.006148343440145254,
+            -0.03248206898570061,
+            -0.015117023140192032,
+            -0.056908346712589264,
+            0.057090409100055695,
+            0.02987913228571415,
+            -0.0642392635345459,
+            -0.01212853193283081,
+            -0.04195745661854744,
+            -0.008033841848373413,
+            -0.05249612778425217,
+            0.05965931713581085,
+            0.08591161668300629,
+            -0.012983623892068863,
+            0.002055486897006631,
+            -0.002928174799308181,
+            -0.023014886304736137,
+            -0.05307631567120552,
+            0.0325687900185585,
+            -0.008586175739765167,
+            -0.005393583793193102,
+            0.009566529653966427,
+            0.06500132381916046,
+            -0.02100509963929653,
+            -0.018470296636223793,
+            0.001247459789738059,
+            0.007388024125248194,
+            0.012469757348299026,
+            0.08475572615861893,
+            0.06918514519929886,
+            0.054265547543764114,
+            0.03292711451649666,
+            -0.08437038213014603,
+            0.07744771242141724,
+            -0.0004291488730814308,
+            -0.020394261926412582,
+            0.039096955209970474,
+            0.015851527452468872,
+            -0.009922537952661514,
+            0.02087295800447464,
+            -0.019477976486086845,
+            -0.06510577350854874,
+            0.008559669367969036,
+            0.015032066963613033,
+            -0.022979427129030228,
+            -0.017166415229439735,
+            -0.014456263743340969,
+            -0.034205030649900436,
+            -0.04903494939208031,
+            0.073653943836689,
+            -0.041798241436481476,
+            0.0035302129108458757,
+            0.031043095514178276,
+            0.038764648139476776,
+            0.03582717105746269,
+            -0.003121789079159498,
+            0.03909862041473389,
+            -0.03283870965242386,
+            0.06343409419059753,
+            0.085169717669487,
+            0.0037416887935250998,
+            0.043896209448575974,
+            -0.02215113304555416,
+            -0.04062772914767265,
+            -0.029482074081897736,
+            0.0013964198296889663,
+            0.04621904715895653,
+            0.030072476714849472,
+            -0.023583346977829933,
+            -0.016047311946749687,
+            -0.04016166180372238,
+            -0.026690224185585976,
+            0.034725841134786606,
+            -0.08011004328727722,
+            -0.023635270074009895,
+            -0.01675681211054325,
+            0.02217511460185051,
+            -0.018720457330346107,
+            0.0413116030395031,
+            -0.0045730252750217915,
+            -0.08402986079454422,
+            0.03641941770911217,
+            0.028000695630908012,
+            0.042173273861408234,
+            0.024761751294136047,
+            -0.051845893263816833,
+            -0.07877497375011444,
+            -0.020710380747914314,
+            -0.035789184272289276,
+            0.04824375733733177,
+            -0.04493764415383339,
+            -0.0014088008319959044,
+            0.09272980690002441,
+            -0.030772028490900993,
+            0.027623610571026802,
+            -0.0008853759500198066,
+            -0.015347420237958431,
+            -0.0006863650633022189,
+            0.02924676053225994,
+            0.03864092007279396,
+            -0.043402496725320816,
+            0.11410719156265259,
+            0.01606914773583412,
+            0.03158045932650566,
+            -0.049648500978946686,
+            -0.026801105588674545,
+            0.013934214599430561,
+            -0.04582132399082184,
+            -0.02133217453956604,
+            0.013296819292008877,
+            0.030687933787703514,
+            0.0014671665849164128,
+            0.005454834550619125,
+            -0.024595070630311966,
+            0.036868833005428314,
+            -0.003586424048990011,
+            -0.007300499361008406,
+            0.00619609747081995,
+            0.004614396020770073,
+            0.06406176835298538,
+            0.010256785899400711,
+            -0.050202082842588425,
+            -0.013155301101505756,
+            -0.04005127400159836,
+            -0.027943719178438187,
+            0.05738724395632744,
+            -0.002920332597568631,
+            -0.00731270294636488,
+            0.04419538751244545,
+            0.024069754406809807,
+            0.012176074087619781,
+            0.004615467507392168,
+            -0.04112132638692856,
+            -0.04844773933291435,
+            -0.012684458866715431,
+            0.0071298484690487385,
+            -0.010914848186075687,
+            -0.03592529892921448,
+            -0.05016973987221718,
+            -0.011797907762229443,
+            -0.043843258172273636,
+            -0.03715396672487259,
+            0.016528192907571793,
+            0.024301515892148018,
+            0.01335576456040144,
+            0.021006477996706963,
+            -0.021391959860920906,
+            0.05299517139792442,
+            0.0070807491429150105,
+            -0.08096124231815338,
+            -0.07334060221910477,
+            -0.034530941396951675,
+            -0.04421507194638252,
+            0.010524646379053593,
+            0.009575314819812775,
+            -0.031711090356111526,
+            0.023479584604501724,
+            -0.04212309420108795,
+            0.016264619305729866,
+            0.03907531499862671,
+            -0.0011187525233253837,
+            -0.03998023644089699,
+            -0.027464834973216057,
+            -0.07113838940858841,
+            -0.028915319591760635,
+            -0.01282753050327301,
+            -0.0033073138911277056,
+            0.026715606451034546,
+            -0.002769897459074855,
+            0.020033732056617737,
+            0.014502385631203651,
+            -0.017903830856084824,
+            0.06932531297206879,
+            0.0432068407535553,
+            0.01685408502817154,
+            0.04834728315472603,
+            -0.009553197771310806,
+            0.019799189642071724,
+            0.01173039898276329,
+            0.04158413037657738,
+            -0.018829666078090668,
+            -0.008410722948610783,
+            0.008009687066078186,
+            0.034592460840940475,
+            0.07790639251470566,
+            -0.022050900384783745,
+            0.04081638529896736,
+            0.046872470527887344,
+            0.0010260086273774505,
+            -0.05322079360485077,
+            0.009096509777009487,
+            -0.06831686198711395,
+            -0.01390997413545847,
+            -0.020475609228014946,
+            0.017393099144101143,
+            -0.007532020565122366,
+            -0.06435851007699966,
+            -0.014785194769501686,
+            0.02654031664133072,
+            0.004753720946609974,
+            0.026440177112817764,
+            -0.028890414163470268,
+            -0.011440729722380638,
+            0.003554105758666992,
+            -0.0022926912643015385,
+            -0.02393224649131298,
+            0.03711748123168945,
+            -0.06023703143000603,
+            -0.008778683841228485,
+            -0.05984162166714668,
+            -0.024247022345662117,
+            -0.036919932812452316,
+            0.05249374359846115,
+            0.03022468276321888,
+            -0.011348876170814037,
+            0.0008303995127789676,
+            0.001597013440914452,
+            -0.015491127036511898,
+            -0.035073015838861465,
+            -0.024477796629071236,
+            -0.030328145250678062,
+            -0.09301470220088959,
+            -0.046440113335847855,
+            0.036719027906656265,
+            -0.021899227052927017,
+            0.04666316881775856,
+            -0.07481305301189423,
+            -0.04928148165345192,
+            -0.01480096485465765,
+            0.0014140848070383072,
+            0.016779841855168343,
+            -0.04318199306726456,
+            0.011910341680049896,
+            -0.04019855335354805,
+            -0.027363713830709457,
+            0.006433602888137102,
+            0.023732252418994904,
+            -0.013081788085401058,
+            0.02489032782614231,
+            0.005415213759988546,
+            -0.058724161237478256,
+            0.032487478107213974,
+            -0.014332194812595844,
+            -0.020952431485056877,
+            0.055405858904123306,
+            -0.02239573374390602,
+            0.016315918415784836,
+            0.04710645601153374,
+            0.006866136100143194,
+            -0.019589263945817947,
+            -0.046199049800634384,
+            0.04977096989750862,
+            -0.03211359679698944,
+            0.06759121268987656,
+            -0.007805021945387125,
+            0.009877636097371578,
+            -0.003194598713889718,
+            -0.0014034705236554146,
+            0.024012917652726173,
+            0.0007609894964843988,
+            0.04028927534818649,
+            0.047299597412347794,
+            0.04644732549786568,
+            0.06253348290920258,
+            -0.03101237863302231,
+            -0.04797065258026123,
+            -0.02459110878407955,
+            -0.06663094460964203,
+            -0.012946722097694874,
+            -0.046321313828229904,
+            -0.03617801144719124,
+            -0.06608668714761734,
+            0.01371682621538639,
+            -0.040183935314416885,
+            0.027353622019290924,
+            -0.013125114142894745,
+            0.020482128486037254,
+            -0.10186963528394699,
+            0.03741387277841568,
+            -0.048566944897174835,
+            0.0017904090927913785,
+            0.0444694422185421,
+            -0.02355058304965496,
+            -0.04245513305068016,
+            0.01599632203578949,
+            -0.00974870752543211,
+            -0.02246273122727871,
+            0.011107604950666428,
+            -0.006354854442179203,
+            -0.08260829746723175,
+            -0.054969724267721176,
+            -0.038703542202711105,
+            -0.02590899169445038,
+            -0.012424441985785961,
+            0.033952418714761734,
+            0.032632969319820404,
+            0.03585505858063698,
+            -0.027734532952308655,
+            -0.05185376852750778,
+            0.005663866177201271,
+            0.01415393128991127,
+            0.007472912315279245,
+            -0.0325092077255249,
+            -0.0008526426972821355,
+            0.05909401550889015,
+            -0.006496420595794916,
+            0.06674317270517349,
+            0.06033811718225479,
+            -0.04705937206745148,
+            0.01221691444516182,
+            -0.005195186473429203,
+            0.017006050795316696,
+            0.015768419951200485,
+            -0.02346021682024002,
+            -0.04318040981888771,
+            -0.00965888798236847,
+            -0.012831253930926323,
+            -0.023086808621883392,
+            -0.043478451669216156,
+            0.02215973101556301,
+            0.01018955372273922,
+            -0.0029477940406650305,
+            -0.026364397257566452,
+            -0.04219489544630051,
+            -0.0690244510769844,
+            0.0017003740649670362,
+            -0.03498053178191185,
+            -0.01891854591667652,
+            -0.020380523055791855,
+            -0.07183944433927536,
+            0.01474913302809,
+            0.012818068265914917,
+            0.02298390306532383,
+            0.006645163521170616,
+            -0.014497633092105389,
+            -0.05751577392220497,
+            -0.01127719134092331,
+            0.014469895511865616,
+            0.039319343864917755,
+            -0.002891098614782095,
+            0.0038161359261721373,
+            -0.0176107045263052,
+            -0.02695712260901928,
+            0.023520348593592644,
+            0.053624920547008514,
+            -0.0472102165222168,
+            -0.021724319085478783,
+            -0.04204733297228813,
+            0.004941252060234547,
+            -0.07744265347719193,
+            -0.028974706307053566,
+            -6.1493665270973e-05,
+            -0.020630594342947006,
+            -0.014794640243053436,
+            -0.045572925359010696,
+            0.03233763575553894,
+            0.00969443004578352,
+            0.03665856271982193,
+            0.027483846992254257,
+            0.074271060526371,
+            -0.07454165071249008,
+            -0.034101732075214386,
+            -0.07216823101043701,
+            -0.001424514572136104,
+            -0.0025912360288202763,
+            -0.002444307319819927,
+            -0.012540637515485287,
+            0.009027975611388683,
+            0.06855443120002747,
+            -0.0013480151537805796,
+            0.027303414419293404,
+            -0.019723499193787575,
+            0.033644214272499084,
+            -0.04313155263662338,
+            -0.016152892261743546,
+            -0.020085612311959267,
+            0.029526935890316963,
+            0.0004591972683556378,
+            -0.013712934218347073,
+            0.015895912423729897,
+            -0.046559300273656845,
+            -0.00015638815239071846,
+            0.0015497541753575206,
+            -0.0015048328787088394,
+            0.06692329794168472,
+            0.0013934546150267124,
+            0.008921030908823013,
+            -0.010347972624003887,
+            -0.039798807352781296,
+            0.06892028450965881,
+            0.021145053207874298,
+            0.007431029342114925,
+            -0.05281573906540871,
+            0.015844792127609253,
+            0.014578046277165413,
+            -0.0020482230465859175,
+            0.03509555384516716,
+            -0.021227506920695305,
+            -0.03619229048490524,
+            0.004116897005587816,
+            0.02835669554769993,
+            -0.0028248224407434464,
+            0.00836214143782854,
+            -0.004688165616244078,
+            0.04566347226500511,
+            -0.0352579727768898,
+            -0.007859165780246258,
+            -0.003958444111049175,
+            0.023938892409205437,
+            0.04262895882129669,
+            -0.02836589328944683,
+            0.0456448458135128,
+            -0.062015753239393234,
+            0.03518408164381981,
+            0.06333593279123306,
+            -0.0155468275770545,
+            0.013991734012961388,
+            0.02207978442311287,
+            0.0032898876816034317,
+            0.05948015674948692,
+            0.010670959949493408,
+            -0.00624996330589056,
+            -0.04401599243283272,
+            -0.0022705462761223316
+        ],
+        "content_embedding": [
+            -0.01892169564962387,
+            0.0662541389465332,
+            0.008976679295301437,
+            -0.03809165209531784,
+            0.02344459854066372,
+            0.012984057888388634,
+            0.016158411279320717,
+            0.0040777078829705715,
+            -0.0321662537753582,
+            -0.0026544055435806513,
+            -0.03179372847080231,
+            0.019741656258702278,
+            0.049423426389694214,
+            0.019327590242028236,
+            0.01367267221212387,
+            -0.042058881372213364,
+            0.023155249655246735,
+            -0.015003002248704433,
+            0.01056167297065258,
+            0.0032619787380099297,
+            -0.014582481235265732,
+            -0.01262009609490633,
+            -0.009695992805063725,
+            -0.025683948770165443,
+            0.010330218821763992,
+            -0.043577518314123154,
+            -0.03799012303352356,
+            0.03159527853131294,
+            -0.046592168509960175,
+            0.03461733087897301,
+            0.029929379001259804,
+            -0.02696100063621998,
+            0.01958872564136982,
+            -0.04882275313138962,
+            -0.04835181683301926,
+            -0.07444816827774048,
+            0.0615590400993824,
+            -0.018079139292240143,
+            -0.02907492406666279,
+            0.03256160393357277,
+            0.052772294729948044,
+            0.0014335751766338944,
+            0.02048010565340519,
+            -0.01859121397137642,
+            0.0436980240046978,
+            -0.028847631067037582,
+            0.06271578371524811,
+            -0.04908007010817528,
+            0.04021253436803818,
+            -0.07390867173671722,
+            0.0029745057690888643,
+            -0.01733274944126606,
+            -0.005066753830760717,
+            -0.006927797570824623,
+            -0.01495048776268959,
+            0.020951012149453163,
+            -0.02161789871752262,
+            0.004997345618903637,
+            0.02517000213265419,
+            -0.03955457732081413,
+            0.038905348628759384,
+            0.008108963258564472,
+            -0.04058837890625,
+            0.03415047749876976,
+            -0.004129728768020868,
+            -0.07600218057632446,
+            -0.008998502045869827,
+            0.012445643544197083,
+            -0.005613638553768396,
+            -0.015701062977313995,
+            0.010493642650544643,
+            -0.01511659286916256,
+            -0.007434363476932049,
+            0.04920893907546997,
+            -0.044436678290367126,
+            -0.015229232609272003,
+            -0.009392009116709232,
+            -0.004889432340860367,
+            -0.03250344097614288,
+            0.05671893432736397,
+            0.03468514233827591,
+            -0.04985000938177109,
+            0.021073583513498306,
+            0.005558345932513475,
+            0.04397028684616089,
+            -0.011105467565357685,
+            -0.0010204907739534974,
+            -0.0013343892060220242,
+            0.010888955555856228,
+            0.11187340319156647,
+            0.05144372954964638,
+            0.014714346267282963,
+            0.03652629256248474,
+            0.08354610204696655,
+            -0.050587598234415054,
+            0.07670528441667557,
+            0.022823045030236244,
+            -0.010303523391485214,
+            -0.00016479991609230638,
+            -0.015029380097985268,
+            -0.010333288460969925,
+            0.03660477325320244,
+            0.013327172957360744,
+            -0.008142965845763683,
+            0.04656663164496422,
+            0.043171610683202744,
+            0.027440473437309265,
+            0.011585040017962456,
+            -0.008035292848944664,
+            -0.0008554590749554336,
+            -0.04715310037136078,
+            -0.013419345021247864,
+            -0.034535810351371765,
+            0.028465399518609047,
+            -0.030552269890904427,
+            0.02954002656042576,
+            0.11263657361268997,
+            -0.060091886669397354,
+            -0.004718341864645481,
+            0.02276463434100151,
+            -0.029855655506253242,
+            1.136395258072298e-05,
+            0.01254600752145052,
+            0.030318304896354675,
+            0.04609473794698715,
+            0.04090471565723419,
+            -0.015202691778540611,
+            -0.025406358763575554,
+            0.01403091847896576,
+            -0.01206378173083067,
+            -0.034794360399246216,
+            0.021181223914027214,
+            -0.041345320641994476,
+            0.026389217004179955,
+            0.04634319990873337,
+            0.05973498523235321,
+            -0.0791369080543518,
+            -0.018549518659710884,
+            0.009269041940569878,
+            0.005099988542497158,
+            0.016017470508813858,
+            0.016928445547819138,
+            0.004272987134754658,
+            -0.03169683367013931,
+            0.008137955330312252,
+            -0.07982300966978073,
+            -0.037415798753499985,
+            -0.0016467635286971927,
+            -0.016258487477898598,
+            0.01855027861893177,
+            0.012749083340168,
+            -0.015595809556543827,
+            0.009437683038413525,
+            0.005881224758923054,
+            0.009153603576123714,
+            -0.035431332886219025,
+            -0.03822671249508858,
+            -0.007053021341562271,
+            0.07195861637592316,
+            0.03834277018904686,
+            0.025282155722379684,
+            0.03235918655991554,
+            -0.040675584226846695,
+            0.06914123892784119,
+            -0.014681060798466206,
+            0.04182145744562149,
+            0.016547678038477898,
+            0.0302575696259737,
+            0.027968881651759148,
+            0.028392894193530083,
+            -0.03601876646280289,
+            0.011166741140186787,
+            0.013932433910667896,
+            -0.024813517928123474,
+            0.04876561462879181,
+            0.03280804678797722,
+            0.020525190979242325,
+            -0.04888831451535225,
+            0.05333299934864044,
+            -0.01227282639592886,
+            0.009397462010383606,
+            -0.062118303030729294,
+            0.020511150360107422,
+            0.03606007248163223,
+            -0.011546325869858265,
+            0.02632950246334076,
+            0.03558770939707756,
+            -0.04729287326335907,
+            -0.00040853474638424814,
+            -0.05594595894217491,
+            0.03343893215060234,
+            -0.03624171018600464,
+            -0.01565496437251568,
+            0.03419746086001396,
+            -0.014939344488084316,
+            -0.0346553735435009,
+            0.02617849037051201,
+            -0.018064821138978004,
+            0.00044916238402947783,
+            -0.029752276837825775,
+            -0.06982599943876266,
+            -0.01529014203697443,
+            -0.10238753259181976,
+            0.056908924132585526,
+            -0.018579944968223572,
+            0.032441046088933945,
+            0.02623467892408371,
+            0.0005816647899337113,
+            0.024393916130065918,
+            -0.0010619793320074677,
+            0.09054756909608841,
+            0.012866330333054066,
+            0.0110749127343297,
+            0.060603830963373184,
+            -0.04485912621021271,
+            -0.035673510283231735,
+            0.00880404282361269,
+            -0.0236192774027586,
+            -0.04651271551847458,
+            0.04936773329973221,
+            0.016861658543348312,
+            -0.026910705491900444,
+            0.02507326751947403,
+            0.0018011556239798665,
+            -0.01599423959851265,
+            0.007061067037284374,
+            -0.028597962111234665,
+            -0.005096979904919863,
+            -0.003091734368354082,
+            -0.008610324002802372,
+            -0.03941959887742996,
+            -0.07249880582094193,
+            0.07896454632282257,
+            -0.01282701175659895,
+            0.03806105628609657,
+            -0.01628866419196129,
+            -0.00032510326127521694,
+            0.007600210607051849,
+            0.012463843449950218,
+            0.07028777152299881,
+            0.024854836985468864,
+            0.00597741175442934,
+            0.012146051973104477,
+            -0.04252159595489502,
+            -0.08857864141464233,
+            0.005069843493402004,
+            -0.002303875982761383,
+            0.007218160200864077,
+            -0.054320499300956726,
+            0.01721455715596676,
+            -0.012323171831667423,
+            0.029316846281290054,
+            0.010660098865628242,
+            0.01619168184697628,
+            0.024796800687909126,
+            -0.06043343245983124,
+            -0.009076021611690521,
+            0.05426326394081116,
+            0.024232488125562668,
+            -0.025832876563072205,
+            0.024366402998566628,
+            -0.04501958563923836,
+            0.04263340309262276,
+            -0.01757700741291046,
+            0.0240378025919199,
+            0.007873878814280033,
+            -0.019204245880246162,
+            -0.04099274054169655,
+            -0.0028695412911474705,
+            -0.02336733788251877,
+            -0.009908018633723259,
+            0.01244357880204916,
+            0.014616346918046474,
+            0.07263968884944916,
+            -0.006017595529556274,
+            0.006593986880034208,
+            -0.017023928463459015,
+            -0.0008568437770009041,
+            0.0393415242433548,
+            -0.03193742036819458,
+            -0.07265064865350723,
+            -0.056716252118349075,
+            -0.06321432441473007,
+            0.0014871162129566073,
+            0.015271728858351707,
+            0.06799189001321793,
+            0.002235528314486146,
+            0.015148743987083435,
+            0.029075419530272484,
+            -0.036075517535209656,
+            0.03699851781129837,
+            0.002699150936678052,
+            0.029273545369505882,
+            0.024833064526319504,
+            0.02166113816201687,
+            0.07822758704423904,
+            0.0907154381275177,
+            -0.015422212891280651,
+            -0.004725399427115917,
+            -0.013691544532775879,
+            0.00014949020987842232,
+            0.003309824038296938,
+            0.019388742744922638,
+            -0.01792132295668125,
+            -0.005919941700994968,
+            -0.009184692986309528,
+            -0.00453580915927887,
+            -0.017324700951576233,
+            0.020368218421936035,
+            0.007512629963457584,
+            -0.05764073505997658,
+            0.01584697514772415,
+            -0.016094518825411797,
+            -0.0366678424179554,
+            -0.02194156125187874,
+            0.053442906588315964,
+            0.04864593967795372,
+            -0.009642759338021278,
+            0.06584249436855316,
+            0.017993653193116188,
+            0.02838297188282013,
+            -0.02758033573627472,
+            -0.018208689987659454,
+            -0.08217029273509979,
+            0.001340706367045641,
+            0.07344162464141846,
+            -0.0014725526561960578,
+            0.027256185188889503,
+            -0.03795681148767471,
+            0.03496084362268448,
+            -0.009351355955004692,
+            0.03554052114486694,
+            -0.0647641122341156,
+            -0.018092816695570946,
+            -0.0003290708118584007,
+            -0.008958869613707066,
+            -0.0006743986159563065,
+            0.02749652974307537,
+            -0.005728874355554581,
+            -0.00014254855341278017,
+            0.02650611102581024,
+            -0.007747439201921225,
+            -0.036285076290369034,
+            -0.04723037779331207,
+            -0.01256555411964655,
+            -0.015652446076273918,
+            -0.0033896011300385,
+            -0.027379868552088737,
+            0.06606956571340561,
+            0.001414530212059617,
+            -0.03816799819469452,
+            0.005582350306212902,
+            -0.0037654521875083447,
+            -0.03315531834959984,
+            -0.03833584487438202,
+            0.005306297447532415,
+            0.06055983901023865,
+            0.017386972904205322,
+            0.017846958711743355,
+            0.002940434729680419,
+            0.06065093353390694,
+            -0.033751003444194794,
+            0.02014659158885479,
+            -0.026745468378067017,
+            0.02349875122308731,
+            0.06887564063072205,
+            0.08784544467926025,
+            0.0348343662917614,
+            0.017027992755174637,
+            0.007463646121323109,
+            0.010731169953942299,
+            -0.015452216379344463,
+            0.0697169378399849,
+            0.06115807220339775,
+            0.05587253347039223,
+            0.0035254100803285837,
+            -0.06922555714845657,
+            -0.00895272009074688,
+            0.04390031844377518,
+            0.003160918829962611,
+            0.0734192356467247,
+            -0.012384983710944653,
+            0.00778034096583724,
+            -0.06225632503628731,
+            0.01105977687984705,
+            -0.019027134403586388,
+            -0.01744268462061882,
+            -0.03861316666007042,
+            -0.026121554896235466,
+            -0.03796643018722534,
+            -0.02607419341802597,
+            -0.00727757578715682,
+            -0.04364367574453354,
+            0.027054548263549805,
+            0.001148495590314269,
+            -0.0051346817053854465,
+            -0.014047800563275814,
+            0.033344950526952744,
+            0.016461240127682686,
+            0.033907197415828705,
+            -0.052207209169864655,
+            -0.058969806879758835,
+            0.019914019852876663,
+            0.04874560981988907,
+            0.0043409536592662334,
+            0.014156220480799675,
+            -0.025425465777516365,
+            -0.03806624561548233,
+            0.027224158868193626,
+            -8.918229286791757e-05,
+            0.04550011456012726,
+            0.02069287933409214,
+            -0.006964664440602064,
+            -0.05213857442140579,
+            0.03515300899744034,
+            -0.02322443015873432,
+            0.07085354626178741,
+            0.010733392089605331,
+            -0.04821530729532242,
+            -0.024944474920630455,
+            0.01349271647632122,
+            0.0064827692694962025,
+            0.021682681515812874,
+            0.03466835618019104,
+            -0.023484358564019203,
+            -0.004177657887339592,
+            0.019195759668946266,
+            0.021642865613102913,
+            0.03591984510421753,
+            -6.837025284767151e-05,
+            0.003064215648919344,
+            0.0067205713130533695,
+            0.024574855342507362,
+            0.03467808663845062,
+            -0.07038415223360062,
+            0.020557953044772148,
+            0.05572228878736496,
+            0.024007081985473633,
+            0.008300675079226494,
+            0.05382058024406433,
+            -0.008657778613269329,
+            -0.04247821494936943,
+            -0.02082398161292076,
+            -0.030047548934817314,
+            -0.0042150202207267284,
+            0.0643019825220108,
+            0.08603832125663757,
+            -0.0032497297506779432,
+            -0.05890907347202301,
+            -0.017683515325188637,
+            -0.0017970707267522812,
+            0.030202442780137062,
+            -0.004163889214396477,
+            -0.005693267099559307,
+            0.07439851015806198,
+            -0.007623215671628714,
+            -0.014011486433446407,
+            -0.06531509011983871,
+            -0.012002935633063316,
+            0.05098460614681244,
+            0.018368106335401535,
+            0.044709816575050354,
+            -0.034841395914554596,
+            0.04669453203678131,
+            -0.006633058190345764,
+            -0.06744810938835144,
+            0.00022071562125347555,
+            -0.02252846583724022,
+            -0.008146141655743122,
+            0.04570293799042702,
+            -0.017073389142751694,
+            0.033481452614068985,
+            0.02024919167160988,
+            -0.00039372473838739097,
+            -0.015125994570553303,
+            0.0035840750206261873,
+            0.03293292224407196,
+            -0.023488696664571762,
+            0.02769201435148716,
+            0.03366998955607414,
+            0.013383373618125916,
+            -0.0062416414730250835,
+            -0.05436183512210846,
+            -0.007013875991106033,
+            -0.0343070924282074,
+            0.008950931020081043,
+            -0.0007773659308440983,
+            0.01631912775337696,
+            -0.01733097992837429,
+            0.007631183601915836,
+            0.022811884060502052,
+            0.05997275933623314,
+            -0.025991076603531837,
+            -0.06607384979724884,
+            -0.0873650386929512,
+            -0.05788758397102356,
+            -0.020700229331851006,
+            0.00862400233745575,
+            0.008653292432427406,
+            -0.05257308855652809,
+            -0.01877412386238575,
+            0.001132996054366231,
+            0.007562611252069473,
+            0.007040517870336771,
+            -0.03939346596598625,
+            -0.0012852386571466923,
+            0.03364014998078346,
+            -0.08792895078659058,
+            0.0003337061498314142,
+            0.04566165804862976,
+            0.022397097200155258,
+            0.07704627513885498,
+            0.011688907630741596,
+            0.06875491887331009,
+            0.031596384942531586,
+            -0.07542278617620468,
+            0.06929827481508255,
+            0.03525209799408913,
+            -0.05507253482937813,
+            0.06310203671455383,
+            0.009202172048389912,
+            0.08802317827939987,
+            0.015267971903085709,
+            0.01631786674261093,
+            -0.08159693330526352,
+            0.011958948336541653,
+            -0.022956276312470436,
+            -0.0045707738026976585,
+            0.06590449810028076,
+            -0.025062261149287224,
+            0.05683448538184166,
+            0.08174461871385574,
+            0.018841996788978577,
+            -0.02901572361588478,
+            0.04103256016969681,
+            -0.06138996779918671,
+            -0.02983909286558628,
+            -0.03850552439689636,
+            -0.018056459724903107,
+            0.00292590050958097,
+            -0.0737059935927391,
+            0.00898703932762146,
+            -0.012909052893519402,
+            -0.00488039618358016,
+            0.019017860293388367,
+            -0.037835441529750824,
+            -0.05031483247876167,
+            0.025473300367593765,
+            -0.009489303454756737,
+            -0.08405261486768723,
+            0.06039801985025406,
+            -0.028819533064961433,
+            0.01564796455204487,
+            -0.07851359248161316,
+            0.00776974530890584,
+            -0.0627446398139,
+            0.043354298919439316,
+            -0.0447402149438858,
+            0.008833021856844425,
+            -0.0005271312547847629,
+            -0.03740326315164566,
+            -0.033597033470869064,
+            0.02730080671608448,
+            -0.030516251921653748,
+            0.03767557814717293,
+            -0.10619816929101944,
+            -0.038678478449583054,
+            0.02232091873884201,
+            0.03868230804800987,
+            0.018831931054592133,
+            -0.05178656801581383,
+            -0.05465080961585045,
+            0.03249572589993477,
+            0.009297838434576988,
+            -0.003563723061233759,
+            -0.04144677892327309,
+            0.0509132519364357,
+            -0.02094709314405918,
+            -0.022470436990261078,
+            -0.04437573254108429,
+            -0.03695523366332054,
+            -0.075083889067173,
+            0.07801777124404907,
+            0.007801617495715618,
+            -0.005376672837883234,
+            0.020604871213436127,
+            -0.06675189733505249,
+            0.0027014226652681828,
+            0.08348087221384048,
+            -0.031110215932130814,
+            -0.02220381610095501,
+            0.021845143288373947,
+            0.03032352775335312,
+            -0.0012008004123345017,
+            -0.03200481832027435,
+            0.049666762351989746,
+            0.005313111934810877,
+            -0.020655009895563126,
+            0.007201225031167269,
+            -0.05322100222110748,
+            -0.03385355696082115,
+            0.010354285128414631,
+            0.04187091067433357,
+            0.006058192811906338,
+            0.005469379480928183,
+            0.041591376066207886,
+            -0.023555509746074677,
+            0.043303441256284714,
+            -0.04954344779253006,
+            -0.033017441630363464,
+            -0.01149839162826538,
+            -0.012791389599442482,
+            0.001670036930590868,
+            -0.012347050942480564,
+            0.0004881276981905103,
+            -0.031120697036385536,
+            0.022906621918082237,
+            -0.050669725984334946,
+            0.04269399866461754,
+            -0.011447146534919739,
+            -0.017906805500388145,
+            -0.06953153014183044,
+            0.04467186704277992,
+            -0.04761233553290367,
+            -0.013187393546104431,
+            0.05690088868141174,
+            -0.042590390890836716,
+            -0.01746809110045433,
+            0.020567748695611954,
+            0.05125907063484192,
+            0.020307395607233047,
+            0.03492629528045654,
+            -0.04882863909006119,
+            -0.03183748573064804,
+            -0.06539574265480042,
+            -0.01744089275598526,
+            -0.02758834883570671,
+            0.0050849285908043385,
+            -0.00035606700112111866,
+            0.023614611476659775,
+            0.01930573768913746,
+            -0.05899752303957939,
+            -0.04627015441656113,
+            0.0068423328921198845,
+            0.03920449689030647,
+            -0.007687605917453766,
+            0.016464397311210632,
+            0.04807426780462265,
+            0.0023120716214179993,
+            0.01973593607544899,
+            0.07794646173715591,
+            0.07625434547662735,
+            -0.03674965724349022,
+            -0.0012999586760997772,
+            0.009016714058816433,
+            0.03811555355787277,
+            0.017517905682325363,
+            0.004926901776343584,
+            -0.07054422050714493,
+            -0.01442575640976429,
+            0.01330371480435133,
+            -0.008963101543486118,
+            -0.009463613852858543,
+            0.0017095000948756933,
+            -0.016330784186720848,
+            -0.017924489453434944,
+            -0.042089130729436874,
+            -0.06883884966373444,
+            -0.042998943477869034,
+            0.014172191731631756,
+            -0.0023317155428230762,
+            -0.027441971004009247,
+            0.004573931451886892,
+            -0.07700463384389877,
+            -0.013737251050770283,
+            0.025464439764618874,
+            -0.02619084157049656,
+            -0.008323452435433865,
+            -0.03393486887216568,
+            -0.04159104451537132,
+            -0.004442669451236725,
+            -0.008337379433214664,
+            0.05703001841902733,
+            -0.05177110433578491,
+            0.002210760721936822,
+            0.005930258426815271,
+            -0.0369490347802639,
+            -0.013454861007630825,
+            -0.004840550944209099,
+            -0.04600533843040466,
+            -0.010599354282021523,
+            -0.008193885907530785,
+            -0.029226260259747505,
+            -0.06824758648872375,
+            0.002242376795038581,
+            -0.00545460032299161,
+            -0.016073163598775864,
+            -0.02212926186621189,
+            0.014335459098219872,
+            0.02033282071352005,
+            -0.01998221129179001,
+            0.06560437381267548,
+            -0.007302496116608381,
+            0.037101101130247116,
+            -0.015349503606557846,
+            -0.0149971479550004,
+            -0.003208655398339033,
+            0.01065454725176096,
+            -0.010318529792129993,
+            0.005211932118982077,
+            -0.007634020410478115,
+            0.007333737798035145,
+            0.04658440127968788,
+            -0.017371229827404022,
+            -0.02044561877846718,
+            0.021157968789339066,
+            -0.005675977561622858,
+            -0.016465574502944946,
+            0.001816042698919773,
+            -0.022665906697511673,
+            0.04769016057252884,
+            -0.02464037574827671,
+            -0.05675514042377472,
+            0.05963050201535225,
+            -0.01688731089234352,
+            -0.05340677872300148,
+            0.010052076540887356,
+            0.02069842256605625,
+            0.028715714812278748,
+            0.009125969372689724,
+            -0.02970687672495842,
+            -0.010313224978744984,
+            -0.03552298620343208,
+            0.04363728687167168,
+            -0.03991911932826042,
+            -0.004784241318702698,
+            -0.044753339141607285,
+            0.01931679993867874,
+            -0.056493211537599564,
+            -0.006617037579417229,
+            0.035743631422519684,
+            -0.053424812853336334,
+            -0.02699253521859646,
+            -0.007951406762003899,
+            -0.009088664315640926,
+            -0.018690962344408035,
+            -0.04115553945302963,
+            0.02701025828719139,
+            0.0571308434009552,
+            -0.029878465458750725,
+            -0.03173048421740532,
+            -0.01149672456085682,
+            -0.0105333486571908,
+            -0.005241425707936287,
+            -0.02809373289346695,
+            0.05968040972948074,
+            0.0010212024208158255,
+            0.042596235871315,
+            0.04825957119464874,
+            -0.003983878996223211,
+            0.016225650906562805,
+            0.015263753943145275,
+            -0.023301145061850548,
+            0.041719190776348114,
+            0.028326746076345444,
+            0.026445787400007248,
+            -0.022935770452022552,
+            0.03078318201005459
+        ],
+        "chunk_ind": 1
+    },
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/ai_platform",
+        "title": "AI Platform",
+        "content": "Build AI Agents powered by the knowledge and workflows specific to your organization.\n\nBeyond Answers\nAgents enabled by generative AI and reasoning capable models are helping teams to automate their work. Danswer is helping teams make it happen. Danswer provides out of the box user chat sessions, attaching custom tools, handling LLM reasoning, code execution, data analysis, referencing internal knowledge, and much more.\n\nDanswer as a platform is not a no-code agent builder. We are made by developers for developers and this gives your team the full flexibility and power to create agents not constrained by blocks and simple logic paths.\n\nFlexibility and Extensibility\nDanswer is open source and completely whitebox. This not only gives transparency to what happens within the system but also means that your team can directly modify the source code to suit your unique needs.",
+        "title_embedding": [
+            0.032763753086328506,
+            0.049961112439632416,
+            0.00777681777253747,
+            -0.009621717967092991,
+            0.03860695660114288,
+            0.035656899213790894,
+            -0.029095029458403587,
+            -0.030549267306923866,
+            -0.028131460770964622,
+            -0.023247526958584785,
+            -0.030750803649425507,
+            0.04233109578490257,
+            0.044790223240852356,
+            0.020764602348208427,
+            -0.011113415472209454,
+            -0.052699606865644455,
+            0.05441703647375107,
+            -0.027375519275665283,
+            0.03858301043510437,
+            0.0015289749717339873,
+            -0.0007870558765716851,
+            -0.013234086334705353,
+            -0.008892231620848179,
+            -0.0269540473818779,
+            0.032256800681352615,
+            0.028824904933571815,
+            0.021423548460006714,
+            0.0196831077337265,
+            0.004699843470007181,
+            0.01062865275889635,
+            -0.03573931008577347,
+            -0.01450167316943407,
+            0.06177164614200592,
+            -0.004766061902046204,
+            -0.011502844281494617,
+            -0.059983331710100174,
+            0.03794373199343681,
+            0.003160010790452361,
+            0.05785837396979332,
+            -0.016349520534276962,
+            0.048589278012514114,
+            0.03928593918681145,
+            -0.027400294318795204,
+            -0.007712628226727247,
+            0.044047582894563675,
+            0.03514353185892105,
+            0.050972215831279755,
+            -0.027322333306074142,
+            0.08146621286869049,
+            -0.041862014681100845,
+            0.034794293344020844,
+            0.0064093489199876785,
+            -0.05552367866039276,
+            -0.06472223997116089,
+            -0.0006224742392078042,
+            0.010324635542929173,
+            -0.00513586075976491,
+            0.006625971291214228,
+            -0.03121061436831951,
+            -0.02010185271501541,
+            0.024356791749596596,
+            0.04554779455065727,
+            -0.04365985095500946,
+            0.038004688918590546,
+            0.026826566085219383,
+            -0.0002007065195357427,
+            0.0025419823359698057,
+            0.022517988458275795,
+            0.004520556423813105,
+            -0.04712541028857231,
+            0.042386990040540695,
+            0.0317973829805851,
+            0.022796982899308205,
+            0.03537650406360626,
+            -0.024706847965717316,
+            -0.05100490525364876,
+            0.013296891935169697,
+            -0.027389265596866608,
+            -0.022103115916252136,
+            0.07237043976783752,
+            0.022473221644759178,
+            -0.08428098261356354,
+            0.0284805316478014,
+            0.014994120225310326,
+            0.0647200271487236,
+            -0.0013714460656046867,
+            -0.02798375114798546,
+            0.004889763426035643,
+            -0.02891303412616253,
+            0.06638259440660477,
+            -0.015550877898931503,
+            0.01490933820605278,
+            0.03998437523841858,
+            0.031558796763420105,
+            -0.09123710542917252,
+            0.03090553544461727,
+            -0.027405250817537308,
+            0.0028605929110199213,
+            -0.01660272665321827,
+            -0.024673976004123688,
+            -0.03330164775252342,
+            0.019906772300601006,
+            0.020785389468073845,
+            -0.02234416827559471,
+            0.0711885541677475,
+            0.010001438669860363,
+            -0.007417359855026007,
+            -0.03474368155002594,
+            0.0117587149143219,
+            -0.030912458896636963,
+            -0.04288865998387337,
+            0.004992801230400801,
+            -0.011203224770724773,
+            0.026435980573296547,
+            -0.019328005611896515,
+            -0.01772245578467846,
+            0.05772961303591728,
+            -0.018587617203593254,
+            0.03977040946483612,
+            0.0511898435652256,
+            0.02799198590219021,
+            -0.021339384838938713,
+            0.016965094953775406,
+            0.08415205776691437,
+            0.010289170779287815,
+            -0.02373247779905796,
+            -0.06358940154314041,
+            0.03165338188409805,
+            -0.013218379579484463,
+            -0.041016921401023865,
+            0.052579861134290695,
+            0.016211217269301414,
+            0.012958381325006485,
+            -0.029191715642809868,
+            -0.0013247805181890726,
+            0.05056416615843773,
+            -0.05472686141729355,
+            -0.05397220700979233,
+            0.07864602655172348,
+            0.044400643557310104,
+            0.011529057286679745,
+            -0.0056294528767466545,
+            0.0019877285230904818,
+            -0.01892041228711605,
+            0.031235355883836746,
+            -0.06018691137433052,
+            0.015224655158817768,
+            0.0035034629981964827,
+            -0.04407024383544922,
+            0.03802705183625221,
+            0.016176624223589897,
+            0.05680167302489281,
+            -0.017375409603118896,
+            -0.01676156371831894,
+            -0.017084985971450806,
+            -0.042274024337530136,
+            -0.07406415045261383,
+            0.020823167636990547,
+            0.04484682157635689,
+            -0.023108867928385735,
+            0.02925572544336319,
+            0.06840821355581284,
+            -0.027610015124082565,
+            0.04234248027205467,
+            -0.02915036305785179,
+            -0.004962626378983259,
+            -0.0017270881216973066,
+            0.023044373840093613,
+            0.037656962871551514,
+            0.04789644852280617,
+            -0.0027900487184524536,
+            0.0004090967122465372,
+            0.014888445846736431,
+            0.009237252175807953,
+            0.036635007709264755,
+            0.015078885480761528,
+            0.046658437699079514,
+            -0.025920215994119644,
+            0.014571646228432655,
+            -0.053589239716529846,
+            0.024663543328642845,
+            -0.0388394258916378,
+            0.0037244234699755907,
+            0.007817366160452366,
+            -0.03352022543549538,
+            -0.0609428733587265,
+            0.04179045185446739,
+            -0.05036167427897453,
+            -0.04099080711603165,
+            0.02920934371650219,
+            -0.037300609052181244,
+            0.010041946545243263,
+            0.025091813877224922,
+            -0.032656773924827576,
+            -0.05137333646416664,
+            -0.038329657167196274,
+            0.03855415806174278,
+            0.006781625561416149,
+            0.02984003536403179,
+            -0.06467068940401077,
+            0.02395613305270672,
+            0.018539344891905785,
+            -0.0718475878238678,
+            0.031203489750623703,
+            -0.057184506207704544,
+            0.02436862140893936,
+            0.02837834134697914,
+            -0.010054084472358227,
+            -0.02551312930881977,
+            0.021066943183541298,
+            0.06444599479436874,
+            0.01263453345745802,
+            -0.018358737230300903,
+            0.010503370314836502,
+            -0.023012487217783928,
+            0.009831788949668407,
+            0.0049070375971496105,
+            -0.022574082016944885,
+            -0.0049112942069768906,
+            -0.01014224998652935,
+            0.055648382753133774,
+            -0.016490083187818527,
+            -0.012448773719370365,
+            -0.014511270448565483,
+            0.027931246906518936,
+            0.024195006117224693,
+            -0.005839435383677483,
+            0.029669128358364105,
+            -0.007521398831158876,
+            0.03150096535682678,
+            -0.01941276341676712,
+            -0.06204359978437424,
+            0.01095200888812542,
+            0.0023097621742635965,
+            0.008341503329575062,
+            0.0100992601364851,
+            -0.039239075034856796,
+            0.04388657584786415,
+            0.015824418514966965,
+            0.06830465793609619,
+            0.009663422591984272,
+            -0.00038048860733397305,
+            0.035620324313640594,
+            -0.011668454855680466,
+            -0.06677736341953278,
+            0.008154943585395813,
+            -0.03417421504855156,
+            -0.022497251629829407,
+            0.01800542138516903,
+            0.0010614683851599693,
+            0.00842749048024416,
+            0.020196812227368355,
+            -0.005975049454718828,
+            0.024395788088440895,
+            -0.01633184403181076,
+            -0.004018640611320734,
+            -0.0018627216340973973,
+            0.058719366788864136,
+            -0.024047864601016045,
+            -0.0032275430858135223,
+            0.07045131176710129,
+            -0.03221508115530014,
+            0.0352499820291996,
+            0.02055438607931137,
+            0.02973576821386814,
+            0.0017980994889512658,
+            0.05022549629211426,
+            0.03819788247346878,
+            -0.005316003691405058,
+            0.011116476729512215,
+            0.019071733579039574,
+            0.03500362113118172,
+            -0.03451540693640709,
+            0.09197302162647247,
+            0.008307289332151413,
+            0.015847783535718918,
+            -0.003909585066139698,
+            -0.04707544669508934,
+            0.01712993159890175,
+            -0.026143768802285194,
+            -0.007809836883097887,
+            -0.02002348005771637,
+            -0.03528841957449913,
+            -0.012745876796543598,
+            0.016280299052596092,
+            0.005661313887685537,
+            0.022872695699334145,
+            0.016736241057515144,
+            -0.048460669815540314,
+            0.012391538359224796,
+            -0.04375111311674118,
+            -0.06501554697751999,
+            -0.0159616582095623,
+            0.009163076989352703,
+            -0.008098017424345016,
+            0.03997795283794403,
+            0.09088447690010071,
+            -0.025736957788467407,
+            -0.01334838755428791,
+            0.015781259164214134,
+            0.010901914909482002,
+            0.021588636562228203,
+            0.011131210252642632,
+            -0.034338608384132385,
+            0.053609222173690796,
+            0.018425501883029938,
+            0.03827910125255585,
+            -0.003314226632937789,
+            0.010824226774275303,
+            0.020308859646320343,
+            -0.11467628926038742,
+            0.04042372852563858,
+            0.01810252107679844,
+            0.03511713072657585,
+            -0.0987866222858429,
+            0.016760295256972313,
+            0.007829226553440094,
+            -0.011888569220900536,
+            0.034833233803510666,
+            -0.009197549894452095,
+            0.005588896572589874,
+            -0.07932842522859573,
+            -0.02078017219901085,
+            -0.03448954597115517,
+            0.0152775589376688,
+            0.08626428246498108,
+            0.03126169368624687,
+            0.04502886161208153,
+            -0.026686420664191246,
+            -0.028234312310814857,
+            0.0049273171462118626,
+            0.023110508918762207,
+            -0.08400018513202667,
+            0.017200743779540062,
+            0.02693784609436989,
+            0.0036261421628296375,
+            -0.018591655418276787,
+            0.005189367104321718,
+            0.0002512435312382877,
+            -0.01673535816371441,
+            0.06507309526205063,
+            0.02960938587784767,
+            0.0194547139108181,
+            -0.045088544487953186,
+            -0.01410599984228611,
+            -0.001771911047399044,
+            0.042333200573921204,
+            -0.015243434347212315,
+            0.027360277250409126,
+            -0.02644488774240017,
+            -0.059026844799518585,
+            0.0013204477727413177,
+            -0.005272903945297003,
+            -0.03697441890835762,
+            -0.03736754506826401,
+            0.06495915353298187,
+            0.004548369906842709,
+            0.004532824270427227,
+            -0.005509661976248026,
+            0.013331729918718338,
+            0.005671144928783178,
+            -0.043852198868989944,
+            0.06886028498411179,
+            -0.0020801422651857138,
+            0.014272121712565422,
+            -0.02358032390475273,
+            0.010091368108987808,
+            -0.013035510666668415,
+            -0.009768063202500343,
+            -0.024086249992251396,
+            0.04728090390563011,
+            -0.024031780660152435,
+            0.032426923513412476,
+            0.06455196440219879,
+            0.08759471774101257,
+            0.009270765818655491,
+            -0.0936349406838417,
+            -0.012462696991860867,
+            -0.019188350066542625,
+            -0.06805568188428879,
+            0.01794586144387722,
+            -0.0007348881918005645,
+            0.0024105070624500513,
+            -0.016566181555390358,
+            0.012622764334082603,
+            -0.03900640457868576,
+            0.010342570021748543,
+            0.011543489061295986,
+            0.01152091845870018,
+            -0.05232607573270798,
+            0.004903953988105059,
+            -0.05708310753107071,
+            -0.04076048359274864,
+            0.016818160191178322,
+            -0.020741824060678482,
+            0.01609313301742077,
+            -0.022479360923171043,
+            0.03654901310801506,
+            0.022170664742588997,
+            0.01575297676026821,
+            -0.011484816670417786,
+            -0.025103436782956123,
+            0.05906060338020325,
+            0.02779274433851242,
+            0.028078753501176834,
+            0.04629473015666008,
+            -0.005719225853681564,
+            -0.06190178170800209,
+            0.006866101641207933,
+            -0.002305209171026945,
+            0.03215618431568146,
+            0.007546067703515291,
+            -0.02738751657307148,
+            -0.04539818689227104,
+            0.04683874174952507,
+            -0.0208493173122406,
+            0.03900844231247902,
+            -0.027456291019916534,
+            -0.028509290888905525,
+            0.013289637863636017,
+            0.0017003221437335014,
+            -0.0198791716247797,
+            0.014913729391992092,
+            0.005401032045483589,
+            -0.04071260988712311,
+            0.02060793526470661,
+            -0.003016189206391573,
+            0.03800947219133377,
+            -0.019319988787174225,
+            -0.024961907416582108,
+            -0.02498740889132023,
+            -0.04191872850060463,
+            -0.042030803859233856,
+            0.013421737588942051,
+            -0.045663248747587204,
+            0.024831216782331467,
+            0.06314653903245926,
+            0.013705547899007797,
+            0.025637097656726837,
+            -0.006122317630797625,
+            0.0041285231709480286,
+            0.050409767776727676,
+            0.007197089493274689,
+            -0.01965370774269104,
+            -0.04048306494951248,
+            0.11998444050550461,
+            0.029942067340016365,
+            0.02599455416202545,
+            -0.057833291590213776,
+            0.0033883019350469112,
+            0.00468824477866292,
+            -0.01925582066178322,
+            -0.01766190119087696,
+            0.011122050695121288,
+            0.04823627695441246,
+            0.018773270770907402,
+            -0.020368080586194992,
+            -0.009206349961459637,
+            0.031074542552232742,
+            0.02497885189950466,
+            0.0031681342516094446,
+            0.015077338553965092,
+            -0.022211533039808273,
+            0.058754149824380875,
+            -0.016073331236839294,
+            -0.014968045987188816,
+            -0.0051240865141153336,
+            -0.06383436918258667,
+            -0.0280417762696743,
+            0.013401271775364876,
+            -0.006949563976377249,
+            -0.009025825187563896,
+            0.03748825564980507,
+            0.04152849316596985,
+            -0.03703063353896141,
+            0.0006073106196708977,
+            -0.019878407940268517,
+            -0.059219732880592346,
+            0.03231174871325493,
+            -0.012458872981369495,
+            -0.0006862205918878317,
+            -0.029703414067626,
+            -0.011737367138266563,
+            -0.01565374620258808,
+            -0.002873011166229844,
+            0.035379018634557724,
+            -0.025712305679917336,
+            0.027225548401474953,
+            -0.011701708659529686,
+            -0.020186487585306168,
+            -0.013381940312683582,
+            0.044779565185308456,
+            0.027129901573061943,
+            -0.03770675137639046,
+            -0.06656532734632492,
+            -0.04852313920855522,
+            -0.07922673970460892,
+            0.042464420199394226,
+            0.08760115504264832,
+            -0.01756269298493862,
+            0.025902874767780304,
+            -0.049739014357328415,
+            0.015325409360229969,
+            0.04406426474452019,
+            0.012947683222591877,
+            -0.022557666525244713,
+            -0.033376943320035934,
+            -0.12034522742033005,
+            0.019998058676719666,
+            0.04397791251540184,
+            0.024618806317448616,
+            -0.013922464102506638,
+            0.031511057168245316,
+            0.03906194865703583,
+            0.011382625438272953,
+            -0.027103818953037262,
+            0.04971625655889511,
+            0.051205385476350784,
+            -0.08501561731100082,
+            0.011972213163971901,
+            -0.018331818282604218,
+            -0.00884521659463644,
+            -0.0015008534537628293,
+            0.0827648937702179,
+            -0.03979771211743355,
+            0.0015674568712711334,
+            -0.014266063459217548,
+            -0.03932151570916176,
+            0.04269920662045479,
+            -0.059784602373838425,
+            0.01841970533132553,
+            0.06251460313796997,
+            -0.02819698490202427,
+            -0.040344759821891785,
+            0.0010407248046249151,
+            -0.034333355724811554,
+            -0.029237672686576843,
+            -0.0001084851028281264,
+            0.06710729002952576,
+            0.019469408318400383,
+            -0.01640215329825878,
+            0.019526075571775436,
+            0.007778842933475971,
+            0.03379968926310539,
+            0.030870657414197922,
+            -0.059688691049814224,
+            -0.05436835065484047,
+            0.053333111107349396,
+            0.004061849322170019,
+            -0.08632408827543259,
+            0.014255499467253685,
+            -0.05555962026119232,
+            0.010840730741620064,
+            -0.05179913341999054,
+            -0.007342956960201263,
+            0.0011719957692548633,
+            0.022990427911281586,
+            0.013041576370596886,
+            -0.026316920295357704,
+            -0.022087475284934044,
+            -0.02786962315440178,
+            0.013592005707323551,
+            0.021783264353871346,
+            -0.059460774064064026,
+            -0.029133567586541176,
+            -0.06166587024927139,
+            -0.055512115359306335,
+            0.004256486427038908,
+            0.0341678261756897,
+            0.011773993261158466,
+            -0.029188869521021843,
+            -0.021346861496567726,
+            -0.036212995648384094,
+            0.025272972881793976,
+            0.02215636521577835,
+            -0.03782811760902405,
+            0.01701144315302372,
+            -0.05073560029268265,
+            -0.06574195623397827,
+            0.012947561219334602,
+            0.003303218400105834,
+            -0.05047185719013214,
+            0.010198806412518024,
+            -0.04323785379528999,
+            -0.04194899648427963,
+            0.02726336568593979,
+            -0.015397109091281891,
+            -0.02849482372403145,
+            0.058862827718257904,
+            -0.0026129265315830708,
+            0.006432596128433943,
+            0.04382907226681709,
+            -0.05114292353391647,
+            -0.02147330716252327,
+            -0.05826929211616516,
+            0.046473387628793716,
+            -0.09205549210309982,
+            0.04540986940264702,
+            -0.006234755739569664,
+            -0.05360054224729538,
+            -0.012155161239206791,
+            -0.030249077826738358,
+            0.02822766825556755,
+            0.013851269148290157,
+            0.027002329006791115,
+            0.09613272547721863,
+            0.035666726529598236,
+            0.03504599630832672,
+            -0.00038134161150082946,
+            -0.06922309845685959,
+            0.016433153301477432,
+            -0.031455833464860916,
+            -0.018132444471120834,
+            -0.02008064091205597,
+            -0.015955988317728043,
+            -0.04022971913218498,
+            -0.00230028061196208,
+            -0.06941505521535873,
+            0.0230435561388731,
+            -0.026967540383338928,
+            0.0354134738445282,
+            -0.08307641744613647,
+            0.055718302726745605,
+            0.0012352125486359,
+            0.017340224236249924,
+            0.02709241770207882,
+            -0.009195402264595032,
+            0.020474854856729507,
+            0.0016901030903682113,
+            0.05093026161193848,
+            -0.02238425612449646,
+            0.011796950362622738,
+            -0.007241291459649801,
+            -0.0334753580391407,
+            -0.04778272658586502,
+            -0.030247407034039497,
+            -0.012979192659258842,
+            0.004056413192301989,
+            0.015001167543232441,
+            0.06737781316041946,
+            0.028164206072688103,
+            0.0028011424001306295,
+            -0.049282923340797424,
+            0.06260383874177933,
+            0.04237203299999237,
+            -0.026161646470427513,
+            0.02427232824265957,
+            0.021224258467555046,
+            0.002963172970339656,
+            -0.049155037850141525,
+            0.033326923847198486,
+            0.07168576121330261,
+            -0.04409810155630112,
+            -0.012802177108824253,
+            0.011941076256334782,
+            0.005057428497821093,
+            -0.04857957363128662,
+            -0.011230324395000935,
+            0.009986268356442451,
+            0.010389930568635464,
+            -0.013448627665638924,
+            -0.04319113492965698,
+            -0.02839748188853264,
+            0.011157489381730556,
+            0.015462666749954224,
+            -0.014774681068956852,
+            -0.035400133579969406,
+            0.003983446396887302,
+            -0.06980624049901962,
+            -0.0019868735689669847,
+            -0.0014860559022054076,
+            -0.017261963337659836,
+            0.03138411417603493,
+            -0.07367079704999924,
+            0.025024767965078354,
+            0.037335801869630814,
+            0.04612639173865318,
+            -0.018027080222964287,
+            -0.015578734688460827,
+            -0.05632679536938667,
+            -0.01690700650215149,
+            0.023824671283364296,
+            0.003364108270034194,
+            -0.0478903129696846,
+            0.014160525053739548,
+            0.0023307709489017725,
+            0.028807908296585083,
+            0.0053710732609033585,
+            -0.007223619148135185,
+            -0.09570229798555374,
+            0.013001752085983753,
+            -0.03882845118641853,
+            -0.018106481060385704,
+            -0.08351759612560272,
+            -0.01296163722872734,
+            -0.0017098417738452554,
+            -0.042986027896404266,
+            0.02120766043663025,
+            -0.00032761419424787164,
+            0.059994783252477646,
+            0.00795682892203331,
+            0.025746053084731102,
+            0.026430919766426086,
+            0.10314885526895523,
+            -0.042013708502054214,
+            -0.01044819038361311,
+            -0.06457454711198807,
+            0.04287077486515045,
+            -0.0233222134411335,
+            -0.011595506221055984,
+            0.008520099334418774,
+            0.021662304177880287,
+            0.04874734953045845,
+            0.03213977813720703,
+            -0.03502868860960007,
+            -0.013689175248146057,
+            0.007175855804234743,
+            -0.06394322961568832,
+            -0.03230760619044304,
+            -0.0520993173122406,
+            0.03424723073840141,
+            -0.01675051636993885,
+            -0.04967552423477173,
+            0.03324288874864578,
+            -0.03822193667292595,
+            -0.015012933872640133,
+            -0.02746376395225525,
+            -0.015637405216693878,
+            0.040449269115924835,
+            -0.0027442676946520805,
+            0.008192671462893486,
+            0.013573664240539074,
+            -0.0065663764253258705,
+            0.07001614570617676,
+            -0.00289558875374496,
+            0.004224210977554321,
+            -0.05637960880994797,
+            -0.010168599896132946,
+            -0.02271331660449505,
+            0.0014612390659749508,
+            0.06994854658842087,
+            -0.00733678275719285,
+            -0.0025255896616727114,
+            0.03514084219932556,
+            0.02634606696665287,
+            -0.016171403229236603,
+            -0.02692556194961071,
+            0.015410004183650017,
+            0.07382199913263321,
+            0.01444800104945898,
+            -0.020071715116500854,
+            0.030701540410518646,
+            0.0056877885945141315,
+            0.011047931388020515,
+            -0.05641033127903938,
+            0.03570398688316345,
+            -0.06379767507314682,
+            0.09488129615783691,
+            0.015704551711678505,
+            -0.0008733674185350537,
+            0.009907273575663567,
+            0.004910382442176342,
+            0.050873469561338425,
+            0.01800096221268177,
+            -0.027450138702988625,
+            -0.001498246449045837,
+            -0.027504686266183853,
+            -0.019632702693343163
+        ],
+        "content_embedding": [
+            -0.0417482890188694,
+            0.0512668639421463,
+            0.0012354102218523622,
+            -0.035204555839300156,
+            0.028333576396107674,
+            0.006138786673545837,
+            0.017678435891866684,
+            0.004378852900117636,
+            -0.022564459592103958,
+            -0.03274708241224289,
+            -0.06855575740337372,
+            0.03446828946471214,
+            0.03136003389954567,
+            -0.016096506267786026,
+            -0.007832110859453678,
+            0.01546874176710844,
+            0.025302864611148834,
+            -0.01542437169700861,
+            0.009685760363936424,
+            0.025153761729598045,
+            0.01136286836117506,
+            -0.03678102046251297,
+            0.01742858625948429,
+            -0.04800569638609886,
+            0.052324045449495316,
+            -0.0188713688403368,
+            -0.017203466966748238,
+            0.04401639476418495,
+            -0.05147295445203781,
+            -0.005816930439323187,
+            0.04151167348027229,
+            0.0020627069752663374,
+            0.008849645033478737,
+            -0.03293370082974434,
+            -0.030744211748242378,
+            -0.025762831792235374,
+            0.07024409621953964,
+            -0.029683783650398254,
+            -0.02081390842795372,
+            0.034864746034145355,
+            0.057659171521663666,
+            0.009455090388655663,
+            -0.001964752795174718,
+            -0.028249403461813927,
+            0.045469045639038086,
+            0.010203365236520767,
+            0.039163172245025635,
+            -0.01693413034081459,
+            0.03357663378119469,
+            -0.016916338354349136,
+            0.007125346455723047,
+            -0.02135808765888214,
+            -0.007920235395431519,
+            -0.014657854102551937,
+            -0.0023566402960568666,
+            0.026274284347891808,
+            -0.0449351891875267,
+            0.006130301393568516,
+            0.0021915079560130835,
+            -0.05063489079475403,
+            0.010083623230457306,
+            0.03967271372675896,
+            -0.047972869127988815,
+            0.011878297664225101,
+            -0.02869013138115406,
+            -0.06947814673185349,
+            0.012776396237313747,
+            -0.022227533161640167,
+            -0.021391209214925766,
+            -0.0071424199268221855,
+            -0.010884602554142475,
+            0.0022353651002049446,
+            0.04208262637257576,
+            0.04585080221295357,
+            -0.028864840045571327,
+            0.014383035711944103,
+            0.0006865983596071601,
+            -0.003945623058825731,
+            -0.024596840143203735,
+            0.02039221115410328,
+            0.05236830934882164,
+            -0.06231372430920601,
+            0.0006878590793348849,
+            0.005045242141932249,
+            0.04543100297451019,
+            -0.022787010297179222,
+            -0.0323825404047966,
+            0.0060617332346737385,
+            -0.0009496629354543984,
+            0.1132081151008606,
+            0.021422259509563446,
+            -0.008516624569892883,
+            0.011941758915781975,
+            0.06050655618309975,
+            -0.06464048475027084,
+            0.0715012326836586,
+            -0.04892478510737419,
+            -0.014262699522078037,
+            0.02197115309536457,
+            -0.02258905954658985,
+            -0.03329572454094887,
+            0.0733470693230629,
+            0.01521797850728035,
+            -0.02922399342060089,
+            0.05403874069452286,
+            -0.0024076823610812426,
+            -0.005156014114618301,
+            -0.0004758739669341594,
+            -0.0009397549438290298,
+            0.022768890485167503,
+            -0.06273472309112549,
+            -0.013565277680754662,
+            -0.038060612976551056,
+            0.03901419788599014,
+            -0.025413114577531815,
+            -0.031085047870874405,
+            0.062427643686532974,
+            -0.05666875094175339,
+            0.018170330673456192,
+            0.03758049011230469,
+            -0.005046131554991007,
+            -0.03363005071878433,
+            0.0071977670304477215,
+            -0.007294844835996628,
+            0.04950850084424019,
+            0.05829211696982384,
+            -0.028599455952644348,
+            -0.00011273028212599456,
+            -0.027114247903227806,
+            -0.04813091829419136,
+            0.03546503558754921,
+            -0.0017865434056147933,
+            -0.06174362823367119,
+            0.015936153009533882,
+            0.05498664081096649,
+            0.06208323314785957,
+            -0.06043750047683716,
+            -0.07075081020593643,
+            0.03265148773789406,
+            0.01779918558895588,
+            -0.004657578654587269,
+            0.013401461765170097,
+            -0.031561195850372314,
+            -0.010674675926566124,
+            0.02138788439333439,
+            -0.059565648436546326,
+            0.003320328425616026,
+            -0.0016824831254780293,
+            -0.021733451634645462,
+            0.048551496118307114,
+            -0.003053524298593402,
+            0.011647860519587994,
+            -0.0014629715587943792,
+            0.059308186173439026,
+            0.0077448501251637936,
+            -0.01239799801260233,
+            -0.039145924150943756,
+            0.016731932759284973,
+            0.062229979783296585,
+            -0.029277512803673744,
+            0.05666857957839966,
+            0.021947506815195084,
+            -0.027742277830839157,
+            0.05703498050570488,
+            -0.02114000730216503,
+            -0.0011631653178483248,
+            0.04833010211586952,
+            0.013655254617333412,
+            0.042764052748680115,
+            0.04422000050544739,
+            -0.010796190239489079,
+            -0.0081519465893507,
+            0.0005064443103037775,
+            -0.007894535548985004,
+            0.01271637249737978,
+            0.0280605535954237,
+            0.023104701191186905,
+            -0.05545410141348839,
+            0.03579716384410858,
+            -0.01674344576895237,
+            0.011995082721114159,
+            -0.04967891052365303,
+            0.018647905439138412,
+            -0.0025427585933357477,
+            -0.05248319357633591,
+            -0.004207089077681303,
+            0.0029677890706807375,
+            -0.08436138927936554,
+            0.011933421716094017,
+            -0.046401966363191605,
+            0.004982754122465849,
+            -0.03336072713136673,
+            0.007464535068720579,
+            -0.02536672353744507,
+            -0.02103051170706749,
+            -0.0247516930103302,
+            0.03470923379063606,
+            0.008188062347471714,
+            0.04575216770172119,
+            -0.04027656093239784,
+            -0.028462760150432587,
+            -0.00641157990321517,
+            -0.1032537892460823,
+            0.015407266095280647,
+            -0.017259350046515465,
+            0.057880233973264694,
+            0.02970932051539421,
+            0.003135938895866275,
+            0.04052228853106499,
+            0.006307818461209536,
+            0.09373948723077774,
+            0.012201530858874321,
+            0.01518191210925579,
+            0.005055180750787258,
+            -0.00017229207151103765,
+            -0.008860277943313122,
+            -0.0009321855613961816,
+            -0.024702103808522224,
+            -0.02220877818763256,
+            0.018036337569355965,
+            0.0461902916431427,
+            -2.3178456103778444e-05,
+            -0.021639293059706688,
+            -0.009496558457612991,
+            -0.0069047678261995316,
+            -0.005369818769395351,
+            -0.038412243127822876,
+            0.0376049242913723,
+            -0.02614714205265045,
+            0.010913437232375145,
+            -0.02533271722495556,
+            -0.08659890294075012,
+            0.05744393169879913,
+            0.012141053564846516,
+            0.060547053813934326,
+            -0.0005550469504669309,
+            -0.01619824767112732,
+            -0.0022558700293302536,
+            0.01814994402229786,
+            0.06237058714032173,
+            0.055474210530519485,
+            -0.02512912079691887,
+            0.010455053299665451,
+            -0.023948650807142258,
+            -0.07459914684295654,
+            -0.006999264471232891,
+            -0.006154322065412998,
+            -0.014305580407381058,
+            -0.042501892894506454,
+            0.04605546593666077,
+            -0.007378050591796637,
+            0.013837042264640331,
+            0.005601659417152405,
+            -0.02454686351120472,
+            0.0228840634226799,
+            -0.010892537422478199,
+            0.0011435768101364374,
+            0.027678560465574265,
+            0.015353331342339516,
+            -0.03731193020939827,
+            0.05862969905138016,
+            -0.02842552959918976,
+            0.03124571032822132,
+            0.02315538190305233,
+            0.012950807809829712,
+            0.026965327560901642,
+            0.009465894661843777,
+            -0.010829408653080463,
+            -0.008594458922743797,
+            0.014982074499130249,
+            0.021298887208104134,
+            -0.018343189731240273,
+            0.01739460788667202,
+            0.07865084707736969,
+            0.02205476351082325,
+            0.015017225407063961,
+            -0.011981618590652943,
+            -0.02248695306479931,
+            0.017631210386753082,
+            -0.02025180496275425,
+            -0.05385996773838997,
+            -0.05477667227387428,
+            -0.042989905923604965,
+            0.004830287769436836,
+            0.03188111260533333,
+            0.048253823071718216,
+            0.0027890182100236416,
+            -0.01684093475341797,
+            0.029284454882144928,
+            -0.014463928528130054,
+            0.029999280348420143,
+            0.013334669172763824,
+            -0.030123639851808548,
+            0.007939296774566174,
+            -0.015909308567643166,
+            0.03652086481451988,
+            0.043923694640398026,
+            -0.03349898010492325,
+            0.016639679670333862,
+            0.007404185365885496,
+            -0.023147881031036377,
+            0.004568914417177439,
+            0.008112411946058273,
+            -0.021877270191907883,
+            -0.0072467140853405,
+            -0.024027734994888306,
+            0.022522028535604477,
+            0.03248016908764839,
+            0.04624137282371521,
+            0.03288194164633751,
+            -0.0706077441573143,
+            0.00647892989218235,
+            -0.013711459934711456,
+            -0.00910367164760828,
+            -0.06070556864142418,
+            0.013195404782891273,
+            0.02949078381061554,
+            -0.04314878582954407,
+            0.03952472656965256,
+            -0.039313577115535736,
+            0.01958983577787876,
+            -0.04745025932788849,
+            0.011169768869876862,
+            -0.07735665887594223,
+            0.012919869273900986,
+            0.08162245899438858,
+            0.04961969330906868,
+            0.02261139266192913,
+            -0.01081178616732359,
+            -0.022089937701821327,
+            0.036029793322086334,
+            0.07065453380346298,
+            -0.050287678837776184,
+            0.009584897197782993,
+            -0.005743148736655712,
+            -0.03252799063920975,
+            -0.029911693185567856,
+            -0.0031824579928070307,
+            -0.0022875897120684385,
+            0.010553253814578056,
+            -0.005088122095912695,
+            -0.019103137776255608,
+            -0.029758833348751068,
+            -0.03040270134806633,
+            -0.05643913522362709,
+            -0.0183008573949337,
+            0.0036066959146410227,
+            -0.010227258317172527,
+            0.03830184414982796,
+            0.008860573172569275,
+            -0.04022029787302017,
+            0.016092464327812195,
+            -0.007906369864940643,
+            -0.0206406619399786,
+            -0.01545781921595335,
+            0.011720928363502026,
+            0.030331697314977646,
+            0.020348263904452324,
+            0.013491041027009487,
+            0.015015012584626675,
+            0.0757412239909172,
+            -0.013692211359739304,
+            0.0554184690117836,
+            -0.03535052016377449,
+            0.027659131214022636,
+            0.062012042850255966,
+            0.05365491285920143,
+            0.02611374668776989,
+            0.03400697186589241,
+            -0.0187185350805521,
+            0.030734656378626823,
+            -0.04378894716501236,
+            0.04222285747528076,
+            0.06321597844362259,
+            0.0926889032125473,
+            0.06395434588193893,
+            -0.045033425092697144,
+            -0.02227518893778324,
+            -0.018914448097348213,
+            -0.024137776345014572,
+            0.06653360277414322,
+            0.03000609017908573,
+            0.016536613926291466,
+            -0.05106441304087639,
+            0.009556908160448074,
+            -0.003165673930197954,
+            -0.02989509329199791,
+            -0.008909299969673157,
+            -0.002428715117275715,
+            -0.038857024163007736,
+            -0.014716073870658875,
+            -0.02291145734488964,
+            -0.03815469145774841,
+            0.018349675461649895,
+            -0.001724440953694284,
+            0.024225711822509766,
+            -0.038882117718458176,
+            0.013145080767571926,
+            0.013105038553476334,
+            0.033219680190086365,
+            -0.04639777913689613,
+            -0.044315461069345474,
+            -0.012929159216582775,
+            0.003259071381762624,
+            0.012331360019743443,
+            -1.7462354662711732e-05,
+            -0.02317662350833416,
+            -0.042660780251026154,
+            0.001802539685741067,
+            -0.041100095957517624,
+            0.04925210401415825,
+            0.047337062656879425,
+            -0.01313596311956644,
+            -0.048633869737386703,
+            0.06100405752658844,
+            -0.024509647861123085,
+            0.06903672963380814,
+            0.026338376104831696,
+            -0.05955340713262558,
+            -0.013524221256375313,
+            -0.007072206120938063,
+            -0.0004094979085493833,
+            0.02331911027431488,
+            0.006079655606299639,
+            -0.027727166190743446,
+            0.01562763936817646,
+            0.011910749599337578,
+            -0.010385152883827686,
+            0.02091721072793007,
+            -0.030102524906396866,
+            -0.014945127069950104,
+            0.007444288115948439,
+            -0.009210431948304176,
+            0.01587914675474167,
+            -0.07968660444021225,
+            0.034870292991399765,
+            0.04423568770289421,
+            0.05101220682263374,
+            -0.0018310192972421646,
+            0.04378198832273483,
+            0.008875945582985878,
+            -0.018744593486189842,
+            -0.010748499073088169,
+            -0.05976865068078041,
+            -0.024797234684228897,
+            0.02921747788786888,
+            0.08715134114027023,
+            -0.014189728535711765,
+            -0.05772070586681366,
+            -0.00013612159818876535,
+            0.034182313829660416,
+            0.02940666675567627,
+            -0.007551911287009716,
+            0.005196248646825552,
+            0.09129910916090012,
+            0.03463520109653473,
+            -0.028487645089626312,
+            -0.054952532052993774,
+            -0.019425109028816223,
+            0.04267658665776253,
+            -0.010463536716997623,
+            -0.022979862987995148,
+            0.003282969817519188,
+            0.032446060329675674,
+            -0.03184691444039345,
+            -0.01494336687028408,
+            0.027125416323542595,
+            -0.03301938623189926,
+            -0.021615097299218178,
+            0.01919432356953621,
+            -0.018361948430538177,
+            0.0440165251493454,
+            0.018785251304507256,
+            0.05379289388656616,
+            -0.06103529781103134,
+            -0.04040123522281647,
+            0.06767034530639648,
+            -0.04255857691168785,
+            0.059002116322517395,
+            0.06269264966249466,
+            0.04158494248986244,
+            0.016211502254009247,
+            -0.046843864023685455,
+            -0.028105739504098892,
+            0.007073850836604834,
+            0.052667438983917236,
+            0.00735336821526289,
+            0.017733542248606682,
+            -0.023568013682961464,
+            -0.007077949587255716,
+            0.01566276140511036,
+            0.048224493861198425,
+            -0.0003875133115798235,
+            -0.046327680349349976,
+            -0.08656812459230423,
+            -0.025123324245214462,
+            -0.034193720668554306,
+            0.03014206700026989,
+            0.05021859332919121,
+            -0.0026385232340544462,
+            0.023799851536750793,
+            -0.009769299067556858,
+            -0.01290298905223608,
+            0.004491783678531647,
+            -0.012223453260958195,
+            -0.00033618492307141423,
+            0.0233011394739151,
+            -0.08696971833705902,
+            0.058488454669713974,
+            0.000664825493004173,
+            0.039359770715236664,
+            0.014214815571904182,
+            0.03424450755119324,
+            0.05592956021428108,
+            0.016471324488520622,
+            -0.059732481837272644,
+            0.06536833196878433,
+            0.024387361481785774,
+            -0.10856911540031433,
+            0.06828989833593369,
+            0.0036337117198854685,
+            0.05830007046461105,
+            0.016170067712664604,
+            0.013002794235944748,
+            -0.11607159674167633,
+            0.0019640743266791105,
+            0.026027854532003403,
+            -0.028382647782564163,
+            0.041647832840681076,
+            -0.005042455159127712,
+            -0.0010717103723436594,
+            0.09709432721138,
+            0.018342554569244385,
+            -0.03699033707380295,
+            0.03425338864326477,
+            -0.07419072836637497,
+            -0.05410637706518173,
+            0.013680101372301579,
+            -0.007827416993677616,
+            -0.007252392824739218,
+            0.016606653109192848,
+            0.015743359923362732,
+            -0.007168450392782688,
+            0.030557913705706596,
+            0.010715801268815994,
+            -0.03387424722313881,
+            -0.059598296880722046,
+            0.061636339873075485,
+            -0.024311335757374763,
+            -0.08930302411317825,
+            0.04300369694828987,
+            -0.052911426872015,
+            0.048922792077064514,
+            -0.07488273829221725,
+            0.0253959558904171,
+            -0.057005614042282104,
+            -0.010324039496481419,
+            -0.03382004797458649,
+            0.01331509929150343,
+            -0.0060559725388884544,
+            0.021830739453434944,
+            0.0004554805636871606,
+            0.06132755056023598,
+            -0.04885099083185196,
+            0.01681993156671524,
+            -0.09306737780570984,
+            -0.03891037777066231,
+            0.03394221141934395,
+            0.03513973951339722,
+            9.119489550357684e-05,
+            -0.009680265560746193,
+            -0.018936453387141228,
+            0.002022465690970421,
+            0.03725491091609001,
+            -0.007916543632745743,
+            -0.05493376404047012,
+            0.06674706935882568,
+            -0.04586830735206604,
+            -0.05310272425413132,
+            -0.002019708277657628,
+            -0.03419820964336395,
+            -0.08405481278896332,
+            0.044505130499601364,
+            -0.022271662950515747,
+            0.008551442995667458,
+            0.024632176384329796,
+            -0.057307109236717224,
+            -0.025764044374227524,
+            0.05102856457233429,
+            -0.01996302232146263,
+            -0.003182733431458473,
+            0.010233199223876,
+            -0.005380541551858187,
+            -0.033068619668483734,
+            -0.038329556584358215,
+            0.041149478405714035,
+            -0.038474202156066895,
+            0.03263046592473984,
+            0.043984752148389816,
+            -0.06405626237392426,
+            -0.04378855600953102,
+            -0.017724154517054558,
+            -0.00023550254991278052,
+            -0.006340715568512678,
+            0.008379276841878891,
+            0.06068692356348038,
+            -0.023048071190714836,
+            0.04665880277752876,
+            -0.026433007791638374,
+            -0.04106089845299721,
+            0.008102682419121265,
+            -0.02919689752161503,
+            -0.002803279785439372,
+            0.00115284975618124,
+            -0.007610488682985306,
+            -0.009425876662135124,
+            0.014759095385670662,
+            -0.07407978177070618,
+            0.040522702038288116,
+            -0.028428586199879646,
+            -0.015484650619328022,
+            -0.08971428871154785,
+            0.04541322588920593,
+            0.01523630227893591,
+            -0.02159152925014496,
+            0.06348283588886261,
+            -0.0762605369091034,
+            0.008550439029932022,
+            -0.0010396456345915794,
+            0.09191705286502838,
+            0.01919129304587841,
+            0.012690366245806217,
+            -0.032078325748443604,
+            -0.03879883140325546,
+            -0.06354136019945145,
+            -0.016241934150457382,
+            -0.013353055343031883,
+            0.013797549530863762,
+            0.03027600795030594,
+            0.05205754190683365,
+            0.018223397433757782,
+            -0.02529638260602951,
+            -0.012619049288332462,
+            0.05183516442775726,
+            0.04441876709461212,
+            -0.0014240458840504289,
+            -0.004662310238927603,
+            0.007740246132016182,
+            -0.023739585652947426,
+            0.008351752534508705,
+            0.04186442866921425,
+            0.06846421957015991,
+            -0.03302106633782387,
+            -0.019061105325818062,
+            0.03688846528530121,
+            0.027123648673295975,
+            -0.008548760786652565,
+            0.006452383007854223,
+            -0.05057734623551369,
+            0.009094422683119774,
+            -0.003088460536673665,
+            -0.01042612362653017,
+            0.03579631447792053,
+            -0.008917502127587795,
+            0.010444638319313526,
+            -0.023657843470573425,
+            -0.03254014626145363,
+            -0.0009533764678053558,
+            0.00684812106192112,
+            0.01948300190269947,
+            0.00943666510283947,
+            -0.010625068098306656,
+            0.02385423146188259,
+            -0.05145318806171417,
+            -0.03215208277106285,
+            0.007343036122620106,
+            0.01264273189008236,
+            0.036680057644844055,
+            0.022073568776249886,
+            -0.06296181678771973,
+            -0.008569572120904922,
+            -0.012322318740189075,
+            0.021164294332265854,
+            -0.051289938390254974,
+            0.0010486009996384382,
+            0.0021613994613289833,
+            0.030476249754428864,
+            0.01092084776610136,
+            -0.054112132638692856,
+            -0.06015515327453613,
+            0.023149874061346054,
+            0.03427460417151451,
+            -0.019571471959352493,
+            -0.07272381335496902,
+            0.009794066660106182,
+            -0.04319072142243385,
+            -0.04802769050002098,
+            -0.0024639740586280823,
+            0.01276618055999279,
+            0.030480578541755676,
+            -0.007069519720971584,
+            0.026940204203128815,
+            -0.013154259882867336,
+            0.05308559536933899,
+            0.0008981192368082702,
+            -0.012286764569580555,
+            -0.010251149535179138,
+            0.056114789098501205,
+            -0.027719540521502495,
+            -0.06385437399148941,
+            -0.01707690954208374,
+            0.03182663023471832,
+            0.04629168286919594,
+            0.003105542855337262,
+            -0.035991836339235306,
+            0.030695278197526932,
+            -0.01389816403388977,
+            0.005694018676877022,
+            0.024141885340213776,
+            -0.056052565574645996,
+            0.07325056940317154,
+            -0.052376989275217056,
+            -0.0827232152223587,
+            0.07083716243505478,
+            -0.011363365687429905,
+            -0.049301743507385254,
+            0.01282532885670662,
+            0.029815899208188057,
+            0.0025471607223153114,
+            0.014735412783920765,
+            -0.017929038032889366,
+            -0.017711512744426727,
+            -0.03859850764274597,
+            0.020923320204019547,
+            -0.024983150884509087,
+            -0.009905354119837284,
+            -0.033428773283958435,
+            0.0033264297526329756,
+            -0.057740144431591034,
+            0.0011588952038437128,
+            0.05510108917951584,
+            -0.042752135545015335,
+            0.00023805272940080613,
+            0.02830038219690323,
+            -0.0023612258955836296,
+            0.004450241569429636,
+            -0.09065061807632446,
+            0.05099336430430412,
+            0.050836946815252304,
+            0.002225160365924239,
+            -0.02620827779173851,
+            -0.0017080202233046293,
+            -0.006798254791647196,
+            -0.06735426932573318,
+            -0.012160968966782093,
+            0.0198799017816782,
+            -0.005785979796200991,
+            0.030539529398083687,
+            0.03791653737425804,
+            -0.01715696230530739,
+            -0.013931870460510254,
+            -0.026593970134854317,
+            0.015033211559057236,
+            0.04166087508201599,
+            -0.0009548550006002188,
+            0.002680635079741478,
+            -0.005521025042980909,
+            -0.005426781252026558
+        ],
+        "chunk_ind": 0
+    },
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/customer_support",
+        "title": "Customer Support",
+        "content": "Help your customer support team instantly answer any question across your entire product.\n\nAI Enabled Support\nCustomer support agents have one of the highest breadth jobs. They field requests that cover the entire surface area of the product and need to help your users find success on extremely short timelines. Because they're not the same people who designed or built the system, they often lack the depth of understanding needed - resulting in delays and escalations to other teams. Modern teams are leveraging AI to help their CS team optimize the speed and quality of these critical customer-facing interactions.\n\nThe Importance of Context\nThere are two critical components of AI copilots for customer support. The first is that the AI system needs to be connected with as much information as possible (not just support tools like Zendesk or Intercom) and that the knowledge needs to be as fresh as possible. Sometimes a fix might even be in places rarely checked by CS such as pull requests in a code repository. The second critical component is the ability of the AI system to break down difficult concepts and convoluted processes into more digestible descriptions and for your team members to be able to chat back and forth with the system to build a better understanding.\n\nDanswer takes care of both of these. The system connects up to over 30+ different applications and the knowledge is pulled in constantly so that the information access is always up to date.",
+        "title_embedding": [
+            0.029702378436923027,
+            0.04087577760219574,
+            0.011759690940380096,
+            -0.015867559239268303,
+            0.051922496408224106,
+            0.04818926751613617,
+            -0.051036059856414795,
+            -0.000768028199672699,
+            -0.011680016294121742,
+            -0.04731091484427452,
+            -0.06819964945316315,
+            0.02706378884613514,
+            0.012011447921395302,
+            -0.0162456426769495,
+            0.00561815220862627,
+            -0.060200855135917664,
+            0.011203722096979618,
+            0.011659571900963783,
+            0.005986262112855911,
+            0.010378050617873669,
+            -0.02058245800435543,
+            -0.007857420481741428,
+            -0.011501888744533062,
+            -0.06029190123081207,
+            -0.005981787107884884,
+            0.02316387929022312,
+            -0.003978169988840818,
+            0.014874234795570374,
+            -0.02605351060628891,
+            0.015183809213340282,
+            0.024635987356305122,
+            0.009090029634535313,
+            0.02292451448738575,
+            -0.051132138818502426,
+            -0.01627650111913681,
+            -0.04312199354171753,
+            0.055974528193473816,
+            -0.007541665807366371,
+            0.05875417962670326,
+            0.014909300021827221,
+            0.04143260419368744,
+            0.013145966455340385,
+            -0.0019000619649887085,
+            -0.014630978927016258,
+            0.036174625158309937,
+            -0.009382152929902077,
+            0.0129568912088871,
+            -0.01105422992259264,
+            0.05389830842614174,
+            -0.05418487638235092,
+            -0.019787268713116646,
+            0.021062593907117844,
+            -0.004908672533929348,
+            -0.05889728665351868,
+            0.005012272857129574,
+            -0.032561108469963074,
+            0.045741673558950424,
+            0.0023587732575833797,
+            -0.029787305742502213,
+            -0.016032742336392403,
+            -7.657184141862672e-06,
+            0.06451895087957382,
+            -0.061427876353263855,
+            0.05008486285805702,
+            0.013032016344368458,
+            -0.008349837735295296,
+            -0.023183466866612434,
+            0.028445789590477943,
+            0.013328451663255692,
+            0.002017433987930417,
+            0.08471205830574036,
+            0.02884836308658123,
+            0.03370589017868042,
+            0.02926166169345379,
+            -0.019738871604204178,
+            -0.06608780473470688,
+            -0.024426797404885292,
+            -0.008213629014790058,
+            -0.017787031829357147,
+            0.02045559138059616,
+            0.04079979658126831,
+            -0.03456271439790726,
+            0.011362932622432709,
+            -0.022804994136095047,
+            0.06335528194904327,
+            0.007143533322960138,
+            -0.038967471569776535,
+            0.01895124651491642,
+            -0.021929487586021423,
+            0.020517535507678986,
+            -0.005601715296506882,
+            0.048027630895376205,
+            0.05165044218301773,
+            0.021509770303964615,
+            -0.05171488970518112,
+            0.022959010675549507,
+            0.008313633501529694,
+            -0.033466871827840805,
+            -0.00873673614114523,
+            -0.07275433838367462,
+            -0.01826190948486328,
+            -0.0008458571974188089,
+            0.03251243755221367,
+            0.0027185927610844374,
+            0.03351648896932602,
+            0.050253089517354965,
+            0.03931482136249542,
+            0.011611105874180794,
+            0.0006847226177342236,
+            -0.018391452729701996,
+            -0.04876922070980072,
+            0.0032669915817677975,
+            0.002616048092022538,
+            0.018911289051175117,
+            0.0035516773350536823,
+            0.04444553330540657,
+            0.0874137431383133,
+            -0.06154795363545418,
+            -0.0007547208806499839,
+            0.05003364384174347,
+            0.047423556447029114,
+            0.010614278726279736,
+            -0.002085448009893298,
+            0.028551766648888588,
+            -0.00320938928052783,
+            -0.0028788738418370485,
+            -0.032166119664907455,
+            0.032534729689359665,
+            0.05165233090519905,
+            -0.02726086415350437,
+            0.032850414514541626,
+            0.021426480263471603,
+            0.008582738228142262,
+            -0.05970913544297218,
+            -0.013117690570652485,
+            0.0327497161924839,
+            -0.04084235802292824,
+            -0.034347862005233765,
+            0.08484583348035812,
+            -0.007416227832436562,
+            0.0817202478647232,
+            0.013180759735405445,
+            0.01752362586557865,
+            -0.011748716235160828,
+            0.006734360009431839,
+            -0.05940733850002289,
+            -0.01464597787708044,
+            0.021172411739826202,
+            0.015961064025759697,
+            0.013145568780601025,
+            -0.03292446956038475,
+            0.05433695763349533,
+            -0.04581886902451515,
+            -0.011024394072592258,
+            -0.013903305865824223,
+            -0.04525483399629593,
+            -0.009453397244215012,
+            -0.01541796326637268,
+            0.024864252656698227,
+            0.016035286709666252,
+            0.04684724658727646,
+            0.04711056500673294,
+            -0.08100881427526474,
+            0.07197079807519913,
+            -0.00942996796220541,
+            -0.04369058832526207,
+            0.02740531787276268,
+            0.027486257255077362,
+            0.035378992557525635,
+            0.05205152928829193,
+            0.000229495475650765,
+            -0.015225332230329514,
+            0.018225882202386856,
+            0.019075268879532814,
+            0.05653514340519905,
+            0.00206256122328341,
+            0.04172705486416817,
+            0.025263279676437378,
+            0.02995399944484234,
+            -0.02344629354774952,
+            -0.00580610940232873,
+            -0.03100346215069294,
+            0.024410588666796684,
+            0.05101815611124039,
+            -0.044721707701683044,
+            -0.03469102457165718,
+            0.015877151861786842,
+            -0.05606666952371597,
+            -0.04264648258686066,
+            8.914931095205247e-05,
+            0.005455580540001392,
+            0.02580810897052288,
+            0.03810019791126251,
+            -0.005504349246621132,
+            -0.03873325511813164,
+            -0.06938246637582779,
+            0.06987633556127548,
+            -0.0071804821491241455,
+            0.025614110752940178,
+            -0.04443173483014107,
+            0.014129945077002048,
+            0.04964412376284599,
+            -0.06445024162530899,
+            0.03494735807180405,
+            0.00042216022848151624,
+            0.03607922047376633,
+            0.014481625519692898,
+            -0.03348603844642639,
+            0.04130083695054054,
+            0.03306325525045395,
+            0.0670546144247055,
+            -0.004632187075912952,
+            -0.02899966388940811,
+            0.028892826288938522,
+            -0.02801397442817688,
+            -0.009044334292411804,
+            -0.0299741979688406,
+            0.006851669866591692,
+            -0.051097121089696884,
+            0.041360609233379364,
+            0.040426693856716156,
+            -0.04066675901412964,
+            0.01568625681102276,
+            -0.016009517014026642,
+            -0.004742924124002457,
+            0.048653047531843185,
+            -0.010993007570505142,
+            0.039591673761606216,
+            0.0007891886634752154,
+            0.0280364528298378,
+            -0.029024146497249603,
+            -0.07559369504451752,
+            0.020083241164684296,
+            -0.02160985954105854,
+            0.00466573191806674,
+            0.017442384734749794,
+            -0.02421753853559494,
+            0.05211988091468811,
+            0.0016645005671307445,
+            0.08051992207765579,
+            0.02786155417561531,
+            0.022991932928562164,
+            0.04568661376833916,
+            -0.03650546818971634,
+            -0.061528630554676056,
+            0.026744728907942772,
+            -0.029581749811768532,
+            -0.04499091953039169,
+            0.002152943518012762,
+            -0.015043909661471844,
+            0.047530874609947205,
+            0.041445743292570114,
+            0.041881438344717026,
+            0.03457123413681984,
+            -0.023201758041977882,
+            -0.05317879468202591,
+            -0.020216727629303932,
+            0.050812073051929474,
+            0.0008769077248871326,
+            -0.01832399144768715,
+            0.026449931785464287,
+            -0.00376958679407835,
+            0.04373340308666229,
+            -0.015004824846982956,
+            0.02940281480550766,
+            -0.028869349509477615,
+            -0.02610083483159542,
+            -0.0077619897201657295,
+            0.03147227317094803,
+            -0.0032539069652557373,
+            0.02559952437877655,
+            0.02357475273311138,
+            0.0001173858399852179,
+            0.047926079481840134,
+            0.03721074387431145,
+            -0.019753489643335342,
+            0.017013119533658028,
+            -0.028762344270944595,
+            -0.005848998669534922,
+            -0.006997200194746256,
+            0.05184704810380936,
+            -0.05036364868283272,
+            0.002436417154967785,
+            0.0003822402795776725,
+            0.003277599113062024,
+            0.03293520584702492,
+            -0.0045876270160079,
+            -0.004722336772829294,
+            -0.019277948886156082,
+            0.0012148021487519145,
+            -0.023630889132618904,
+            -0.011700155213475227,
+            -0.006263254676014185,
+            0.02274380624294281,
+            0.004808057565242052,
+            0.03601561859250069,
+            0.1043647825717926,
+            -0.014201398938894272,
+            -0.016617566347122192,
+            -0.02272864058613777,
+            -0.030905725434422493,
+            0.0010793384863063693,
+            -0.049122776836156845,
+            -0.005753105040639639,
+            0.01338914968073368,
+            0.027050666511058807,
+            0.04214894399046898,
+            -0.023005545139312744,
+            0.031917206943035126,
+            0.015109232626855373,
+            -0.07634111493825912,
+            0.024369796738028526,
+            -0.00647472171112895,
+            0.043127499520778656,
+            -0.0673207938671112,
+            0.0822305828332901,
+            0.06597486138343811,
+            0.004127069376409054,
+            0.06724239140748978,
+            0.004546293988823891,
+            0.03501193970441818,
+            -0.03256544470787048,
+            0.02815675362944603,
+            -0.018141930922865868,
+            0.008695757016539574,
+            0.030166303738951683,
+            -0.009897452779114246,
+            0.03613714128732681,
+            -0.013987270183861256,
+            -0.02539311721920967,
+            -0.06444346904754639,
+            -0.01528739370405674,
+            -0.06564117968082428,
+            -0.029204169288277626,
+            0.03283213824033737,
+            -0.018580380827188492,
+            0.026825398206710815,
+            -0.012654704041779041,
+            -0.0018552436958998442,
+            -0.01754637062549591,
+            0.08191259950399399,
+            0.0534062460064888,
+            -0.027856973931193352,
+            -0.053807660937309265,
+            -0.02944841794669628,
+            -0.014591488987207413,
+            -0.0004364093765616417,
+            -0.01691609062254429,
+            0.0792316198348999,
+            0.004102316685020924,
+            -0.03579891845583916,
+            -0.0108651639893651,
+            -0.00966869480907917,
+            -0.00033933919621631503,
+            -0.026145832613110542,
+            -0.016428470611572266,
+            0.030308935791254044,
+            -0.02421530708670616,
+            -0.01272093690931797,
+            0.0026039716321974993,
+            0.029183251783251762,
+            -0.015125368721783161,
+            0.07109745591878891,
+            0.02079625427722931,
+            0.018054857850074768,
+            -0.00201214081607759,
+            0.028579678386449814,
+            -0.011699595488607883,
+            0.010970978997647762,
+            -0.008872047066688538,
+            0.03169122710824013,
+            -0.06434084475040436,
+            0.03283708542585373,
+            0.002920384518802166,
+            0.1117773950099945,
+            0.0207917969673872,
+            -0.07546871900558472,
+            -0.0013152466854080558,
+            -0.009336701594293118,
+            0.00034176796907559037,
+            0.06051360070705414,
+            -0.0378379300236702,
+            -0.01082307007163763,
+            -0.009045763872563839,
+            -0.042135000228881836,
+            -0.04675054922699928,
+            0.0008745589875616133,
+            0.022651556879281998,
+            0.016891758888959885,
+            -0.06758315861225128,
+            0.03011692874133587,
+            -0.0689280554652214,
+            -0.0287728663533926,
+            -0.016613522544503212,
+            0.009947648271918297,
+            -0.009978469461202621,
+            0.016907479614019394,
+            0.00691134762018919,
+            0.04193537309765816,
+            -0.0010126983979716897,
+            -0.01135191135108471,
+            -0.04884914308786392,
+            0.05164073780179024,
+            0.06193321943283081,
+            -0.01663290709257126,
+            0.0512203685939312,
+            0.00277346046641469,
+            -0.0766502171754837,
+            0.0011106275487691164,
+            -0.008470023050904274,
+            0.03819086030125618,
+            -0.006837388966232538,
+            -0.03457418084144592,
+            -0.06866854429244995,
+            0.05483240634202957,
+            -0.016624240204691887,
+            0.028569897636771202,
+            -0.01400308683514595,
+            0.0378500260412693,
+            0.007686481345444918,
+            0.030080482363700867,
+            -0.05881612002849579,
+            0.015524756163358688,
+            0.030225753784179688,
+            -0.02950134687125683,
+            0.01465617585927248,
+            0.0009167538373731077,
+            0.056613512337207794,
+            -0.02706410363316536,
+            -0.0473414771258831,
+            -0.006343611981719732,
+            0.011811802163720131,
+            0.007573770359158516,
+            0.021041858941316605,
+            -0.014327406883239746,
+            0.01859954372048378,
+            0.06863977015018463,
+            -0.002199358306825161,
+            -0.03532129153609276,
+            0.009050965309143066,
+            0.02409159019589424,
+            -0.0025098449550569057,
+            -0.00499211298301816,
+            0.00033862097188830376,
+            -0.059937484562397,
+            0.10898157954216003,
+            0.0318506620824337,
+            0.0058680190704762936,
+            -0.03671310096979141,
+            -0.03349997103214264,
+            -0.0349581353366375,
+            -0.023411044850945473,
+            -0.011138128116726875,
+            0.00608166866004467,
+            0.08696430921554565,
+            -0.008161027915775776,
+            0.045368045568466187,
+            -0.01864445023238659,
+            0.035301174968481064,
+            0.003979773260653019,
+            0.016739632934331894,
+            0.011675872839987278,
+            0.025817174464464188,
+            0.03272102400660515,
+            0.013721601106226444,
+            -0.04690241813659668,
+            0.05665350705385208,
+            -0.05676185339689255,
+            0.013739561662077904,
+            0.020678944885730743,
+            -0.03532474488019943,
+            0.013464651070535183,
+            0.04246523231267929,
+            0.017674343660473824,
+            -0.005077862646430731,
+            -0.019556084647774696,
+            -0.009594413451850414,
+            -0.04825031757354736,
+            0.00016230896289926022,
+            0.003143883077427745,
+            0.031157106161117554,
+            -0.0332491435110569,
+            -0.010140872560441494,
+            -0.04249225929379463,
+            -0.028256090357899666,
+            -0.008995918557047844,
+            0.021160980686545372,
+            0.03130994364619255,
+            -0.001629085629247129,
+            0.02819039300084114,
+            0.009931858628988266,
+            0.05051739886403084,
+            0.006850008387118578,
+            -0.018732454627752304,
+            -0.09201951324939728,
+            -0.042829085141420364,
+            -0.054845187813043594,
+            0.021562401205301285,
+            0.05139476805925369,
+            -0.011137720197439194,
+            0.021749140694737434,
+            -0.01837606355547905,
+            0.017083071172237396,
+            0.023444999009370804,
+            -0.009212017990648746,
+            -0.011377239599823952,
+            -0.018595905974507332,
+            -0.09953112155199051,
+            0.0019816216081380844,
+            -0.004408092238008976,
+            -0.027098996564745903,
+            -0.002341427141800523,
+            0.029412943869829178,
+            0.041135817766189575,
+            0.015357858501374722,
+            0.0017108122119680047,
+            0.028364799916744232,
+            0.004185053985565901,
+            -0.04822831228375435,
+            0.0948278158903122,
+            -0.0414741188287735,
+            0.03572544828057289,
+            0.04517536610364914,
+            0.009942572563886642,
+            -0.025769881904125214,
+            -0.017749540507793427,
+            -0.03137620911002159,
+            -0.00617972994223237,
+            0.04517003893852234,
+            -0.03762981668114662,
+            0.0358721986413002,
+            0.11737086623907089,
+            0.0004563555121421814,
+            -0.06275733560323715,
+            -0.011418106034398079,
+            -0.08354005962610245,
+            -0.048093460500240326,
+            -0.030154218897223473,
+            -0.010961515828967094,
+            0.007697841618210077,
+            -0.08613990992307663,
+            0.01947987824678421,
+            0.017181523144245148,
+            0.02698543854057789,
+            0.040134966373443604,
+            -0.03223738074302673,
+            -0.03745822235941887,
+            0.054219458252191544,
+            -0.03571298345923424,
+            -0.035921428352594376,
+            0.04604002833366394,
+            -0.04030536487698555,
+            -0.032255616039037704,
+            -0.06959861516952515,
+            0.032114237546920776,
+            -0.027767114341259003,
+            0.019928939640522003,
+            0.022700339555740356,
+            -0.04375129193067551,
+            -0.05712258815765381,
+            -0.02559071220457554,
+            0.006574000231921673,
+            -0.025430524721741676,
+            -0.028035728260874748,
+            -0.04453514888882637,
+            -0.10015997290611267,
+            -0.0672021210193634,
+            0.012007188983261585,
+            0.014830735512077808,
+            0.00970692653208971,
+            -0.051091041415929794,
+            -0.031854890286922455,
+            -0.025864001363515854,
+            -0.016826078295707703,
+            -0.0026011785957962275,
+            -0.05528291314840317,
+            0.04440443217754364,
+            -0.03783581778407097,
+            -0.09876326471567154,
+            0.029767369851469994,
+            -0.0023010883014649153,
+            -0.05689188838005066,
+            0.007344242185354233,
+            0.009551416151225567,
+            -0.08018877357244492,
+            0.007934950292110443,
+            -0.03663663938641548,
+            -0.0009275389602407813,
+            0.026911364868283272,
+            -0.001246148720383644,
+            0.054533813148736954,
+            0.009721122682094574,
+            0.005318093113601208,
+            -0.00535608222708106,
+            0.008591657504439354,
+            0.04166155681014061,
+            -0.03877246752381325,
+            0.01399280782788992,
+            0.01457316055893898,
+            -0.02407732978463173,
+            -0.006206234451383352,
+            0.07004162669181824,
+            0.01619933731853962,
+            -0.004103302024304867,
+            0.045894261449575424,
+            0.03686122968792915,
+            0.04804258793592453,
+            0.05579492822289467,
+            -0.0030228029936552048,
+            -0.07259590923786163,
+            0.023546412587165833,
+            -0.0011336577590554953,
+            0.04003886505961418,
+            -0.0139979999512434,
+            -0.00017333473078906536,
+            -0.05454326048493385,
+            -0.021969035267829895,
+            -0.03901325911283493,
+            0.012417349964380264,
+            -0.05304381251335144,
+            0.004690664820373058,
+            -0.06269649416208267,
+            0.037584088742733,
+            -0.039430856704711914,
+            0.01138926949352026,
+            0.024504829198122025,
+            -0.023247476667165756,
+            -0.001942053553648293,
+            0.01666364073753357,
+            0.005435148254036903,
+            -0.026095639914274216,
+            -0.002955301431939006,
+            -0.043733760714530945,
+            -0.06700831651687622,
+            -0.06828623265028,
+            0.045247308909893036,
+            -0.02567214146256447,
+            -0.03503000736236572,
+            0.0028738975524902344,
+            0.007734893821179867,
+            0.041422292590141296,
+            -0.01760552078485489,
+            -0.016545895487070084,
+            -0.007150533143430948,
+            0.02324298955500126,
+            0.009319701232016087,
+            0.003866465063765645,
+            0.025515582412481308,
+            0.03634219616651535,
+            -0.033345021307468414,
+            0.020966341719031334,
+            0.07540836185216904,
+            -0.0002573730598669499,
+            0.05161430686712265,
+            0.0003421941655687988,
+            -0.016416313126683235,
+            -0.018457459285855293,
+            -0.04053647443652153,
+            -0.008631067350506783,
+            0.011850157752633095,
+            0.014364494942128658,
+            -0.029469167813658714,
+            -0.0497945100069046,
+            0.008882390335202217,
+            0.04006727412343025,
+            0.010511534288525581,
+            -0.03620539978146553,
+            -0.030285054817795753,
+            -0.10150802135467529,
+            -0.01594187133014202,
+            -0.05266118794679642,
+            -0.017643682658672333,
+            0.017300395295023918,
+            -0.080828458070755,
+            0.027883851900696754,
+            0.0260021835565567,
+            -0.013791227713227272,
+            0.01261923462152481,
+            0.02038010023534298,
+            -0.04244862496852875,
+            0.011010567657649517,
+            0.011758117005228996,
+            0.0030098427087068558,
+            0.003606629790738225,
+            0.0020418025087565184,
+            -0.0004243037255946547,
+            0.03312380611896515,
+            0.02103457599878311,
+            0.03972248733043671,
+            -0.07605717331171036,
+            -0.039676181972026825,
+            -0.07193399965763092,
+            -0.030211182311177254,
+            -0.0786738321185112,
+            -0.025149084627628326,
+            -0.0017661137972027063,
+            -0.017345670610666275,
+            0.027870142832398415,
+            -0.003350367769598961,
+            0.060104407370090485,
+            0.020051708444952965,
+            0.014986025169491768,
+            0.0056610992178320885,
+            0.0593392550945282,
+            -0.0053368182852864265,
+            -0.025954807177186012,
+            -0.07451668381690979,
+            -0.0021227921824902296,
+            -0.03457536920905113,
+            0.010495556518435478,
+            -0.0450577586889267,
+            -0.016477687284350395,
+            0.05702868103981018,
+            0.025161782279610634,
+            0.016737932339310646,
+            -0.00856244657188654,
+            0.01180358324199915,
+            -0.04093103110790253,
+            -0.03350433334708214,
+            -0.01662720926105976,
+            0.002310116309672594,
+            0.0026625224854797125,
+            -0.04226106405258179,
+            0.02971433289349079,
+            -0.07220850884914398,
+            -0.0009123267373070121,
+            -0.02786707505583763,
+            -0.025111757218837738,
+            0.033243875950574875,
+            0.03572067618370056,
+            -0.0019114067545160651,
+            -0.030592206865549088,
+            -0.021260922774672508,
+            0.06805034726858139,
+            0.00013317271077539772,
+            0.006557960994541645,
+            -0.0329759456217289,
+            0.006780629511922598,
+            0.00866342056542635,
+            -0.01449753437191248,
+            0.028198137879371643,
+            0.019570309668779373,
+            -0.036116212606430054,
+            0.058676715940237045,
+            0.021564209833741188,
+            -0.026323653757572174,
+            -0.02302497997879982,
+            0.02744974195957184,
+            0.05593085661530495,
+            0.02073318511247635,
+            -0.074001245200634,
+            -0.0184424240142107,
+            0.031868427991867065,
+            0.018198778852820396,
+            -0.0450170636177063,
+            0.030543111264705658,
+            -0.06377965956926346,
+            0.04590768367052078,
+            -0.005907150451093912,
+            -0.01748581975698471,
+            0.006950956769287586,
+            0.05506323277950287,
+            0.04594920575618744,
+            -0.015593858435750008,
+            -0.017087753862142563,
+            0.029356854036450386,
+            -0.06531023979187012,
+            0.005519233178347349
+        ],
+        "content_embedding": [
+            0.019554156810045242,
+            0.04886673390865326,
+            0.00547438021749258,
+            -0.02931770123541355,
+            0.024625789374113083,
+            0.008072949014604092,
+            0.03052533231675625,
+            -0.006179450079798698,
+            -0.0333777479827404,
+            -0.021061548963189125,
+            -0.05983448028564453,
+            0.025203319266438484,
+            0.032834798097610474,
+            0.013029776513576508,
+            -0.011046950705349445,
+            -0.031452618539333344,
+            -8.028985030250624e-05,
+            0.0007977305795066059,
+            -0.009920830838382244,
+            -0.0018231356516480446,
+            -0.006695937365293503,
+            -0.014926698990166187,
+            -0.009614776819944382,
+            -0.05784115567803383,
+            0.02651236765086651,
+            -0.011027690954506397,
+            -0.009276495315134525,
+            0.04284600168466568,
+            -0.05643690750002861,
+            0.04219788312911987,
+            0.05048283189535141,
+            0.004557965788990259,
+            -0.01679980754852295,
+            -0.040684137493371964,
+            -0.044776126742362976,
+            -0.018855446949601173,
+            0.06871335208415985,
+            -0.014481916092336178,
+            -0.024082450196146965,
+            0.04807353392243385,
+            0.04378245398402214,
+            -0.0010759941069409251,
+            0.014099782332777977,
+            -0.037625934928655624,
+            0.05659622326493263,
+            0.01868855021893978,
+            0.025368744507431984,
+            -0.03537469357252121,
+            0.014213587157428265,
+            -0.0705343633890152,
+            -0.016253503039479256,
+            -0.005150542128831148,
+            -0.017522070556879044,
+            -0.03862348571419716,
+            0.00953003577888012,
+            0.016891248524188995,
+            -0.007589798420667648,
+            0.006262748036533594,
+            0.01169696543365717,
+            -0.05435675010085106,
+            0.03128333017230034,
+            0.07449059188365936,
+            -0.043514277786016464,
+            0.022657765075564384,
+            0.01074683852493763,
+            -0.051405169069767,
+            0.00816179346293211,
+            -0.014555123634636402,
+            -0.01839461177587509,
+            -0.00691940588876605,
+            0.009614893235266209,
+            -0.0071630412712693214,
+            0.02593475580215454,
+            0.07137756794691086,
+            -0.049324654042720795,
+            -0.04460940882563591,
+            -0.004007663112133741,
+            -0.018305329605937004,
+            -0.029537511989474297,
+            0.03624692186713219,
+            0.04646339640021324,
+            -0.053126320242881775,
+            -0.008358806371688843,
+            -0.001911070430651307,
+            0.05718495324254036,
+            -0.02093559131026268,
+            -0.028956729918718338,
+            0.005082732532173395,
+            -0.028617851436138153,
+            0.1116873174905777,
+            0.006924519315361977,
+            0.05388922244310379,
+            0.04239538311958313,
+            0.03629518672823906,
+            -0.07756507396697998,
+            0.08572934567928314,
+            -0.034708015620708466,
+            -0.052686456590890884,
+            -0.03376411274075508,
+            -0.011243571527302265,
+            -0.028565097600221634,
+            0.03532436862587929,
+            0.025328388437628746,
+            -0.009712263941764832,
+            0.0416463203728199,
+            0.02975877933204174,
+            0.04073808714747429,
+            0.045180853456258774,
+            0.01522457879036665,
+            0.0021614122670143843,
+            -0.0602865032851696,
+            -0.024015003815293312,
+            -0.032889459282159805,
+            0.020978014916181564,
+            -0.04493942856788635,
+            0.012820002622902393,
+            0.0644337609410286,
+            -0.019941547885537148,
+            -0.004083186853677034,
+            0.018924430012702942,
+            -0.017330998554825783,
+            0.0008878704975359142,
+            0.02702835015952587,
+            -0.026265576481819153,
+            0.04109559580683708,
+            -0.009575187228620052,
+            -0.020085688680410385,
+            -5.433974365587346e-05,
+            0.018562229350209236,
+            -0.02393198385834694,
+            -0.030048735439777374,
+            0.01821220852434635,
+            -0.029501279816031456,
+            -0.01944204978644848,
+            -0.01614498719573021,
+            0.03840102627873421,
+            -0.04210539907217026,
+            -0.050642915070056915,
+            0.026918280869722366,
+            -0.008575397543609142,
+            0.011357792653143406,
+            0.015665695071220398,
+            0.01620817743241787,
+            -0.0165872685611248,
+            0.028748027980327606,
+            -0.10036404430866241,
+            -0.04525408893823624,
+            0.010306733660399914,
+            -0.04815903678536415,
+            0.025287121534347534,
+            -0.010369544848799706,
+            0.023846469819545746,
+            0.016379185020923615,
+            0.019874077290296555,
+            0.004489564802497625,
+            -0.009458004496991634,
+            -0.032719556242227554,
+            -0.0319439135491848,
+            0.06740261614322662,
+            0.005906871519982815,
+            0.024023521691560745,
+            0.04657802730798721,
+            -0.08182766288518906,
+            0.062149085104465485,
+            -0.0061394101940095425,
+            0.014895983040332794,
+            0.029691752046346664,
+            0.0031379619613289833,
+            0.04791149124503136,
+            0.06459061801433563,
+            -0.012314707040786743,
+            0.016569096595048904,
+            -0.01054114755243063,
+            -0.027581606060266495,
+            0.02896907925605774,
+            0.0048512346111238,
+            0.04427911341190338,
+            -0.026665540412068367,
+            0.07613496482372284,
+            -0.023571502417325974,
+            -0.004409146960824728,
+            -0.036679890006780624,
+            0.0016537840710952878,
+            0.024527747184038162,
+            -0.02983722649514675,
+            0.0021592022385448217,
+            0.005430649966001511,
+            -0.08198896050453186,
+            -0.017340485006570816,
+            -0.014370240271091461,
+            0.012875599786639214,
+            -0.008910057134926319,
+            -0.013228043913841248,
+            -0.0023926664143800735,
+            -0.015292157419025898,
+            -0.03927253186702728,
+            0.07715654373168945,
+            -0.012554320506751537,
+            0.03217530623078346,
+            -0.042463112622499466,
+            -0.03743144869804382,
+            0.008893481455743313,
+            -0.0666876956820488,
+            0.02818153239786625,
+            -0.009054000489413738,
+            0.03337392210960388,
+            0.03328379616141319,
+            0.009759150445461273,
+            0.01269217487424612,
+            -0.02173757553100586,
+            0.07456912100315094,
+            0.006997218355536461,
+            0.007162875030189753,
+            0.040701836347579956,
+            -0.04538433253765106,
+            -0.0032951829489320517,
+            0.002400761004537344,
+            -0.04169681295752525,
+            -0.05523253232240677,
+            0.06444490700960159,
+            0.03810356929898262,
+            -0.035072412341833115,
+            0.04415622353553772,
+            -0.02322838269174099,
+            -0.009917438961565495,
+            0.008140898309648037,
+            -0.02388846129179001,
+            0.0034457307774573565,
+            -0.0054973880760371685,
+            0.024084730073809624,
+            -0.007739664521068335,
+            -0.06571771949529648,
+            0.07359053194522858,
+            -0.011194998398423195,
+            0.0211270023137331,
+            0.017875710502266884,
+            -0.01821357198059559,
+            0.03745369240641594,
+            0.023267023265361786,
+            0.07556653767824173,
+            0.042081139981746674,
+            0.01543054636567831,
+            0.038234904408454895,
+            -0.019599558785557747,
+            -0.08676281571388245,
+            -0.013498742133378983,
+            -0.011986842378973961,
+            -0.014071784913539886,
+            -0.03772296756505966,
+            0.009192361496388912,
+            -0.020469751209020615,
+            -0.004528891295194626,
+            0.003913218155503273,
+            0.013755199499428272,
+            0.009065939113497734,
+            -0.049990858882665634,
+            0.030110390856862068,
+            0.05044790729880333,
+            0.020461000502109528,
+            -0.036168310791254044,
+            0.04875757917761803,
+            -0.023571951314806938,
+            0.04313709959387779,
+            0.02649555914103985,
+            0.0055029122158885,
+            0.0138795655220747,
+            -0.0009625149541534483,
+            -0.019915221258997917,
+            -0.0024100886657834053,
+            -0.00426845159381628,
+            0.0008668623049743474,
+            0.014360995963215828,
+            0.01953921653330326,
+            0.05843087658286095,
+            0.010600714012980461,
+            -0.01941727101802826,
+            0.016723858192563057,
+            -0.027911949902772903,
+            0.011009825393557549,
+            -0.0070054735988378525,
+            -0.02280472218990326,
+            -0.0902462899684906,
+            -0.029843643307685852,
+            -0.005726281087845564,
+            -0.007387514691799879,
+            0.05724000930786133,
+            -0.016834275797009468,
+            0.015216480940580368,
+            0.02209043875336647,
+            0.004264513496309519,
+            0.019342858344316483,
+            0.018849363550543785,
+            -0.04794900864362717,
+            0.02397482842206955,
+            -0.007828679867088795,
+            0.020298736169934273,
+            0.09897984564304352,
+            -0.04056645929813385,
+            -0.011319068260490894,
+            0.0034366592299193144,
+            -0.0073711141012609005,
+            -0.005176732316613197,
+            0.022680383175611496,
+            -0.01522906869649887,
+            0.002589346142485738,
+            -0.016521241515874863,
+            0.021019242703914642,
+            -0.004480182658880949,
+            0.009419859386980534,
+            0.013903859071433544,
+            -0.05053260177373886,
+            0.0012486587511375546,
+            0.017442021518945694,
+            -0.016688739880919456,
+            -0.034772295504808426,
+            0.03298048675060272,
+            0.04604269936680794,
+            -0.008270222693681717,
+            0.024096740409731865,
+            0.022777941077947617,
+            0.03941553831100464,
+            -0.030161242932081223,
+            -0.01781023107469082,
+            -0.03533001244068146,
+            0.02088671736419201,
+            0.0896298736333847,
+            0.0061553712002933025,
+            0.054883576929569244,
+            -0.04094908386468887,
+            0.04382561892271042,
+            -0.003113290062174201,
+            0.015607516281306744,
+            -0.04469290003180504,
+            -0.013229981996119022,
+            -0.0026423041708767414,
+            -0.017652839422225952,
+            0.005086386110633612,
+            0.035375673323869705,
+            9.725322161102667e-05,
+            -0.026332346722483635,
+            0.01958891935646534,
+            0.041558533906936646,
+            -0.02570311166346073,
+            -0.04806942865252495,
+            -0.035452235490083694,
+            -0.015692519024014473,
+            -0.0027702273800969124,
+            0.022831231355667114,
+            0.05247980356216431,
+            0.0010726081673055887,
+            -0.025438379496335983,
+            0.006880710367113352,
+            0.009276037104427814,
+            -0.021758491173386574,
+            -0.008571256883442402,
+            0.003312689019367099,
+            0.032590776681900024,
+            0.02640446089208126,
+            0.0034450399689376354,
+            0.028627941384911537,
+            0.07457757741212845,
+            -0.012303084135055542,
+            0.045139290392398834,
+            -3.2679530704626814e-05,
+            0.024825602769851685,
+            0.04402294382452965,
+            0.025363540276885033,
+            0.03840187191963196,
+            0.019677763804793358,
+            -0.03521053493022919,
+            0.036523785442113876,
+            -0.06001857668161392,
+            0.006586587987840176,
+            0.022668125107884407,
+            0.06063239648938179,
+            0.015586800873279572,
+            -0.08306828886270523,
+            -0.04289577156305313,
+            -0.0050475094467401505,
+            -0.0309798214584589,
+            0.05810924991965294,
+            -0.017818300053477287,
+            -0.01088999304920435,
+            -0.017884155735373497,
+            0.04356890171766281,
+            -0.023094575852155685,
+            -0.04477296397089958,
+            -0.0007801170577295125,
+            0.018146106973290443,
+            -0.05633535981178284,
+            0.006709645502269268,
+            -0.037334758788347244,
+            -0.055724598467350006,
+            0.00994165986776352,
+            -0.009849119931459427,
+            -0.027259083464741707,
+            -0.02565668150782585,
+            0.005235382355749607,
+            0.016267497092485428,
+            0.00393668282777071,
+            -0.05648971349000931,
+            -0.05955129489302635,
+            0.026542942970991135,
+            0.040565431118011475,
+            -0.02225298061966896,
+            0.0017030639573931694,
+            -0.02689032256603241,
+            -0.029025251045823097,
+            0.030817490071058273,
+            -0.007936912588775158,
+            0.05566547438502312,
+            0.0174697358161211,
+            -0.014709461480379105,
+            -0.07380940765142441,
+            0.07026955485343933,
+            -0.024563433602452278,
+            0.05333513021469116,
+            0.020963717252016068,
+            -0.015575055032968521,
+            -0.04304461553692818,
+            0.00822180975228548,
+            -0.013204170390963554,
+            -0.0028262599371373653,
+            0.015431943349540234,
+            -0.025627007707953453,
+            0.0006762628327123821,
+            -0.02078782208263874,
+            0.009704814292490482,
+            -0.006950112525373697,
+            -0.020425891503691673,
+            0.044901806861162186,
+            0.020927794277668,
+            0.009534145705401897,
+            0.004958992823958397,
+            -0.037563592195510864,
+            0.03806327283382416,
+            0.0783824622631073,
+            0.011150919832289219,
+            -0.024385575205087662,
+            0.03461897745728493,
+            0.02127663977444172,
+            -0.012272517196834087,
+            -0.01546854991465807,
+            -0.06705902516841888,
+            -0.01649612747132778,
+            0.06068763509392738,
+            0.07799869775772095,
+            0.0014717800077050924,
+            -0.04836009815335274,
+            -0.026833070442080498,
+            0.013509837910532951,
+            0.04280327260494232,
+            0.009658309631049633,
+            -0.007854060269892216,
+            0.09166036546230316,
+            -0.00760420598089695,
+            0.024465130642056465,
+            -0.041591379791498184,
+            0.007116211112588644,
+            0.05567977577447891,
+            0.01807284727692604,
+            0.028304288163781166,
+            -0.014866949990391731,
+            0.026369474828243256,
+            0.008983064442873001,
+            -0.02317068539559841,
+            0.03937782347202301,
+            -0.01901034638285637,
+            -0.0021325594279915094,
+            0.030924763530492783,
+            -0.020755570381879807,
+            0.030001072213053703,
+            0.03809978440403938,
+            0.0334426648914814,
+            -0.042968180030584335,
+            -0.012311535887420177,
+            0.03697645664215088,
+            -0.041293930262327194,
+            0.01696925237774849,
+            0.03560850769281387,
+            0.03847989812493324,
+            -0.037730954587459564,
+            -0.05352506786584854,
+            -0.02746652066707611,
+            -0.05294184759259224,
+            -0.017248092219233513,
+            -0.005418767221271992,
+            0.01951681822538376,
+            -0.017932193353772163,
+            -0.007422131486237049,
+            0.03827866166830063,
+            0.05701953545212746,
+            -0.02210610918700695,
+            0.010034722276031971,
+            -0.07954911887645721,
+            -0.0485968291759491,
+            -0.028629625216126442,
+            0.009054362773895264,
+            0.02398092858493328,
+            -0.009973667562007904,
+            0.0011409823782742023,
+            -0.011182617396116257,
+            0.0020028105936944485,
+            0.006942914333194494,
+            -0.039862822741270065,
+            -0.0066703930497169495,
+            -0.004236259963363409,
+            -0.1050848662853241,
+            0.025538505986332893,
+            0.029989799484610558,
+            0.01211432833224535,
+            0.04559238627552986,
+            0.050545401871204376,
+            0.05476491525769234,
+            0.01163802482187748,
+            -0.054837070405483246,
+            0.07410066574811935,
+            0.0024028397165238857,
+            -0.10432479530572891,
+            0.07078853249549866,
+            -0.012614017352461815,
+            0.06030529364943504,
+            0.054839566349983215,
+            0.014005501754581928,
+            -0.1118561178445816,
+            0.009275965392589569,
+            -0.023663705214858055,
+            -0.028527697548270226,
+            0.0584726445376873,
+            -0.045622922480106354,
+            0.03320262208580971,
+            0.11574956774711609,
+            0.007655338849872351,
+            -0.07191741466522217,
+            0.03859880194067955,
+            -0.06297247856855392,
+            -0.018405890092253685,
+            -0.028816718608140945,
+            -0.0126464469358325,
+            0.0202946774661541,
+            -0.03483844920992851,
+            0.024250855669379234,
+            -0.006263037212193012,
+            0.02315174601972103,
+            -0.0037849699147045612,
+            -0.0359908789396286,
+            -0.037620242685079575,
+            0.05056930333375931,
+            -0.006831671576946974,
+            -0.10494183748960495,
+            0.07645728439092636,
+            -0.028021620586514473,
+            0.009692930616438389,
+            -0.10232461243867874,
+            0.05202733352780342,
+            -0.05352945625782013,
+            0.02073156274855137,
+            -0.017188169062137604,
+            0.0020886484999209642,
+            -0.04133611172437668,
+            -0.044309552758932114,
+            0.018564041703939438,
+            0.026704275980591774,
+            -0.055197130888700485,
+            0.007287430576980114,
+            -0.1245347261428833,
+            -0.037194203585386276,
+            0.06665091216564178,
+            0.061706364154815674,
+            0.01804385520517826,
+            -0.013962237164378166,
+            -0.024994580075144768,
+            -0.011294208467006683,
+            -0.009245212189853191,
+            0.029879143461585045,
+            -0.046995047479867935,
+            0.02158532105386257,
+            -0.002750945743173361,
+            -0.08373189717531204,
+            0.006320012733340263,
+            -0.029947226867079735,
+            -0.09511808305978775,
+            0.03186006098985672,
+            0.0028628590516746044,
+            -0.02251911163330078,
+            0.03152289241552353,
+            -0.07237773388624191,
+            0.013269931077957153,
+            0.09650041908025742,
+            -0.045900944620370865,
+            -0.006541605107486248,
+            0.006844623479992151,
+            0.03238837793469429,
+            -0.020313216373324394,
+            -0.013520441018044949,
+            0.000403873244067654,
+            -0.024447409436106682,
+            -0.014962681569159031,
+            0.006884200032800436,
+            -0.04582136869430542,
+            -0.02838127687573433,
+            0.02936256304383278,
+            0.04077419266104698,
+            0.02266734093427658,
+            0.013416043482720852,
+            0.0345231331884861,
+            0.052821315824985504,
+            0.028492338955402374,
+            -0.01292923279106617,
+            -0.03745116665959358,
+            0.0028044944629073143,
+            0.006625712383538485,
+            0.017212992534041405,
+            -0.00239459122531116,
+            -0.002529381774365902,
+            -0.029911531135439873,
+            0.029142336919903755,
+            -0.07905209809541702,
+            0.01719854585826397,
+            -0.03854485601186752,
+            -0.0116807846352458,
+            -0.046873632818460464,
+            0.05275706946849823,
+            -0.04152297228574753,
+            -0.03727864846587181,
+            0.048614900559186935,
+            -0.010790652595460415,
+            -0.0024877521209418774,
+            0.012286793440580368,
+            0.08292551338672638,
+            0.03208302706480026,
+            0.02867130935192108,
+            -0.039545897394418716,
+            -0.02800118923187256,
+            -0.07673710584640503,
+            0.005896291229873896,
+            -0.030191265046596527,
+            -0.0187264122068882,
+            -0.03647911176085472,
+            0.0371234230697155,
+            0.05115009844303131,
+            -0.04013253375887871,
+            -0.03304912894964218,
+            0.05173036456108093,
+            0.04860515147447586,
+            -0.00717319268733263,
+            0.045631736516952515,
+            0.04671349748969078,
+            0.004948618821799755,
+            0.009837846271693707,
+            0.04381090775132179,
+            0.0918705016374588,
+            -0.005869758781045675,
+            0.02249985560774803,
+            0.010875782929360867,
+            0.02956228516995907,
+            0.0036174776032567024,
+            0.011353887617588043,
+            -0.05186513066291809,
+            0.0173268411308527,
+            0.011808891780674458,
+            -0.00798418466001749,
+            -0.00929324608296156,
+            -0.004763087723404169,
+            0.010464239865541458,
+            -0.001494695316068828,
+            -0.024331238120794296,
+            -0.042497288435697556,
+            -0.049635156989097595,
+            0.006581253372132778,
+            -0.05040008947253227,
+            -0.01882144808769226,
+            -0.02419630065560341,
+            -0.06420613825321198,
+            0.020046783611178398,
+            0.024303266778588295,
+            -0.009806456044316292,
+            -0.003666533390060067,
+            0.0018573087872937322,
+            -0.02440156601369381,
+            0.0059090061113238335,
+            -0.03926969692111015,
+            0.011777290143072605,
+            -0.032393939793109894,
+            -0.0015500712906941772,
+            0.019816264510154724,
+            0.0037668957374989986,
+            -0.0033931683283299208,
+            -0.01164526678621769,
+            -0.07331964373588562,
+            -0.008798380382359028,
+            -0.016916625201702118,
+            -0.034663647413253784,
+            -0.05818398669362068,
+            0.015174336731433868,
+            -0.010697754099965096,
+            -0.04179975762963295,
+            0.0064012641087174416,
+            0.01987038180232048,
+            0.019733907654881477,
+            -0.0013441460905596614,
+            0.05315450206398964,
+            -0.021319502964615822,
+            0.05351290851831436,
+            0.052106473594903946,
+            -0.010152475908398628,
+            -0.017445174977183342,
+            0.030323222279548645,
+            0.02796917036175728,
+            0.0026626174803823233,
+            -0.03001641295850277,
+            0.014342408627271652,
+            0.03061019256711006,
+            -0.00294340867549181,
+            -0.018154315650463104,
+            0.02443081885576248,
+            0.003663803683593869,
+            -0.023388244211673737,
+            -0.0018819351680576801,
+            0.0010939103085547686,
+            0.03557095676660538,
+            -0.053037356585264206,
+            -0.06498610228300095,
+            0.04878298565745354,
+            -0.03413922339677811,
+            -0.02763182483613491,
+            0.0009715812630020082,
+            0.00486012501642108,
+            0.03292006626725197,
+            0.023539533838629723,
+            0.011763699352741241,
+            -0.026951594278216362,
+            -0.03602864220738411,
+            0.04788520559668541,
+            -0.04133124649524689,
+            -0.013166938908398151,
+            -0.037222955375909805,
+            0.014670289121568203,
+            -0.056680355221033096,
+            -0.008314837701618671,
+            0.04227377846837044,
+            -0.04986898973584175,
+            -0.03443481773138046,
+            -0.00174588686786592,
+            0.0027100981678813696,
+            -0.001079584937542677,
+            -0.06199319660663605,
+            0.03100454993546009,
+            0.06752201914787292,
+            0.028809374198317528,
+            -0.06289442628622055,
+            -0.005715612787753344,
+            -0.052428603172302246,
+            -0.01548265665769577,
+            -0.048164043575525284,
+            0.04651368409395218,
+            0.00703784916549921,
+            0.0493292361497879,
+            0.024252086877822876,
+            -0.03811171278357506,
+            0.030049748718738556,
+            0.02749229036271572,
+            -0.0005660666502080858,
+            0.009821311570703983,
+            0.01306783128529787,
+            0.01775788515806198,
+            -0.051085181534290314,
+            0.028779184445738792
+        ],
+        "chunk_ind": 0
+    },
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/sales",
+        "title": "Sales",
+        "content": "Keep your team up to date on every conversation and update so they can close.\n\nRecall Every Detail\nBeing able to instantly revisit every detail of any call without reading transcripts is helping Sales teams provide more tailored pitches, build stronger relationships, and close more deals. Instead of searching and reading through hours of transcripts in preparation for a call, your team can now ask Danswer \"What specific features was ACME interested in seeing for the demo\". Since your team doesn't have time to read every transcript prior to a call, Danswer provides a more thorough summary because it can instantly parse hundreds of pages and distill out the relevant information. Even for fast lookups it becomes much more convenient - for example to brush up on connection building topics by asking \"What rapport building topic did we chat about in the last call with ACME\".\n\nKnow Every Product Update\nIt is impossible for Sales teams to keep up with every product update. Because of this, when a prospect has a question that the Sales team does not know, they have no choice but to rely on the Product and Engineering orgs to get an authoritative answer. Not only is this distracting to the other teams, it also slows down the time to respond to the prospect (and as we know, time is the biggest killer of deals). With Danswer, it is even possible to get answers live on call because of how fast accessing information becomes. A question like \"Have we shipped the Microsoft AD integration yet?\" can now be answered in seconds meaning that prospects can get answers while on the call instead of asynchronously and sales cycles are reduced as a result.",
+        "title_embedding": [
+            0.008453648537397385,
+            0.049128592014312744,
+            0.0009390072082169354,
+            -0.011420674622058868,
+            0.009472657926380634,
+            0.05824451148509979,
+            -0.04129518195986748,
+            -0.018892904743552208,
+            -0.007598293945193291,
+            -0.03804052621126175,
+            -0.003747896058484912,
+            0.005537204910069704,
+            0.026590371504426003,
+            -0.02672540210187435,
+            0.02651272714138031,
+            -0.033856429159641266,
+            0.03809495270252228,
+            -0.013283955864608288,
+            -0.00013869917893316597,
+            0.020309027284383774,
+            -0.011650309897959232,
+            -0.02103874459862709,
+            0.01066586747765541,
+            -0.057243604212999344,
+            0.031903959810733795,
+            -0.012924387119710445,
+            -0.012852400541305542,
+            0.01951044611632824,
+            0.018149282783269882,
+            -0.01314238179475069,
+            0.01411629281938076,
+            -0.009169652126729488,
+            0.017607972025871277,
+            -0.0685962364077568,
+            -0.03812728449702263,
+            -0.06783904880285263,
+            0.009403989650309086,
+            0.0073580555617809296,
+            0.024488259106874466,
+            0.0005618860013782978,
+            0.0418679341673851,
+            0.01953919045627117,
+            -0.05312833935022354,
+            -0.024998614564538002,
+            0.017944931983947754,
+            -0.004367160610854626,
+            0.023977765813469887,
+            0.012059491127729416,
+            0.049591515213251114,
+            -0.05067070946097374,
+            0.001997043378651142,
+            0.0024496091064065695,
+            -0.029425427317619324,
+            -0.01783224567770958,
+            0.0016368982614949346,
+            0.006662128958851099,
+            0.0406189002096653,
+            0.011542750522494316,
+            0.023075561970472336,
+            -0.012605642899870872,
+            0.010134617798030376,
+            0.054372530430555344,
+            -0.008662662468850613,
+            0.016197390854358673,
+            0.015719175338745117,
+            -0.024816671386361122,
+            -0.006610441952943802,
+            0.015418685972690582,
+            -0.011234346777200699,
+            -0.003028685925528407,
+            0.09540237486362457,
+            0.026426734402775764,
+            0.03904794156551361,
+            0.025213684886693954,
+            -0.01716603711247444,
+            -0.03701924905180931,
+            0.0097318384796381,
+            0.003443458816036582,
+            -0.022713838145136833,
+            0.029555244371294975,
+            -0.0012461059959605336,
+            -0.02693314291536808,
+            0.027585584670305252,
+            0.00919498410075903,
+            0.0784342810511589,
+            -0.004147294908761978,
+            0.0058822669088840485,
+            -0.03257093206048012,
+            -0.0194808728992939,
+            -0.015468079596757889,
+            -0.007020206656306982,
+            0.017711102962493896,
+            0.09179206192493439,
+            0.07245390862226486,
+            -0.08327075839042664,
+            -0.01777547225356102,
+            -0.0036551833618432283,
+            0.005220785271376371,
+            -0.013074155896902084,
+            -0.05137576162815094,
+            -0.03984086588025093,
+            -0.001254269853234291,
+            0.0388508178293705,
+            -0.026559771969914436,
+            0.0941508412361145,
+            0.028184799477458,
+            0.030329154804348946,
+            -0.015901995822787285,
+            -0.05033569037914276,
+            -0.040444258600473404,
+            -0.08671051263809204,
+            0.016047460958361626,
+            -0.015745067968964577,
+            0.036006249487400055,
+            -0.019317714497447014,
+            0.0028998597990721464,
+            0.08916892111301422,
+            -0.06901372224092484,
+            0.05473657324910164,
+            0.06781236082315445,
+            0.016306273639202118,
+            0.0011640831362456083,
+            0.0008445090497843921,
+            0.06246241182088852,
+            -0.020153285935521126,
+            -0.011525464244186878,
+            -0.043366242200136185,
+            0.036528658121824265,
+            0.012839822098612785,
+            -0.0585474967956543,
+            0.04394524171948433,
+            0.017970293760299683,
+            0.0273651871830225,
+            -0.006580607499927282,
+            0.04960521310567856,
+            0.04129025712609291,
+            -0.039038222283124924,
+            -0.007922167889773846,
+            0.06417153030633926,
+            0.00870921928435564,
+            0.04419026896357536,
+            0.03394830971956253,
+            0.04194091632962227,
+            -0.009943729266524315,
+            -0.026762165129184723,
+            -0.06321611255407333,
+            0.018493760377168655,
+            0.02112971432507038,
+            -0.008362037129700184,
+            0.0030741533264517784,
+            0.02977512590587139,
+            0.007839385420084,
+            -0.030427763238549232,
+            -0.005435082130134106,
+            -0.023782288655638695,
+            -0.06198855862021446,
+            -0.006325197406113148,
+            0.03481286019086838,
+            0.01096314936876297,
+            -0.008224114775657654,
+            0.016886647790670395,
+            0.018134206533432007,
+            -0.02693709172308445,
+            0.08969569951295853,
+            -0.03854775056242943,
+            -0.036120977252721786,
+            0.003924157004803419,
+            0.05902013182640076,
+            0.02604551427066326,
+            0.04082872346043587,
+            -0.006722352001816034,
+            -0.011970511637628078,
+            -0.014307579025626183,
+            0.0019104834645986557,
+            0.05876787751913071,
+            -0.002502535469830036,
+            0.027572669088840485,
+            0.027537895366549492,
+            -0.026239819824695587,
+            -0.02237943559885025,
+            0.026839105412364006,
+            -0.04806261509656906,
+            0.022188611328601837,
+            0.05301826074719429,
+            -0.017422696575522423,
+            -0.04489205405116081,
+            0.04667934030294418,
+            -0.03202678635716438,
+            -0.022968970239162445,
+            0.019313516095280647,
+            -0.06724600493907928,
+            0.023997649550437927,
+            0.02574523165822029,
+            -0.021134337410330772,
+            -0.018166225403547287,
+            -0.02959403023123741,
+            0.038194961845874786,
+            0.009773771278560162,
+            0.026523802429437637,
+            -0.014497771859169006,
+            0.009200031869113445,
+            0.01488631684333086,
+            -0.10185936093330383,
+            0.018728939816355705,
+            -0.01575741171836853,
+            0.02251303941011429,
+            0.02899281494319439,
+            -0.04970388114452362,
+            0.007716581225395203,
+            0.06781720370054245,
+            0.07015375792980194,
+            -0.02934109978377819,
+            -0.006221897434443235,
+            0.012976646423339844,
+            -0.00737345777451992,
+            -0.02422930672764778,
+            -0.03612032160162926,
+            0.003943525720387697,
+            -0.02053997851908207,
+            0.01488402672111988,
+            -0.016384800896048546,
+            -0.02631048858165741,
+            0.0029128696769475937,
+            0.0012878051493316889,
+            0.029553934931755066,
+            0.06647666543722153,
+            -0.014463554136455059,
+            0.04724975302815437,
+            0.01416710950434208,
+            0.0259545985609293,
+            -0.019878843799233437,
+            -0.04123354330658913,
+            -0.003678504843264818,
+            -0.015237071551382542,
+            -0.037017468363046646,
+            -0.014126508496701717,
+            0.037044424563646317,
+            0.041153766214847565,
+            0.034035731106996536,
+            0.0518031120300293,
+            -0.004720885772258043,
+            -0.01163511723279953,
+            0.07213476300239563,
+            -0.06036211922764778,
+            -0.03499453514814377,
+            -0.006943386513739824,
+            -0.06392820924520493,
+            -0.013479998335242271,
+            0.01668090932071209,
+            0.030153054744005203,
+            0.04787909612059593,
+            0.042566943913698196,
+            0.01869821362197399,
+            0.05060578137636185,
+            -0.005738548934459686,
+            0.0004935020115226507,
+            -0.04920157790184021,
+            0.05485580489039421,
+            -0.02717270515859127,
+            -0.026000261306762695,
+            0.017413917928934097,
+            0.00194182014092803,
+            0.06368009001016617,
+            -0.021680809557437897,
+            0.011896755546331406,
+            0.008441813290119171,
+            -0.009322874248027802,
+            0.004058067686855793,
+            0.003404452698305249,
+            -0.0070596663281321526,
+            -0.01350175030529499,
+            0.027827585116028786,
+            -0.017853371798992157,
+            0.05100760981440544,
+            -0.01331804133951664,
+            -0.021219315007328987,
+            0.012195413932204247,
+            -0.04513333737850189,
+            0.022477995604276657,
+            0.004410626832395792,
+            0.033304695039987564,
+            0.023220136761665344,
+            0.00041832958231680095,
+            0.007724999450147152,
+            0.0359807088971138,
+            0.010411631315946579,
+            0.0007441110792569816,
+            -0.018354782834649086,
+            -0.030612032860517502,
+            0.04444800317287445,
+            -0.004541076719760895,
+            -0.012099254876375198,
+            0.03223736584186554,
+            -0.017639242112636566,
+            0.012390367686748505,
+            0.055463794618844986,
+            0.09133724123239517,
+            -0.028237899765372276,
+            -0.026783155277371407,
+            -0.029024433344602585,
+            0.014482105150818825,
+            0.05629871413111687,
+            -0.03724139928817749,
+            0.008170249871909618,
+            0.06597549468278885,
+            0.051776643842458725,
+            0.042193781584501266,
+            -0.01338224858045578,
+            0.03543481230735779,
+            0.0065676262602210045,
+            -0.04679378867149353,
+            0.048750247806310654,
+            -0.01348006259649992,
+            0.06560897082090378,
+            -0.10058096796274185,
+            0.06226775795221329,
+            0.06525543332099915,
+            -0.020321687683463097,
+            0.05926727131009102,
+            -0.023439910262823105,
+            0.00998155027627945,
+            -0.04136430844664574,
+            0.04513855278491974,
+            -0.07410337775945663,
+            -0.0032536713406443596,
+            0.022534336894750595,
+            -0.0035887588746845722,
+            0.018703486770391464,
+            -0.023037323728203773,
+            -0.03570957109332085,
+            -0.03149940446019173,
+            0.01058092713356018,
+            -0.08196881413459778,
+            -0.012937279418110847,
+            0.02611234411597252,
+            0.03242015466094017,
+            0.00964296329766512,
+            -0.03003847971558571,
+            -0.02878165803849697,
+            0.005552087444812059,
+            0.11100566387176514,
+            0.006707212887704372,
+            0.007847320288419724,
+            -0.04757271707057953,
+            -0.010918735526502132,
+            0.007332456298172474,
+            -0.04022597521543503,
+            -0.03945135325193405,
+            0.08289318531751633,
+            -0.049061018973588943,
+            -0.04947024583816528,
+            -0.030500037595629692,
+            -0.03648613020777702,
+            0.007221090141683817,
+            -0.023051844909787178,
+            0.06497090309858322,
+            0.024345578625798225,
+            -0.0074218385852873325,
+            -0.04062318801879883,
+            -0.020212918519973755,
+            -0.009461181238293648,
+            -0.04491201415657997,
+            0.05126942694187164,
+            -0.005242756102234125,
+            0.024492694064974785,
+            -0.02291315235197544,
+            0.06517285853624344,
+            -0.006112807895988226,
+            0.004548671655356884,
+            0.009358521550893784,
+            0.0066603804007172585,
+            -0.005717182531952858,
+            0.046729590743780136,
+            0.04008319228887558,
+            0.09077014029026031,
+            0.03511488437652588,
+            -0.05282759666442871,
+            0.020438214763998985,
+            -0.01378707680851221,
+            -0.005117158405482769,
+            0.07433145493268967,
+            0.0034097072202712297,
+            -0.011192821897566319,
+            0.0009265196276828647,
+            -0.03159809112548828,
+            -0.033578045666217804,
+            -0.012528836727142334,
+            -0.006292750593274832,
+            0.0452519953250885,
+            -0.025647340342402458,
+            -0.026395585387945175,
+            -0.044332105666399,
+            -0.0012870433274656534,
+            0.016866305842995644,
+            -0.00726186903193593,
+            -0.014325585216283798,
+            0.02150380238890648,
+            0.05446008965373039,
+            0.01817481219768524,
+            0.017272990196943283,
+            7.192481280071661e-05,
+            -0.021787019446492195,
+            0.03518282249569893,
+            0.04129958152770996,
+            0.005599076859652996,
+            0.07016170769929886,
+            0.0068466924130916595,
+            -0.038150086998939514,
+            0.022464951500296593,
+            -0.007263584528118372,
+            0.04023060202598572,
+            -0.006662019528448582,
+            -0.03398700803518295,
+            -0.027063554152846336,
+            -0.014334858395159245,
+            0.00031888100784271955,
+            0.03320762887597084,
+            -0.0263507217168808,
+            -0.01863865926861763,
+            0.018559059128165245,
+            0.06845609098672867,
+            -0.0037615702021867037,
+            0.023087816312909126,
+            -0.019276361912488937,
+            -0.03351914510130882,
+            0.021971892565488815,
+            0.041575655341148376,
+            0.05621027201414108,
+            -0.003078967332839966,
+            -0.06297048181295395,
+            -0.05009821802377701,
+            -0.026463210582733154,
+            0.0035874273162335157,
+            0.021911393851041794,
+            -0.074904665350914,
+            -0.0012849566992372274,
+            0.06580246239900589,
+            -0.0096419183537364,
+            -0.01183160487562418,
+            0.002244731178507209,
+            -0.02129410021007061,
+            -0.004490557126700878,
+            0.006305266637355089,
+            0.020787451416254044,
+            -0.028946323320269585,
+            0.09907153993844986,
+            0.06419308483600616,
+            -0.018514622002840042,
+            -0.03974919393658638,
+            -0.055583421140909195,
+            -0.04144161939620972,
+            -0.01479779276996851,
+            -0.015063298866152763,
+            -0.05278000980615616,
+            0.056262142956256866,
+            0.0039010541513562202,
+            0.025815758854150772,
+            -0.01457720622420311,
+            0.017469312995672226,
+            0.032789044082164764,
+            0.010338534601032734,
+            0.009348046034574509,
+            -0.002339842962101102,
+            0.023598607629537582,
+            0.01676766760647297,
+            -0.03503762558102608,
+            0.033228978514671326,
+            -0.03216487169265747,
+            -0.027531251311302185,
+            0.05846886709332466,
+            -0.00979926623404026,
+            0.011551604606211185,
+            0.026247017085552216,
+            0.00776244793087244,
+            -0.042052820324897766,
+            0.003744697431102395,
+            -0.013622709549963474,
+            -0.021054048091173172,
+            -0.022621311247348785,
+            0.03077824041247368,
+            0.019676415249705315,
+            -0.02765408344566822,
+            -0.013561422936618328,
+            -0.015634974464774132,
+            -0.03614448755979538,
+            0.014710169285535812,
+            0.00825627613812685,
+            0.014769040048122406,
+            0.006793464533984661,
+            -0.010395821183919907,
+            -0.0022474846336990595,
+            0.033902380615472794,
+            0.003390782279893756,
+            -0.04533020406961441,
+            -0.09353062510490417,
+            -0.027594659477472305,
+            -0.026893651112914085,
+            0.00822615996003151,
+            0.03448451682925224,
+            -0.013120760209858418,
+            0.03301888331770897,
+            -0.03717275336384773,
+            -0.017613839358091354,
+            0.03131122142076492,
+            0.01355862244963646,
+            -0.016891464591026306,
+            -0.005221182014793158,
+            -0.09999044984579086,
+            -0.0072242445312440395,
+            0.023685455322265625,
+            -0.006333169527351856,
+            0.05389386788010597,
+            0.0006377844838425517,
+            0.06066382676362991,
+            -0.010544034652411938,
+            -0.03069271147251129,
+            0.046539101749658585,
+            0.04315992072224617,
+            -0.04527072235941887,
+            0.08369841426610947,
+            -0.05231470242142677,
+            -0.02663319930434227,
+            -0.017284002155065536,
+            0.05720992013812065,
+            -0.02959314174950123,
+            -0.053442198783159256,
+            -0.055300384759902954,
+            -0.0034046657383441925,
+            0.058179739862680435,
+            -0.0067557781003415585,
+            0.048700254410505295,
+            0.06980213522911072,
+            0.0022220234386622906,
+            -0.02891203574836254,
+            -0.03402455151081085,
+            -0.07090416550636292,
+            -0.02885468490421772,
+            -0.033339668065309525,
+            0.057542525231838226,
+            0.035563718527555466,
+            -0.0376402772963047,
+            0.01505962759256363,
+            0.025728864595294,
+            0.001696597901172936,
+            0.04947248846292496,
+            -0.0798964574933052,
+            -0.02692596986889839,
+            -0.012759744189679623,
+            -0.02195296436548233,
+            -0.014371627941727638,
+            0.02381875552237034,
+            -0.01423177681863308,
+            -0.014195146039128304,
+            -0.09804418683052063,
+            -0.0008884363924153149,
+            -0.0418919213116169,
+            0.015419455245137215,
+            -0.0015723679680377245,
+            -0.008167393505573273,
+            -0.027652231976389885,
+            -0.01364823617041111,
+            0.042369287461042404,
+            -0.020887810736894608,
+            -0.01855718344449997,
+            -0.030453767627477646,
+            -0.08889546990394592,
+            -0.042489275336265564,
+            -0.003145430004224181,
+            -0.0007042307406663895,
+            0.016261309385299683,
+            -0.04196145758032799,
+            -0.02786160260438919,
+            0.00031149861752055585,
+            0.0020716730505228043,
+            -0.018168985843658447,
+            -0.035522907972335815,
+            0.06329862773418427,
+            -0.06277810037136078,
+            -0.04981480538845062,
+            0.05988100543618202,
+            0.0031491960398852825,
+            -0.03463412821292877,
+            -0.010109111666679382,
+            -0.013390148058533669,
+            -0.08232187479734421,
+            0.018557677045464516,
+            -0.023832205682992935,
+            -0.021515224128961563,
+            0.03051081858575344,
+            -0.021489854902029037,
+            0.009503633715212345,
+            0.025148555636405945,
+            -0.023579541593790054,
+            -0.035016197711229324,
+            -0.022730164229869843,
+            0.04465099051594734,
+            -0.04341805726289749,
+            0.011980813927948475,
+            0.024123655632138252,
+            -0.026239709928631783,
+            -0.017752202227711678,
+            0.027042675763368607,
+            0.023839112371206284,
+            0.01306204218417406,
+            0.039557792246341705,
+            0.07731491327285767,
+            0.02771804668009281,
+            0.07320678234100342,
+            -0.008236434310674667,
+            -0.025150621309876442,
+            0.0035144551657140255,
+            -0.045307569205760956,
+            -0.004989498760551214,
+            0.006890833377838135,
+            0.013798183761537075,
+            -0.04717986658215523,
+            0.00230599008500576,
+            -0.06974467635154724,
+            0.013648996129631996,
+            -0.05875125527381897,
+            -0.0020281318575143814,
+            -0.07060255855321884,
+            0.04474693909287453,
+            -0.010507912375032902,
+            0.01920556277036667,
+            0.037952445447444916,
+            -0.04831290617585182,
+            -0.030323892831802368,
+            0.017083611339330673,
+            0.01788332499563694,
+            -0.019379939883947372,
+            0.0296696275472641,
+            -0.0202578566968441,
+            -0.05725785344839096,
+            -0.07489712536334991,
+            0.023742130026221275,
+            -0.07134415209293365,
+            -0.011462788097560406,
+            0.0046195476315915585,
+            0.04435937851667404,
+            0.01344655267894268,
+            -0.003911314997822046,
+            -0.03020038641989231,
+            -0.0032732610125094652,
+            0.03007005713880062,
+            0.006368617527186871,
+            -0.03210403770208359,
+            0.00835089199244976,
+            0.05988067761063576,
+            -0.03537531942129135,
+            0.05247778445482254,
+            0.03723180294036865,
+            -0.008693824522197247,
+            0.04847349226474762,
+            0.016112500801682472,
+            0.011540782637894154,
+            -0.0065521071664988995,
+            -0.03243750333786011,
+            -0.011966057121753693,
+            0.017163656651973724,
+            -0.0029253605753183365,
+            -0.053153038024902344,
+            0.0018134346464648843,
+            0.01733018085360527,
+            0.029417017474770546,
+            0.030433885753154755,
+            0.0021621473133563995,
+            -0.027712296694517136,
+            -0.05925380811095238,
+            -0.022185055539011955,
+            -0.0350322499871254,
+            -0.02007930353283882,
+            0.010399214923381805,
+            -0.055177975445985794,
+            0.0007819311576895416,
+            0.024769598618149757,
+            0.03780986741185188,
+            0.03521614894270897,
+            -0.01817735843360424,
+            -0.08278614282608032,
+            -0.021156983450055122,
+            0.03359638899564743,
+            -0.023659229278564453,
+            -0.007879458367824554,
+            0.0292595736682415,
+            -0.035273004323244095,
+            0.032482825219631195,
+            0.02688293345272541,
+            0.023407144472002983,
+            -0.047480449080467224,
+            0.0006359686376526952,
+            -0.04895651713013649,
+            0.011627614498138428,
+            -0.07718108594417572,
+            0.010565578006207943,
+            -0.01866811513900757,
+            0.01029923651367426,
+            -0.023772811517119408,
+            -0.032370492815971375,
+            0.05088132247328758,
+            0.011682837270200253,
+            0.03289812430739403,
+            0.017765464261174202,
+            0.0604407899081707,
+            -0.03875206410884857,
+            -0.05453289672732353,
+            -0.05849386751651764,
+            -0.008108421228826046,
+            -0.036137521266937256,
+            -0.003845603670924902,
+            -0.010756440460681915,
+            0.01515593845397234,
+            0.09156721830368042,
+            0.02919408679008484,
+            0.024247899651527405,
+            -0.020837178453803062,
+            0.04395196586847305,
+            -0.10095755755901337,
+            -0.07707840204238892,
+            -0.04705304652452469,
+            0.007648217957466841,
+            -0.015342561528086662,
+            -0.02814168483018875,
+            0.039529476314783096,
+            -0.06335531175136566,
+            -0.03782089054584503,
+            -0.032349780201911926,
+            -0.011073637753725052,
+            0.03126451373100281,
+            0.01734590344130993,
+            -0.0038809722755104303,
+            0.013081631623208523,
+            -0.03124905936419964,
+            0.0832752138376236,
+            -0.007435368373990059,
+            0.00989855732768774,
+            0.004071374889463186,
+            -0.021534224972128868,
+            -0.022376252338290215,
+            0.0033940861467272043,
+            0.01537957414984703,
+            -0.01530678290873766,
+            -0.006626737304031849,
+            0.0722239539027214,
+            0.023105483502149582,
+            -0.048958491533994675,
+            -0.031564872711896896,
+            0.020468993112444878,
+            0.02797403372824192,
+            0.02208004891872406,
+            -0.06905028969049454,
+            -0.0069040716625750065,
+            0.05185015872120857,
+            0.020353762432932854,
+            -0.059334978461265564,
+            0.019217371940612793,
+            -0.06793943047523499,
+            0.052697695791721344,
+            0.039136022329330444,
+            -0.03286914899945259,
+            0.0017800497589632869,
+            0.031667277216911316,
+            0.0455632358789444,
+            -0.028096599504351616,
+            0.007715262472629547,
+            0.005349942483007908,
+            -0.051207322627305984,
+            -0.04513049125671387
+        ],
+        "content_embedding": [
+            0.018702084198594093,
+            0.03602918982505798,
+            -0.0030462138820439577,
+            -0.044632311910390854,
+            -0.00032779801404103637,
+            0.013867323286831379,
+            0.028261501342058182,
+            -0.0397375151515007,
+            -0.04266185685992241,
+            -0.01854686811566353,
+            -0.00980929471552372,
+            0.005383333191275597,
+            0.025814494118094444,
+            0.020457584410905838,
+            -0.01165740005671978,
+            -0.02068958804011345,
+            0.024463720619678497,
+            -0.029769178479909897,
+            0.00032462665694765747,
+            0.020778311416506767,
+            -0.009875921532511711,
+            -0.03926593065261841,
+            -0.007392906118184328,
+            -0.02128470689058304,
+            0.020180456340312958,
+            -0.007395976223051548,
+            -0.01573384366929531,
+            0.01813557744026184,
+            -0.04103250429034233,
+            0.025176111608743668,
+            0.07708748430013657,
+            -0.03301733359694481,
+            -0.00411647092550993,
+            -0.03946784511208534,
+            -0.06054544821381569,
+            -0.040751539170742035,
+            0.01922212913632393,
+            -0.027164554223418236,
+            -0.07051867246627808,
+            0.03071393072605133,
+            0.07258772104978561,
+            -0.026042146608233452,
+            0.00869719311594963,
+            -0.028085211291909218,
+            0.0623227022588253,
+            -0.018381644040346146,
+            0.07613946497440338,
+            -0.037470318377017975,
+            0.022420862689614296,
+            -0.05576684698462486,
+            0.008394862525165081,
+            -0.032719686627388,
+            -0.01822705753147602,
+            -0.0010964440880343318,
+            0.01320287398993969,
+            0.03199688717722893,
+            0.02982492372393608,
+            0.013676099479198456,
+            0.04192396625876427,
+            -0.007906809449195862,
+            0.009779189713299274,
+            0.014214487746357918,
+            -0.0091244550421834,
+            0.0358707420527935,
+            0.003965431824326515,
+            -0.10266417264938354,
+            0.007566090207546949,
+            0.001176235033199191,
+            -0.027059122920036316,
+            -0.011146960780024529,
+            0.010613090358674526,
+            0.0269278846681118,
+            0.00038031316944397986,
+            0.024177612736821175,
+            -0.004191671498119831,
+            0.005235857795923948,
+            0.018077049404382706,
+            -0.04018911346793175,
+            -0.05125276744365692,
+            0.01798614114522934,
+            0.022944867610931396,
+            -0.04374289512634277,
+            0.003367446828633547,
+            0.026103869080543518,
+            0.03640212118625641,
+            -0.006004476919770241,
+            0.011275619268417358,
+            0.016332507133483887,
+            0.0004818506713490933,
+            0.04315895587205887,
+            0.022076765075325966,
+            -0.004544341471046209,
+            0.03210305795073509,
+            0.0906452164053917,
+            -0.07215604186058044,
+            0.08494149893522263,
+            0.006179131101816893,
+            -0.004107883665710688,
+            -0.012479269877076149,
+            -0.034221433103084564,
+            -0.017944667488336563,
+            0.04593302682042122,
+            0.019292891025543213,
+            -0.0031508891843259335,
+            0.06966886669397354,
+            0.062235988676548004,
+            0.01879720948636532,
+            -6.745052814949304e-05,
+            0.013703186996281147,
+            -0.003644032636657357,
+            -0.04458186402916908,
+            -0.0017342075007036328,
+            -0.033848460763692856,
+            0.019645417109131813,
+            -0.02943187765777111,
+            0.001105084316805005,
+            0.11609244346618652,
+            -0.05156521871685982,
+            0.009876714088022709,
+            0.005161592271178961,
+            -0.032977450639009476,
+            -0.04834079369902611,
+            0.021944768726825714,
+            -0.012638481333851814,
+            0.04569210857152939,
+            0.015415391884744167,
+            -0.03170562908053398,
+            -0.0031294531654566526,
+            0.015164556913077831,
+            -0.034650497138500214,
+            0.006696060299873352,
+            0.006991597358137369,
+            -0.05405446141958237,
+            0.002478727139532566,
+            0.03736428543925285,
+            0.006255546119064093,
+            -0.023612871766090393,
+            -0.04719111695885658,
+            0.019092371687293053,
+            -0.007389509119093418,
+            0.005412441678345203,
+            0.0032002630177885294,
+            0.014097358100116253,
+            0.0011166664771735668,
+            -0.0012068386422470212,
+            -0.0596468411386013,
+            0.0030182863119989634,
+            0.018532730638980865,
+            0.020043276250362396,
+            -0.0014203430619090796,
+            0.03718654438853264,
+            -0.04137871786952019,
+            0.003067273646593094,
+            0.060497768223285675,
+            -0.002445906400680542,
+            -0.05149608850479126,
+            -0.011358898133039474,
+            -0.0057965232990682125,
+            0.04786103963851929,
+            0.05521485581994057,
+            0.03300704434514046,
+            0.01832137256860733,
+            -0.03220272436738014,
+            0.05887257307767868,
+            -0.04280361905694008,
+            0.030713768675923347,
+            0.031198250129818916,
+            0.018273506313562393,
+            0.021060051396489143,
+            0.018141141161322594,
+            -0.01044323481619358,
+            0.012220986187458038,
+            0.011005178093910217,
+            -0.008985857479274273,
+            0.05407913029193878,
+            -0.010373812168836594,
+            0.015498371794819832,
+            -0.04114103317260742,
+            0.02436467818915844,
+            -0.033720944076776505,
+            0.04162474721670151,
+            -0.03209234029054642,
+            0.007773025427013636,
+            0.03150646388530731,
+            -0.042852289974689484,
+            -0.0062582893297076225,
+            0.04668346792459488,
+            -0.06638985872268677,
+            -0.005832660011947155,
+            -0.022002393379807472,
+            -0.015554124489426613,
+            -0.017163358628749847,
+            -0.04198216274380684,
+            -0.01709570921957493,
+            0.026661567389965057,
+            -0.049418482929468155,
+            0.06740570068359375,
+            0.0159238763153553,
+            0.0023050543386489153,
+            -0.031238939613103867,
+            -0.03321292996406555,
+            -0.004760078154504299,
+            -0.07937376946210861,
+            0.03254229202866554,
+            -0.023319443687796593,
+            0.04906806722283363,
+            0.05458753556013107,
+            -0.013899387791752815,
+            0.03574313595890999,
+            0.011882249265909195,
+            0.11678190529346466,
+            0.0007563747349195182,
+            0.025212422013282776,
+            0.023760458454489708,
+            -0.021716047078371048,
+            -0.017915191128849983,
+            -0.02478560246527195,
+            -0.028406206518411636,
+            -0.06178540363907814,
+            0.044959306716918945,
+            -0.003325885394588113,
+            -0.02079332433640957,
+            -0.010518986731767654,
+            -0.00242405547760427,
+            -0.0030814141500741243,
+            0.0010505993850529194,
+            0.0034556719474494457,
+            0.005322635173797607,
+            -0.02364535629749298,
+            -0.0034255431964993477,
+            -0.04123266786336899,
+            -0.09191295504570007,
+            0.03907715901732445,
+            0.017849568277597427,
+            0.003202608088031411,
+            -0.009892004542052746,
+            0.00447180075570941,
+            0.02633223496377468,
+            0.010955878533422947,
+            0.08653970807790756,
+            0.020712584257125854,
+            0.0020712309051305056,
+            0.02159838192164898,
+            -0.03148637339472771,
+            -0.1106131300330162,
+            -0.0034493962302803993,
+            0.008495570160448551,
+            0.025898300111293793,
+            -0.01585080474615097,
+            0.047963947057724,
+            0.03191608935594559,
+            0.05672791600227356,
+            0.017725899815559387,
+            -0.011000119149684906,
+            0.05266193300485611,
+            -0.02026527188718319,
+            -0.0076444013975560665,
+            0.04474569857120514,
+            0.0030594514682888985,
+            -0.04981522262096405,
+            -0.0012618869077414274,
+            -0.028191188350319862,
+            0.06203592196106911,
+            -0.04548441618680954,
+            -0.01024117786437273,
+            0.012013573199510574,
+            -0.03531227633357048,
+            -0.0303136445581913,
+            0.01046642567962408,
+            -0.029064299538731575,
+            -0.015392802655696869,
+            0.02021191082894802,
+            0.015328207053244114,
+            0.07215247303247452,
+            -0.024603676050901413,
+            -0.0021844934672117233,
+            0.01121720764786005,
+            -0.03952696546912193,
+            0.057082369923591614,
+            -0.007885781116783619,
+            -0.05230427160859108,
+            -0.0490812249481678,
+            -0.031762074679136276,
+            -0.009603463113307953,
+            -0.008093117736279964,
+            0.030981115996837616,
+            -0.0013626269064843655,
+            -0.019603300839662552,
+            0.025847315788269043,
+            0.012290321290493011,
+            0.007945788092911243,
+            0.011359087191522121,
+            0.01893901824951172,
+            0.03544235974550247,
+            0.01802144944667816,
+            0.07854204624891281,
+            0.07568025588989258,
+            -0.05122705176472664,
+            0.008560816757380962,
+            -0.03897644579410553,
+            0.024606050923466682,
+            0.03792334720492363,
+            0.01617903634905815,
+            -0.04735874757170677,
+            0.003156541381031275,
+            -0.011881450191140175,
+            -0.026287615299224854,
+            0.030718199908733368,
+            0.04659629613161087,
+            -0.006789658684283495,
+            -0.03779527172446251,
+            -0.0030539771541953087,
+            -0.05919982120394707,
+            -0.03511202707886696,
+            -0.079665407538414,
+            0.0580061711370945,
+            0.07525473088026047,
+            0.02381243370473385,
+            0.04075026884675026,
+            0.011406723409891129,
+            0.020479097962379456,
+            -0.004844597075134516,
+            -0.012301536276936531,
+            -0.07029860466718674,
+            0.0063995844684541225,
+            0.03451430797576904,
+            0.023998068645596504,
+            0.03119623102247715,
+            -0.0481903962790966,
+            0.03359334170818329,
+            0.02563287690281868,
+            0.04894624277949333,
+            -0.05896732583642006,
+            -0.020169634371995926,
+            0.01319917943328619,
+            0.00783664919435978,
+            0.0051937587559223175,
+            0.0014421058585867286,
+            -0.026831623166799545,
+            -0.0031287523452192545,
+            0.024796785786747932,
+            -0.0008400659426115453,
+            -0.03314085304737091,
+            -0.038561608642339706,
+            -0.013552311807870865,
+            -0.012201554141938686,
+            -0.010810038074851036,
+            -0.025452986359596252,
+            0.058491192758083344,
+            -0.017493925988674164,
+            -0.04302553832530975,
+            0.00978845451027155,
+            0.0001076174812624231,
+            -0.042208705097436905,
+            -0.011103725992143154,
+            0.019692320376634598,
+            0.035941820591688156,
+            0.02046988718211651,
+            -0.013415461406111717,
+            0.013622494414448738,
+            0.03867186978459358,
+            -0.019314907491207123,
+            0.018686568364501,
+            -0.026298167183995247,
+            0.052582357078790665,
+            0.027494613081216812,
+            0.046435534954071045,
+            0.03811647742986679,
+            0.0486551970243454,
+            0.0019721186254173517,
+            -0.017199190333485603,
+            -0.006901210639625788,
+            0.025136850774288177,
+            0.0804467722773552,
+            0.061168037354946136,
+            0.017717817798256874,
+            -0.06603220850229263,
+            -0.05064086616039276,
+            0.039705704897642136,
+            -0.024581512436270714,
+            0.09781734645366669,
+            -0.030609596520662308,
+            0.006824797950685024,
+            -0.004317844286561012,
+            0.0027715987525880337,
+            0.000943489489145577,
+            -0.013181749731302261,
+            -0.025556521490216255,
+            -0.028432460501790047,
+            -0.03751988708972931,
+            -0.019560452550649643,
+            -0.0136410528793931,
+            -0.024382753297686577,
+            0.02857314422726631,
+            -0.011391760781407356,
+            -0.0005634031840600073,
+            0.03159620240330696,
+            -0.005198314320296049,
+            -0.006265239790081978,
+            0.025611309334635735,
+            -0.058613672852516174,
+            -0.04532675817608833,
+            -0.021873218938708305,
+            0.02903951145708561,
+            -0.0121288001537323,
+            0.04538734629750252,
+            -0.027941465377807617,
+            -0.07353822141885757,
+            0.03773269057273865,
+            -0.00512319291010499,
+            0.028662901371717453,
+            0.04792957380414009,
+            -0.01053295936435461,
+            -0.035430196672677994,
+            0.009273026138544083,
+            0.004318219143897295,
+            0.08100441098213196,
+            0.014380274340510368,
+            -0.03378414362668991,
+            -0.01987980492413044,
+            0.021573858335614204,
+            0.02855539321899414,
+            -0.007542841136455536,
+            0.004633236676454544,
+            0.008639613166451454,
+            0.006394797004759312,
+            0.019281607121229172,
+            -0.015256315469741821,
+            -0.00148143304977566,
+            -0.02985287643969059,
+            0.025533605366945267,
+            -0.011111553758382797,
+            0.01765139028429985,
+            0.05442074313759804,
+            -0.09088895469903946,
+            0.032542143017053604,
+            0.0574481226503849,
+            0.027539772912859917,
+            0.028780700638890266,
+            0.04139337316155434,
+            -0.014601831324398518,
+            -0.04883953556418419,
+            -0.015341846272349358,
+            -0.03965975344181061,
+            -0.000157194648636505,
+            0.0576823353767395,
+            0.0886307954788208,
+            0.008708767592906952,
+            -0.03148962929844856,
+            0.00118768191896379,
+            -0.009285139851272106,
+            0.0349595844745636,
+            0.00010961518273688853,
+            -0.007748626638203859,
+            0.08073285222053528,
+            -0.026761949062347412,
+            0.008831962943077087,
+            -0.04131530225276947,
+            -0.00733856251463294,
+            0.06809361279010773,
+            0.03708426281809807,
+            0.06835067272186279,
+            0.006261076312512159,
+            0.015920374542474747,
+            -0.004048558417707682,
+            -0.02677253447473049,
+            0.07650309801101685,
+            -0.023148853331804276,
+            -0.057494066655635834,
+            0.014478741213679314,
+            -0.007499868981540203,
+            0.02524508722126484,
+            0.04229635000228882,
+            0.017856169492006302,
+            -0.048948079347610474,
+            -0.014559978619217873,
+            0.05829133093357086,
+            -0.007394919637590647,
+            0.008901085704565048,
+            0.03540206328034401,
+            0.011715879663825035,
+            0.03447958827018738,
+            -0.05490283668041229,
+            -0.0033728398848325014,
+            -0.029942180961370468,
+            -0.0025663028936833143,
+            0.02124219387769699,
+            0.02587033249437809,
+            -0.03495795652270317,
+            -0.01766275428235531,
+            0.05129474401473999,
+            0.050688110291957855,
+            -0.04483504965901375,
+            -0.01242926623672247,
+            -0.10075337439775467,
+            -0.039148375391960144,
+            -0.01708081364631653,
+            0.022980742156505585,
+            0.0012766321888193488,
+            -0.05624091997742653,
+            0.02920171432197094,
+            -0.0004628047754522413,
+            -0.012556084431707859,
+            -0.004911895841360092,
+            -0.006203844211995602,
+            0.011994684115052223,
+            -0.005147894844412804,
+            -0.0653131902217865,
+            0.021533435210585594,
+            0.027332814410328865,
+            0.010824107564985752,
+            0.06882979720830917,
+            0.002402055310085416,
+            0.06736285239458084,
+            0.007376556750386953,
+            -0.09907388687133789,
+            0.0738959014415741,
+            0.011251496151089668,
+            -0.089520663022995,
+            0.07383604347705841,
+            -0.02708776667714119,
+            0.039623651653528214,
+            0.001983445603400469,
+            0.034104056656360626,
+            -0.10747380554676056,
+            -0.01417585276067257,
+            -0.04512251541018486,
+            0.001210794085636735,
+            0.05437818914651871,
+            -0.004397509153932333,
+            0.03336326405405998,
+            0.06369500607252121,
+            0.014758906327188015,
+            -0.009938295930624008,
+            0.027304060757160187,
+            -0.025614865124225616,
+            -0.019298158586025238,
+            -0.03774742782115936,
+            -0.021663375198841095,
+            0.004094315692782402,
+            -0.05966729298233986,
+            0.025774789974093437,
+            0.004207789432257414,
+            0.03211497142910957,
+            0.0222456231713295,
+            -0.07142850011587143,
+            -0.02816791646182537,
+            0.008917749859392643,
+            -0.03328888863325119,
+            -0.04815923050045967,
+            0.055692847818136215,
+            0.005043115001171827,
+            0.007406118791550398,
+            -0.10431766510009766,
+            0.003935595508664846,
+            -0.07654271274805069,
+            0.018002459779381752,
+            -0.026796353980898857,
+            -0.01480060163885355,
+            -0.003071046667173505,
+            -0.031164491549134254,
+            -0.004783581010997295,
+            0.028996651992201805,
+            -0.00432590302079916,
+            0.03827083855867386,
+            -0.10670920461416245,
+            -0.009844367392361164,
+            0.05061553791165352,
+            0.044921379536390305,
+            -0.021305931732058525,
+            -0.010570063255727291,
+            -0.037161700427532196,
+            0.03244778513908386,
+            -0.026579100638628006,
+            -0.021481862291693687,
+            -0.0803975760936737,
+            0.0602957159280777,
+            -0.030482472851872444,
+            -0.016915978863835335,
+            -0.03414126858115196,
+            -0.07902093976736069,
+            -0.05861777812242508,
+            0.02578902617096901,
+            -0.006971773691475391,
+            0.021272379904985428,
+            0.004250700585544109,
+            -0.04617677628993988,
+            -0.025576869025826454,
+            0.09362083673477173,
+            -0.02747775986790657,
+            -0.010427952744066715,
+            -0.005847673863172531,
+            0.03786874935030937,
+            -0.023494398221373558,
+            -0.03599749505519867,
+            0.06915943324565887,
+            -0.0005259242025204003,
+            -0.020210636779665947,
+            0.012060794048011303,
+            -0.03892034292221069,
+            -0.03120974451303482,
+            0.03798247501254082,
+            0.08222217112779617,
+            -0.03596770018339157,
+            -0.001334474771283567,
+            0.06480739265680313,
+            0.016418466344475746,
+            0.017459729686379433,
+            -0.05252225697040558,
+            -0.05017365887761116,
+            -0.013104243203997612,
+            -0.023724595084786415,
+            0.006762322038412094,
+            -0.005946440156549215,
+            0.006083739455789328,
+            -0.014293180778622627,
+            0.041052985936403275,
+            -0.02417348138988018,
+            0.03206375241279602,
+            -0.05389661714434624,
+            0.0052835363894701,
+            -0.038879118859767914,
+            0.0735245794057846,
+            -0.004067298024892807,
+            -0.007775747217237949,
+            0.03442619740962982,
+            -0.013299554586410522,
+            -0.01858234964311123,
+            0.015830783173441887,
+            0.050536882132291794,
+            0.039121512323617935,
+            0.025488585233688354,
+            -0.083254374563694,
+            -0.0376444049179554,
+            -0.03673558309674263,
+            0.02858356013894081,
+            -0.0017373028676956892,
+            -0.029856612905859947,
+            -0.020456591621041298,
+            0.014330082572996616,
+            0.026495426893234253,
+            -0.029473086819052696,
+            -0.05005846545100212,
+            0.036553170531988144,
+            0.03604103624820709,
+            -0.014557436108589172,
+            0.0075491974130272865,
+            0.02816123701632023,
+            0.022223982959985733,
+            -0.010599660687148571,
+            0.0738152265548706,
+            0.09043020755052567,
+            -0.0195071529597044,
+            0.03981706127524376,
+            0.04508437216281891,
+            0.03942303732037544,
+            0.016333166509866714,
+            -0.007340291049331427,
+            -0.041841596364974976,
+            -0.014305119402706623,
+            -0.005970897153019905,
+            -0.04999639838933945,
+            -0.00753607414662838,
+            -0.026936067268252373,
+            0.02390979416668415,
+            -0.02593693509697914,
+            -0.015126893296837807,
+            -0.035575494170188904,
+            -0.01576480083167553,
+            0.01482314057648182,
+            0.01960604451596737,
+            0.012122674845159054,
+            0.018187053501605988,
+            -0.04843643680214882,
+            -0.032472606748342514,
+            -0.002006485592573881,
+            -0.003907614853233099,
+            0.012371492572128773,
+            -0.03494970500469208,
+            -0.04294227808713913,
+            0.004812099505215883,
+            -0.02547234669327736,
+            0.028849929571151733,
+            -0.021960295736789703,
+            -0.0013683459255844355,
+            0.030571121722459793,
+            -0.04714812710881233,
+            -0.0034763696603477,
+            -0.03908781707286835,
+            -0.04101671278476715,
+            0.013097001239657402,
+            0.004385901615023613,
+            -0.033829864114522934,
+            -0.04254792630672455,
+            0.015273491851985455,
+            -0.040665704756975174,
+            -0.00920754112303257,
+            -0.041413065046072006,
+            -0.013588172383606434,
+            -0.0017439010553061962,
+            -0.030785854905843735,
+            0.07103034108877182,
+            -0.01929519884288311,
+            0.025452272966504097,
+            -0.022381870076060295,
+            -0.028560174629092216,
+            -0.011340905912220478,
+            -0.008002392016351223,
+            -0.013147140853106976,
+            0.021906575188040733,
+            -0.04703265056014061,
+            0.00766343716531992,
+            0.06170996278524399,
+            -0.004122643731534481,
+            -0.01931242272257805,
+            0.03596718981862068,
+            0.0322248749434948,
+            -0.026486199349164963,
+            -0.03052559122443199,
+            -0.03129229322075844,
+            0.024310404434800148,
+            -0.029317326843738556,
+            -0.07851212471723557,
+            0.04514205455780029,
+            -0.020682433620095253,
+            -0.04681077226996422,
+            0.008951415307819843,
+            0.04007868468761444,
+            0.028472354635596275,
+            0.009118284098803997,
+            -0.013431325554847717,
+            -0.020488806068897247,
+            -0.027376288548111916,
+            0.035840798169374466,
+            -0.03870074450969696,
+            -0.002007996430620551,
+            -0.017398731783032417,
+            0.031902845948934555,
+            -0.07956399768590927,
+            -0.04125808924436569,
+            0.01867605932056904,
+            -0.08004764467477798,
+            -0.005094117484986782,
+            0.03083234466612339,
+            -0.01335862372070551,
+            -0.04482260346412659,
+            -0.033836718648672104,
+            0.02282416820526123,
+            0.06287918239831924,
+            0.010162614285945892,
+            -0.05158773064613342,
+            0.0008301119669340551,
+            -0.03881796821951866,
+            0.002004651352763176,
+            -0.022358564659953117,
+            0.022438282147049904,
+            0.04948568344116211,
+            0.03556858375668526,
+            0.04151606187224388,
+            -0.03065376915037632,
+            0.0035080660600215197,
+            -0.004210236947983503,
+            -0.02430005557835102,
+            0.02775806188583374,
+            0.0647825375199318,
+            0.003446039743721485,
+            -0.015345090068876743,
+            -0.004865385591983795
+        ],
+        "chunk_ind": 0
+    },
+    {
+        "url": "https://docs.danswer.dev/more/use_cases/operations",
+        "title": "Operations",
+        "content": "Double the productivity of your Ops teams like IT, HR, etc.\n\nAutomatically Resolve Tickets\nModern teams are leveraging AI to auto-resolve up to 50% of tickets. Whether it is an employee asking about benefits details or how to set up the VPN for remote work, Danswer can help your team help themselves. This frees up your team to do the real impactful work of landing star candidates or improving your internal processes.\n\nAI Aided Onboarding\nOne of the periods where your team needs the most help is when they're just ramping up. Instead of feeling lost in dozens of new tools, Danswer gives them a single place where they can ask about anything in natural language. Whether it's how to set up their work environment or what their onboarding goals are, Danswer can walk them through every step with the help of Generative AI. This lets your team feel more empowered and gives time back to the more seasoned members of your team to focus on moving the needle.",
+        "title_embedding": [
+            0.010730741545557976,
+            0.018373621627688408,
+            -0.0013679212424904108,
+            0.0001091610174626112,
+            0.034599218517541885,
+            0.038814906030893326,
+            -0.03269535303115845,
+            -0.015120825730264187,
+            -0.011999477632343769,
+            -0.023377608507871628,
+            -0.003536652075126767,
+            0.019120972603559494,
+            0.032052282243967056,
+            -0.03316797688603401,
+            0.002971385605633259,
+            -0.06660863012075424,
+            0.015637297183275223,
+            -0.004499480128288269,
+            -0.01167437992990017,
+            0.024017684161663055,
+            0.011193061247467995,
+            0.02649473212659359,
+            0.04822992533445358,
+            -0.022897351533174515,
+            0.04229900613427162,
+            0.004849326331168413,
+            -0.0013035786105319858,
+            -0.022214103490114212,
+            -0.03940191864967346,
+            -0.010781657882034779,
+            0.049329955130815506,
+            -0.010857120156288147,
+            0.04519270732998848,
+            -0.025993647053837776,
+            -0.03483803570270538,
+            -0.08306694775819778,
+            0.023532472550868988,
+            0.01766788400709629,
+            0.07221196591854095,
+            -0.001823332509957254,
+            0.003104567062109709,
+            0.05640452727675438,
+            -0.04483344778418541,
+            -0.04334229975938797,
+            0.019754929468035698,
+            -0.011862428858876228,
+            0.03863349184393883,
+            0.01501200906932354,
+            0.03863223269581795,
+            -0.05570034310221672,
+            -0.0267505943775177,
+            0.03793827444314957,
+            -0.015061624348163605,
+            -0.025455573573708534,
+            0.015559123829007149,
+            -0.01849287934601307,
+            0.04038143530488014,
+            0.026783155277371407,
+            0.0023170525673776865,
+            -0.005963196512311697,
+            0.04137548804283142,
+            0.01764686405658722,
+            -0.02246052585542202,
+            -0.012901525013148785,
+            -0.017714571207761765,
+            -0.01027537789195776,
+            -0.016164785251021385,
+            -0.007979510352015495,
+            0.021984701976180077,
+            -0.009949913248419762,
+            0.08931540697813034,
+            0.019962448626756668,
+            0.03341870754957199,
+            0.028858954086899757,
+            0.0011826930567622185,
+            -0.00010822620242834091,
+            0.021924806758761406,
+            -0.009182920679450035,
+            -0.018256863579154015,
+            0.04718794673681259,
+            0.020425673574209213,
+            0.0002538118860684335,
+            0.05454721301794052,
+            0.019339540973305702,
+            0.09129136055707932,
+            -0.010062575340270996,
+            0.011977903544902802,
+            -0.022689297795295715,
+            -0.02022380940616131,
+            0.013067576102912426,
+            0.005217134952545166,
+            0.037848640233278275,
+            0.09128513187170029,
+            0.052562836557626724,
+            -0.08711232244968414,
+            -0.0015501821180805564,
+            0.00441542686894536,
+            -0.025076182559132576,
+            -0.014110158197581768,
+            -0.06030100956559181,
+            -0.0010345132322981954,
+            0.018056612461805344,
+            0.03524528071284294,
+            -0.01293126679956913,
+            0.04700767621397972,
+            0.06564126163721085,
+            0.010113431140780449,
+            0.018003467470407486,
+            -0.05646204203367233,
+            -0.02047823928296566,
+            -0.048007529228925705,
+            0.05380301922559738,
+            -0.01607179455459118,
+            0.0008504731231369078,
+            -0.015249709598720074,
+            -0.005599239841103554,
+            0.054663073271512985,
+            -0.036346085369586945,
+            0.024314023554325104,
+            0.0768347755074501,
+            0.03234424442052841,
+            0.008295328356325626,
+            -0.0015371616464108229,
+            0.043760448694229126,
+            -0.0018124273046851158,
+            -0.017262862995266914,
+            -0.042646538466215134,
+            0.025395702570676804,
+            0.03709862381219864,
+            -0.03921937197446823,
+            0.050630342215299606,
+            0.024703586474061012,
+            0.022064829245209694,
+            -0.008263661526143551,
+            0.03165263310074806,
+            0.017121389508247375,
+            -0.06672775000333786,
+            -0.06984685361385345,
+            0.11498068273067474,
+            -0.02806312032043934,
+            0.025353405624628067,
+            0.006591377779841423,
+            -0.011244958266615868,
+            -0.014589745551347733,
+            0.031125560402870178,
+            -0.06834094226360321,
+            -0.015911657363176346,
+            0.0176913570612669,
+            -0.022801805287599564,
+            0.01242455281317234,
+            0.001349485362879932,
+            0.05671858787536621,
+            -0.024975799024105072,
+            -0.019439268857240677,
+            -0.0038488772697746754,
+            -0.057438429445028305,
+            -0.004414469934999943,
+            0.018468070775270462,
+            0.006111294496804476,
+            0.018633801490068436,
+            0.04350016638636589,
+            0.04317209869623184,
+            -0.058885347098112106,
+            0.06605420261621475,
+            -0.060788847506046295,
+            -0.059009939432144165,
+            0.003782198065891862,
+            0.04133265092968941,
+            0.019440937787294388,
+            0.0046022264286875725,
+            0.010991339571774006,
+            -0.0554840974509716,
+            0.04083068668842316,
+            0.007237149402499199,
+            0.04218628630042076,
+            0.015044232830405235,
+            0.0565854087471962,
+            0.03397437930107117,
+            0.017036354169249535,
+            -0.02862199954688549,
+            -0.0019827275536954403,
+            -0.04851892590522766,
+            0.020672423765063286,
+            0.029321348294615746,
+            -0.04698231443762779,
+            -0.06201909855008125,
+            0.01672869734466076,
+            -0.06657315790653229,
+            -0.061089128255844116,
+            -0.006803641561418772,
+            -0.041214216500520706,
+            0.025210469961166382,
+            0.03659403696656227,
+            -0.056864507496356964,
+            -0.017966432496905327,
+            -0.04572960361838341,
+            0.03334927558898926,
+            0.0010855993023142219,
+            0.009142755530774593,
+            -0.045417286455631256,
+            0.0508459098637104,
+            -0.0020350103732198477,
+            -0.08134196698665619,
+            -0.014234581962227821,
+            -0.018346119672060013,
+            0.027286984026432037,
+            0.01077864971011877,
+            0.025067729875445366,
+            -0.0010993028990924358,
+            0.055933188647031784,
+            0.08128975331783295,
+            -0.00964485201984644,
+            0.02629624865949154,
+            0.02482009492814541,
+            -0.004299841821193695,
+            -0.02121540531516075,
+            -0.012340475805103779,
+            -0.010211183689534664,
+            -0.033044300973415375,
+            0.01627231389284134,
+            0.0025289515033364296,
+            -0.024867739528417587,
+            -0.02439618855714798,
+            0.031935419887304306,
+            0.015503033064305782,
+            0.058931007981300354,
+            0.0018315770430490375,
+            0.05652814358472824,
+            -0.01652582921087742,
+            -0.009255263954401016,
+            -0.03914093226194382,
+            -0.015466556884348392,
+            0.04188372567296028,
+            -0.027017222717404366,
+            -0.022300412878394127,
+            0.016426892951130867,
+            0.017171800136566162,
+            0.07701553404331207,
+            0.043874118477106094,
+            0.07433117181062698,
+            0.026014234870672226,
+            0.013816924765706062,
+            0.014636811800301075,
+            -0.036465343087911606,
+            -0.03994021564722061,
+            0.003269175998866558,
+            -0.03902558237314224,
+            -0.0039056213572621346,
+            0.01340826041996479,
+            -0.012667474336922169,
+            0.013740241527557373,
+            0.014225244522094727,
+            -0.01617649756371975,
+            0.045204829424619675,
+            0.0021076719276607037,
+            -0.03156042471528053,
+            -0.051285773515701294,
+            0.050252512097358704,
+            -0.03781714290380478,
+            0.017089596018195152,
+            0.028835022822022438,
+            0.005575904157012701,
+            0.05654308199882507,
+            -0.004466162994503975,
+            0.05050184950232506,
+            0.011318539269268513,
+            -0.049033407121896744,
+            0.022067567333579063,
+            0.011711984872817993,
+            0.005116294138133526,
+            -0.02025405503809452,
+            -0.010977067984640598,
+            -0.030916348099708557,
+            0.05374428257346153,
+            0.00808737613260746,
+            -0.044140078127384186,
+            0.030092595145106316,
+            -0.01702306419610977,
+            -0.020538438111543655,
+            0.00427399855107069,
+            0.002780117094516754,
+            -0.005801517982035875,
+            -0.04212724789977074,
+            0.024279015138745308,
+            -6.510222738143057e-05,
+            -0.020652174949645996,
+            -0.006957880686968565,
+            0.0016846335493028164,
+            -0.0663430467247963,
+            -0.0335371308028698,
+            2.869481068046298e-05,
+            -0.037348829209804535,
+            0.039201609790325165,
+            -0.000545984017662704,
+            0.02819826453924179,
+            0.04861782118678093,
+            0.08528425544500351,
+            0.012239249423146248,
+            0.014359706081449986,
+            -0.019882500171661377,
+            -0.03053932823240757,
+            0.04383694753050804,
+            -0.05820803344249725,
+            -0.0014366158284246922,
+            0.06986244767904282,
+            0.06611118465662003,
+            0.024454524740576744,
+            -0.02059408277273178,
+            -0.016752762719988823,
+            -0.01459463406354189,
+            -0.02889612317085266,
+            0.04970743879675865,
+            -0.028303874656558037,
+            0.03978912904858589,
+            -0.07312510907649994,
+            0.05429210141301155,
+            0.00571118388324976,
+            -0.02056923136115074,
+            0.053339049220085144,
+            -0.018249599263072014,
+            -0.021281961351633072,
+            -0.05355891212821007,
+            0.0478244312107563,
+            -0.06067035347223282,
+            -0.0008842060924507678,
+            0.04420148581266403,
+            0.000588231545407325,
+            0.056371595710515976,
+            0.014279269613325596,
+            -0.05001707002520561,
+            -0.037117116153240204,
+            0.030044561251997948,
+            -0.05617158114910126,
+            0.02152038738131523,
+            0.017715860158205032,
+            0.02034214325249195,
+            0.021688375622034073,
+            -0.024712584912776947,
+            -0.03572659194469452,
+            -0.03006441332399845,
+            0.11338905245065689,
+            0.029146766290068626,
+            -0.01698526367545128,
+            -0.05355465039610863,
+            -0.0036588346119970083,
+            -0.0032888834830373526,
+            -0.022690337151288986,
+            -0.05653419718146324,
+            0.053955987095832825,
+            -0.010936236009001732,
+            -0.05121518298983574,
+            -0.04233774170279503,
+            -0.0650610476732254,
+            -0.009094701148569584,
+            -0.00337587739340961,
+            0.05269891023635864,
+            0.05860234051942825,
+            0.015521558932960033,
+            -0.038103096187114716,
+            -0.029688136652112007,
+            -0.011934547685086727,
+            -0.05070113763213158,
+            0.025371648371219635,
+            0.03601797670125961,
+            0.021862944588065147,
+            -0.020703352987766266,
+            0.05289195850491524,
+            -0.004169228952378035,
+            -0.03900706395506859,
+            -0.0029100535903126,
+            0.003359412308782339,
+            -0.03696342185139656,
+            0.007729679811745882,
+            0.005611894652247429,
+            0.04680318757891655,
+            0.002203753450885415,
+            -0.057359859347343445,
+            0.0586426742374897,
+            0.027003217488527298,
+            -0.030223455280065536,
+            0.06817735731601715,
+            0.03476058319211006,
+            -0.011411039158701897,
+            -5.9986756241414696e-05,
+            -0.03633617237210274,
+            -0.016408616676926613,
+            0.00833839550614357,
+            0.0022074412554502487,
+            0.048158638179302216,
+            -0.031786687672138214,
+            -0.028205247595906258,
+            -0.056731242686510086,
+            -0.023744143545627594,
+            -0.00782334990799427,
+            0.03123593144118786,
+            -0.016950296238064766,
+            0.01794753596186638,
+            0.04291892051696777,
+            0.0099559361115098,
+            -0.0012914348626509309,
+            -0.004629518836736679,
+            -0.05153423175215721,
+            0.058259084820747375,
+            0.056788213551044464,
+            -0.0333746112883091,
+            0.030548732727766037,
+            0.01471715047955513,
+            -0.002818142296746373,
+            -0.013207555748522282,
+            0.022568998858332634,
+            0.025705108419060707,
+            -0.014197800308465958,
+            -0.02527414821088314,
+            0.0009442049195058644,
+            0.005861984565854073,
+            0.00919109396636486,
+            0.012627449817955494,
+            0.017443764954805374,
+            -0.0162491612136364,
+            0.05474800989031792,
+            0.02047792077064514,
+            0.0003552198759280145,
+            0.0005291366251185536,
+            0.003957713954150677,
+            -0.073692187666893,
+            0.044466596096754074,
+            0.02759671024978161,
+            0.03048691712319851,
+            0.024890316650271416,
+            -0.04379572719335556,
+            -0.004758962895721197,
+            -0.012311465106904507,
+            0.003943922929465771,
+            0.035470347851514816,
+            -0.06701556593179703,
+            -0.0010001214686781168,
+            0.06592956185340881,
+            0.008649672381579876,
+            0.005259859841316938,
+            -0.00453479727730155,
+            -0.026313234120607376,
+            0.0128184137865901,
+            -0.04696577414870262,
+            0.0357656292617321,
+            -0.007445288822054863,
+            0.10806939005851746,
+            0.0789022147655487,
+            -0.01642726920545101,
+            -0.042041581124067307,
+            -0.015044954605400562,
+            -0.020660532638430595,
+            -0.01043805480003357,
+            -0.011654903180897236,
+            -0.05754747614264488,
+            0.06964936852455139,
+            0.03990045189857483,
+            -0.0017825361574068666,
+            -0.0382373072206974,
+            0.005986799951642752,
+            0.022415796294808388,
+            -0.011907496489584446,
+            -0.015296644531190395,
+            0.005779241677373648,
+            0.051009099930524826,
+            -0.013554011471569538,
+            -0.036914244294166565,
+            -0.018791811540722847,
+            -0.03514847159385681,
+            -0.050969723612070084,
+            0.07429437339305878,
+            -0.014117387123405933,
+            0.01858820766210556,
+            0.029953083023428917,
+            0.013503451831638813,
+            0.024872425943613052,
+            -0.01447504572570324,
+            -0.03305073082447052,
+            -0.03757826238870621,
+            0.009820172563195229,
+            0.004212466534227133,
+            0.009773447178304195,
+            -0.03176327049732208,
+            -0.00981978140771389,
+            -0.018437808379530907,
+            -0.016843365505337715,
+            0.04063236713409424,
+            0.03664008155465126,
+            -0.017759712412953377,
+            0.017491042613983154,
+            0.017961829900741577,
+            -0.007718573324382305,
+            0.08410634845495224,
+            0.04713455215096474,
+            -0.030763784423470497,
+            -0.07868245244026184,
+            -0.04293506219983101,
+            -0.02663402259349823,
+            0.06502995640039444,
+            0.04134368151426315,
+            -0.0075789024122059345,
+            0.037559330463409424,
+            -0.022960234433412552,
+            -0.03559660166501999,
+            0.02433147467672825,
+            0.004160139709711075,
+            -0.006933990400284529,
+            -0.018853498622775078,
+            -0.07674053311347961,
+            -0.015214351005852222,
+            -0.0031465317588299513,
+            -0.0032662826124578714,
+            0.020626401528716087,
+            -0.023296812549233437,
+            0.01533068809658289,
+            -0.018831759691238403,
+            -0.013347934931516647,
+            0.05832105129957199,
+            0.016617508605122566,
+            -0.06054726615548134,
+            0.020946228876709938,
+            -0.05081603676080704,
+            0.0005762121290899813,
+            -0.013293956406414509,
+            0.05424816533923149,
+            -0.028400346636772156,
+            -0.008744322694838047,
+            -0.027219830080866814,
+            -0.009028765372931957,
+            0.08349941670894623,
+            -0.02220912277698517,
+            -0.0035634897649288177,
+            0.039803411811590195,
+            -0.009209544397890568,
+            -0.05272921174764633,
+            -0.03965644910931587,
+            -0.05518830195069313,
+            -0.0076417475938797,
+            0.011989743448793888,
+            0.07631900161504745,
+            0.03853122144937515,
+            -0.03482687100768089,
+            0.010916730388998985,
+            -0.012828757055103779,
+            0.04919871687889099,
+            0.05019025877118111,
+            -0.04743821546435356,
+            -0.024848056957125664,
+            0.03214584290981293,
+            -0.03514641523361206,
+            -0.030496522784233093,
+            -0.005360030569136143,
+            -0.03538995608687401,
+            -0.016368992626667023,
+            -0.07339470833539963,
+            0.022023534402251244,
+            -0.012556820176541805,
+            0.018365638330578804,
+            0.025001555681228638,
+            -0.029118210077285767,
+            -0.046666670590639114,
+            0.008231466636061668,
+            0.015543444082140923,
+            -0.018029138445854187,
+            0.007146908901631832,
+            -0.009827123954892159,
+            -0.09648268669843674,
+            -0.03867226839065552,
+            -0.0066386335529387,
+            -0.03292228281497955,
+            0.022209322080016136,
+            -0.048995133489370346,
+            -0.03118632733821869,
+            -0.017475144937634468,
+            0.005314778070896864,
+            -0.010616753250360489,
+            -0.04724809527397156,
+            0.027169331908226013,
+            0.003503959160298109,
+            -0.06886278837919235,
+            0.022409209981560707,
+            0.008452179841697216,
+            -0.024392019957304,
+            -0.020817982032895088,
+            -0.004606961738318205,
+            -0.08660633116960526,
+            0.031076667830348015,
+            0.020726440474390984,
+            -0.011223231442272663,
+            0.05015091598033905,
+            0.021451715379953384,
+            0.049609262496232986,
+            0.05560477077960968,
+            -0.05192512646317482,
+            -0.016848105937242508,
+            0.009753277525305748,
+            0.03756638243794441,
+            -0.03334583342075348,
+            0.040159404277801514,
+            0.01897178590297699,
+            -0.056337252259254456,
+            0.018127072602510452,
+            8.343596709892154e-05,
+            0.02721443772315979,
+            -0.037742555141448975,
+            0.01802193559706211,
+            0.09791397303342819,
+            -0.025166938081383705,
+            0.07462649047374725,
+            -0.027766922488808632,
+            -0.06332777440547943,
+            -0.005818391218781471,
+            -0.018811773508787155,
+            -0.03425326570868492,
+            0.003983109258115292,
+            -0.034356050193309784,
+            -0.04613350331783295,
+            0.006219684612005949,
+            -0.02750561013817787,
+            0.03812394291162491,
+            -0.029214290902018547,
+            -0.008362852968275547,
+            -0.046344004571437836,
+            0.015400147996842861,
+            -0.027044160291552544,
+            0.019668098539114,
+            0.01860121265053749,
+            -0.02408520132303238,
+            -0.07096672058105469,
+            -0.0003313044144306332,
+            0.024360060691833496,
+            -0.02565479651093483,
+            0.033272501081228256,
+            -0.009740769863128662,
+            -0.07754906266927719,
+            -0.005022569093853235,
+            0.01834244467318058,
+            -0.0670025572180748,
+            0.0031950040720403194,
+            0.016223441809415817,
+            0.05652153119444847,
+            0.05896124988794327,
+            -0.05402825400233269,
+            -0.007856715470552444,
+            0.010221654549241066,
+            -0.005885730497539043,
+            0.01112558413296938,
+            -0.03978840634226799,
+            -0.008038174360990524,
+            0.048503343015909195,
+            -0.04076062887907028,
+            0.05717281252145767,
+            0.02310008741915226,
+            -0.015716947615146637,
+            0.00578808831050992,
+            0.005534487310796976,
+            0.01627301797270775,
+            0.012666025198996067,
+            -0.037932366132736206,
+            -0.02889685146510601,
+            0.00509311119094491,
+            -0.015608384273946285,
+            -0.059495046734809875,
+            -0.02494220808148384,
+            0.024446364492177963,
+            0.03732331097126007,
+            -0.006206210236996412,
+            0.03822293132543564,
+            -0.030338769778609276,
+            -0.10873781144618988,
+            -0.021372010931372643,
+            -0.04941859468817711,
+            -0.0004265901807229966,
+            -0.011848388239741325,
+            -0.040232446044683456,
+            0.02904931642115116,
+            -0.0006163326324895024,
+            0.04501201957464218,
+            -0.0009105035569518805,
+            -0.034277040511369705,
+            -0.028672119602560997,
+            0.0012552812695503235,
+            0.04327942058444023,
+            0.0058600720949471,
+            0.007959491573274136,
+            -0.0052106245420873165,
+            -0.01613856852054596,
+            0.029123952612280846,
+            0.007203509099781513,
+            0.006210403982549906,
+            -0.053780049085617065,
+            -0.003420298220589757,
+            -0.03657878562808037,
+            0.019055450335144997,
+            -0.07376986742019653,
+            -0.019189076498150826,
+            -0.007890082895755768,
+            -0.022230826318264008,
+            0.006225301884114742,
+            0.01237472239881754,
+            0.06595351547002792,
+            0.04492981359362602,
+            0.01869170181453228,
+            0.012387770228087902,
+            0.05166162550449371,
+            -0.06800039112567902,
+            -0.03918451815843582,
+            -0.056165844202041626,
+            0.02326592244207859,
+            -0.015541363507509232,
+            0.033293239772319794,
+            -0.007216084748506546,
+            -0.012664951384067535,
+            0.05342225730419159,
+            0.009652439504861832,
+            0.027964351698756218,
+            -0.016909338533878326,
+            0.03330600634217262,
+            -0.060126710683107376,
+            -0.07517267763614655,
+            -0.025813739746809006,
+            -0.024271255359053612,
+            0.011216769926249981,
+            -0.020943764597177505,
+            0.01686793565750122,
+            -0.06828945130109787,
+            -0.018023250624537468,
+            -0.004829781129956245,
+            -0.022270847111940384,
+            0.030936313793063164,
+            -0.013894669711589813,
+            0.0368037149310112,
+            -0.05242981016635895,
+            -0.05051284283399582,
+            0.06604990363121033,
+            0.009301775135099888,
+            -0.014957845211029053,
+            -0.04281012713909149,
+            -0.0006833449588157237,
+            -0.005769087467342615,
+            -0.010068363510072231,
+            0.058508969843387604,
+            0.01898365654051304,
+            -0.018636951223015785,
+            0.04689044877886772,
+            0.056091975420713425,
+            -0.03881967067718506,
+            -0.019658103585243225,
+            -0.0003990831028204411,
+            0.07142409682273865,
+            0.012017408385872841,
+            -0.04087359085679054,
+            0.03531723469495773,
+            0.030207011848688126,
+            0.005139552056789398,
+            -0.07084347307682037,
+            0.009759706445038319,
+            -0.074960857629776,
+            0.05481172725558281,
+            0.028797954320907593,
+            -0.016217226162552834,
+            0.03127933293581009,
+            0.06848310679197311,
+            0.0331764742732048,
+            -0.007261150050908327,
+            -0.023036431521177292,
+            -0.015215406194329262,
+            -0.056041885167360306,
+            -0.05146646127104759
+        ],
+        "content_embedding": [
+            -0.035467296838760376,
+            0.05443365499377251,
+            0.004722591955214739,
+            -0.022551164031028748,
+            0.013528825715184212,
+            -0.01865273527801037,
+            0.02473974972963333,
+            -0.0093984454870224,
+            -0.04820428788661957,
+            -0.011309967376291752,
+            -0.041853927075862885,
+            0.02753269486129284,
+            0.023483864963054657,
+            0.0086299953982234,
+            -0.004760670010000467,
+            -0.011422916315495968,
+            0.02231433242559433,
+            -0.018113020807504654,
+            0.027575815096497536,
+            0.008628769777715206,
+            0.031078867614269257,
+            -0.022500742226839066,
+            0.011097921058535576,
+            -0.020555853843688965,
+            0.041947007179260254,
+            0.007781036198139191,
+            -0.03356451168656349,
+            0.05067972093820572,
+            -0.05009249970316887,
+            0.013387891463935375,
+            0.04737459868192673,
+            -0.02956528402864933,
+            -0.012283756397664547,
+            -0.02492530830204487,
+            -0.051903702318668365,
+            -0.020648062229156494,
+            0.07008657604455948,
+            -0.05525917932391167,
+            -0.005968133453279734,
+            0.042117420583963394,
+            0.04600752145051956,
+            -0.023857053369283676,
+            0.024955181404948235,
+            -0.019478371366858482,
+            0.027009692043066025,
+            -0.0035888778511434793,
+            0.050264179706573486,
+            -0.026319395750761032,
+            0.02974606677889824,
+            -0.03574950620532036,
+            -0.0011404261458665133,
+            0.00908538419753313,
+            -0.026853032410144806,
+            -0.01625720039010048,
+            -0.011155915446579456,
+            0.012969470582902431,
+            -0.03395452722907066,
+            0.004224491771310568,
+            0.027397319674491882,
+            -0.02826162986457348,
+            0.042576149106025696,
+            0.04274202510714531,
+            -0.023343440145254135,
+            0.031393349170684814,
+            0.003865004051476717,
+            -0.068922258913517,
+            0.021687647327780724,
+            -0.006593589670956135,
+            0.014760294929146767,
+            -0.007734894752502441,
+            -0.0031233499757945538,
+            -0.014138679951429367,
+            0.01479700393974781,
+            0.05785622447729111,
+            -0.04781193286180496,
+            -0.02295715920627117,
+            -0.02882302924990654,
+            -0.018427176401019096,
+            -0.018964029848575592,
+            0.06096252053976059,
+            0.05383418872952461,
+            -0.0870966985821724,
+            0.0031813366804271936,
+            0.01873805560171604,
+            0.046315208077430725,
+            -0.016668183729052544,
+            0.007771935313940048,
+            -0.008389935828745365,
+            -0.03101789578795433,
+            0.12752030789852142,
+            0.03457779064774513,
+            0.03240315988659859,
+            0.048814207315444946,
+            0.05700814723968506,
+            -0.06329526007175446,
+            0.057182129472494125,
+            -0.027157682925462723,
+            -0.008035550825297832,
+            -0.02922128140926361,
+            -0.033899255096912384,
+            -0.013119292445480824,
+            0.05552409961819649,
+            0.04266372323036194,
+            -0.003065067809075117,
+            0.03790399059653282,
+            0.01804651878774166,
+            0.018934324383735657,
+            0.04061003401875496,
+            0.03233874961733818,
+            0.017353640869259834,
+            -0.045026157051324844,
+            0.0002423059631837532,
+            0.00812580157071352,
+            0.017658300697803497,
+            -0.007964730262756348,
+            0.01015512179583311,
+            0.04046032205224037,
+            -0.06913582980632782,
+            0.004168798215687275,
+            0.03317571058869362,
+            0.012463781051337719,
+            -0.020389260724186897,
+            -0.022882815450429916,
+            -0.015693804249167442,
+            0.0500093474984169,
+            0.05442529916763306,
+            -0.05275300145149231,
+            -0.0008568991324864328,
+            -0.009641895070672035,
+            -0.003380047157406807,
+            -0.019793419167399406,
+            0.0063031697645783424,
+            -0.03327865153551102,
+            0.016263391822576523,
+            0.03218098729848862,
+            0.022281551733613014,
+            -0.06236790120601654,
+            -0.06712637841701508,
+            0.013925489969551563,
+            0.01688770391047001,
+            0.01467123068869114,
+            0.029035737738013268,
+            -0.013510127551853657,
+            -0.0371820330619812,
+            0.01489016879349947,
+            -0.05226032808423042,
+            -0.021125929430127144,
+            -0.002014611614868045,
+            -0.05400311201810837,
+            0.019783688709139824,
+            -0.0014786357060074806,
+            -0.01895768567919731,
+            0.01083068735897541,
+            -0.005890357308089733,
+            0.003382777562364936,
+            -0.04273455590009689,
+            -0.0478648841381073,
+            0.006365248002111912,
+            0.04027433693408966,
+            0.022969869896769524,
+            0.06722807139158249,
+            0.02113204449415207,
+            -0.03740633279085159,
+            0.0656556561589241,
+            -0.014887429773807526,
+            0.022357333451509476,
+            0.036207813769578934,
+            0.009992213919758797,
+            0.03665810078382492,
+            0.07260408997535706,
+            -0.005702183116227388,
+            -0.00880548357963562,
+            0.033025965094566345,
+            -0.017093362286686897,
+            0.039024271070957184,
+            0.04158668965101242,
+            0.008214588277041912,
+            -0.017436640337109566,
+            0.05074054002761841,
+            -0.021646566689014435,
+            0.0577220693230629,
+            -0.06182146817445755,
+            0.03490613400936127,
+            0.009574614465236664,
+            -0.05135552957653999,
+            -0.016593866050243378,
+            0.001374077401123941,
+            -0.0582745335996151,
+            0.009521838277578354,
+            -0.04114346206188202,
+            0.05998831242322922,
+            -0.017294712364673615,
+            -0.017998410388827324,
+            -0.036417942494153976,
+            -0.014111478812992573,
+            -0.035168007016181946,
+            0.04580182209610939,
+            0.006420421414077282,
+            0.012049577198922634,
+            -0.03691839054226875,
+            -0.041549112647771835,
+            -0.02593182772397995,
+            -0.07608001679182053,
+            -0.004325924441218376,
+            -0.029293090105056763,
+            0.05871257558465004,
+            0.04561365023255348,
+            -0.018353786319494247,
+            0.018815817311406136,
+            -0.02768997848033905,
+            0.10190171003341675,
+            0.05626858025789261,
+            -0.006779504008591175,
+            0.005354198161512613,
+            -0.053908295929431915,
+            -0.03453999012708664,
+            0.02781379222869873,
+            -0.04989396408200264,
+            -0.03950505331158638,
+            0.02282608300447464,
+            0.006389955058693886,
+            -0.007375086657702923,
+            -0.016243990510702133,
+            0.028544900938868523,
+            -0.020701216533780098,
+            0.012176213786005974,
+            -0.019373498857021332,
+            0.023940887302160263,
+            -0.03986368328332901,
+            -0.0040043736808001995,
+            -0.02535220980644226,
+            -0.08186554163694382,
+            0.07860240340232849,
+            0.004978376906365156,
+            0.06400016695261002,
+            -0.02991490252315998,
+            -0.03288387134671211,
+            0.03286135569214821,
+            0.0247455183416605,
+            0.08134172111749649,
+            0.062203265726566315,
+            0.017080431804060936,
+            -0.003969072364270687,
+            -0.024894852191209793,
+            -0.07480036467313766,
+            0.03882874175906181,
+            -0.0074541885405778885,
+            -0.011322571896016598,
+            -0.06285038590431213,
+            0.004618136677891016,
+            -0.019962741062045097,
+            0.01853892207145691,
+            0.002305575180798769,
+            0.023541608825325966,
+            0.017216432839632034,
+            -0.044929757714271545,
+            -0.022532327100634575,
+            0.05122198164463043,
+            0.0008766956743784249,
+            -0.025474006310105324,
+            0.04030180349946022,
+            -0.013362268917262554,
+            0.049265045672655106,
+            0.001589711057022214,
+            0.014848759397864342,
+            0.0022126054391264915,
+            -0.028914116322994232,
+            -0.009881545789539814,
+            -0.010318109765648842,
+            0.006385906133800745,
+            0.010556558147072792,
+            0.007666149642318487,
+            0.016665013507008553,
+            0.09090837836265564,
+            0.008256189525127411,
+            -0.006583006586879492,
+            0.0044986652210354805,
+            -0.0336960032582283,
+            0.047732625156641006,
+            -0.007069372106343508,
+            -0.044969744980335236,
+            -0.0907006487250328,
+            -0.04223865643143654,
+            0.007469010539352894,
+            0.006147805601358414,
+            0.04827409237623215,
+            0.03519561514258385,
+            0.032267000526189804,
+            0.05173507332801819,
+            -0.016001909971237183,
+            0.034578241407871246,
+            -0.02854917198419571,
+            -0.01587686315178871,
+            0.03134807571768761,
+            -0.010930678807199001,
+            0.044245973229408264,
+            0.1186295673251152,
+            -0.031642355024814606,
+            0.01669827103614807,
+            -0.026775898411870003,
+            -0.002936996053904295,
+            -0.013815718702971935,
+            -0.009453569538891315,
+            -0.035879991948604584,
+            -0.02232815884053707,
+            -0.009286822751164436,
+            -0.01117252279073,
+            0.014098073355853558,
+            0.023366371169686317,
+            0.022420832887291908,
+            -0.029833031818270683,
+            0.0013570807641372085,
+            -0.0211170744150877,
+            -0.027633074671030045,
+            -0.02915397845208645,
+            0.023663034662604332,
+            0.04199281334877014,
+            -0.0311698317527771,
+            0.025238486006855965,
+            -0.00992826372385025,
+            0.005087476689368486,
+            -0.050041183829307556,
+            -0.018602682277560234,
+            -0.06774407625198364,
+            -0.019117988646030426,
+            0.08245334029197693,
+            0.030311768874526024,
+            0.020432988181710243,
+            -0.03738946095108986,
+            0.04251522198319435,
+            0.002886879490688443,
+            0.0938342958688736,
+            -0.05836429446935654,
+            -0.0232597254216671,
+            0.0074686286970973015,
+            -0.0020157117396593094,
+            -0.013439277186989784,
+            0.02590363286435604,
+            0.0034541902132332325,
+            0.002955070696771145,
+            -0.0020802158396691084,
+            -0.011827156879007816,
+            -0.06622112542390823,
+            -0.05226997658610344,
+            -0.031827233731746674,
+            0.0012936017010360956,
+            0.01702217012643814,
+            -0.016136569902300835,
+            0.04939497634768486,
+            0.006943605840206146,
+            -0.05098084360361099,
+            0.03143058344721794,
+            -0.004094736184924841,
+            -0.011557313613593578,
+            0.000757173984311521,
+            -0.01120754610747099,
+            0.036905039101839066,
+            0.02395678497850895,
+            0.009881307370960712,
+            0.05312298238277435,
+            0.05778184533119202,
+            0.02520277164876461,
+            0.020175758749246597,
+            -0.025740133598446846,
+            0.04891965910792351,
+            0.05155428871512413,
+            0.04089348018169403,
+            0.06249197572469711,
+            0.02368168905377388,
+            -0.03247880935668945,
+            0.019493652507662773,
+            -0.06181112304329872,
+            0.019057979807257652,
+            0.037210095673799515,
+            0.02910085767507553,
+            0.02495957538485527,
+            -0.08179862797260284,
+            -0.007498551160097122,
+            0.036072876304388046,
+            -0.016061626374721527,
+            0.0725645050406456,
+            0.005645937751978636,
+            0.0032921379897743464,
+            -0.029446475207805634,
+            0.020205944776535034,
+            0.002388844033703208,
+            -0.029442399740219116,
+            -0.031475961208343506,
+            0.024486446753144264,
+            -0.038434699177742004,
+            -0.05131153389811516,
+            0.00129043054766953,
+            -0.01692604087293148,
+            -0.007422945462167263,
+            -0.02433120459318161,
+            0.004650108516216278,
+            -0.04251663386821747,
+            -0.009143602102994919,
+            0.017507102340459824,
+            0.042100246995687485,
+            -0.06103592365980148,
+            -0.0508011095225811,
+            -0.000937039265409112,
+            0.025160834193229675,
+            -0.048878248780965805,
+            0.010422220453619957,
+            -0.014773974195122719,
+            -0.06574267894029617,
+            0.0027621325571089983,
+            -0.0019821953028440475,
+            0.006184928119182587,
+            0.024707674980163574,
+            -0.022308405488729477,
+            -0.06509386748075485,
+            0.04186487942934036,
+            0.0023416660260409117,
+            0.0650840550661087,
+            0.03807358071208,
+            -0.024585191160440445,
+            -0.017596496269106865,
+            0.005341595038771629,
+            0.03675152733922005,
+            0.06293662637472153,
+            0.010854244232177734,
+            -0.050199203193187714,
+            -0.037359531968832016,
+            0.017929432913661003,
+            0.023822667077183723,
+            0.019726712256669998,
+            -0.00759292533621192,
+            0.043509598821401596,
+            -0.0014670701930299401,
+            -0.0006681938539259136,
+            -0.0055070724338293076,
+            -0.07182206958532333,
+            0.037307076156139374,
+            0.06350742280483246,
+            0.049223095178604126,
+            0.017340589314699173,
+            0.05529596656560898,
+            0.023639194667339325,
+            -0.02478986792266369,
+            -0.02248029224574566,
+            -0.042737238109111786,
+            -0.0018032155930995941,
+            0.05576873943209648,
+            0.12722158432006836,
+            0.004959811456501484,
+            -0.033451229333877563,
+            -0.007337609305977821,
+            0.018852578476071358,
+            0.031502317637205124,
+            0.013375848531723022,
+            -0.0066598327830433846,
+            0.07771285623311996,
+            -0.017693838104605675,
+            -0.030503049492836,
+            -0.04393269121646881,
+            0.013323146849870682,
+            0.010107941925525665,
+            0.02004137821495533,
+            0.0377974770963192,
+            0.013478322885930538,
+            0.024949608370661736,
+            -0.01634461060166359,
+            -0.015226340852677822,
+            0.013924108818173409,
+            0.0038409747648984194,
+            -0.00358059024438262,
+            -0.005057516973465681,
+            -0.008272752165794373,
+            0.04373026266694069,
+            -0.0014998909318819642,
+            0.009924792684614658,
+            -0.040317751467227936,
+            -0.04541180655360222,
+            0.06625904142856598,
+            -0.028080880641937256,
+            0.04732294753193855,
+            0.0047000702470541,
+            0.02857903018593788,
+            0.004553706850856543,
+            -0.04188435524702072,
+            0.023083623498678207,
+            -0.060619525611400604,
+            0.01963491179049015,
+            -0.008608276024460793,
+            0.0034778753761202097,
+            -0.016133952885866165,
+            0.009059683419764042,
+            -0.0009118590969592333,
+            0.08675801753997803,
+            0.004535067826509476,
+            -0.021998068317770958,
+            -0.0789642184972763,
+            -0.033289894461631775,
+            -0.04486677423119545,
+            0.014945252798497677,
+            0.04513613134622574,
+            -0.05311649665236473,
+            0.009399711154401302,
+            0.004345519933849573,
+            -0.004021052736788988,
+            0.01709410734474659,
+            -0.020986782386898994,
+            -0.011685016565024853,
+            -0.02048366330564022,
+            -0.08524532616138458,
+            -0.004114300478249788,
+            0.040531281381845474,
+            -0.0005771859432570636,
+            0.02984555996954441,
+            -0.0002479814866092056,
+            0.04756562039256096,
+            -0.013039377517998219,
+            -0.09513615071773529,
+            0.07444311678409576,
+            0.0044719018042087555,
+            -0.09768522530794144,
+            0.04403488337993622,
+            0.013910059817135334,
+            0.06657753884792328,
+            0.026994489133358,
+            0.03657658398151398,
+            -0.11561834812164307,
+            0.02878704108297825,
+            -0.012593223713338375,
+            -0.01532658003270626,
+            0.06045927479863167,
+            -0.04569881781935692,
+            -0.0029045850969851017,
+            0.06762480735778809,
+            0.012874988839030266,
+            -0.011422640644013882,
+            0.025211291387677193,
+            -0.07570745050907135,
+            -0.018061399459838867,
+            -0.033531878143548965,
+            -0.010049374774098396,
+            0.02582205832004547,
+            -0.015443898737430573,
+            0.029427431523799896,
+            -0.02071801945567131,
+            0.02054932527244091,
+            0.017246615141630173,
+            -0.07276910543441772,
+            -0.053859222680330276,
+            0.060189153999090195,
+            -0.04336293414235115,
+            -0.06396458297967911,
+            0.08002400398254395,
+            -0.041166432201862335,
+            0.000430541840614751,
+            -0.10547704994678497,
+            -0.014112395234405994,
+            -0.06070064380764961,
+            0.01796649396419525,
+            -0.045275188982486725,
+            -0.0018861661665141582,
+            -0.0022482818458229303,
+            0.004191190470010042,
+            6.745498831151053e-05,
+            0.07350871711969376,
+            -0.01797996647655964,
+            0.03183342143893242,
+            -0.10409794747829437,
+            -0.0291685052216053,
+            0.02071727253496647,
+            0.021267961710691452,
+            -0.021560702472925186,
+            -0.05026571452617645,
+            -0.013422243297100067,
+            -0.0011607048800215125,
+            0.016948888078331947,
+            -0.01588856242597103,
+            -0.05063013359904289,
+            0.05952488258481026,
+            -0.05575632303953171,
+            -0.06906414031982422,
+            -0.004353572614490986,
+            -0.02773641049861908,
+            -0.043097492307424545,
+            0.03103402815759182,
+            0.02164989709854126,
+            0.0013185666175559163,
+            0.02606332302093506,
+            -0.059723641723394394,
+            -0.008657965809106827,
+            0.06580374389886856,
+            -0.020771000534296036,
+            -0.022305399179458618,
+            0.008068420924246311,
+            -0.004975682124495506,
+            -0.033461254090070724,
+            -0.040884874761104584,
+            0.052932899445295334,
+            -0.0054899416863918304,
+            -0.03155453875660896,
+            0.002439886098727584,
+            -0.0361575223505497,
+            -0.03652369976043701,
+            -0.010043974034488201,
+            0.01681465655565262,
+            3.9381829992635176e-05,
+            0.011527255177497864,
+            0.06904088705778122,
+            -0.005501871462911367,
+            0.0259085800498724,
+            -0.021282166242599487,
+            -0.03796657174825668,
+            -0.002881726250052452,
+            -0.018672630190849304,
+            -0.003463461296632886,
+            -0.008101037703454494,
+            -0.019035371020436287,
+            -0.0025111068971455097,
+            0.03926572576165199,
+            -0.0513470396399498,
+            0.04829537495970726,
+            -0.001188569120131433,
+            -0.0121685229241848,
+            -0.059901442378759384,
+            0.07364466786384583,
+            0.006562754046171904,
+            0.00707247294485569,
+            0.028408123180270195,
+            -0.02494397945702076,
+            -0.04187498614192009,
+            0.0066386316902935505,
+            0.06244710460305214,
+            0.02900586649775505,
+            0.04932861402630806,
+            -0.04402685537934303,
+            -0.006739918142557144,
+            -0.0502609983086586,
+            -0.0015801729168742895,
+            -0.026301531121134758,
+            -0.024203499779105186,
+            -0.019028285518288612,
+            0.055076178163290024,
+            0.009030332788825035,
+            -0.04907704144716263,
+            -0.019399652257561684,
+            0.009713590145111084,
+            0.05042042210698128,
+            -0.00020382895309012383,
+            -0.010405965149402618,
+            0.01872927136719227,
+            0.002546734409406781,
+            0.020958390086889267,
+            0.0634453296661377,
+            0.04931068792939186,
+            -0.014592095278203487,
+            0.0075549716129899025,
+            0.02017839439213276,
+            0.03344761207699776,
+            -0.005005223676562309,
+            0.01818416453897953,
+            -0.05528895929455757,
+            0.03879536688327789,
+            0.018610917031764984,
+            -0.029319677501916885,
+            -0.00493574095889926,
+            0.01762193627655506,
+            0.008898349478840828,
+            -0.017192110419273376,
+            -0.03400791808962822,
+            -0.026591692119836807,
+            -0.03768239915370941,
+            0.007602880708873272,
+            -0.020310858264565468,
+            -0.0036565649788826704,
+            -0.00616755336523056,
+            -0.057577136904001236,
+            0.008387535810470581,
+            0.021555650979280472,
+            -0.01923108845949173,
+            -0.019822189584374428,
+            -0.03861076384782791,
+            -0.04258895292878151,
+            0.0005390863516367972,
+            -0.009946192614734173,
+            0.04911184310913086,
+            -0.05009220167994499,
+            0.00297548552043736,
+            0.019344164058566093,
+            0.005506082437932491,
+            0.018321573734283447,
+            -0.027131471782922745,
+            -0.052699681371450424,
+            -0.02292790077626705,
+            0.0568309910595417,
+            -0.008538461290299892,
+            -0.05772045999765396,
+            0.018903164193034172,
+            -0.03690820932388306,
+            -0.034110669046640396,
+            -0.008381159976124763,
+            0.03926640748977661,
+            0.04370100051164627,
+            -0.007475440856069326,
+            0.06952399015426636,
+            -0.0031064660288393497,
+            0.040785301476716995,
+            -0.008955440483987331,
+            -0.016698094084858894,
+            -0.007412049453705549,
+            0.022290483117103577,
+            0.006028760224580765,
+            -0.019992463290691376,
+            -0.04154061898589134,
+            -0.023284243419766426,
+            0.04972238838672638,
+            0.0008079080143943429,
+            -0.0057194954715669155,
+            0.037808094173669815,
+            -0.00983867421746254,
+            -0.030191699042916298,
+            0.014910571277141571,
+            0.0004001195775344968,
+            0.08586109429597855,
+            -0.014850648120045662,
+            -0.07815773040056229,
+            0.05393945425748825,
+            -0.019953783601522446,
+            0.0016224493738263845,
+            0.018219690769910812,
+            0.014311570674180984,
+            0.05567210912704468,
+            0.004363455809652805,
+            0.01872050203382969,
+            -0.05933142080903053,
+            -0.04872509092092514,
+            0.02652469463646412,
+            -0.04346488043665886,
+            -0.027931908145546913,
+            -0.03355146571993828,
+            0.004589339718222618,
+            -0.05587214604020119,
+            -0.015419036149978638,
+            0.04789341986179352,
+            -0.059666525572538376,
+            0.00552733987569809,
+            0.012681040912866592,
+            0.007240649312734604,
+            -0.0045613935217261314,
+            -0.060716547071933746,
+            0.03223521634936333,
+            0.024270178750157356,
+            -0.025784391909837723,
+            -0.01736401580274105,
+            -0.0014227400533854961,
+            -0.011367680504918098,
+            -0.035415612161159515,
+            -0.01793254353106022,
+            0.033805977553129196,
+            -0.0080083217471838,
+            0.021929381415247917,
+            0.012236963026225567,
+            0.002641203347593546,
+            0.0067292568273842335,
+            -0.007680798415094614,
+            -0.02231515571475029,
+            0.023524953052401543,
+            0.010132606141269207,
+            0.0030864113941788673,
+            -0.03816894069314003,
+            -0.0007171767647378147
+        ],
+        "chunk_ind": 0
+    }
+]
\ No newline at end of file
diff --git a/backend/danswer/seeding/load_docs.py b/backend/danswer/seeding/load_docs.py
new file mode 100644
index 00000000000..2756e7ddf61
--- /dev/null
+++ b/backend/danswer/seeding/load_docs.py
@@ -0,0 +1,212 @@
+import datetime
+import json
+import os
+from typing import cast
+
+from sqlalchemy.orm import Session
+
+from danswer.access.models import default_public_access
+from danswer.configs.constants import DEFAULT_BOOST
+from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import KV_DOCUMENTS_SEEDED_KEY
+from danswer.configs.model_configs import DEFAULT_DOCUMENT_ENCODER_MODEL
+from danswer.connectors.models import Document
+from danswer.connectors.models import IndexAttemptMetadata
+from danswer.connectors.models import InputType
+from danswer.connectors.models import Section
+from danswer.db.connector import check_connectors_exist
+from danswer.db.connector import create_connector
+from danswer.db.connector_credential_pair import add_credential_to_connector
+from danswer.db.credentials import PUBLIC_CREDENTIAL_ID
+from danswer.db.document import check_docs_exist
+from danswer.db.enums import AccessType
+from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.index_attempt import mock_successful_index_attempt
+from danswer.db.search_settings import get_current_search_settings
+from danswer.document_index.factory import get_default_document_index
+from danswer.indexing.indexing_pipeline import index_doc_batch_prepare
+from danswer.indexing.models import ChunkEmbedding
+from danswer.indexing.models import DocMetadataAwareIndexChunk
+from danswer.key_value_store.factory import get_kv_store
+from danswer.key_value_store.interface import KvKeyNotFoundError
+from danswer.server.documents.models import ConnectorBase
+from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
+
+
+logger = setup_logger()
+
+
+def _create_indexable_chunks(
+    preprocessed_docs: list[dict],
+) -> tuple[list[Document], list[DocMetadataAwareIndexChunk]]:
+    ids_to_documents = {}
+    chunks = []
+    for preprocessed_doc in preprocessed_docs:
+        document = Document(
+            id=preprocessed_doc["url"],  # For Web connector, the URL is the ID
+            # The section is not really used past this point since we have already done the other processing
+            # for the chunking and embedding.
+            sections=[
+                Section(text=preprocessed_doc["content"], link=preprocessed_doc["url"])
+            ],
+            source=DocumentSource.WEB,
+            semantic_identifier=preprocessed_doc["title"],
+            metadata={},
+            doc_updated_at=None,
+            primary_owners=[],
+            secondary_owners=[],
+        )
+        if preprocessed_doc["chunk_ind"] == 0:
+            ids_to_documents[document.id] = document
+
+        chunk = DocMetadataAwareIndexChunk(
+            chunk_id=preprocessed_doc["chunk_ind"],
+            blurb=preprocessed_doc["content"]
+            .split(".", 1)[0]
+            .split("!", 1)[0]
+            .split("?", 1)[0],
+            content=preprocessed_doc["content"],
+            source_links={0: preprocessed_doc["url"]},
+            section_continuation=False,
+            source_document=document,
+            title_prefix=preprocessed_doc["title"],
+            metadata_suffix_semantic="",
+            metadata_suffix_keyword="",
+            mini_chunk_texts=None,
+            large_chunk_reference_ids=[],
+            embeddings=ChunkEmbedding(
+                full_embedding=preprocessed_doc["content_embedding"],
+                mini_chunk_embeddings=[],
+            ),
+            title_embedding=preprocessed_doc["title_embedding"],
+            tenant_id=None,
+            access=default_public_access,
+            document_sets=set(),
+            boost=DEFAULT_BOOST,
+        )
+        chunks.append(chunk)
+
+    return list(ids_to_documents.values()), chunks
+
+
+def seed_initial_documents(db_session: Session) -> None:
+    """
+    Seed initial documents so users don't have an empty index to start
+
+    Documents are only loaded if:
+    - This is the first setup (if the user deletes the docs, we don't load them again)
+    - The index is empty, there are no docs and no (non-default) connectors
+    - The user has not updated the embedding models
+        - If they do, then we have to actually index the website
+        - If the embedding model is already updated on server startup, they're not a new user
+
+    Note that regardless of any search settings, the default documents are always loaded with
+    the predetermined chunk sizes and single pass embedding.
+
+    Steps are as follows:
+    - Check if this needs to run
+    - Create the connector representing this
+    - Create the cc-pair (attaching the public credential) and mocking values like the last success
+    - Indexing the documents into Postgres
+    - Indexing the documents into Vespa
+    - Create a fake index attempt with fake times
+    """
+    logger.info("Seeding initial documents")
+
+    kv_store = get_kv_store()
+    try:
+        kv_store.load(KV_DOCUMENTS_SEEDED_KEY)
+        logger.info("Documents already seeded, skipping")
+        return
+    except KvKeyNotFoundError:
+        pass
+
+    if check_docs_exist(db_session):
+        logger.info("Documents already exist, skipping")
+        return
+
+    if check_connectors_exist(db_session):
+        logger.info("Connectors already exist, skipping")
+        return
+
+    search_settings = get_current_search_settings(db_session)
+    if search_settings.model_name != DEFAULT_DOCUMENT_ENCODER_MODEL:
+        logger.info("Embedding model has been updated, skipping")
+        return
+
+    document_index = get_default_document_index(
+        primary_index_name=search_settings.index_name, secondary_index_name=None
+    )
+
+    # Create a connector so the user can delete it if they want
+    # or reindex it with a new search model if they want
+    connector_data = ConnectorBase(
+        name="Sample Use Cases",
+        source=DocumentSource.WEB,
+        input_type=InputType.LOAD_STATE,
+        connector_specific_config={
+            "base_url": "https://docs.danswer.dev/more/use_cases",
+            "web_connector_type": "recursive",
+        },
+        refresh_freq=None,  # Never refresh by default
+        prune_freq=None,
+        indexing_start=None,
+    )
+
+    connector = create_connector(db_session, connector_data)
+    connector_id = cast(int, connector.id)
+
+    last_index_time = datetime.datetime.now(datetime.timezone.utc)
+
+    result = add_credential_to_connector(
+        db_session=db_session,
+        user=None,
+        connector_id=connector_id,
+        credential_id=PUBLIC_CREDENTIAL_ID,
+        access_type=AccessType.PUBLIC,
+        cc_pair_name=connector_data.name,
+        groups=None,
+        initial_status=ConnectorCredentialPairStatus.PAUSED,
+        last_successful_index_time=last_index_time,
+    )
+    cc_pair_id = cast(int, result.data)
+
+    initial_docs_path = os.path.join(
+        os.getcwd(), "danswer", "seeding", "initial_docs.json"
+    )
+    processed_docs = json.load(open(initial_docs_path))
+
+    docs, chunks = _create_indexable_chunks(processed_docs)
+
+    index_doc_batch_prepare(
+        document_batch=docs,
+        index_attempt_metadata=IndexAttemptMetadata(
+            connector_id=connector_id,
+            credential_id=PUBLIC_CREDENTIAL_ID,
+        ),
+        db_session=db_session,
+        ignore_time_skip=True,  # Doesn't actually matter here
+    )
+
+    # In this case since there are no other connectors running in the background
+    # and this is a fresh deployment, there is no need to grab any locks
+    logger.info(
+        "Indexing seeding documents into Vespa "
+        "(Vespa may take a few seconds to become ready after receiving the schema)"
+    )
+
+    # Retries here because the index may take a few seconds to become ready
+    # as we just sent over the Vespa schema and there is a slight delay
+    index_with_retries = retry_builder()(document_index.index)
+    index_with_retries(chunks=chunks)
+
+    # Mock a run for the UI even though it did not actually call out to anything
+    mock_successful_index_attempt(
+        connector_credential_pair_id=cc_pair_id,
+        search_settings_id=search_settings.id,
+        docs_indexed=len(docs),
+        db_session=db_session,
+    )
+
+    kv_store.store(KV_DOCUMENTS_SEEDED_KEY, True)
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index 747fe809451..a27580a2b15 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -40,6 +40,7 @@
 from danswer.natural_language_processing.search_nlp_models import warm_up_cross_encoder
 from danswer.search.models import SavedSearchSettings
 from danswer.search.retrieval.search_runner import download_nltk_data
+from danswer.seeding.load_docs import seed_initial_documents
 from danswer.server.manage.llm.models import LLMProviderUpsertRequest
 from danswer.server.settings.store import load_settings
 from danswer.server.settings.store import store_settings
@@ -58,6 +59,12 @@
 
 
 def setup_danswer(db_session: Session) -> None:
+    """
+    Setup Danswer for a particular tenant. In the Single Tenant case, it will set it up for the default schema
+    on server startup. In the MT case, it will be called when the tenant is created.
+
+    The Tenant Service calls the tenants/create endpoint which runs this.
+    """
     check_index_swap(db_session=db_session)
     search_settings = get_current_search_settings(db_session)
     secondary_search_settings = get_secondary_search_settings(db_session)
@@ -107,7 +114,8 @@ def setup_danswer(db_session: Session) -> None:
     if not MULTI_TENANT:
         mark_reindex_flag(db_session)
 
-    # ensure Vespa is setup correctly
+    # Ensure Vespa is setup correctly, this step is relatively near the end because Vespa
+    # takes a bit of time to start up
     logger.notice("Verifying Document Index(s) is/are available.")
     document_index = get_default_document_index(
         primary_index_name=search_settings.index_name,
@@ -139,6 +147,8 @@ def setup_danswer(db_session: Session) -> None:
     # update multipass indexing setting based on GPU availability
     update_default_multipass_indexing(db_session)
 
+    seed_initial_documents(db_session)
+
 
 def translate_saved_search_settings(db_session: Session) -> None:
     kv_store = get_kv_store()
@@ -311,11 +321,13 @@ def update_default_multipass_indexing(db_session: Session) -> None:
 
 
 def setup_multitenant_danswer() -> None:
+    # For Managed Vespa, the schema is sent over via the Vespa Console manually.
     if not MANAGED_VESPA:
         setup_vespa_multitenant(SUPPORTED_EMBEDDING_MODELS)
 
 
 def setup_vespa_multitenant(supported_indices: list[SupportedEmbeddingModel]) -> bool:
+    # This is for local testing
     WAIT_SECONDS = 5
     VESPA_ATTEMPTS = 5
     for x in range(VESPA_ATTEMPTS):
diff --git a/backend/danswer/connectors/cross_connector_utils/retry_wrapper.py b/backend/danswer/utils/retry_wrapper.py
similarity index 79%
rename from backend/danswer/connectors/cross_connector_utils/retry_wrapper.py
rename to backend/danswer/utils/retry_wrapper.py
index 7312d1349f7..2d6d79ca5eb 100644
--- a/backend/danswer/connectors/cross_connector_utils/retry_wrapper.py
+++ b/backend/danswer/utils/retry_wrapper.py
@@ -22,18 +22,18 @@ def retry_builder(
     jitter: tuple[float, float] | float = 1,
 ) -> Callable[[F], F]:
     """Builds a generic wrapper/decorator for calls to external APIs that
-    may fail due to rate limiting, flakes, or other reasons. Applies expontential
+    may fail due to rate limiting, flakes, or other reasons. Applies exponential
     backoff with jitter to retry the call."""
 
-    @retry(
-        tries=tries,
-        delay=delay,
-        max_delay=max_delay,
-        backoff=backoff,
-        jitter=jitter,
-        logger=cast(Logger, logger),
-    )
     def retry_with_default(func: F) -> F:
+        @retry(
+            tries=tries,
+            delay=delay,
+            max_delay=max_delay,
+            backoff=backoff,
+            jitter=jitter,
+            logger=cast(Logger, logger),
+        )
         def wrapped_func(*args: list, **kwargs: dict[str, Any]) -> Any:
             return func(*args, **kwargs)
 
diff --git a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
index 19dbb845323..f1e805d46d7 100644
--- a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
@@ -9,7 +9,6 @@
 from sqlalchemy.orm import Session
 
 from danswer.access.models import ExternalAccess
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.factory import instantiate_connector
 from danswer.connectors.google_drive.connector_auth import (
     get_google_drive_creds,
@@ -19,6 +18,7 @@
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 from ee.danswer.db.document import upsert_document_external_perms__no_commit
 
 # Google Drive APIs are quite flakey and may 500 for an
diff --git a/backend/ee/danswer/external_permissions/google_drive/group_sync.py b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
index 7bb919d4686..ab0f62a886b 100644
--- a/backend/ee/danswer/external_permissions/google_drive/group_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
@@ -7,7 +7,6 @@
 from googleapiclient.errors import HttpError  # type: ignore
 from sqlalchemy.orm import Session
 
-from danswer.connectors.cross_connector_utils.retry_wrapper import retry_builder
 from danswer.connectors.google_drive.connector_auth import (
     get_google_drive_creds,
 )
@@ -15,6 +14,7 @@
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 from ee.danswer.db.external_perm import ExternalUserGroup
 from ee.danswer.db.external_perm import replace_user__ext_group_for_cc_pair__no_commit
 
diff --git a/backend/scripts/document_seeding_prep.py b/backend/scripts/document_seeding_prep.py
new file mode 100644
index 00000000000..d853b68ccb4
--- /dev/null
+++ b/backend/scripts/document_seeding_prep.py
@@ -0,0 +1,240 @@
+# This script preps the documents used for initially seeding the index. It handles the embedding so that the
+# documents can be added to the index with minimal processing.
+import json
+
+from pydantic import BaseModel
+from sentence_transformers import SentenceTransformer  # type: ignore
+
+
+class SeedPresaveDocument(BaseModel):
+    url: str
+    title: str
+    content: str
+    title_embedding: list[float]
+    content_embedding: list[float]
+    chunk_ind: int = 0
+
+
+# Be sure to use the default embedding model
+model = SentenceTransformer("nomic-ai/nomic-embed-text-v1", trust_remote_code=True)
+tokenizer = model.tokenizer
+
+# This is easier than cleaning up the crawl, needs to be updated if the sites are changed
+overview_title = "Use Cases Overview"
+overview = (
+    "How to leverage Danswer in your organization\n\n"
+    "Danswer Overview\n"
+    "Danswer is the AI Assistant connected to your organization's docs, apps, and people. "
+    "Danswer makes Generative AI more versatile for work by enabling new types of questions like "
+    '"What is the most common feature request we\'ve heard from customers this month". '
+    "Whereas other AI systems have no context of your team and are generally unhelpful with work related questions, "
+    "Danswer makes it possible to ask these questions in natural language and get back answers in seconds.\n\n"
+    "Danswer can connect to +30 different tools and the use cases are not limited to the ones in the following pages. "
+    "The highlighted use cases are for inspiration and come from feedback gathered from our users and customers.\n\n\n"
+    "Common Getting Started Questions:\n\n"
+    "Why are these docs connected in my Danswer deployment?\n"
+    "Answer: This is just an example of how connectors work in Danswer. You can connect up your own team's knowledge "
+    "and you will be able to ask questions unique to your organization. Danswer will keep all of the knowledge up to date "
+    "and in sync with your connected applications.\n\n"
+    "Is my data being sent anywhere when I connect it up to Danswer?\n"
+    "Answer: No! Danswer is built with data security as our highest priority. We open sourced it so our users can know "
+    "exactly what is going on with their data. By default all of the document processing happens within Danswer. "
+    "The only time it is sent outward is for the GenAI call to generate answers.\n\n"
+    "Where is the feature for auto sync-ing document level access permissions from all connected sources?\n"
+    "Answer: This falls under the Enterprise Edition set of Danswer features built on top of the MIT/community edition. "
+    "If you are on Danswer Cloud, you have access to them by default. If you're running it yourself, reach out to the "
+    "Danswer team to receive access."
+)
+
+enterprise_search_title = "Enterprise Search"
+enterprise_search_1 = (
+    "Value of Enterprise Search with Danswer\n\n"
+    "What is Enterprise Search and why is it Important?\n"
+    "An Enterprise Search system gives team members a single place to access all of the disparate knowledge "
+    "of an organization. Critical information is saved across a host of channels like call transcripts with "
+    "prospects, engineering design docs, IT runbooks, customer support email exchanges, project management "
+    "tickets, and more. As fast moving teams scale up, information gets spread out and more disorganized.\n\n"
+    "Since it quickly becomes infeasible to check across every source, decisions get made on incomplete "
+    "information, employee satisfaction decreases, and the most valuable members of your team are tied up "
+    "with constant distractions as junior teammates are unable to unblock themselves. Danswer solves this "
+    "problem by letting anyone on the team access all of the knowledge across your organization in a "
+    "permissioned and secure way. Users can ask questions in natural language and get back answers and "
+    "documents across all of the connected sources instantly.\n\n"
+    "What's the real cost?\n"
+    "A typical knowledge worker spends over 2 hours a week on search, but more than that, the cost of "
+    "incomplete or incorrect information can be extremely high. Customer support/success that isn't able "
+    "to find the reference to similar cases could cause hours or even days of delay leading to lower "
+    "customer satisfaction or in the worst case - churn. An account exec not realizing that a prospect had "
+    "previously mentioned a specific need could lead to lost deals. An engineer not realizing a similar "
+    "feature had previously been built could result in weeks of wasted development time and tech debt with "
+    "duplicate implementation. With a lack of knowledge, your whole organization is navigating in the dark "
+    "- inefficient and mistake prone."
+)
+
+enterprise_search_2 = (
+    "More than Search\n"
+    "When analyzing the entire corpus of knowledge within your company is as easy as asking a question "
+    "in a search bar, your entire team can stay informed and up to date. Danswer also makes it trivial "
+    "to identify where knowledge is well documented and where it is lacking. Team members who are centers "
+    "of knowledge can begin to effectively document their expertise since it is no longer being thrown into "
+    "a black hole. All of this allows the organization to achieve higher efficiency and drive business outcomes.\n\n"
+    "With Generative AI, the entire user experience has evolved as well. For example, instead of just finding similar "
+    "cases for your customer support team to reference, Danswer breaks down the issue and explains it so that even "
+    "the most junior members can understand it. This in turn lets them give the most holistic and technically accurate "
+    "response possible to your customers. On the other end, even the super stars of your sales team will not be able "
+    "to review 10 hours of transcripts before hopping on that critical call, but Danswer can easily parse through it "
+    "in mere seconds and give crucial context to help your team close."
+)
+
+ai_platform_title = "AI Platform"
+ai_platform = (
+    "Build AI Agents powered by the knowledge and workflows specific to your organization.\n\n"
+    "Beyond Answers\n"
+    "Agents enabled by generative AI and reasoning capable models are helping teams to automate their work. "
+    "Danswer is helping teams make it happen. Danswer provides out of the box user chat sessions, attaching custom tools, "
+    "handling LLM reasoning, code execution, data analysis, referencing internal knowledge, and much more.\n\n"
+    "Danswer as a platform is not a no-code agent builder. We are made by developers for developers and this gives your "
+    "team the full flexibility and power to create agents not constrained by blocks and simple logic paths.\n\n"
+    "Flexibility and Extensibility\n"
+    "Danswer is open source and completely whitebox. This not only gives transparency to what happens within the system "
+    "but also means that your team can directly modify the source code to suit your unique needs."
+)
+
+customer_support_title = "Customer Support"
+customer_support = (
+    "Help your customer support team instantly answer any question across your entire product.\n\n"
+    "AI Enabled Support\n"
+    "Customer support agents have one of the highest breadth jobs. They field requests that cover the entire surface "
+    "area of the product and need to help your users find success on extremely short timelines. "
+    "Because they're not the same people who designed or built the system, they often lack the depth of understanding "
+    "needed - resulting in delays and escalations to other teams. Modern teams are leveraging AI to help their CS team "
+    "optimize the speed and quality of these critical customer-facing interactions.\n\n"
+    "The Importance of Context\n"
+    "There are two critical components of AI copilots for customer support. The first is that the AI system needs to be "
+    "connected with as much information as possible (not just support tools like Zendesk or Intercom) and that the "
+    "knowledge needs to be as fresh as possible. Sometimes a fix might even be in places rarely checked by CS such as "
+    "pull requests in a code repository. The second critical component is the ability of the AI system to break down "
+    "difficult concepts and convoluted processes into more digestible descriptions and for your team members to be able "
+    "to chat back and forth with the system to build a better understanding.\n\n"
+    "Danswer takes care of both of these. The system connects up to over 30+ different applications and the knowledge is "
+    "pulled in constantly so that the information access is always up to date."
+)
+
+sales_title = "Sales"
+sales = (
+    "Keep your team up to date on every conversation and update so they can close.\n\n"
+    "Recall Every Detail\n"
+    "Being able to instantly revisit every detail of any call without reading transcripts is helping Sales teams provide "
+    "more tailored pitches, build stronger relationships, and close more deals. Instead of searching and reading through "
+    'hours of transcripts in preparation for a call, your team can now ask Danswer "What specific features was ACME '
+    "interested in seeing for the demo\". Since your team doesn't have time to read every transcript prior to a call, "
+    "Danswer provides a more thorough summary because it can instantly parse hundreds of pages and distill out the relevant "
+    "information. Even for fast lookups it becomes much more convenient - for example to brush up on connection building "
+    'topics by asking "What rapport building topic did we chat about in the last call with ACME".\n\n'
+    "Know Every Product Update\n"
+    "It is impossible for Sales teams to keep up with every product update. Because of this, when a prospect has a question "
+    "that the Sales team does not know, they have no choice but to rely on the Product and Engineering orgs to get an "
+    "authoritative answer. Not only is this distracting to the other teams, it also slows down the time to respond to the "
+    "prospect (and as we know, time is the biggest killer of deals). With Danswer, it is even possible to get answers live "
+    'on call because of how fast accessing information becomes. A question like "Have we shipped the Microsoft AD '
+    'integration yet?" can now be answered in seconds meaning that prospects can get answers while on the call instead of '
+    "asynchronously and sales cycles are reduced as a result."
+)
+
+operations_title = "Operations"
+operations = (
+    "Double the productivity of your Ops teams like IT, HR, etc.\n\n"
+    "Automatically Resolve Tickets\n"
+    "Modern teams are leveraging AI to auto-resolve up to 50% of tickets. Whether it is an employee asking about benefits "
+    "details or how to set up the VPN for remote work, Danswer can help your team help themselves. This frees up your team to "
+    "do the real impactful work of landing star candidates or improving your internal processes.\n\n"
+    "AI Aided Onboarding\n"
+    "One of the periods where your team needs the most help is when they're just ramping up. Instead of feeling lost in dozens "
+    "of new tools, Danswer gives them a single place where they can ask about anything in natural language. Whether it's how to "
+    "set up their work environment or what their onboarding goals are, Danswer can walk them through every step with the help "
+    "of Generative AI. This lets your team feel more empowered and gives time back to the more seasoned members of your team to "
+    "focus on moving the needle."
+)
+
+# For simplicity, we're not adding any metadata suffix here. Generally there is none for the Web connector anyway
+overview_doc = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/overview",
+    title=overview_title,
+    content=overview,
+    title_embedding=model.encode(f"search_document: {overview_title}"),
+    content_embedding=model.encode(f"search_document: {overview_title}\n{overview}"),
+)
+
+enterprise_search_doc = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/enterprise_search",
+    title=enterprise_search_title,
+    content=enterprise_search_1,
+    title_embedding=model.encode(f"search_document: {enterprise_search_title}"),
+    content_embedding=model.encode(
+        f"search_document: {enterprise_search_title}\n{enterprise_search_1}"
+    ),
+)
+
+enterprise_search_doc_2 = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/enterprise_search",
+    title=enterprise_search_title,
+    content=enterprise_search_2,
+    title_embedding=model.encode(f"search_document: {enterprise_search_title}"),
+    content_embedding=model.encode(
+        f"search_document: {enterprise_search_title}\n{enterprise_search_2}"
+    ),
+    chunk_ind=1,
+)
+
+ai_platform_doc = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/ai_platform",
+    title=ai_platform_title,
+    content=ai_platform,
+    title_embedding=model.encode(f"search_document: {ai_platform_title}"),
+    content_embedding=model.encode(
+        f"search_document: {ai_platform_title}\n{ai_platform}"
+    ),
+)
+
+customer_support_doc = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/customer_support",
+    title=customer_support_title,
+    content=customer_support,
+    title_embedding=model.encode(f"search_document: {customer_support_title}"),
+    content_embedding=model.encode(
+        f"search_document: {customer_support_title}\n{customer_support}"
+    ),
+)
+
+sales_doc = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/sales",
+    title=sales_title,
+    content=sales,
+    title_embedding=model.encode(f"search_document: {sales_title}"),
+    content_embedding=model.encode(f"search_document: {sales_title}\n{sales}"),
+)
+
+operations_doc = SeedPresaveDocument(
+    url="https://docs.danswer.dev/more/use_cases/operations",
+    title=operations_title,
+    content=operations,
+    title_embedding=model.encode(f"search_document: {operations_title}"),
+    content_embedding=model.encode(
+        f"search_document: {operations_title}\n{operations}"
+    ),
+)
+
+documents = [
+    overview_doc,
+    enterprise_search_doc,
+    enterprise_search_doc_2,
+    ai_platform_doc,
+    customer_support_doc,
+    sales_doc,
+    operations_doc,
+]
+
+documents_dict = [doc.model_dump() for doc in documents]
+
+with open("./backend/danswer/seeding/initial_docs.json", "w") as json_file:
+    json.dump(documents_dict, json_file, indent=4)
diff --git a/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py b/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py
index db399328917..f17c4211066 100644
--- a/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py
+++ b/backend/tests/integration/connector_job_tests/slack/slack_api_utils.py
@@ -73,9 +73,7 @@ def _clear_slack_conversation_members(
         if member_id == admin_user_id:
             continue
         try:
-            make_slack_api_call_w_retries(
-                slack_client.conversations_kick, channel=channel_id, user=member_id
-            )
+            slack_client.conversations_kick(channel=channel_id, user=member_id)
             print(f"Kicked member: {member_id}")
         except Exception as e:
             if "cant_kick_self" in str(e):
@@ -83,9 +81,7 @@ def _clear_slack_conversation_members(
             print(f"Error kicking member: {e}")
             print(member_id)
     try:
-        make_slack_api_call_w_retries(
-            slack_client.conversations_unarchive, channel=channel_id
-        )
+        slack_client.conversations_unarchive(channel=channel_id)
         channel["is_archived"] = False
     except Exception:
         # Channel is already unarchived
@@ -98,11 +94,7 @@ def _add_slack_conversation_members(
     channel_id = _get_slack_channel_id(channel)
     for user_id in member_ids:
         try:
-            make_slack_api_call_w_retries(
-                slack_client.conversations_invite,
-                channel=channel_id,
-                users=user_id,
-            )
+            slack_client.conversations_invite(channel=channel_id, users=user_id)
         except Exception as e:
             if "already_in_channel" in str(e):
                 continue
@@ -129,11 +121,7 @@ def _delete_slack_conversation_messages(
             try:
                 if not (ts := message.get("ts")):
                     raise ValueError("Message timestamp is missing")
-                make_slack_api_call_w_retries(
-                    slack_client.chat_delete,
-                    channel=channel_id,
-                    ts=ts,
-                )
+                slack_client.chat_delete(channel=channel_id, ts=ts)
             except Exception as e:
                 print(f"Error deleting message: {e}")
                 print(message)
@@ -165,16 +153,12 @@ def _build_slack_channel_from_name(
         )
 
     try:
-        channel_response = make_slack_api_call_w_retries(
-            slack_client.conversations_unarchive,
-            channel=channel_response["channel"]["id"],
-        )
+        slack_client.conversations_unarchive(channel=channel_response["channel"]["id"])
     except Exception:
         # Channel is already unarchived
         pass
     try:
-        channel_response = make_slack_api_call_w_retries(
-            slack_client.conversations_invite,
+        slack_client.conversations_invite(
             channel=channel_response["channel"]["id"],
             users=[admin_user_id],
         )
@@ -302,10 +286,6 @@ def cleanup_after_test(
             # "done" in the channel name indicates that this channel is free to be used for a new test
             new_name = f"done_{str(uuid4())}"
             try:
-                make_slack_api_call_w_retries(
-                    slack_client.conversations_rename,
-                    channel=channel["id"],
-                    name=new_name,
-                )
+                slack_client.conversations_rename(channel=channel["id"], name=new_name)
             except SlackApiError as e:
                 print(f"Error renaming channel {channel['id']}: {e}")

From 10b5b556581ae77bca79e0a90a239ade8325a728 Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 18:31:10 -0700
Subject: [PATCH 205/376] entry["id"] can apparently be a number, so convert to
 str

---
 backend/danswer/connectors/loopio/connector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/connectors/loopio/connector.py b/backend/danswer/connectors/loopio/connector.py
index e10bed87617..d3bdfe503f7 100644
--- a/backend/danswer/connectors/loopio/connector.py
+++ b/backend/danswer/connectors/loopio/connector.py
@@ -161,7 +161,7 @@ def _process_entries(
                 ]
                 doc_batch.append(
                     Document(
-                        id=entry["id"],
+                        id=str(entry["id"]),
                         sections=[Section(link=link, text=content_text)],
                         source=DocumentSource.LOOPIO,
                         semantic_identifier=questions[0],

From eae1dad0fa5c438c16a267b2caadfe3f634a05f5 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 24 Oct 2024 18:32:53 -0700
Subject: [PATCH 206/376] Silence unnecessary debug log (#2908)

* silence log

* silence
---
 backend/danswer/db/engine.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index c03a47d7f44..b071a41e8c3 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -365,9 +365,6 @@ def set_search_path_on_checkout(
     if tenant_id and is_valid_schema_name(tenant_id):
         with dbapi_conn.cursor() as cursor:
             cursor.execute(f'SET search_path TO "{tenant_id}"')
-            logger.debug(
-                f"Set search_path to {tenant_id} for connection {connection_record}"
-            )
 
 
 def get_session_generator_with_tenant() -> Generator[Session, None, None]:

From 863f00f0151c973bcac4e7e345289cdc96a85c39 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 19:13:09 -0700
Subject: [PATCH 207/376] Auto Backport Partial (#2910)

---
 .github/pull_request_template.md              | 30 ++++++-----
 .github/workflows/pr-backport-autotrigger.yml | 52 +++++++++++++++++++
 2 files changed, 69 insertions(+), 13 deletions(-)
 create mode 100644 .github/workflows/pr-backport-autotrigger.yml

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index e57283f0377..c49908b1d54 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -6,20 +6,24 @@
 [Describe the tests you ran to verify your changes]
 
 
-## Accepted Risk
-[Any know risks or failure modes to point out to reviewers]
+## Accepted Risk (provide if relevant)
+N/A
 
 
-## Related Issue(s)
-[If applicable, link to the issue(s) this PR addresses]
+## Related Issue(s) (provide if relevant)
+N/A
 
 
-## Checklist:
-- [ ] All of the automated tests pass
-- [ ] All PR comments are addressed and marked resolved
-- [ ] If there are migrations, they have been rebased to latest main
-- [ ] If there are new dependencies, they are added to the requirements
-- [ ] If there are new environment variables, they are added to all of the deployment methods
-- [ ] If there are new APIs that don't require auth, they are added to PUBLIC_ENDPOINT_SPECS
-- [ ] Docker images build and basic functionalities work
-- [ ] Author has done a final read through of the PR right before merge
+## Mental Checklist:
+- All of the automated tests pass
+- All PR comments are addressed and marked resolved
+- If there are migrations, they have been rebased to latest main
+- If there are new dependencies, they are added to the requirements
+- If there are new environment variables, they are added to all of the deployment methods
+- If there are new APIs that don't require auth, they are added to PUBLIC_ENDPOINT_SPECS
+- Docker images build and basic functionalities work
+- Author has done a final read through of the PR right before merge
+
+## Backporting (check the box to trigger backport action)
+Note: You have to check that the action passes, otherwise resolve the conflicts manually and tag the patches.
+[ ] This PR should be backported (make sure to check that the backport attempt succeeds)
diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
new file mode 100644
index 00000000000..4ba5136a82c
--- /dev/null
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -0,0 +1,52 @@
+name: Backport on Merge
+
+on:
+  pull_request:
+    types: [closed]
+
+jobs:
+  backport:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Check for Backport Checkbox
+        id: checkbox-check
+        run: |
+          PR_BODY="${{ github.event.pull_request.body }}"
+          if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
+            echo "::set-output name=backport::true"
+          else
+            echo "::set-output name=backport::false"
+          fi
+
+      - name: List and sort release branches
+        id: list-branches
+        run: |
+          git fetch --all
+          BRANCHES=$(git branch -r | grep 'origin/release/v' | sed 's|origin/release/v||' | sort -Vr)
+          BETA=$(echo "$BRANCHES" | head -n 1)
+          STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
+          echo "::set-output name=beta::$BETA"
+          echo "::set-output name=stable::$STABLE"
+
+      - name: Trigger Backport
+        if: steps.checkbox-check.outputs.backport == 'true'
+        run: |
+          echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
+          # Fetch all history for all branches and tags
+          git fetch --prune --unshallow
+          # Checkout the beta branch
+          git checkout ${{ steps.list-branches.outputs.beta }}
+          # Cherry-pick the last commit from the merged PR
+          git cherry-pick ${{ github.event.pull_request.merge_commit_sha }}
+          # Push the changes to the beta branch
+          git push origin ${{ steps.list-branches.outputs.beta }}
+          # Checkout the stable branch
+          git checkout ${{ steps.list-branches.outputs.stable }}
+          # Cherry-pick the last commit from the merged PR
+          git cherry-pick ${{ github.event.pull_request.merge_commit_sha }}
+          # Push the changes to the stable branch
+          git push origin ${{ steps.list-branches.outputs.stable }}

From a9314948661951fb70c0f3fb43418bc283f9a9bd Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 19:17:11 -0700
Subject: [PATCH 208/376] Harmless Backport (#2911)

---
 backend/danswer/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/__init__.py b/backend/danswer/__init__.py
index e2d480be4e6..ea33fd83292 100644
--- a/backend/danswer/__init__.py
+++ b/backend/danswer/__init__.py
@@ -1,3 +1,3 @@
 import os
 
-__version__ = os.environ.get("DANSWER_VERSION", "") or "0.3-dev"
+__version__ = os.environ.get("DANSWER_VERSION", "") or "Development"

From eaa8ae7399eebc2253e4ffba3cc74549b47ddf37 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 24 Oct 2024 19:43:57 -0700
Subject: [PATCH 209/376] Bugfix/connector deletion lockout (#2901)

* first cut at deletion hardening

* clean up logging

* remove commented code
---
 .../danswer/background/celery/apps/primary.py |   4 +
 .../danswer/background/celery/celery_redis.py |  25 ++++
 .../danswer/background/celery/celery_utils.py |  10 +-
 .../celery/tasks/connector_deletion/tasks.py  | 111 ++++++++++++++----
 .../background/celery/tasks/indexing/tasks.py |  87 ++++++++++----
 .../background/celery/tasks/pruning/tasks.py  |  31 +++--
 .../shared/RedisConnectorDeletionFenceData.py |   8 ++
 .../background/celery/tasks/shared/tasks.py   |  10 --
 .../background/celery/tasks/vespa/tasks.py    |  36 ++++--
 .../background/indexing/run_indexing.py       |  31 ++++-
 .../danswer/server/manage/administrative.py   |  21 ++--
 11 files changed, 281 insertions(+), 93 deletions(-)
 create mode 100644 backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py

diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 23e25fa92b8..983b76773ed 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -17,6 +17,7 @@
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_redis import RedisConnectorPruning
+from danswer.background.celery.celery_redis import RedisConnectorStop
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
@@ -161,6 +162,9 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
         for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
             r.delete(key)
 
+    for key in r.scan_iter(RedisConnectorStop.FENCE_PREFIX + "*"):
+        r.delete(key)
+
 
 # @worker_process_init.connect
 # def on_worker_process_init(sender: Any, **kwargs: Any) -> None:
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index 1ea5e3b1765..e412b0bf733 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -313,6 +313,8 @@ def generate_tasks(
         lock: redis.lock.Lock,
         tenant_id: str | None,
     ) -> int | None:
+        """Returns None if the cc_pair doesn't exist.
+        Otherwise, returns an int with the number of generated tasks."""
         last_lock_time = time.monotonic()
 
         async_results = []
@@ -540,6 +542,29 @@ def is_indexing(self, redis_client: Redis) -> bool:
         return False
 
 
+class RedisConnectorStop(RedisObjectHelper):
+    """Used to signal any running tasks for a connector to stop. We should refactor
+    connector related redis helpers into a single class.
+    """
+
+    PREFIX = "connectorstop"
+    FENCE_PREFIX = PREFIX + "_fence"  # a fence for the entire indexing process
+    TASKSET_PREFIX = PREFIX + "_taskset"  # stores a list of prune tasks id's
+
+    def __init__(self, id: int) -> None:
+        super().__init__(str(id))
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock | None,
+        tenant_id: str | None,
+    ) -> int | None:
+        return None
+
+
 def celery_get_queue_length(queue: str, r: Redis) -> int:
     """This is a redis specific way to get the length of a celery queue.
     It is priority aware and knows how to count across the multiple redis lists
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index b1e9c2113e2..18038a349de 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -1,4 +1,3 @@
-from collections.abc import Callable
 from datetime import datetime
 from datetime import timezone
 from typing import Any
@@ -6,6 +5,7 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
 from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
     rate_limit_builder,
@@ -79,7 +79,7 @@ def document_batch_to_ids(
 
 def extract_ids_from_runnable_connector(
     runnable_connector: BaseConnector,
-    progress_callback: Callable[[int], None] | None = None,
+    callback: RunIndexingCallbackInterface | None = None,
 ) -> set[str]:
     """
     If the PruneConnector hasnt been implemented for the given connector, just pull
@@ -110,8 +110,10 @@ def extract_ids_from_runnable_connector(
             max_calls=MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE, period=60
         )(document_batch_to_ids)
     for doc_batch in doc_batch_generator:
-        if progress_callback:
-            progress_callback(len(doc_batch))
+        if callback:
+            if callback.should_stop():
+                raise RuntimeError("Stop signal received")
+            callback.progress(len(doc_batch))
         all_connector_doc_ids.update(doc_batch_processing_func(doc_batch))
 
     return all_connector_doc_ids
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index f6a59d03e3a..59d236cde31 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -1,3 +1,6 @@
+from datetime import datetime
+from datetime import timezone
+
 import redis
 from celery import Celery
 from celery import shared_task
@@ -8,6 +11,12 @@
 
 from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import RedisConnectorDeletion
+from danswer.background.celery.celery_redis import RedisConnectorIndexing
+from danswer.background.celery.celery_redis import RedisConnectorPruning
+from danswer.background.celery.celery_redis import RedisConnectorStop
+from danswer.background.celery.tasks.shared.RedisConnectorDeletionFenceData import (
+    RedisConnectorDeletionFenceData,
+)
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
@@ -15,9 +24,15 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
+from danswer.db.search_settings import get_all_search_settings
 from danswer.redis.redis_pool import get_redis_client
 
 
+class TaskDependencyError(RuntimeError):
+    """Raised to the caller to indicate dependent tasks are running that would interfere
+    with connector deletion."""
+
+
 @shared_task(
     name="check_for_connector_deletion_task",
     soft_time_limit=JOB_TIMEOUT,
@@ -37,17 +52,30 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> N
         if not lock_beat.acquire(blocking=False):
             return
 
+        # collect cc_pair_ids
         cc_pair_ids: list[int] = []
         with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = get_connector_credential_pairs(db_session)
             for cc_pair in cc_pairs:
                 cc_pair_ids.append(cc_pair.id)
 
+        # try running cleanup on the cc_pair_ids
         for cc_pair_id in cc_pair_ids:
             with get_session_with_tenant(tenant_id) as db_session:
-                try_generate_document_cc_pair_cleanup_tasks(
-                    self.app, cc_pair_id, db_session, r, lock_beat, tenant_id
-                )
+                rcs = RedisConnectorStop(cc_pair_id)
+                try:
+                    try_generate_document_cc_pair_cleanup_tasks(
+                        self.app, cc_pair_id, db_session, r, lock_beat, tenant_id
+                    )
+                except TaskDependencyError as e:
+                    # this means we wanted to start deleting but dependent tasks were running
+                    # Leave a stop signal to clear indexing and pruning tasks more quickly
+                    task_logger.info(str(e))
+                    r.set(rcs.fence_key, cc_pair_id)
+                else:
+                    # clear the stop signal if it exists ... no longer needed
+                    r.delete(rcs.fence_key)
+
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
@@ -70,6 +98,10 @@ def try_generate_document_cc_pair_cleanup_tasks(
     """Returns an int if syncing is needed. The int represents the number of sync tasks generated.
     Note that syncing can still be required even if the number of sync tasks generated is zero.
     Returns None if no syncing is required.
+
+    Will raise TaskDependencyError if dependent tasks such as indexing and pruning are
+    still running. In our case, the caller reacts by setting a stop signal in Redis to
+    exit those tasks as quickly as possible.
     """
 
     lock_beat.reacquire()
@@ -90,28 +122,63 @@ def try_generate_document_cc_pair_cleanup_tasks(
     if cc_pair.status != ConnectorCredentialPairStatus.DELETING:
         return None
 
-    # add tasks to celery and build up the task set to monitor in redis
-    r.delete(rcd.taskset_key)
-
-    # Add all documents that need to be updated into the queue
-    task_logger.info(
-        f"RedisConnectorDeletion.generate_tasks starting. cc_pair_id={cc_pair.id}"
+    # set a basic fence to start
+    fence_value = RedisConnectorDeletionFenceData(
+        num_tasks=None,
+        submitted=datetime.now(timezone.utc),
     )
-    tasks_generated = rcd.generate_tasks(app, db_session, r, lock_beat, tenant_id)
-    if tasks_generated is None:
+    r.set(rcd.fence_key, fence_value.model_dump_json())
+
+    try:
+        # do not proceed if connector indexing or connector pruning are running
+        search_settings_list = get_all_search_settings(db_session)
+        for search_settings in search_settings_list:
+            rci = RedisConnectorIndexing(cc_pair_id, search_settings.id)
+            if r.get(rci.fence_key):
+                raise TaskDependencyError(
+                    f"Connector deletion - Delayed (indexing in progress): "
+                    f"cc_pair={cc_pair_id} "
+                    f"search_settings={search_settings.id}"
+                )
+
+        rcp = RedisConnectorPruning(cc_pair_id)
+        if r.get(rcp.fence_key):
+            raise TaskDependencyError(
+                f"Connector deletion - Delayed (pruning in progress): "
+                f"cc_pair={cc_pair_id}"
+            )
+
+        # add tasks to celery and build up the task set to monitor in redis
+        r.delete(rcd.taskset_key)
+
+        # Add all documents that need to be updated into the queue
+        task_logger.info(
+            f"RedisConnectorDeletion.generate_tasks starting. cc_pair={cc_pair_id}"
+        )
+        tasks_generated = rcd.generate_tasks(app, db_session, r, lock_beat, tenant_id)
+        if tasks_generated is None:
+            raise ValueError("RedisConnectorDeletion.generate_tasks returned None")
+    except TaskDependencyError:
+        r.delete(rcd.fence_key)
+        raise
+    except Exception:
+        task_logger.exception("Unexpected exception")
+        r.delete(rcd.fence_key)
         return None
+    else:
+        # Currently we are allowing the sync to proceed with 0 tasks.
+        # It's possible for sets/groups to be generated initially with no entries
+        # and they still need to be marked as up to date.
+        # if tasks_generated == 0:
+        #     return 0
 
-    # Currently we are allowing the sync to proceed with 0 tasks.
-    # It's possible for sets/groups to be generated initially with no entries
-    # and they still need to be marked as up to date.
-    # if tasks_generated == 0:
-    #     return 0
+        task_logger.info(
+            f"RedisConnectorDeletion.generate_tasks finished. "
+            f"cc_pair={cc_pair_id} tasks_generated={tasks_generated}"
+        )
 
-    task_logger.info(
-        f"RedisConnectorDeletion.generate_tasks finished. "
-        f"cc_pair_id={cc_pair.id} tasks_generated={tasks_generated}"
-    )
+        # set this only after all tasks have been added
+        fence_value.num_tasks = tasks_generated
+        r.set(rcd.fence_key, fence_value.model_dump_json())
 
-    # set this only after all tasks have been added
-    r.set(rcd.fence_key, tasks_generated)
     return tasks_generated
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index bdd55f77f32..980266ec878 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -5,6 +5,7 @@
 from typing import cast
 from uuid import uuid4
 
+import redis
 from celery import Celery
 from celery import shared_task
 from celery import Task
@@ -13,12 +14,15 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.apps.app_base import task_logger
+from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.background.celery.celery_redis import RedisConnectorIndexing
+from danswer.background.celery.celery_redis import RedisConnectorStop
 from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
     RedisConnectorIndexingFenceData,
 )
 from danswer.background.indexing.job_client import SimpleJobClient
 from danswer.background.indexing.run_indexing import run_indexing_entrypoint
+from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
 from danswer.configs.constants import CELERY_INDEXING_LOCK_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
@@ -50,6 +54,30 @@
 logger = setup_logger()
 
 
+class RunIndexingCallback(RunIndexingCallbackInterface):
+    def __init__(
+        self,
+        stop_key: str,
+        generator_progress_key: str,
+        redis_lock: redis.lock.Lock,
+        redis_client: Redis,
+    ):
+        super().__init__()
+        self.redis_lock: redis.lock.Lock = redis_lock
+        self.stop_key: str = stop_key
+        self.generator_progress_key: str = generator_progress_key
+        self.redis_client = redis_client
+
+    def should_stop(self) -> bool:
+        if self.redis_client.exists(self.stop_key):
+            return True
+        return False
+
+    def progress(self, amount: int) -> None:
+        self.redis_lock.reacquire()
+        self.redis_client.incrby(self.generator_progress_key, amount)
+
+
 @shared_task(
     name="check_for_indexing",
     soft_time_limit=300,
@@ -262,6 +290,10 @@ def try_creating_indexing_task(
             return None
 
         # skip indexing if the cc_pair is deleting
+        rcd = RedisConnectorDeletion(cc_pair.id)
+        if r.exists(rcd.fence_key):
+            return None
+
         db_session.refresh(cc_pair)
         if cc_pair.status == ConnectorCredentialPairStatus.DELETING:
             return None
@@ -308,13 +340,8 @@ def try_creating_indexing_task(
             raise RuntimeError("send_task for connector_indexing_proxy_task failed.")
 
         # now fill out the fence with the rest of the data
-        fence_value = RedisConnectorIndexingFenceData(
-            index_attempt_id=index_attempt_id,
-            started=None,
-            submitted=datetime.now(timezone.utc),
-            celery_task_id=result.id,
-        )
-
+        fence_value.index_attempt_id = index_attempt_id
+        fence_value.celery_task_id = result.id
         r.set(rci.fence_key, fence_value.model_dump_json())
     except Exception:
         r.delete(rci.fence_key)
@@ -400,6 +427,22 @@ def connector_indexing_task(
 
     r = get_redis_client(tenant_id=tenant_id)
 
+    rcd = RedisConnectorDeletion(cc_pair_id)
+    if r.exists(rcd.fence_key):
+        raise RuntimeError(
+            f"Indexing will not start because connector deletion is in progress: "
+            f"cc_pair={cc_pair_id} "
+            f"fence={rcd.fence_key}"
+        )
+
+    rcs = RedisConnectorStop(cc_pair_id)
+    if r.exists(rcs.fence_key):
+        raise RuntimeError(
+            f"Indexing will not start because a connector stop signal was detected: "
+            f"cc_pair={cc_pair_id} "
+            f"fence={rcs.fence_key}"
+        )
+
     rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
 
     while True:
@@ -409,7 +452,7 @@ def connector_indexing_task(
             task_logger.info(
                 f"connector_indexing_task: fence_value not found: fence={rci.fence_key}"
             )
-            raise
+            raise RuntimeError(f"Fence not found: fence={rci.fence_key}")
 
         try:
             fence_json = fence_value.decode("utf-8")
@@ -443,17 +486,20 @@ def connector_indexing_task(
     if not acquired:
         task_logger.warning(
             f"Indexing task already running, exiting...: "
-            f"cc_pair_id={cc_pair_id} search_settings_id={search_settings_id}"
+            f"cc_pair={cc_pair_id} search_settings={search_settings_id}"
         )
         # r.set(rci.generator_complete_key, HTTPStatus.CONFLICT.value)
         return None
 
+    fence_data.started = datetime.now(timezone.utc)
+    r.set(rci.fence_key, fence_data.model_dump_json())
+
     try:
         with get_session_with_tenant(tenant_id) as db_session:
             attempt = get_index_attempt(db_session, index_attempt_id)
             if not attempt:
                 raise ValueError(
-                    f"Index attempt not found: index_attempt_id={index_attempt_id}"
+                    f"Index attempt not found: index_attempt={index_attempt_id}"
                 )
 
             cc_pair = get_connector_credential_pair_from_id(
@@ -462,31 +508,31 @@ def connector_indexing_task(
             )
 
             if not cc_pair:
-                raise ValueError(f"cc_pair not found: cc_pair_id={cc_pair_id}")
+                raise ValueError(f"cc_pair not found: cc_pair={cc_pair_id}")
 
             if not cc_pair.connector:
                 raise ValueError(
-                    f"Connector not found: connector_id={cc_pair.connector_id}"
+                    f"Connector not found: cc_pair={cc_pair_id} connector={cc_pair.connector_id}"
                 )
 
             if not cc_pair.credential:
                 raise ValueError(
-                    f"Credential not found: credential_id={cc_pair.credential_id}"
+                    f"Credential not found: cc_pair={cc_pair_id} credential={cc_pair.credential_id}"
                 )
 
             rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
 
-            # Define the callback function
-            def redis_increment_callback(amount: int) -> None:
-                lock.reacquire()
-                r.incrby(rci.generator_progress_key, amount)
+            # define a callback class
+            callback = RunIndexingCallback(
+                rcs.fence_key, rci.generator_progress_key, lock, r
+            )
 
             run_indexing_entrypoint(
                 index_attempt_id,
                 tenant_id,
                 cc_pair_id,
                 is_ee,
-                progress_callback=redis_increment_callback,
+                callback=callback,
             )
 
             # get back the total number of indexed docs and return it
@@ -499,9 +545,10 @@ def redis_increment_callback(amount: int) -> None:
 
             r.set(rci.generator_complete_key, HTTPStatus.OK.value)
     except Exception as e:
-        task_logger.exception(f"Failed to run indexing for cc_pair_id={cc_pair_id}.")
+        task_logger.exception(f"Indexing failed: cc_pair={cc_pair_id}")
         if attempt:
-            mark_attempt_failed(attempt, db_session, failure_reason=str(e))
+            with get_session_with_tenant(tenant_id) as db_session:
+                mark_attempt_failed(attempt, db_session, failure_reason=str(e))
 
         r.delete(rci.generator_lock_key)
         r.delete(rci.generator_progress_key)
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index 9f290d6f23a..2e68986e839 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -11,8 +11,11 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.apps.app_base import task_logger
+from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.background.celery.celery_redis import RedisConnectorPruning
+from danswer.background.celery.celery_redis import RedisConnectorStop
 from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector
+from danswer.background.celery.tasks.indexing.tasks import RunIndexingCallback
 from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_PRUNING_LOCK_TIMEOUT
@@ -168,6 +171,10 @@ def try_creating_prune_generator_task(
             return None
 
         # skip pruning if the cc_pair is deleting
+        rcd = RedisConnectorDeletion(cc_pair.id)
+        if r.exists(rcd.fence_key):
+            return None
+
         db_session.refresh(cc_pair)
         if cc_pair.status == ConnectorCredentialPairStatus.DELETING:
             return None
@@ -234,7 +241,7 @@ def connector_pruning_generator_task(
     acquired = lock.acquire(blocking=False)
     if not acquired:
         task_logger.warning(
-            f"Pruning task already running, exiting...: cc_pair_id={cc_pair_id}"
+            f"Pruning task already running, exiting...: cc_pair={cc_pair_id}"
         )
         return None
 
@@ -252,11 +259,6 @@ def connector_pruning_generator_task(
                 )
                 return
 
-            # Define the callback function
-            def redis_increment_callback(amount: int) -> None:
-                lock.reacquire()
-                r.incrby(rcp.generator_progress_key, amount)
-
             runnable_connector = instantiate_connector(
                 db_session,
                 cc_pair.connector.source,
@@ -265,9 +267,14 @@ def redis_increment_callback(amount: int) -> None:
                 cc_pair.credential,
             )
 
+            rcs = RedisConnectorStop(cc_pair_id)
+
+            callback = RunIndexingCallback(
+                rcs.fence_key, rcp.generator_progress_key, lock, r
+            )
             # a list of docs in the source
             all_connector_doc_ids: set[str] = extract_ids_from_runnable_connector(
-                runnable_connector, redis_increment_callback
+                runnable_connector, callback
             )
 
             # a list of docs in our local index
@@ -285,7 +292,7 @@ def redis_increment_callback(amount: int) -> None:
 
             task_logger.info(
                 f"Pruning set collected: "
-                f"cc_pair_id={cc_pair.id} "
+                f"cc_pair={cc_pair_id} "
                 f"docs_to_remove={len(doc_ids_to_remove)} "
                 f"doc_source={cc_pair.connector.source}"
             )
@@ -293,7 +300,7 @@ def redis_increment_callback(amount: int) -> None:
             rcp.documents_to_prune = set(doc_ids_to_remove)
 
             task_logger.info(
-                f"RedisConnectorPruning.generate_tasks starting. cc_pair_id={cc_pair.id}"
+                f"RedisConnectorPruning.generate_tasks starting. cc_pair_id={cc_pair_id}"
             )
             tasks_generated = rcp.generate_tasks(
                 self.app, db_session, r, None, tenant_id
@@ -303,12 +310,14 @@ def redis_increment_callback(amount: int) -> None:
 
             task_logger.info(
                 f"RedisConnectorPruning.generate_tasks finished. "
-                f"cc_pair_id={cc_pair.id} tasks_generated={tasks_generated}"
+                f"cc_pair={cc_pair_id} tasks_generated={tasks_generated}"
             )
 
             r.set(rcp.generator_complete_key, tasks_generated)
     except Exception as e:
-        task_logger.exception(f"Failed to run pruning for connector id {connector_id}.")
+        task_logger.exception(
+            f"Failed to run pruning: cc_pair={cc_pair_id} connector={connector_id}"
+        )
 
         r.delete(rcp.generator_progress_key)
         r.delete(rcp.taskset_key)
diff --git a/backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py b/backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py
new file mode 100644
index 00000000000..1c664d14b4b
--- /dev/null
+++ b/backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py
@@ -0,0 +1,8 @@
+from datetime import datetime
+
+from pydantic import BaseModel
+
+
+class RedisConnectorDeletionFenceData(BaseModel):
+    num_tasks: int | None
+    submitted: datetime
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 7ce43454aa3..52a49d467e9 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -1,9 +1,6 @@
-from datetime import datetime
-
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
-from pydantic import BaseModel
 
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.apps.app_base import task_logger
@@ -23,13 +20,6 @@
 DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES = 3
 
 
-class RedisConnectorIndexingFenceData(BaseModel):
-    index_attempt_id: int | None
-    started: datetime | None
-    submitted: datetime
-    celery_task_id: str | None
-
-
 @shared_task(
     name="document_by_cc_pair_cleanup_task",
     bind=True,
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 812074b91e9..fcc4d2aa5bc 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -23,6 +23,9 @@
 from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_redis import RedisDocumentSet
 from danswer.background.celery.celery_redis import RedisUserGroup
+from danswer.background.celery.tasks.shared.RedisConnectorDeletionFenceData import (
+    RedisConnectorDeletionFenceData,
+)
 from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
     RedisConnectorIndexingFenceData,
 )
@@ -368,7 +371,7 @@ def monitor_document_set_taskset(
 
     count = cast(int, r.scard(rds.taskset_key))
     task_logger.info(
-        f"Document set sync progress: document_set_id={document_set_id} "
+        f"Document set sync progress: document_set={document_set_id} "
         f"remaining={count} initial={initial_count}"
     )
     if count > 0:
@@ -383,12 +386,12 @@ def monitor_document_set_taskset(
             # if there are no connectors, then delete the document set.
             delete_document_set(document_set_row=document_set, db_session=db_session)
             task_logger.info(
-                f"Successfully deleted document set with ID: '{document_set_id}'!"
+                f"Successfully deleted document set: document_set={document_set_id}"
             )
         else:
             mark_document_set_as_synced(document_set_id, db_session)
             task_logger.info(
-                f"Successfully synced document set with ID: '{document_set_id}'!"
+                f"Successfully synced document set: document_set={document_set_id}"
             )
 
     r.delete(rds.taskset_key)
@@ -408,19 +411,29 @@ def monitor_connector_deletion_taskset(
 
     rcd = RedisConnectorDeletion(cc_pair_id)
 
-    fence_value = r.get(rcd.fence_key)
+    # read related data and evaluate/print task progress
+    fence_value = cast(bytes, r.get(rcd.fence_key))
     if fence_value is None:
         return
 
     try:
-        initial_count = int(cast(int, fence_value))
+        fence_json = fence_value.decode("utf-8")
+        fence_data = RedisConnectorDeletionFenceData.model_validate_json(
+            cast(str, fence_json)
+        )
     except ValueError:
-        task_logger.error("The value is not an integer.")
+        task_logger.exception(
+            "monitor_ccpair_indexing_taskset: fence_data not decodeable."
+        )
+        raise
+
+    # the fence is setting up but isn't ready yet
+    if fence_data.num_tasks is None:
         return
 
     count = cast(int, r.scard(rcd.taskset_key))
     task_logger.info(
-        f"Connector deletion progress: cc_pair={cc_pair_id} remaining={count} initial={initial_count}"
+        f"Connector deletion progress: cc_pair={cc_pair_id} remaining={count} initial={fence_data.num_tasks}"
     )
     if count > 0:
         return
@@ -483,7 +496,7 @@ def monitor_connector_deletion_taskset(
             )
             if not connector or not len(connector.credentials):
                 task_logger.info(
-                    "Found no credentials left for connector, deleting connector"
+                    "Connector deletion - Found no credentials left for connector, deleting connector"
                 )
                 db_session.delete(connector)
             db_session.commit()
@@ -493,17 +506,17 @@ def monitor_connector_deletion_taskset(
             error_message = f"Error: {str(e)}\n\nStack Trace:\n{stack_trace}"
             add_deletion_failure_message(db_session, cc_pair_id, error_message)
             task_logger.exception(
-                f"Failed to run connector_deletion. "
+                f"Connector deletion exceptioned: "
                 f"cc_pair={cc_pair_id} connector={cc_pair.connector_id} credential={cc_pair.credential_id}"
             )
             raise e
 
     task_logger.info(
-        f"Successfully deleted cc_pair: "
+        f"Connector deletion succeeded: "
         f"cc_pair={cc_pair_id} "
         f"connector={cc_pair.connector_id} "
         f"credential={cc_pair.credential_id} "
-        f"docs_deleted={initial_count}"
+        f"docs_deleted={fence_data.num_tasks}"
     )
 
     r.delete(rcd.taskset_key)
@@ -618,6 +631,7 @@ def monitor_ccpair_indexing_taskset(
         return
 
     # Read result state BEFORE generator_complete_key to avoid a race condition
+    # never use any blocking methods on the result from inside a task!
     result: AsyncResult = AsyncResult(fence_data.celery_task_id)
     result_state = result.state
 
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index cb507390452..d95a6a70d57 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -1,6 +1,7 @@
 import time
 import traceback
-from collections.abc import Callable
+from abc import ABC
+from abc import abstractmethod
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -41,6 +42,19 @@
 INDEXING_TRACER_NUM_PRINT_ENTRIES = 5
 
 
+class RunIndexingCallbackInterface(ABC):
+    """Defines a callback interface to be passed to
+    to run_indexing_entrypoint."""
+
+    @abstractmethod
+    def should_stop(self) -> bool:
+        """Signal to stop the looping function in flight."""
+
+    @abstractmethod
+    def progress(self, amount: int) -> None:
+        """Send progress updates to the caller."""
+
+
 def _get_connector_runner(
     db_session: Session,
     attempt: IndexAttempt,
@@ -92,7 +106,7 @@ def _run_indexing(
     db_session: Session,
     index_attempt: IndexAttempt,
     tenant_id: str | None,
-    progress_callback: Callable[[int], None] | None = None,
+    callback: RunIndexingCallbackInterface | None = None,
 ) -> None:
     """
     1. Get documents which are either new or updated from specified application
@@ -206,6 +220,11 @@ def _run_indexing(
                 # index being built. We want to populate it even for paused connectors
                 # Often paused connectors are sources that aren't updated frequently but the
                 # contents still need to be initially pulled.
+                if callback:
+                    if callback.should_stop():
+                        raise RuntimeError("Connector stop signal detected")
+
+                # TODO: should we move this into the above callback instead?
                 db_session.refresh(db_cc_pair)
                 if (
                     (
@@ -263,8 +282,8 @@ def _run_indexing(
                 # be inaccurate
                 db_session.commit()
 
-                if progress_callback:
-                    progress_callback(len(doc_batch))
+                if callback:
+                    callback.progress(len(doc_batch))
 
                 # This new value is updated every batch, so UI can refresh per batch update
                 update_docs_indexed(
@@ -394,7 +413,7 @@ def run_indexing_entrypoint(
     tenant_id: str | None,
     connector_credential_pair_id: int,
     is_ee: bool = False,
-    progress_callback: Callable[[int], None] | None = None,
+    callback: RunIndexingCallbackInterface | None = None,
 ) -> None:
     try:
         if is_ee:
@@ -417,7 +436,7 @@ def run_indexing_entrypoint(
                 f"credentials='{attempt.connector_credential_pair.connector_id}'"
             )
 
-            _run_indexing(db_session, attempt, tenant_id, progress_callback)
+            _run_indexing(db_session, attempt, tenant_id, callback)
 
             logger.info(
                 f"Indexing finished for tenant {tenant_id}: "
diff --git a/backend/danswer/server/manage/administrative.py b/backend/danswer/server/manage/administrative.py
index d16aa59c4cb..1ceeb776abc 100644
--- a/backend/danswer/server/manage/administrative.py
+++ b/backend/danswer/server/manage/administrative.py
@@ -19,7 +19,6 @@
 from danswer.db.connector_credential_pair import (
     update_connector_credential_pair_from_id,
 )
-from danswer.db.deletion_attempt import check_deletion_attempt_is_allowed
 from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.enums import ConnectorCredentialPairStatus
@@ -175,15 +174,19 @@ def create_deletion_attempt_for_connector_id(
         cc_pair_id=cc_pair.id, db_session=db_session, include_secondary_index=True
     )
 
+    # TODO(rkuo): 2024-10-24 - check_deletion_attempt_is_allowed shouldn't be necessary
+    # any more due to background locking improvements.
+    # Remove the below permanently if everything is behaving for 30 days.
+
     # Check if the deletion attempt should be allowed
-    deletion_attempt_disallowed_reason = check_deletion_attempt_is_allowed(
-        connector_credential_pair=cc_pair, db_session=db_session
-    )
-    if deletion_attempt_disallowed_reason:
-        raise HTTPException(
-            status_code=400,
-            detail=deletion_attempt_disallowed_reason,
-        )
+    # deletion_attempt_disallowed_reason = check_deletion_attempt_is_allowed(
+    #     connector_credential_pair=cc_pair, db_session=db_session
+    # )
+    # if deletion_attempt_disallowed_reason:
+    #     raise HTTPException(
+    #         status_code=400,
+    #         detail=deletion_attempt_disallowed_reason,
+    #     )
 
     # mark as deleting
     update_connector_credential_pair_from_id(

From 07a4e112a4d89122eac41e70fb12b486f23e7e35 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 20:25:36 -0700
Subject: [PATCH 210/376] Dev Experience (#2912)

---
 .github/pull_request_template.md              |  2 +-
 .github/workflows/pr-backport-autotrigger.yml | 77 ++++++++++++++-----
 2 files changed, 60 insertions(+), 19 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index c49908b1d54..8287f9b5300 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -26,4 +26,4 @@ N/A
 
 ## Backporting (check the box to trigger backport action)
 Note: You have to check that the action passes, otherwise resolve the conflicts manually and tag the patches.
-[ ] This PR should be backported (make sure to check that the backport attempt succeeds)
+- [ ] This PR should be backported (make sure to check that the backport attempt succeeds)
diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 4ba5136a82c..aaa071e6a94 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -10,43 +10,84 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
 
-      - name: Check for Backport Checkbox
-        id: checkbox-check
+      - name: Set up Git
         run: |
-          PR_BODY="${{ github.event.pull_request.body }}"
-          if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
-            echo "::set-output name=backport::true"
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Check for Backport Label
+        id: check-label
+        run: |
+          labels="${{ github.event.pull_request.labels.*.name }}"
+          if [[ "$labels" == *"backport"* ]]; then
+            echo "backport=true" >> $GITHUB_OUTPUT
           else
-            echo "::set-output name=backport::false"
+            echo "backport=false" >> $GITHUB_OUTPUT
           fi
 
       - name: List and sort release branches
         id: list-branches
         run: |
-          git fetch --all
-          BRANCHES=$(git branch -r | grep 'origin/release/v' | sed 's|origin/release/v||' | sort -Vr)
+          BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/release/* | sed 's|origin/release/||' | sort -Vr)
           BETA=$(echo "$BRANCHES" | head -n 1)
           STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
-          echo "::set-output name=beta::$BETA"
-          echo "::set-output name=stable::$STABLE"
+          echo "beta=$BETA" >> $GITHUB_OUTPUT
+          echo "stable=$STABLE" >> $GITHUB_OUTPUT
+
+          # Fetch latest tags for beta and stable
+          # Beta tag is in the format vX.Y.Z-beta.W
+          # Stable tag is in the format vX.Y.Z
+          LATEST_BETA_TAG=$(git tag -l "v*.*.0-beta.*" | sort -Vr | head -n 1)
+          LATEST_STABLE_TAG=$(git tag -l "v*.*.*" | grep -v -- "-beta" | sort -Vr | head -n 1)
+          
+          # Increment latest beta tag
+          NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '.' '{print $1 "." $2 ".0-beta." ($4+1)}')
+          # Increment latest stable tag
+          NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
+          echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT
+          echo "latest_stable_tag=$LATEST_STABLE_TAG" >> $GITHUB_OUTPUT
+          echo "new_beta_tag=$NEW_BETA_TAG" >> $GITHUB_OUTPUT
+          echo "new_stable_tag=$NEW_STABLE_TAG" >> $GITHUB_OUTPUT
+
+      - name: Echo branch and tag information
+        run: |
+          echo "Beta branch: ${{ steps.list-branches.outputs.beta }}"
+          echo "Stable branch: ${{ steps.list-branches.outputs.stable }}"
+          echo "Latest beta tag: ${{ steps.list-branches.outputs.latest_beta_tag }}"
+          echo "Latest stable tag: ${{ steps.list-branches.outputs.latest_stable_tag }}"
+          echo "New beta tag: ${{ steps.list-branches.outputs.new_beta_tag }}"
+          echo "New stable tag: ${{ steps.list-branches.outputs.new_stable_tag }}"
 
       - name: Trigger Backport
-        if: steps.checkbox-check.outputs.backport == 'true'
+        if: steps.check-label.outputs.backport == 'true'
         run: |
+          set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
-          # Fetch all history for all branches and tags
-          git fetch --prune --unshallow
           # Checkout the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
           # Cherry-pick the last commit from the merged PR
-          git cherry-pick ${{ github.event.pull_request.merge_commit_sha }}
-          # Push the changes to the beta branch
+          git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
+            echo "Cherry-pick to beta failed due to conflicts."
+            exit 1
+          }
+          # Create new beta tag
+          git tag ${{ steps.list-branches.outputs.new_beta_tag }}
+          # Push the changes and tag to the beta branch
           git push origin ${{ steps.list-branches.outputs.beta }}
+          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
           # Checkout the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
           # Cherry-pick the last commit from the merged PR
-          git cherry-pick ${{ github.event.pull_request.merge_commit_sha }}
-          # Push the changes to the stable branch
+          git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
+            echo "Cherry-pick to stable failed due to conflicts."
+            exit 1
+          }
+          # Create new stable tag
+          git tag ${{ steps.list-branches.outputs.new_stable_tag }}
+          # Push the changes and tag to the stable branch
           git push origin ${{ steps.list-branches.outputs.stable }}
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}

From 55b9111410c7dace2638722150ebe53590af87d4 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 20:43:16 -0700
Subject: [PATCH 211/376] Harmless Backport (#2914)

---
 backend/danswer/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/__init__.py b/backend/danswer/__init__.py
index ea33fd83292..9feb2146b7e 100644
--- a/backend/danswer/__init__.py
+++ b/backend/danswer/__init__.py
@@ -1,3 +1,3 @@
 import os
 
-__version__ = os.environ.get("DANSWER_VERSION", "") or "Development"
+__version__ = os.environ.get("DANSWER_VERSION", "") or "Dev"

From 9c0f927e165ca277085e9b1193022b30b6b25ef7 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 20:53:48 -0700
Subject: [PATCH 212/376] Workflow (#2915)

---
 .github/workflows/pr-backport-autotrigger.yml | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index aaa071e6a94..2d49c39402e 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -12,18 +12,18 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 0  # Fetch all history for all branches and tags
 
       - name: Set up Git
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
 
-      - name: Check for Backport Label
-        id: check-label
+      - name: Check for Backport Checkbox
+        id: checkbox-check
         run: |
-          labels="${{ github.event.pull_request.labels.*.name }}"
-          if [[ "$labels" == *"backport"* ]]; then
+          PR_BODY="${{ github.event.pull_request.body }}"
+          if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
             echo "backport=true" >> $GITHUB_OUTPUT
           else
             echo "backport=false" >> $GITHUB_OUTPUT
@@ -32,20 +32,17 @@ jobs:
       - name: List and sort release branches
         id: list-branches
         run: |
+          git fetch --all --tags
           BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/release/* | sed 's|origin/release/||' | sort -Vr)
           BETA=$(echo "$BRANCHES" | head -n 1)
           STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
           echo "beta=$BETA" >> $GITHUB_OUTPUT
           echo "stable=$STABLE" >> $GITHUB_OUTPUT
-
           # Fetch latest tags for beta and stable
-          # Beta tag is in the format vX.Y.Z-beta.W
-          # Stable tag is in the format vX.Y.Z
           LATEST_BETA_TAG=$(git tag -l "v*.*.0-beta.*" | sort -Vr | head -n 1)
           LATEST_STABLE_TAG=$(git tag -l "v*.*.*" | grep -v -- "-beta" | sort -Vr | head -n 1)
-          
           # Increment latest beta tag
-          NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '.' '{print $1 "." $2 ".0-beta." ($4+1)}')
+          NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 ".0-beta." ($NF+1)}')
           # Increment latest stable tag
           NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
           echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT
@@ -63,13 +60,15 @@ jobs:
           echo "New stable tag: ${{ steps.list-branches.outputs.new_stable_tag }}"
 
       - name: Trigger Backport
-        if: steps.check-label.outputs.backport == 'true'
+        if: steps.checkbox-check.outputs.backport == 'true'
         run: |
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
+          # Fetch all history for all branches and tags
+          git fetch --prune --unshallow
           # Checkout the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
-          # Cherry-pick the last commit from the merged PR
+          # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to beta failed due to conflicts."
             exit 1
@@ -81,7 +80,7 @@ jobs:
           git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
           # Checkout the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
-          # Cherry-pick the last commit from the merged PR
+          # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to stable failed due to conflicts."
             exit 1

From d7a30b01d2d6b38111bd0bef6ae9bb7ee1ce594a Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 24 Oct 2024 20:56:59 -0700
Subject: [PATCH 213/376] Harmless Backport (#2916)

---
 backend/danswer/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/__init__.py b/backend/danswer/__init__.py
index 9feb2146b7e..ea33fd83292 100644
--- a/backend/danswer/__init__.py
+++ b/backend/danswer/__init__.py
@@ -1,3 +1,3 @@
 import os
 
-__version__ = os.environ.get("DANSWER_VERSION", "") or "Dev"
+__version__ = os.environ.get("DANSWER_VERSION", "") or "Development"

From 4a47e9a841805a14b4fc9178faea1933fc1e2933 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Thu, 24 Oct 2024 22:38:46 -0700
Subject: [PATCH 214/376] Add strict json mode (#2917)

---
 backend/danswer/chat/process_message.py       |   1 +
 backend/danswer/llm/answering/models.py       |   4 +
 backend/danswer/llm/chat_llm.py               |  19 ++-
 backend/danswer/llm/custom_llm.py             |   2 +
 backend/danswer/llm/interfaces.py             |  12 +-
 .../danswer/server/query_and_chat/models.py   |   4 +
 .../server/query_and_chat/chat_backend.py     |   4 +-
 .../danswer/server/query_and_chat/models.py   |   6 +
 .../scripts/add_connector_creation_script.py  | 148 ++++++++++++++++++
 backend/tests/integration/conftest.py         |  10 ++
 .../tests/dev_apis/test_simple_chat_api.py    |  87 ++++++++++
 11 files changed, 291 insertions(+), 6 deletions(-)
 create mode 100644 backend/scripts/add_connector_creation_script.py

diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index ea4e7be93d4..f58a34c3243 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -672,6 +672,7 @@ def stream_chat_message_objects(
                     all_docs_useful=selected_db_search_docs is not None
                 ),
                 document_pruning_config=document_pruning_config,
+                structured_response_format=new_msg_req.structured_response_format,
             ),
             prompt_config=prompt_config,
             llm=(
diff --git a/backend/danswer/llm/answering/models.py b/backend/danswer/llm/answering/models.py
index fb5fa9c313e..87c1297fe92 100644
--- a/backend/danswer/llm/answering/models.py
+++ b/backend/danswer/llm/answering/models.py
@@ -116,6 +116,10 @@ class AnswerStyleConfig(BaseModel):
     document_pruning_config: DocumentPruningConfig = Field(
         default_factory=DocumentPruningConfig
     )
+    # forces the LLM to return a structured response, see
+    # https://platform.openai.com/docs/guides/structured-outputs/introduction
+    # right now, only used by the simple chat API
+    structured_response_format: dict | None = None
 
     @model_validator(mode="after")
     def check_quotes_and_citation(self) -> "AnswerStyleConfig":
diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
index d50f8253182..d450fff0a63 100644
--- a/backend/danswer/llm/chat_llm.py
+++ b/backend/danswer/llm/chat_llm.py
@@ -280,6 +280,7 @@ def _completion(
         tools: list[dict] | None,
         tool_choice: ToolChoiceOptions | None,
         stream: bool,
+        structured_response_format: dict | None = None,
     ) -> litellm.ModelResponse | litellm.CustomStreamWrapper:
         if isinstance(prompt, list):
             prompt = [
@@ -313,6 +314,11 @@ def _completion(
                 # NOTE: we can't pass this in if tools are not specified
                 # or else OpenAI throws an error
                 **({"parallel_tool_calls": False} if tools else {}),
+                **(
+                    {"response_format": structured_response_format}
+                    if structured_response_format
+                    else {}
+                ),
                 **self._model_kwargs,
             )
         except Exception as e:
@@ -336,12 +342,16 @@ def _invoke_implementation(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> BaseMessage:
         if LOG_DANSWER_MODEL_INTERACTIONS:
             self.log_model_configs()
 
         response = cast(
-            litellm.ModelResponse, self._completion(prompt, tools, tool_choice, False)
+            litellm.ModelResponse,
+            self._completion(
+                prompt, tools, tool_choice, False, structured_response_format
+            ),
         )
         choice = response.choices[0]
         if hasattr(choice, "message"):
@@ -354,18 +364,21 @@ def _stream_implementation(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> Iterator[BaseMessage]:
         if LOG_DANSWER_MODEL_INTERACTIONS:
             self.log_model_configs()
 
         if DISABLE_LITELLM_STREAMING:
-            yield self.invoke(prompt)
+            yield self.invoke(prompt, tools, tool_choice, structured_response_format)
             return
 
         output = None
         response = cast(
             litellm.CustomStreamWrapper,
-            self._completion(prompt, tools, tool_choice, True),
+            self._completion(
+                prompt, tools, tool_choice, True, structured_response_format
+            ),
         )
         try:
             for part in response:
diff --git a/backend/danswer/llm/custom_llm.py b/backend/danswer/llm/custom_llm.py
index 4a5ba7857c3..6b80406cf2f 100644
--- a/backend/danswer/llm/custom_llm.py
+++ b/backend/danswer/llm/custom_llm.py
@@ -80,6 +80,7 @@ def _invoke_implementation(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> BaseMessage:
         return self._execute(prompt)
 
@@ -88,5 +89,6 @@ def _stream_implementation(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> Iterator[BaseMessage]:
         yield self._execute(prompt)
diff --git a/backend/danswer/llm/interfaces.py b/backend/danswer/llm/interfaces.py
index 6cb58e46c6b..7deee11dfa6 100644
--- a/backend/danswer/llm/interfaces.py
+++ b/backend/danswer/llm/interfaces.py
@@ -88,11 +88,14 @@ def invoke(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> BaseMessage:
         self._precall(prompt)
         # TODO add a postcall to log model outputs independent of concrete class
         # implementation
-        return self._invoke_implementation(prompt, tools, tool_choice)
+        return self._invoke_implementation(
+            prompt, tools, tool_choice, structured_response_format
+        )
 
     @abc.abstractmethod
     def _invoke_implementation(
@@ -100,6 +103,7 @@ def _invoke_implementation(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> BaseMessage:
         raise NotImplementedError
 
@@ -108,11 +112,14 @@ def stream(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> Iterator[BaseMessage]:
         self._precall(prompt)
         # TODO add a postcall to log model outputs independent of concrete class
         # implementation
-        return self._stream_implementation(prompt, tools, tool_choice)
+        return self._stream_implementation(
+            prompt, tools, tool_choice, structured_response_format
+        )
 
     @abc.abstractmethod
     def _stream_implementation(
@@ -120,5 +127,6 @@ def _stream_implementation(
         prompt: LanguageModelInput,
         tools: list[dict] | None = None,
         tool_choice: ToolChoiceOptions | None = None,
+        structured_response_format: dict | None = None,
     ) -> Iterator[BaseMessage]:
         raise NotImplementedError
diff --git a/backend/danswer/server/query_and_chat/models.py b/backend/danswer/server/query_and_chat/models.py
index 42f4100a24b..1ca14f9283f 100644
--- a/backend/danswer/server/query_and_chat/models.py
+++ b/backend/danswer/server/query_and_chat/models.py
@@ -108,6 +108,10 @@ class CreateChatMessageRequest(ChunkContext):
     # used for seeded chats to kick off the generation of an AI answer
     use_existing_user_message: bool = False
 
+    # forces the LLM to return a structured response, see
+    # https://platform.openai.com/docs/guides/structured-outputs/introduction
+    structured_response_format: dict | None = None
+
     @model_validator(mode="after")
     def check_search_doc_ids_or_retrieval_options(self) -> "CreateChatMessageRequest":
         if self.search_doc_ids is None and self.retrieval_options is None:
diff --git a/backend/ee/danswer/server/query_and_chat/chat_backend.py b/backend/ee/danswer/server/query_and_chat/chat_backend.py
index dd637dcf081..b25ed8357d9 100644
--- a/backend/ee/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/ee/danswer/server/query_and_chat/chat_backend.py
@@ -176,6 +176,7 @@ def handle_simplified_chat_message(
         chunks_above=0,
         chunks_below=0,
         full_doc=chat_message_req.full_doc,
+        structured_response_format=chat_message_req.structured_response_format,
     )
 
     packets = stream_chat_message_objects(
@@ -202,7 +203,7 @@ def handle_send_message_simple_with_history(
         raise HTTPException(status_code=400, detail="Messages cannot be zero length")
 
     # This is a sanity check to make sure the chat history is valid
-    # It must start with a user message and alternate between user and assistant
+    # It must start with a user message and alternate beteen user and assistant
     expected_role = MessageType.USER
     for msg in req.messages:
         if not msg.message:
@@ -296,6 +297,7 @@ def handle_send_message_simple_with_history(
         chunks_above=0,
         chunks_below=0,
         full_doc=req.full_doc,
+        structured_response_format=req.structured_response_format,
     )
 
     packets = stream_chat_message_objects(
diff --git a/backend/ee/danswer/server/query_and_chat/models.py b/backend/ee/danswer/server/query_and_chat/models.py
index 052be683e9e..4baf17ac8ce 100644
--- a/backend/ee/danswer/server/query_and_chat/models.py
+++ b/backend/ee/danswer/server/query_and_chat/models.py
@@ -48,6 +48,9 @@ class BasicCreateChatMessageRequest(ChunkContext):
     query_override: str | None = None
     # If search_doc_ids provided, then retrieval options are unused
     search_doc_ids: list[int] | None = None
+    # only works if using an OpenAI model. See the following for more details:
+    # https://platform.openai.com/docs/guides/structured-outputs/introduction
+    structured_response_format: dict | None = None
 
 
 class BasicCreateChatMessageWithHistoryRequest(ChunkContext):
@@ -60,6 +63,9 @@ class BasicCreateChatMessageWithHistoryRequest(ChunkContext):
     skip_rerank: bool | None = None
     # If search_doc_ids provided, then retrieval options are unused
     search_doc_ids: list[int] | None = None
+    # only works if using an OpenAI model. See the following for more details:
+    # https://platform.openai.com/docs/guides/structured-outputs/introduction
+    structured_response_format: dict | None = None
 
 
 class SimpleDoc(BaseModel):
diff --git a/backend/scripts/add_connector_creation_script.py b/backend/scripts/add_connector_creation_script.py
new file mode 100644
index 00000000000..9a1944080c0
--- /dev/null
+++ b/backend/scripts/add_connector_creation_script.py
@@ -0,0 +1,148 @@
+from typing import Any
+from typing import Dict
+
+import requests
+
+API_SERVER_URL = "http://localhost:3000"  # Adjust this to your Danswer server URL
+HEADERS = {"Content-Type": "application/json"}
+API_KEY = "danswer-api-key"  # API key here, if auth is enabled
+
+
+def create_connector(
+    name: str,
+    source: str,
+    input_type: str,
+    connector_specific_config: Dict[str, Any],
+    is_public: bool = True,
+    groups: list[int] | None = None,
+) -> Dict[str, Any]:
+    connector_update_request = {
+        "name": name,
+        "source": source,
+        "input_type": input_type,
+        "connector_specific_config": connector_specific_config,
+        "is_public": is_public,
+        "groups": groups or [],
+    }
+
+    response = requests.post(
+        url=f"{API_SERVER_URL}/api/manage/admin/connector",
+        json=connector_update_request,
+        headers=HEADERS,
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def create_credential(
+    name: str,
+    source: str,
+    credential_json: Dict[str, Any],
+    is_public: bool = True,
+    groups: list[int] | None = None,
+) -> Dict[str, Any]:
+    credential_request = {
+        "name": name,
+        "source": source,
+        "credential_json": credential_json,
+        "admin_public": is_public,
+        "groups": groups or [],
+    }
+
+    response = requests.post(
+        url=f"{API_SERVER_URL}/api/manage/credential",
+        json=credential_request,
+        headers=HEADERS,
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def create_cc_pair(
+    connector_id: int,
+    credential_id: int,
+    name: str,
+    access_type: str = "public",
+    groups: list[int] | None = None,
+) -> Dict[str, Any]:
+    cc_pair_request = {
+        "name": name,
+        "access_type": access_type,
+        "groups": groups or [],
+    }
+
+    response = requests.put(
+        url=f"{API_SERVER_URL}/api/manage/connector/{connector_id}/credential/{credential_id}",
+        json=cc_pair_request,
+        headers=HEADERS,
+    )
+    response.raise_for_status()
+    return response.json()
+
+
+def main() -> None:
+    # Create a Web connector
+    web_connector = create_connector(
+        name="Example Web Connector",
+        source="web",
+        input_type="load_state",
+        connector_specific_config={
+            "base_url": "https://example.com",
+            "web_connector_type": "recursive",
+        },
+    )
+    print(f"Created Web Connector: {web_connector}")
+
+    # Create a credential for the Web connector
+    web_credential = create_credential(
+        name="Example Web Credential",
+        source="web",
+        credential_json={},  # Web connectors typically don't need credentials
+        is_public=True,
+    )
+    print(f"Created Web Credential: {web_credential}")
+
+    # Create CC pair for Web connector
+    web_cc_pair = create_cc_pair(
+        connector_id=web_connector["id"],
+        credential_id=web_credential["id"],
+        name="Example Web CC Pair",
+        access_type="public",
+    )
+    print(f"Created Web CC Pair: {web_cc_pair}")
+
+    # Create a GitHub connector
+    github_connector = create_connector(
+        name="Example GitHub Connector",
+        source="github",
+        input_type="poll",
+        connector_specific_config={
+            "repo_owner": "example-owner",
+            "repo_name": "example-repo",
+            "include_prs": True,
+            "include_issues": True,
+        },
+    )
+    print(f"Created GitHub Connector: {github_connector}")
+
+    # Create a credential for the GitHub connector
+    github_credential = create_credential(
+        name="Example GitHub Credential",
+        source="github",
+        credential_json={"github_access_token": "your_github_access_token_here"},
+        is_public=True,
+    )
+    print(f"Created GitHub Credential: {github_credential}")
+
+    # Create CC pair for GitHub connector
+    github_cc_pair = create_cc_pair(
+        connector_id=github_connector["id"],
+        credential_id=github_credential["id"],
+        name="Example GitHub CC Pair",
+        access_type="public",
+    )
+    print(f"Created GitHub CC Pair: {github_cc_pair}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py
index 77d9e0e7022..7b4d55cf3d6 100644
--- a/backend/tests/integration/conftest.py
+++ b/backend/tests/integration/conftest.py
@@ -6,7 +6,9 @@
 
 from danswer.db.engine import get_session_context_manager
 from danswer.db.search_settings import get_current_search_settings
+from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.reset import reset_all
+from tests.integration.common_utils.test_models import DATestUser
 from tests.integration.common_utils.vespa import vespa_fixture
 
 
@@ -44,3 +46,11 @@ def vespa_client(db_session: Session) -> vespa_fixture:
 @pytest.fixture
 def reset() -> None:
     reset_all()
+
+
+@pytest.fixture
+def new_admin_user() -> DATestUser | None:
+    try:
+        return UserManager.create(name="admin_user")
+    except Exception:
+        return None
diff --git a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
index 0a4e7b40b57..e773414b3af 100644
--- a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
+++ b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
@@ -1,7 +1,10 @@
+import json
+
 import requests
 
 from danswer.configs.constants import MessageType
 from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.constants import GENERAL_HEADERS
 from tests.integration.common_utils.constants import NUM_DOCS
 from tests.integration.common_utils.managers.api_key import APIKeyManager
 from tests.integration.common_utils.managers.cc_pair import CCPairManager
@@ -145,3 +148,87 @@ def test_using_reference_docs_with_simple_with_history_api_flow(reset: None) ->
     # This ensures the the document we think we are referencing when we send the search_doc_ids in the second
     # message is the document that we expect it to be
     assert response_json["top_documents"][0]["document_id"] == cc_pair_1.documents[2].id
+
+
+def test_send_message_simple_with_history_strict_json(
+    reset: None,
+    new_admin_user: DATestUser | None,
+) -> None:
+    # create connectors
+    cc_pair_1: DATestCCPair = CCPairManager.create_from_scratch(
+        user_performing_action=new_admin_user,
+    )
+    api_key: DATestAPIKey = APIKeyManager.create(
+        user_performing_action=new_admin_user,
+    )
+    LLMProviderManager.create(user_performing_action=new_admin_user)
+    cc_pair_1.documents = DocumentManager.seed_dummy_docs(
+        cc_pair=cc_pair_1,
+        num_docs=NUM_DOCS,
+        api_key=api_key,
+    )
+
+    response = requests.post(
+        f"{API_SERVER_URL}/chat/send-message-simple-with-history",
+        json={
+            "messages": [
+                {
+                    "message": "List the names of the first three US presidents in JSON format",
+                    "role": MessageType.USER.value,
+                }
+            ],
+            "persona_id": 0,
+            "prompt_id": 0,
+            "structured_response_format": {
+                "type": "json_object",
+                "schema": {
+                    "type": "object",
+                    "properties": {
+                        "presidents": {
+                            "type": "array",
+                            "items": {"type": "string"},
+                            "description": "List of the first three US presidents",
+                        }
+                    },
+                    "required": ["presidents"],
+                },
+            },
+        },
+        headers=new_admin_user.headers if new_admin_user else GENERAL_HEADERS,
+    )
+    assert response.status_code == 200
+
+    response_json = response.json()
+
+    # Check that the answer is present
+    assert "answer" in response_json
+    assert response_json["answer"] is not None
+
+    # helper
+    def clean_json_string(json_string: str) -> str:
+        return json_string.strip().removeprefix("```json").removesuffix("```").strip()
+
+    # Attempt to parse the answer as JSON
+    try:
+        clean_answer = clean_json_string(response_json["answer"])
+        parsed_answer = json.loads(clean_answer)
+        assert isinstance(parsed_answer, dict)
+        assert "presidents" in parsed_answer
+        assert isinstance(parsed_answer["presidents"], list)
+        assert len(parsed_answer["presidents"]) == 3
+        for president in parsed_answer["presidents"]:
+            assert isinstance(president, str)
+    except json.JSONDecodeError:
+        assert False, "The answer is not a valid JSON object"
+
+    # Check that the answer_citationless is also valid JSON
+    assert "answer_citationless" in response_json
+    assert response_json["answer_citationless"] is not None
+    try:
+        clean_answer_citationless = clean_json_string(
+            response_json["answer_citationless"]
+        )
+        parsed_answer_citationless = json.loads(clean_answer_citationless)
+        assert isinstance(parsed_answer_citationless, dict)
+    except json.JSONDecodeError:
+        assert False, "The answer_citationless is not a valid JSON object"

From 4ca38201d14409bd9b94bf7b9ffe81f9096d44f4 Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Thu, 24 Oct 2024 22:43:20 -0700
Subject: [PATCH 215/376] Fix IT fixture ordering

---
 backend/tests/integration/conftest.py                           | 2 +-
 .../tests/integration/tests/dev_apis/test_simple_chat_api.py    | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py
index 7b4d55cf3d6..f3d194e22b1 100644
--- a/backend/tests/integration/conftest.py
+++ b/backend/tests/integration/conftest.py
@@ -49,7 +49,7 @@ def reset() -> None:
 
 
 @pytest.fixture
-def new_admin_user() -> DATestUser | None:
+def new_admin_user(reset: None) -> DATestUser | None:
     try:
         return UserManager.create(name="admin_user")
     except Exception:
diff --git a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
index e773414b3af..fd7db7098bd 100644
--- a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
+++ b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
@@ -151,7 +151,6 @@ def test_using_reference_docs_with_simple_with_history_api_flow(reset: None) ->
 
 
 def test_send_message_simple_with_history_strict_json(
-    reset: None,
     new_admin_user: DATestUser | None,
 ) -> None:
     # create connectors

From 76415aff41110b0079d80595eec56898b6fbb1c8 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 25 Oct 2024 10:59:43 -0700
Subject: [PATCH 216/376] Ensure proper modal fallback (#2906)

* modal fallback

* nit

* k

* k
---
 .../modals/ProviderCreationModal.tsx          |  1 -
 .../prompt-library/modals/AddPromptModal.tsx  |  6 +-
 .../prompt-library/modals/EditPromptModal.tsx | 15 ++---
 web/src/app/chat/modal/FeedbackModal.tsx      |  6 +-
 .../chat/modal/MakePublicAssistantModal.tsx   |  6 +-
 .../app/chat/modal/SetDefaultModelModal.tsx   |  9 +--
 .../app/chat/modal/ShareChatSessionModal.tsx  |  6 +-
 web/src/components/Modal.tsx                  |  6 +-
 .../components/modals/DeleteEntityModal.tsx   |  6 +-
 .../components/modals/GenericConfirmModal.tsx |  6 +-
 web/src/components/modals/ModalWrapper.tsx    | 63 -------------------
 .../components/modals/NoAssistantModal.tsx    |  6 +-
 12 files changed, 35 insertions(+), 101 deletions(-)
 delete mode 100644 web/src/components/modals/ModalWrapper.tsx

diff --git a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
index cf9d5844397..8229d199675 100644
--- a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
@@ -173,7 +173,6 @@ export function ProviderCreationModal({
 
   return (
     <Modal
-      width="max-w-3xl"
       title={`Configure ${selectedProvider.provider_type}`}
       onOutsideClick={onCancel}
       icon={selectedProvider.icon}
diff --git a/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx b/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
index be83a19e926..eaa3c7d27c5 100644
--- a/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
+++ b/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
@@ -1,12 +1,12 @@
 import React from "react";
 import { Formik, Form } from "formik";
 import * as Yup from "yup";
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
 import { Button } from "@tremor/react";
 
 import { BookstackIcon } from "@/components/icons/icons";
 import { AddPromptModalProps } from "../interfaces";
 import { TextFormField } from "@/components/admin/connectors/Field";
+import { Modal } from "@/components/Modal";
 
 const AddPromptSchema = Yup.object().shape({
   title: Yup.string().required("Title is required"),
@@ -15,7 +15,7 @@ const AddPromptSchema = Yup.object().shape({
 
 const AddPromptModal = ({ onClose, onSubmit }: AddPromptModalProps) => {
   return (
-    <ModalWrapper onClose={onClose} modalClassName="max-w-xl">
+    <Modal onOutsideClick={onClose} width="max-w-xl">
       <Formik
         initialValues={{
           title: "",
@@ -57,7 +57,7 @@ const AddPromptModal = ({ onClose, onSubmit }: AddPromptModalProps) => {
           </Form>
         )}
       </Formik>
-    </ModalWrapper>
+    </Modal>
   );
 };
 
diff --git a/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx b/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
index bd20ce5e453..996b70b9fb4 100644
--- a/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
+++ b/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
@@ -1,8 +1,9 @@
 import React from "react";
 import { Formik, Form, Field, ErrorMessage } from "formik";
 import * as Yup from "yup";
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
+import { Modal } from "@/components/Modal";
 import { Button, Textarea, TextInput } from "@tremor/react";
+
 import { useInputPrompt } from "../hooks";
 import { EditPromptModalProps } from "../interfaces";
 
@@ -25,20 +26,20 @@ const EditPromptModal = ({
 
   if (error)
     return (
-      <ModalWrapper onClose={onClose} modalClassName="max-w-xl">
+      <Modal onOutsideClick={onClose} width="max-w-xl">
         <p>Failed to load prompt data</p>
-      </ModalWrapper>
+      </Modal>
     );
 
   if (!promptData)
     return (
-      <ModalWrapper onClose={onClose} modalClassName="max-w-xl">
+      <Modal onOutsideClick={onClose} width="max-w-xl">
         <p>Loading...</p>
-      </ModalWrapper>
+      </Modal>
     );
 
   return (
-    <ModalWrapper onClose={onClose} modalClassName="max-w-xl">
+    <Modal onOutsideClick={onClose} width="max-w-xl">
       <Formik
         initialValues={{
           prompt: promptData.prompt,
@@ -131,7 +132,7 @@ const EditPromptModal = ({
           </Form>
         )}
       </Formik>
-    </ModalWrapper>
+    </Modal>
   );
 };
 
diff --git a/web/src/app/chat/modal/FeedbackModal.tsx b/web/src/app/chat/modal/FeedbackModal.tsx
index 64feffefc73..39c3253b76a 100644
--- a/web/src/app/chat/modal/FeedbackModal.tsx
+++ b/web/src/app/chat/modal/FeedbackModal.tsx
@@ -2,7 +2,7 @@
 
 import { useState } from "react";
 import { FeedbackType } from "../types";
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
+import { Modal } from "@/components/Modal";
 import { FilledLikeIcon } from "@/components/icons/icons";
 
 const predefinedPositiveFeedbackOptions =
@@ -49,7 +49,7 @@ export const FeedbackModal = ({
       : predefinedNegativeFeedbackOptions;
 
   return (
-    <ModalWrapper onClose={onClose} modalClassName="max-w-3xl">
+    <Modal onOutsideClick={onClose} width="max-w-3xl">
       <>
         <h2 className="text-2xl text-emphasis font-bold mb-4 flex">
           <div className="mr-1 my-auto">
@@ -112,6 +112,6 @@ export const FeedbackModal = ({
           </button>
         </div>
       </>
-    </ModalWrapper>
+    </Modal>
   );
 };
diff --git a/web/src/app/chat/modal/MakePublicAssistantModal.tsx b/web/src/app/chat/modal/MakePublicAssistantModal.tsx
index a234050a52b..757cf060e80 100644
--- a/web/src/app/chat/modal/MakePublicAssistantModal.tsx
+++ b/web/src/app/chat/modal/MakePublicAssistantModal.tsx
@@ -1,4 +1,4 @@
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
+import { Modal } from "@/components/Modal";
 import { Button, Divider, Text } from "@tremor/react";
 
 export function MakePublicAssistantModal({
@@ -11,7 +11,7 @@ export function MakePublicAssistantModal({
   onClose: () => void;
 }) {
   return (
-    <ModalWrapper onClose={onClose} modalClassName="max-w-3xl">
+    <Modal onOutsideClick={onClose} width="max-w-3xl">
       <div className="space-y-6">
         <h2 className="text-2xl font-bold text-emphasis">
           {isPublic ? "Public Assistant" : "Make Assistant Public"}
@@ -67,6 +67,6 @@ export function MakePublicAssistantModal({
           </div>
         )}
       </div>
-    </ModalWrapper>
+    </Modal>
   );
 }
diff --git a/web/src/app/chat/modal/SetDefaultModelModal.tsx b/web/src/app/chat/modal/SetDefaultModelModal.tsx
index 5a47d9e66f2..61190120dc2 100644
--- a/web/src/app/chat/modal/SetDefaultModelModal.tsx
+++ b/web/src/app/chat/modal/SetDefaultModelModal.tsx
@@ -1,5 +1,5 @@
 import { Dispatch, SetStateAction, useEffect, useRef } from "react";
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
+import { Modal } from "@/components/Modal";
 import { Text } from "@tremor/react";
 import { getDisplayNameForModel, LlmOverride } from "@/lib/hooks";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
@@ -123,10 +123,7 @@ export function SetDefaultModelModal({
   );
 
   return (
-    <ModalWrapper
-      onClose={onClose}
-      modalClassName="rounded-lg  bg-white max-w-xl"
-    >
+    <Modal onOutsideClick={onClose} width="rounded-lg  bg-white max-w-xl">
       <>
         <div className="flex mb-4">
           <h2 className="text-2xl text-emphasis font-bold flex my-auto">
@@ -203,6 +200,6 @@ export function SetDefaultModelModal({
           </div>
         </div>
       </>
-    </ModalWrapper>
+    </Modal>
   );
 }
diff --git a/web/src/app/chat/modal/ShareChatSessionModal.tsx b/web/src/app/chat/modal/ShareChatSessionModal.tsx
index 16a9147b52a..1b797e77ab9 100644
--- a/web/src/app/chat/modal/ShareChatSessionModal.tsx
+++ b/web/src/app/chat/modal/ShareChatSessionModal.tsx
@@ -1,5 +1,5 @@
 import { useState } from "react";
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
+import { Modal } from "@/components/Modal";
 import { Button, Callout, Divider, Text } from "@tremor/react";
 import { Spinner } from "@/components/Spinner";
 import { ChatSessionSharedStatus } from "../interfaces";
@@ -57,7 +57,7 @@ export function ShareChatSessionModal({
   );
 
   return (
-    <ModalWrapper onClose={onClose} modalClassName="max-w-3xl">
+    <Modal onOutsideClick={onClose} width="max-w-3xl">
       <>
         <div className="flex mb-4">
           <h2 className="text-2xl text-emphasis font-bold flex my-auto">
@@ -154,6 +154,6 @@ export function ShareChatSessionModal({
           )}
         </div>
       </>
-    </ModalWrapper>
+    </Modal>
   );
 }
diff --git a/web/src/components/Modal.tsx b/web/src/components/Modal.tsx
index 169e85025d1..0f354a264f9 100644
--- a/web/src/components/Modal.tsx
+++ b/web/src/components/Modal.tsx
@@ -54,9 +54,9 @@ export function Modal({
             e.stopPropagation();
           }
         }}
-        className={`bg-background text-emphasis rounded shadow-2xl 
+        className={`bg-background  text-emphasis rounded shadow-2xl 
           transform transition-all duration-300 ease-in-out
-          ${width ?? "w-11/12 max-w-5xl"}
+          ${width ?? "w-11/12 max-w-4xl"}
           ${noPadding ? "" : "p-10"}
           ${className || ""}`}
       >
@@ -88,7 +88,7 @@ export function Modal({
               {!hideDividerForTitle && <Divider />}
             </>
           )}
-          {children}
+          <div className="max-h-[60vh] overflow-y-scroll">{children}</div>
         </div>
       </div>
     </div>
diff --git a/web/src/components/modals/DeleteEntityModal.tsx b/web/src/components/modals/DeleteEntityModal.tsx
index 5ef76f9c851..85cda2fd4d5 100644
--- a/web/src/components/modals/DeleteEntityModal.tsx
+++ b/web/src/components/modals/DeleteEntityModal.tsx
@@ -1,6 +1,6 @@
 import { FiTrash, FiX } from "react-icons/fi";
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
 import { BasicClickable } from "@/components/BasicClickable";
+import { Modal } from "../Modal";
 
 export const DeleteEntityModal = ({
   onClose,
@@ -16,7 +16,7 @@ export const DeleteEntityModal = ({
   additionalDetails?: string;
 }) => {
   return (
-    <ModalWrapper onClose={onClose}>
+    <Modal onOutsideClick={onClose}>
       <>
         <div className="flex mb-4">
           <h2 className="my-auto text-2xl font-bold">Delete {entityType}?</h2>
@@ -37,6 +37,6 @@ export const DeleteEntityModal = ({
           </div>
         </div>
       </>
-    </ModalWrapper>
+    </Modal>
   );
 };
diff --git a/web/src/components/modals/GenericConfirmModal.tsx b/web/src/components/modals/GenericConfirmModal.tsx
index fe6c2b020ae..893ae2f6b9e 100644
--- a/web/src/components/modals/GenericConfirmModal.tsx
+++ b/web/src/components/modals/GenericConfirmModal.tsx
@@ -1,5 +1,5 @@
 import { FiCheck } from "react-icons/fi";
-import { ModalWrapper } from "./ModalWrapper";
+import { Modal } from "@/components/Modal";
 import { BasicClickable } from "@/components/BasicClickable";
 
 export const GenericConfirmModal = ({
@@ -16,7 +16,7 @@ export const GenericConfirmModal = ({
   onConfirm: () => void;
 }) => {
   return (
-    <ModalWrapper onClose={onClose}>
+    <Modal onOutsideClick={onClose}>
       <div className="max-w-full">
         <div className="flex mb-4">
           <h2 className="my-auto text-2xl font-bold whitespace-normal overflow-wrap-normal">
@@ -37,6 +37,6 @@ export const GenericConfirmModal = ({
           </div>
         </div>
       </div>
-    </ModalWrapper>
+    </Modal>
   );
 };
diff --git a/web/src/components/modals/ModalWrapper.tsx b/web/src/components/modals/ModalWrapper.tsx
deleted file mode 100644
index f69ff0e2b6e..00000000000
--- a/web/src/components/modals/ModalWrapper.tsx
+++ /dev/null
@@ -1,63 +0,0 @@
-"use client";
-import { XIcon } from "@/components/icons/icons";
-import { isEventWithinRef } from "@/lib/contains";
-import { useRef } from "react";
-
-export const ModalWrapper = ({
-  children,
-  bgClassName,
-  modalClassName,
-  onClose,
-}: {
-  children: JSX.Element;
-  bgClassName?: string;
-  modalClassName?: string;
-  onClose?: () => void;
-}) => {
-  const modalRef = useRef<HTMLDivElement>(null);
-
-  const handleMouseDown = (e: React.MouseEvent<HTMLDivElement>) => {
-    if (
-      onClose &&
-      modalRef.current &&
-      !modalRef.current.contains(e.target as Node) &&
-      !isEventWithinRef(e.nativeEvent, modalRef)
-    ) {
-      onClose();
-    }
-  };
-  return (
-    <div
-      onMouseDown={handleMouseDown}
-      className={`fixed inset-0 bg-black bg-opacity-25 backdrop-blur-sm 
-        flex items-center justify-center z-50 transition-opacity duration-300 ease-in-out
-        ${bgClassName || ""}`}
-    >
-      <div
-        ref={modalRef}
-        onClick={(e) => {
-          if (onClose) {
-            e.stopPropagation();
-          }
-        }}
-        className={`bg-background text-emphasis p-10 rounded shadow-2xl 
-          w-11/12 max-w-3xl transform transition-all duration-300 ease-in-out
-          relative ${modalClassName || ""}`}
-      >
-        {onClose && (
-          <div className="absolute top-2 right-2">
-            <button
-              onClick={onClose}
-              className="cursor-pointer text-text-500 hover:text-text-700 transition-colors duration-200 p-2"
-              aria-label="Close modal"
-            >
-              <XIcon className="w-5 h-5" />
-            </button>
-          </div>
-        )}
-
-        <div className="flex w-full flex-col justify-stretch">{children}</div>
-      </div>
-    </div>
-  );
-};
diff --git a/web/src/components/modals/NoAssistantModal.tsx b/web/src/components/modals/NoAssistantModal.tsx
index 0eed8876629..94d20aadba8 100644
--- a/web/src/components/modals/NoAssistantModal.tsx
+++ b/web/src/components/modals/NoAssistantModal.tsx
@@ -1,8 +1,8 @@
-import { ModalWrapper } from "@/components/modals/ModalWrapper";
+import { Modal } from "@/components/Modal";
 
 export const NoAssistantModal = ({ isAdmin }: { isAdmin: boolean }) => {
   return (
-    <ModalWrapper modalClassName="bg-white max-w-2xl rounded-lg shadow-xl text-center">
+    <Modal width="bg-white max-w-2xl rounded-lg shadow-xl text-center">
       <>
         <h2 className="text-3xl font-bold text-gray-800 mb-4">
           No Assistant Available
@@ -32,6 +32,6 @@ export const NoAssistantModal = ({ isAdmin }: { isAdmin: boolean }) => {
           </p>
         )}
       </>
-    </ModalWrapper>
+    </Modal>
   );
 };

From a348caa9b114b04f36b6284aba21d459fc7dc9a8 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Fri, 25 Oct 2024 14:12:11 -0400
Subject: [PATCH 217/376] Add pagination & Remove req.obj from connectors.tsx

---
 .../danswer/connectors/freshdesk/connector.py | 51 +++++++++++++------
 web/src/lib/connectors/connectors.tsx         |  4 +-
 2 files changed, 36 insertions(+), 19 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 9275ee26d04..89a47b4714d 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -5,7 +5,7 @@
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
-from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector, LoadConnector
+from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector, LoadConnector, SecondsSinceUnixEpoch
 from danswer.connectors.models import ConnectorMissingCredentialError, Document, Section
 from danswer.utils.logger import setup_logger
 
@@ -45,20 +45,34 @@ def _fetch_tickets(self, start: datetime, end: datetime) -> List[dict]:
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
         
-        #convert start and end time to the format required by Freshdesk API
         start_time = start.strftime("%Y-%m-%dT%H:%M:%SZ")
-        end_time = end.strftime("%Y-%m-%dT%H:%M:%SZ")
 
-        freshdesk_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets?include=description&updated_since={start_time}&updated_before={end_time}"
-        response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
-        response.raise_for_status()  # raises exception when not a 2xx response
-        
-        if response.status_code!= 204:
-            tickets = json.loads(response.content)
-            logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API")
-            return tickets
-        else:
-            return []
+        all_tickets = []
+        page = 1
+        per_page = 50
+
+        while True:
+            freshdesk_url = (
+                f"https://{self.domain}.freshdesk.com/api/v2/tickets"
+                f"?include=description&updated_since={start_time}"
+                f"&per_page={per_page}&page={page}"
+            )
+            response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
+            response.raise_for_status()
+            
+            if response.status_code != 204:
+                tickets = json.loads(response.content)
+                all_tickets.extend(tickets)
+                logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API (Page {page})")
+
+                if len(tickets) < per_page:
+                    break
+                
+                page += 1
+            else:
+                break
+
+        return all_tickets
 
     def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
         tickets = self._fetch_tickets(start, end)
@@ -107,7 +121,6 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
                         if isinstance(value, str) and key not in ["description", "description_text"]
                     },
                 )
-
                 doc_batch.append(doc)
 
             if len(doc_batch) >= self.batch_size:
@@ -117,5 +130,11 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
         if doc_batch:
             yield doc_batch
 
-    def poll_source(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
-        yield from self._process_tickets(start, end)
+    def load_from_state(self) -> GenerateDocumentsOutput:
+        return self._fetch_tickets()
+
+    def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:
+        start_datetime = datetime.fromtimestamp(start)
+        end_datetime = datetime.fromtimestamp(end)
+
+        yield from self._process_tickets(start_datetime, end_datetime)
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index dd06a3436a1..d98b7afffac 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -1186,9 +1186,7 @@ export interface AsanaConfig {
   asana_team_id?: string;
 }
 
-export interface FreshdeskConfig {
-  requested_objects?: string[];
-}
+export interface FreshdeskConfig {}
 
 
 export interface MediaWikiConfig extends MediaWikiBaseConfig {

From bd6311968476c667ca4d1436fb29d9b3492677a8 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Fri, 25 Oct 2024 11:19:54 -0700
Subject: [PATCH 218/376] Fix structured outputs (#2923)

* Fix structured outputs

* Add back rest
---
 ...533f0_make_last_attempt_status_nullable.py |  6 +++
 backend/danswer/llm/answering/answer.py       |  6 ++-
 .../tests/dev_apis/test_simple_chat_api.py    | 47 +++++++++----------
 3 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py b/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py
index a6938e365c6..db7b330c3e0 100644
--- a/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py
+++ b/backend/alembic/versions/b082fec533f0_make_last_attempt_status_nullable.py
@@ -31,6 +31,12 @@ def upgrade() -> None:
 
 
 def downgrade() -> None:
+    # First, update any null values to a default value
+    op.execute(
+        "UPDATE connector_credential_pair SET last_attempt_status = 'NOT_STARTED' WHERE last_attempt_status IS NULL"
+    )
+
+    # Then, make the column non-nullable
     op.alter_column(
         "connector_credential_pair",
         "last_attempt_status",
diff --git a/backend/danswer/llm/answering/answer.py b/backend/danswer/llm/answering/answer.py
index 4648e0fe821..d2aeb1b14c4 100644
--- a/backend/danswer/llm/answering/answer.py
+++ b/backend/danswer/llm/answering/answer.py
@@ -237,6 +237,7 @@ def _raw_output_for_explicit_tool_calling_llms(
                 prompt=prompt,
                 tools=final_tool_definitions if final_tool_definitions else None,
                 tool_choice="required" if self.force_use_tool.force_use else None,
+                structured_response_format=self.answer_style_config.structured_response_format,
             ):
                 if isinstance(message, AIMessageChunk) and (
                     message.tool_call_chunks or message.tool_calls
@@ -331,7 +332,10 @@ def _process_llm_stream(
         tool_choice: ToolChoiceOptions | None = None,
     ) -> Iterator[str | StreamStopInfo]:
         for message in self.llm.stream(
-            prompt=prompt, tools=tools, tool_choice=tool_choice
+            prompt=prompt,
+            tools=tools,
+            tool_choice=tool_choice,
+            structured_response_format=self.answer_style_config.structured_response_format,
         ):
             if isinstance(message, AIMessageChunk):
                 if message.content:
diff --git a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
index fd7db7098bd..c37d1a6235d 100644
--- a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
+++ b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
@@ -154,42 +154,38 @@ def test_send_message_simple_with_history_strict_json(
     new_admin_user: DATestUser | None,
 ) -> None:
     # create connectors
-    cc_pair_1: DATestCCPair = CCPairManager.create_from_scratch(
-        user_performing_action=new_admin_user,
-    )
-    api_key: DATestAPIKey = APIKeyManager.create(
-        user_performing_action=new_admin_user,
-    )
     LLMProviderManager.create(user_performing_action=new_admin_user)
-    cc_pair_1.documents = DocumentManager.seed_dummy_docs(
-        cc_pair=cc_pair_1,
-        num_docs=NUM_DOCS,
-        api_key=api_key,
-    )
 
     response = requests.post(
         f"{API_SERVER_URL}/chat/send-message-simple-with-history",
         json={
+            # intentionally not relevant prompt to ensure that the
+            # structured response format is actually used
             "messages": [
                 {
-                    "message": "List the names of the first three US presidents in JSON format",
+                    "message": "What is green?",
                     "role": MessageType.USER.value,
                 }
             ],
             "persona_id": 0,
             "prompt_id": 0,
             "structured_response_format": {
-                "type": "json_object",
-                "schema": {
-                    "type": "object",
-                    "properties": {
-                        "presidents": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                            "description": "List of the first three US presidents",
-                        }
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "presidents",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "presidents": {
+                                "type": "array",
+                                "items": {"type": "string"},
+                                "description": "List of the first three US presidents",
+                            }
+                        },
+                        "required": ["presidents"],
+                        "additionalProperties": False,
                     },
-                    "required": ["presidents"],
+                    "strict": True,
                 },
             },
         },
@@ -211,14 +207,17 @@ def clean_json_string(json_string: str) -> str:
     try:
         clean_answer = clean_json_string(response_json["answer"])
         parsed_answer = json.loads(clean_answer)
+
+        # NOTE: do not check content, just the structure
         assert isinstance(parsed_answer, dict)
         assert "presidents" in parsed_answer
         assert isinstance(parsed_answer["presidents"], list)
-        assert len(parsed_answer["presidents"]) == 3
         for president in parsed_answer["presidents"]:
             assert isinstance(president, str)
     except json.JSONDecodeError:
-        assert False, "The answer is not a valid JSON object"
+        assert (
+            False
+        ), f"The answer is not a valid JSON object - '{response_json['answer']}'"
 
     # Check that the answer_citationless is also valid JSON
     assert "answer_citationless" in response_json

From 9b147ae437f698e717317c5d8c384347315fba93 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 25 Oct 2024 11:47:17 -0700
Subject: [PATCH 219/376] Tenant integration tests (#2913)

* check for index swap

* initial bones

* kk

* k

* k:

* nit

* nit

* rebase + update

* nit

* minior update

* k

* minor integration test fixes

* nit

* ensure we build test docker image

* remove one space

* k

* ensure we wipe volumes

* remove log

* typo

* nit

* k

* k
---
 .github/workflows/pr-Integration-tests.yml    |  65 ++-
 backend/danswer/auth/users.py                 |   5 +-
 .../background/celery/apps/indexing.py        |  28 -
 backend/danswer/background/update.py          | 494 ------------------
 backend/danswer/configs/app_configs.py        |   3 +
 backend/danswer/db/swap_index.py              |   3 +-
 .../danswer/server/danswer_api/ingestion.py   |   4 +
 backend/danswer/server/settings/api.py        |   2 -
 .../server/middleware/tenant_tracking.py      |   1 -
 backend/tests/integration/Dockerfile          |   3 +-
 .../integration/common_utils/managers/chat.py |   2 +-
 .../common_utils/managers/credential.py       |   1 +
 .../common_utils/managers/tenant.py           |  82 +++
 .../integration/common_utils/managers/user.py |  16 +-
 .../tests/integration/common_utils/reset.py   | 112 +++-
 backend/tests/integration/conftest.py         |   6 +
 .../integration/multitenant_tests/cc_Pair     |   0
 .../syncing/test_search_permissions.py        | 150 ++++++
 .../tenants/test_tenant_creation.py           |  41 ++
 .../tests/dev_apis/test_simple_chat_api.py    |   1 +
 20 files changed, 479 insertions(+), 540 deletions(-)
 delete mode 100755 backend/danswer/background/update.py
 create mode 100644 backend/tests/integration/common_utils/managers/tenant.py
 create mode 100644 backend/tests/integration/multitenant_tests/cc_Pair
 create mode 100644 backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py
 create mode 100644 backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py

diff --git a/.github/workflows/pr-Integration-tests.yml b/.github/workflows/pr-Integration-tests.yml
index 0e4856f7194..1f28866d6ee 100644
--- a/.github/workflows/pr-Integration-tests.yml
+++ b/.github/workflows/pr-Integration-tests.yml
@@ -72,7 +72,7 @@ jobs:
           load: true
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/model-server/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
-
+      
       - name: Build integration test Docker image
         uses: ./.github/actions/custom-build-and-push
         with:
@@ -85,7 +85,58 @@ jobs:
           cache-from: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }}
           cache-to: type=s3,prefix=cache/${{ github.repository }}/integration-tests/integration/,region=${{ env.RUNS_ON_AWS_REGION }},bucket=${{ env.RUNS_ON_S3_BUCKET_CACHE }},mode=max
 
-      - name: Start Docker containers
+      # Start containers for multi-tenant tests
+      - name: Start Docker containers for multi-tenant tests
+        run: |
+          cd deployment/docker_compose
+          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
+          MULTI_TENANT=true \
+          AUTH_TYPE=basic \
+          REQUIRE_EMAIL_VERIFICATION=false \
+          DISABLE_TELEMETRY=true \
+          IMAGE_TAG=test \
+          docker compose -f docker-compose.dev.yml -p danswer-stack up -d
+        id: start_docker_multi_tenant
+
+      # In practice, `cloud` Auth type would require OAUTH credentials to be set.
+      - name: Run Multi-Tenant Integration Tests
+        run: |
+          echo "Running integration tests..."
+          docker run --rm --network danswer-stack_default \
+            --name test-runner \
+            -e POSTGRES_HOST=relational_db \
+            -e POSTGRES_USER=postgres \
+            -e POSTGRES_PASSWORD=password \
+            -e POSTGRES_DB=postgres \
+            -e VESPA_HOST=index \
+            -e REDIS_HOST=cache \
+            -e API_SERVER_HOST=api_server \
+            -e OPENAI_API_KEY=${OPENAI_API_KEY} \
+            -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
+            -e TEST_WEB_HOSTNAME=test-runner \
+            -e AUTH_TYPE=cloud \
+            -e MULTI_TENANT=true \
+            danswer/danswer-integration:test \
+            /app/tests/integration/multitenant_tests
+        continue-on-error: true
+        id: run_multitenant_tests
+
+      - name: Check multi-tenant test results
+        run: |
+          if [ ${{ steps.run_tests.outcome }} == 'failure' ]; then
+            echo "Integration tests failed. Exiting with error."
+            exit 1
+          else
+            echo "All integration tests passed successfully."
+          fi 
+
+      - name: Stop multi-tenant Docker containers
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
+
+
+      - name: Start Docker containers 
         run: |
           cd deployment/docker_compose
           ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=true \
@@ -130,7 +181,7 @@ jobs:
           done
           echo "Finished waiting for service."
 
-      - name: Run integration tests
+      - name: Run Standard Integration Tests
         run: |
           echo "Running integration tests..."
           docker run --rm --network danswer-stack_default \
@@ -145,7 +196,8 @@ jobs:
             -e OPENAI_API_KEY=${OPENAI_API_KEY} \
             -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \
             -e TEST_WEB_HOSTNAME=test-runner \
-            danswer/danswer-integration:test
+            danswer/danswer-integration:test \
+            /app/tests/integration/tests
         continue-on-error: true
         id: run_tests
 
@@ -158,6 +210,11 @@ jobs:
             echo "All integration tests passed successfully."
           fi
 
+      - name: Stop Docker containers
+        run: |
+          cd deployment/docker_compose
+          docker compose -f docker-compose.dev.yml -p danswer-stack down -v
+
       - name: Save Docker logs
         if: success() || failure()
         run: |
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index f4f069a5bac..49e9f5f8736 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -58,6 +58,7 @@
 from danswer.auth.schemas import UserUpdate
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import DISABLE_AUTH
+from danswer.configs.app_configs import DISABLE_VERIFICATION
 from danswer.configs.app_configs import EMAIL_FROM
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
@@ -133,7 +134,9 @@ def get_display_email(email: str | None, space_less: bool = False) -> str:
 def user_needs_to_be_verified() -> bool:
     # all other auth types besides basic should require users to be
     # verified
-    return AUTH_TYPE != AuthType.BASIC or REQUIRE_EMAIL_VERIFICATION
+    return not DISABLE_VERIFICATION and (
+        AUTH_TYPE != AuthType.BASIC or REQUIRE_EMAIL_VERIFICATION
+    )
 
 
 def verify_email_is_invited(email: str) -> None:
diff --git a/backend/danswer/background/celery/apps/indexing.py b/backend/danswer/background/celery/apps/indexing.py
index 5e51ebc8c54..a981694e12e 100644
--- a/backend/danswer/background/celery/apps/indexing.py
+++ b/backend/danswer/background/celery/apps/indexing.py
@@ -8,18 +8,11 @@
 from celery.signals import worker_init
 from celery.signals import worker_ready
 from celery.signals import worker_shutdown
-from sqlalchemy.orm import Session
 
 import danswer.background.celery.apps.app_base as app_base
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_APP_NAME
 from danswer.db.engine import SqlEngine
-from danswer.db.search_settings import get_current_search_settings
-from danswer.db.swap_index import check_index_swap
-from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
-from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
-from shared_configs.configs import MODEL_SERVER_PORT
 
 
 logger = setup_logger()
@@ -67,27 +60,6 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_APP_NAME)
     SqlEngine.init_engine(pool_size=8, max_overflow=0)
 
-    # TODO: why is this necessary for the indexer to do?
-    engine = SqlEngine.get_engine()
-    with Session(engine) as db_session:
-        check_index_swap(db_session=db_session)
-        search_settings = get_current_search_settings(db_session)
-
-        # So that the first time users aren't surprised by really slow speed of first
-        # batch of documents indexed
-        if search_settings.provider_type is None:
-            logger.notice("Running a first inference to warm up embedding model")
-            embedding_model = EmbeddingModel.from_db_model(
-                search_settings=search_settings,
-                server_host=INDEXING_MODEL_SERVER_HOST,
-                server_port=MODEL_SERVER_PORT,
-            )
-
-            warm_up_bi_encoder(
-                embedding_model=embedding_model,
-            )
-            logger.notice("First inference complete.")
-
     app_base.wait_for_redis(sender, **kwargs)
     app_base.on_secondary_worker_init(sender, **kwargs)
 
diff --git a/backend/danswer/background/update.py b/backend/danswer/background/update.py
deleted file mode 100755
index b408f289724..00000000000
--- a/backend/danswer/background/update.py
+++ /dev/null
@@ -1,494 +0,0 @@
-# TODO(rkuo): delete after background indexing via celery is fully vetted
-# import logging
-# import time
-# from datetime import datetime
-# import dask
-# from dask.distributed import Client
-# from dask.distributed import Future
-# from distributed import LocalCluster
-# from sqlalchemy import text
-# from sqlalchemy.exc import ProgrammingError
-# from sqlalchemy.orm import Session
-# from danswer.background.indexing.dask_utils import ResourceLogger
-# from danswer.background.indexing.job_client import SimpleJob
-# from danswer.background.indexing.job_client import SimpleJobClient
-# from danswer.background.indexing.run_indexing import run_indexing_entrypoint
-# from danswer.configs.app_configs import CLEANUP_INDEXING_JOBS_TIMEOUT
-# from danswer.configs.app_configs import DASK_JOB_CLIENT_ENABLED
-# from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
-# from danswer.configs.app_configs import MULTI_TENANT
-# from danswer.configs.app_configs import NUM_INDEXING_WORKERS
-# from danswer.configs.app_configs import NUM_SECONDARY_INDEXING_WORKERS
-# from danswer.configs.constants import DocumentSource
-# from danswer.configs.constants import POSTGRES_INDEXER_APP_NAME
-# from danswer.configs.constants import TENANT_ID_PREFIX
-# from danswer.db.connector import fetch_connectors
-# from danswer.db.connector_credential_pair import fetch_connector_credential_pairs
-# from danswer.db.engine import get_db_current_time
-# from danswer.db.engine import get_session_with_tenant
-# from danswer.db.engine import get_sqlalchemy_engine
-# from danswer.db.engine import SqlEngine
-# from danswer.db.index_attempt import create_index_attempt
-# from danswer.db.index_attempt import get_index_attempt
-# from danswer.db.index_attempt import get_inprogress_index_attempts
-# from danswer.db.index_attempt import get_last_attempt_for_cc_pair
-# from danswer.db.index_attempt import get_not_started_index_attempts
-# from danswer.db.index_attempt import mark_attempt_failed
-# from danswer.db.models import ConnectorCredentialPair
-# from danswer.db.models import IndexAttempt
-# from danswer.db.models import IndexingStatus
-# from danswer.db.models import IndexModelStatus
-# from danswer.db.models import SearchSettings
-# from danswer.db.search_settings import get_current_search_settings
-# from danswer.db.search_settings import get_secondary_search_settings
-# from danswer.db.swap_index import check_index_swap
-# from danswer.document_index.vespa.index import VespaIndex
-# from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
-# from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
-# from danswer.utils.logger import setup_logger
-# from danswer.utils.variable_functionality import global_version
-# from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
-# from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
-# from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
-# from shared_configs.configs import LOG_LEVEL
-# logger = setup_logger()
-# # If the indexing dies, it's most likely due to resource constraints,
-# # restarting just delays the eventual failure, not useful to the user
-# dask.config.set({"distributed.scheduler.allowed-failures": 0})
-# _UNEXPECTED_STATE_FAILURE_REASON = (
-#     "Stopped mid run, likely due to the background process being killed"
-# )
-# def _should_create_new_indexing(
-#     cc_pair: ConnectorCredentialPair,
-#     last_index: IndexAttempt | None,
-#     search_settings_instance: SearchSettings,
-#     secondary_index_building: bool,
-#     db_session: Session,
-# ) -> bool:
-#     connector = cc_pair.connector
-#     # don't kick off indexing for `NOT_APPLICABLE` sources
-#     if connector.source == DocumentSource.NOT_APPLICABLE:
-#         return False
-#     # User can still manually create single indexing attempts via the UI for the
-#     # currently in use index
-#     if DISABLE_INDEX_UPDATE_ON_SWAP:
-#         if (
-#             search_settings_instance.status == IndexModelStatus.PRESENT
-#             and secondary_index_building
-#         ):
-#             return False
-#     # When switching over models, always index at least once
-#     if search_settings_instance.status == IndexModelStatus.FUTURE:
-#         if last_index:
-#             # No new index if the last index attempt succeeded
-#             # Once is enough. The model will never be able to swap otherwise.
-#             if last_index.status == IndexingStatus.SUCCESS:
-#                 return False
-#             # No new index if the last index attempt is waiting to start
-#             if last_index.status == IndexingStatus.NOT_STARTED:
-#                 return False
-#             # No new index if the last index attempt is running
-#             if last_index.status == IndexingStatus.IN_PROGRESS:
-#                 return False
-#         else:
-#             if (
-#                 connector.id == 0 or connector.source == DocumentSource.INGESTION_API
-#             ):  # Ingestion API
-#                 return False
-#         return True
-#     # If the connector is paused or is the ingestion API, don't index
-#     # NOTE: during an embedding model switch over, the following logic
-#     # is bypassed by the above check for a future model
-#     if (
-#         not cc_pair.status.is_active()
-#         or connector.id == 0
-#         or connector.source == DocumentSource.INGESTION_API
-#     ):
-#         return False
-#     if not last_index:
-#         return True
-#     if connector.refresh_freq is None:
-#         return False
-#     # Only one scheduled/ongoing job per connector at a time
-#     # this prevents cases where
-#     # (1) the "latest" index_attempt is scheduled so we show
-#     #     that in the UI despite another index_attempt being in-progress
-#     # (2) multiple scheduled index_attempts at a time
-#     if (
-#         last_index.status == IndexingStatus.NOT_STARTED
-#         or last_index.status == IndexingStatus.IN_PROGRESS
-#     ):
-#         return False
-#     current_db_time = get_db_current_time(db_session)
-#     time_since_index = current_db_time - last_index.time_updated
-#     return time_since_index.total_seconds() >= connector.refresh_freq
-# def _mark_run_failed(
-#     db_session: Session, index_attempt: IndexAttempt, failure_reason: str
-# ) -> None:
-#     """Marks the `index_attempt` row as failed + updates the `
-#     connector_credential_pair` to reflect that the run failed"""
-#     logger.warning(
-#         f"Marking in-progress attempt 'connector: {index_attempt.connector_credential_pair.connector_id}, "
-#         f"credential: {index_attempt.connector_credential_pair.credential_id}' as failed due to {failure_reason}"
-#     )
-#     mark_attempt_failed(
-#         index_attempt=index_attempt,
-#         db_session=db_session,
-#         failure_reason=failure_reason,
-#     )
-# """Main funcs"""
-# def create_indexing_jobs(
-#     existing_jobs: dict[int, Future | SimpleJob], tenant_id: str | None
-# ) -> None:
-#     """Creates new indexing jobs for each connector / credential pair which is:
-#     1. Enabled
-#     2. `refresh_frequency` time has passed since the last indexing run for this pair
-#     3. There is not already an ongoing indexing attempt for this pair
-#     """
-#     with get_session_with_tenant(tenant_id) as db_session:
-#         ongoing: set[tuple[int | None, int]] = set()
-#         for attempt_id in existing_jobs:
-#             attempt = get_index_attempt(
-#                 db_session=db_session, index_attempt_id=attempt_id
-#             )
-#             if attempt is None:
-#                 logger.error(
-#                     f"Unable to find IndexAttempt for ID '{attempt_id}' when creating "
-#                     "indexing jobs"
-#                 )
-#                 continue
-#             ongoing.add(
-#                 (
-#                     attempt.connector_credential_pair_id,
-#                     attempt.search_settings_id,
-#                 )
-#             )
-#         # Get the primary search settings
-#         primary_search_settings = get_current_search_settings(db_session)
-#         search_settings = [primary_search_settings]
-#         # Check for secondary search settings
-#         secondary_search_settings = get_secondary_search_settings(db_session)
-#         if secondary_search_settings is not None:
-#             # If secondary settings exist, add them to the list
-#             search_settings.append(secondary_search_settings)
-#         all_connector_credential_pairs = fetch_connector_credential_pairs(db_session)
-#         for cc_pair in all_connector_credential_pairs:
-#             for search_settings_instance in search_settings:
-#                 # Check if there is an ongoing indexing attempt for this connector credential pair
-#                 if (cc_pair.id, search_settings_instance.id) in ongoing:
-#                     continue
-#                 last_attempt = get_last_attempt_for_cc_pair(
-#                     cc_pair.id, search_settings_instance.id, db_session
-#                 )
-#                 if not _should_create_new_indexing(
-#                     cc_pair=cc_pair,
-#                     last_index=last_attempt,
-#                     search_settings_instance=search_settings_instance,
-#                     secondary_index_building=len(search_settings) > 1,
-#                     db_session=db_session,
-#                 ):
-#                     continue
-#                 create_index_attempt(
-#                     cc_pair.id, search_settings_instance.id, db_session
-#                 )
-# def cleanup_indexing_jobs(
-#     existing_jobs: dict[int, Future | SimpleJob],
-#     tenant_id: str | None,
-#     timeout_hours: int = CLEANUP_INDEXING_JOBS_TIMEOUT,
-# ) -> dict[int, Future | SimpleJob]:
-#     existing_jobs_copy = existing_jobs.copy()
-#     # clean up completed jobs
-#     with get_session_with_tenant(tenant_id) as db_session:
-#         for attempt_id, job in existing_jobs.items():
-#             index_attempt = get_index_attempt(
-#                 db_session=db_session, index_attempt_id=attempt_id
-#             )
-#             # do nothing for ongoing jobs that haven't been stopped
-#             if not job.done():
-#                 if not index_attempt:
-#                     continue
-#                 if not index_attempt.is_finished():
-#                     continue
-#             if job.status == "error":
-#                 logger.error(job.exception())
-#             job.release()
-#             del existing_jobs_copy[attempt_id]
-#             if not index_attempt:
-#                 logger.error(
-#                     f"Unable to find IndexAttempt for ID '{attempt_id}' when cleaning "
-#                     "up indexing jobs"
-#                 )
-#                 continue
-#             if (
-#                 index_attempt.status == IndexingStatus.IN_PROGRESS
-#                 or job.status == "error"
-#             ):
-#                 _mark_run_failed(
-#                     db_session=db_session,
-#                     index_attempt=index_attempt,
-#                     failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
-#                 )
-#         # clean up in-progress jobs that were never completed
-#         try:
-#             connectors = fetch_connectors(db_session)
-#             for connector in connectors:
-#                 in_progress_indexing_attempts = get_inprogress_index_attempts(
-#                     connector.id, db_session
-#                 )
-#                 for index_attempt in in_progress_indexing_attempts:
-#                     if index_attempt.id in existing_jobs:
-#                         # If index attempt is canceled, stop the run
-#                         if index_attempt.status == IndexingStatus.FAILED:
-#                             existing_jobs[index_attempt.id].cancel()
-#                         # check to see if the job has been updated in last `timeout_hours` hours, if not
-#                         # assume it to frozen in some bad state and just mark it as failed. Note: this relies
-#                         # on the fact that the `time_updated` field is constantly updated every
-#                         # batch of documents indexed
-#                         current_db_time = get_db_current_time(db_session=db_session)
-#                         time_since_update = current_db_time - index_attempt.time_updated
-#                         if time_since_update.total_seconds() > 60 * 60 * timeout_hours:
-#                             existing_jobs[index_attempt.id].cancel()
-#                             _mark_run_failed(
-#                                 db_session=db_session,
-#                                 index_attempt=index_attempt,
-#                                 failure_reason="Indexing run frozen - no updates in the last three hours. "
-#                                 "The run will be re-attempted at next scheduled indexing time.",
-#                             )
-#                     else:
-#                         # If job isn't known, simply mark it as failed
-#                         _mark_run_failed(
-#                             db_session=db_session,
-#                             index_attempt=index_attempt,
-#                             failure_reason=_UNEXPECTED_STATE_FAILURE_REASON,
-#                         )
-#         except ProgrammingError:
-#             logger.debug(f"No Connector Table exists for: {tenant_id}")
-#     return existing_jobs_copy
-# def kickoff_indexing_jobs(
-#     existing_jobs: dict[int, Future | SimpleJob],
-#     client: Client | SimpleJobClient,
-#     secondary_client: Client | SimpleJobClient,
-#     tenant_id: str | None,
-# ) -> dict[int, Future | SimpleJob]:
-#     existing_jobs_copy = existing_jobs.copy()
-#     current_session = get_session_with_tenant(tenant_id)
-#     # Don't include jobs waiting in the Dask queue that just haven't started running
-#     # Also (rarely) don't include for jobs that started but haven't updated the indexing tables yet
-#     with current_session as db_session:
-#         # get_not_started_index_attempts orders its returned results from oldest to newest
-#         # we must process attempts in a FIFO manner to prevent connector starvation
-#         new_indexing_attempts = [
-#             (attempt, attempt.search_settings)
-#             for attempt in get_not_started_index_attempts(db_session)
-#             if attempt.id not in existing_jobs
-#         ]
-#     logger.debug(f"Found {len(new_indexing_attempts)} new indexing task(s).")
-#     if not new_indexing_attempts:
-#         return existing_jobs
-#     indexing_attempt_count = 0
-#     primary_client_full = False
-#     secondary_client_full = False
-#     for attempt, search_settings in new_indexing_attempts:
-#         if primary_client_full and secondary_client_full:
-#             break
-#         use_secondary_index = (
-#             search_settings.status == IndexModelStatus.FUTURE
-#             if search_settings is not None
-#             else False
-#         )
-#         if attempt.connector_credential_pair.connector is None:
-#             logger.warning(
-#                 f"Skipping index attempt as Connector has been deleted: {attempt}"
-#             )
-#             with current_session as db_session:
-#                 mark_attempt_failed(
-#                     attempt, db_session, failure_reason="Connector is null"
-#                 )
-#             continue
-#         if attempt.connector_credential_pair.credential is None:
-#             logger.warning(
-#                 f"Skipping index attempt as Credential has been deleted: {attempt}"
-#             )
-#             with current_session as db_session:
-#                 mark_attempt_failed(
-#                     attempt, db_session, failure_reason="Credential is null"
-#                 )
-#             continue
-#         if not use_secondary_index:
-#             if not primary_client_full:
-#                 run = client.submit(
-#                     run_indexing_entrypoint,
-#                     attempt.id,
-#                     tenant_id,
-#                     attempt.connector_credential_pair_id,
-#                     global_version.is_ee_version(),
-#                     pure=False,
-#                 )
-#                 if not run:
-#                     primary_client_full = True
-#         else:
-#             if not secondary_client_full:
-#                 run = secondary_client.submit(
-#                     run_indexing_entrypoint,
-#                     attempt.id,
-#                     tenant_id,
-#                     attempt.connector_credential_pair_id,
-#                     global_version.is_ee_version(),
-#                     pure=False,
-#                 )
-#                 if not run:
-#                     secondary_client_full = True
-#         if run:
-#             if indexing_attempt_count == 0:
-#                 logger.info(
-#                     f"Indexing dispatch starts: pending={len(new_indexing_attempts)}"
-#                 )
-#             indexing_attempt_count += 1
-#             secondary_str = " (secondary index)" if use_secondary_index else ""
-#             logger.info(
-#                 f"Indexing dispatched{secondary_str}: "
-#                 f"attempt_id={attempt.id} "
-#                 f"connector='{attempt.connector_credential_pair.connector.name}' "
-#                 f"config='{attempt.connector_credential_pair.connector.connector_specific_config}' "
-#                 f"credentials='{attempt.connector_credential_pair.credential_id}'"
-#             )
-#             existing_jobs_copy[attempt.id] = run
-#     if indexing_attempt_count > 0:
-#         logger.info(
-#             f"Indexing dispatch results: "
-#             f"initial_pending={len(new_indexing_attempts)} "
-#             f"started={indexing_attempt_count} "
-#             f"remaining={len(new_indexing_attempts) - indexing_attempt_count}"
-#         )
-#     return existing_jobs_copy
-# def get_all_tenant_ids() -> list[str] | list[None]:
-#     if not MULTI_TENANT:
-#         return [None]
-#     with get_session_with_tenant(tenant_id="public") as session:
-#         result = session.execute(
-#             text(
-#                 """
-#             SELECT schema_name
-#             FROM information_schema.schemata
-#             WHERE schema_name NOT IN ('pg_catalog', 'information_schema', 'public')"""
-#             )
-#         )
-#         tenant_ids = [row[0] for row in result]
-#     valid_tenants = [
-#         tenant
-#         for tenant in tenant_ids
-#         if tenant is None or tenant.startswith(TENANT_ID_PREFIX)
-#     ]
-#     return valid_tenants
-# def update_loop(
-#     delay: int = 10,
-#     num_workers: int = NUM_INDEXING_WORKERS,
-#     num_secondary_workers: int = NUM_SECONDARY_INDEXING_WORKERS,
-# ) -> None:
-#     if not MULTI_TENANT:
-#         # We can use this function as we are certain only the public schema exists
-#         # (explicitly for the non-`MULTI_TENANT` case)
-#         engine = get_sqlalchemy_engine()
-#         with Session(engine) as db_session:
-#             check_index_swap(db_session=db_session)
-#             search_settings = get_current_search_settings(db_session)
-#             # So that the first time users aren't surprised by really slow speed of first
-#             # batch of documents indexed
-#             if search_settings.provider_type is None:
-#                 logger.notice("Running a first inference to warm up embedding model")
-#                 embedding_model = EmbeddingModel.from_db_model(
-#                     search_settings=search_settings,
-#                     server_host=INDEXING_MODEL_SERVER_HOST,
-#                     server_port=INDEXING_MODEL_SERVER_PORT,
-#                 )
-#                 warm_up_bi_encoder(
-#                     embedding_model=embedding_model,
-#                 )
-#                 logger.notice("First inference complete.")
-#     client_primary: Client | SimpleJobClient
-#     client_secondary: Client | SimpleJobClient
-#     if DASK_JOB_CLIENT_ENABLED:
-#         cluster_primary = LocalCluster(
-#             n_workers=num_workers,
-#             threads_per_worker=1,
-#             silence_logs=logging.ERROR,
-#         )
-#         cluster_secondary = LocalCluster(
-#             n_workers=num_secondary_workers,
-#             threads_per_worker=1,
-#             silence_logs=logging.ERROR,
-#         )
-#         client_primary = Client(cluster_primary)
-#         client_secondary = Client(cluster_secondary)
-#         if LOG_LEVEL.lower() == "debug":
-#             client_primary.register_worker_plugin(ResourceLogger())
-#     else:
-#         client_primary = SimpleJobClient(n_workers=num_workers)
-#         client_secondary = SimpleJobClient(n_workers=num_secondary_workers)
-#     existing_jobs: dict[str | None, dict[int, Future | SimpleJob]] = {}
-#     logger.notice("Startup complete. Waiting for indexing jobs...")
-#     while True:
-#         start = time.time()
-#         start_time_utc = datetime.utcfromtimestamp(start).strftime("%Y-%m-%d %H:%M:%S")
-#         logger.debug(f"Running update, current UTC time: {start_time_utc}")
-#         if existing_jobs:
-#             logger.debug(
-#                 "Found existing indexing jobs: "
-#                 f"{[(tenant_id, list(jobs.keys())) for tenant_id, jobs in existing_jobs.items()]}"
-#             )
-#         try:
-#             tenants = get_all_tenant_ids()
-#             for tenant_id in tenants:
-#                 try:
-#                     logger.debug(
-#                         f"Processing {'index attempts' if tenant_id is None else f'tenant {tenant_id}'}"
-#                     )
-#                     with get_session_with_tenant(tenant_id) as db_session:
-#                         index_to_expire = check_index_swap(db_session=db_session)
-#                         if index_to_expire and tenant_id and MULTI_TENANT:
-#                             VespaIndex.delete_entries_by_tenant_id(
-#                                 tenant_id=tenant_id,
-#                                 index_name=index_to_expire.index_name,
-#                             )
-#                         if not MULTI_TENANT:
-#                             search_settings = get_current_search_settings(db_session)
-#                             if search_settings.provider_type is None:
-#                                 logger.notice(
-#                                     "Running a first inference to warm up embedding model"
-#                                 )
-#                                 embedding_model = EmbeddingModel.from_db_model(
-#                                     search_settings=search_settings,
-#                                     server_host=INDEXING_MODEL_SERVER_HOST,
-#                                     server_port=INDEXING_MODEL_SERVER_PORT,
-#                                 )
-#                                 warm_up_bi_encoder(embedding_model=embedding_model)
-#                                 logger.notice("First inference complete.")
-#                     tenant_jobs = existing_jobs.get(tenant_id, {})
-#                     tenant_jobs = cleanup_indexing_jobs(
-#                         existing_jobs=tenant_jobs, tenant_id=tenant_id
-#                     )
-#                     create_indexing_jobs(existing_jobs=tenant_jobs, tenant_id=tenant_id)
-#                     tenant_jobs = kickoff_indexing_jobs(
-#                         existing_jobs=tenant_jobs,
-#                         client=client_primary,
-#                         secondary_client=client_secondary,
-#                         tenant_id=tenant_id,
-#                     )
-#                     existing_jobs[tenant_id] = tenant_jobs
-#                 except Exception as e:
-#                     logger.exception(
-#                         f"Failed to process tenant {tenant_id or 'default'}: {e}"
-#                     )
-#         except Exception as e:
-#             logger.exception(f"Failed to run update due to {e}")
-#         sleep_time = delay - (time.time() - start)
-#         if sleep_time > 0:
-#             time.sleep(sleep_time)
-# def update__main() -> None:
-#     set_is_ee_based_on_env_variable()
-#     # initialize the Postgres connection pool
-#     SqlEngine.set_app_name(POSTGRES_INDEXER_APP_NAME)
-#     logger.notice("Starting indexing service")
-#     update_loop()
-# if __name__ == "__main__":
-#     update__main()
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index caf7a103b94..5be2b68d2a4 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -43,6 +43,9 @@
 AUTH_TYPE = AuthType((os.environ.get("AUTH_TYPE") or AuthType.DISABLED.value).lower())
 DISABLE_AUTH = AUTH_TYPE == AuthType.DISABLED
 
+# Necessary for cloud integration tests
+DISABLE_VERIFICATION = os.environ.get("DISABLE_VERIFICATION", "").lower() == "true"
+
 # Encryption key secret is used to encrypt connector credentials, api keys, and other sensitive
 # information. This provides an extra layer of security on top of Postgres access controls
 # and is available in Danswer EE
diff --git a/backend/danswer/db/swap_index.py b/backend/danswer/db/swap_index.py
index 415ade5df00..a52b2c37d35 100644
--- a/backend/danswer/db/swap_index.py
+++ b/backend/danswer/db/swap_index.py
@@ -42,7 +42,6 @@ def check_index_swap(db_session: Session) -> SearchSettings | None:
         logger.error("More unique indexings than cc pairs, should not occur")
 
     if cc_pair_count == 0 or cc_pair_count == unique_cc_indexings:
-        # Swap indices
         now_old_search_settings = get_current_search_settings(db_session)
         update_search_settings_status(
             search_settings=now_old_search_settings,
@@ -69,4 +68,6 @@ def check_index_swap(db_session: Session) -> SearchSettings | None:
 
             if MULTI_TENANT:
                 return now_old_search_settings
+    else:
+        logger.warning("No need to swap indices")
     return None
diff --git a/backend/danswer/server/danswer_api/ingestion.py b/backend/danswer/server/danswer_api/ingestion.py
index cea3ec86575..bae316535c7 100644
--- a/backend/danswer/server/danswer_api/ingestion.py
+++ b/backend/danswer/server/danswer_api/ingestion.py
@@ -9,6 +9,7 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
 from danswer.db.document import get_documents_by_cc_pair
 from danswer.db.document import get_ingestion_documents
+from danswer.db.engine import get_current_tenant_id
 from danswer.db.engine import get_session
 from danswer.db.models import User
 from danswer.db.search_settings import get_current_search_settings
@@ -67,6 +68,7 @@ def upsert_ingestion_doc(
     doc_info: IngestionDocument,
     _: User | None = Depends(api_key_dep),
     db_session: Session = Depends(get_session),
+    tenant_id: str = Depends(get_current_tenant_id),
 ) -> IngestionResult:
     doc_info.document.from_ingestion_api = True
 
@@ -101,6 +103,7 @@ def upsert_ingestion_doc(
         document_index=curr_doc_index,
         ignore_time_skip=True,
         db_session=db_session,
+        tenant_id=tenant_id,
     )
 
     new_doc, __chunk_count = indexing_pipeline(
@@ -134,6 +137,7 @@ def upsert_ingestion_doc(
             document_index=sec_doc_index,
             ignore_time_skip=True,
             db_session=db_session,
+            tenant_id=tenant_id,
         )
 
         sec_ind_pipeline(
diff --git a/backend/danswer/server/settings/api.py b/backend/danswer/server/settings/api.py
index 7ff4de55c2a..4f598a18353 100644
--- a/backend/danswer/server/settings/api.py
+++ b/backend/danswer/server/settings/api.py
@@ -102,8 +102,6 @@ def get_settings_notifications(
 
     try:
         # Need a transaction in order to prevent under-counting current notifications
-        db_session.begin()
-
         reindex_notifs = get_notifications(
             user=user, notif_type=NotificationType.REINDEX, db_session=db_session
         )
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index eba02d0428b..f7a4ab0b6a3 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -22,7 +22,6 @@ async def set_tenant_id(
     ) -> Response:
         try:
             logger.info(f"Request route: {request.url.path}")
-
             if not MULTI_TENANT:
                 tenant_id = POSTGRES_DEFAULT_SCHEMA
             else:
diff --git a/backend/tests/integration/Dockerfile b/backend/tests/integration/Dockerfile
index 02cdcad0b44..3eecb0d5683 100644
--- a/backend/tests/integration/Dockerfile
+++ b/backend/tests/integration/Dockerfile
@@ -83,4 +83,5 @@ COPY ./tests/integration /app/tests/integration
 
 ENV PYTHONPATH=/app
 
-CMD ["pytest", "-s", "/app/tests/integration"]
+ENTRYPOINT ["pytest", "-s"]
+CMD ["/app/tests/integration", "--ignore=/app/tests/integration/multitenant_tests"]
\ No newline at end of file
diff --git a/backend/tests/integration/common_utils/managers/chat.py b/backend/tests/integration/common_utils/managers/chat.py
index a8643e9e83d..a2edb32caec 100644
--- a/backend/tests/integration/common_utils/managers/chat.py
+++ b/backend/tests/integration/common_utils/managers/chat.py
@@ -23,7 +23,7 @@
 class ChatSessionManager:
     @staticmethod
     def create(
-        persona_id: int = -1,
+        persona_id: int = 0,
         description: str = "Test chat session",
         user_performing_action: DATestUser | None = None,
     ) -> DATestChatSession:
diff --git a/backend/tests/integration/common_utils/managers/credential.py b/backend/tests/integration/common_utils/managers/credential.py
index 8f729e4b06c..8c8a59d4856 100644
--- a/backend/tests/integration/common_utils/managers/credential.py
+++ b/backend/tests/integration/common_utils/managers/credential.py
@@ -32,6 +32,7 @@ def create(
             "curator_public": curator_public,
             "groups": groups or [],
         }
+
         response = requests.post(
             url=f"{API_SERVER_URL}/manage/credential",
             json=credential_request,
diff --git a/backend/tests/integration/common_utils/managers/tenant.py b/backend/tests/integration/common_utils/managers/tenant.py
new file mode 100644
index 00000000000..76fd16471f8
--- /dev/null
+++ b/backend/tests/integration/common_utils/managers/tenant.py
@@ -0,0 +1,82 @@
+from datetime import datetime
+from datetime import timedelta
+
+import jwt
+import requests
+
+from danswer.server.manage.models import AllUsersResponse
+from danswer.server.models import FullUserSnapshot
+from danswer.server.models import InvitedUserSnapshot
+from tests.integration.common_utils.constants import API_SERVER_URL
+from tests.integration.common_utils.constants import GENERAL_HEADERS
+from tests.integration.common_utils.test_models import DATestUser
+
+
+def generate_auth_token() -> str:
+    payload = {
+        "iss": "control_plane",
+        "exp": datetime.utcnow() + timedelta(minutes=5),
+        "iat": datetime.utcnow(),
+        "scope": "tenant:create",
+    }
+    token = jwt.encode(payload, "", algorithm="HS256")
+    return token
+
+
+class TenantManager:
+    @staticmethod
+    def create(
+        tenant_id: str | None = None,
+        initial_admin_email: str | None = None,
+    ) -> dict[str, str]:
+        body = {
+            "tenant_id": tenant_id,
+            "initial_admin_email": initial_admin_email,
+        }
+
+        token = generate_auth_token()
+        headers = {
+            "Authorization": f"Bearer {token}",
+            "X-API-KEY": "",
+            "Content-Type": "application/json",
+        }
+
+        response = requests.post(
+            url=f"{API_SERVER_URL}/tenants/create",
+            json=body,
+            headers=headers,
+        )
+
+        response.raise_for_status()
+
+        return response.json()
+
+    @staticmethod
+    def get_all_users(
+        user_performing_action: DATestUser | None = None,
+    ) -> AllUsersResponse:
+        response = requests.get(
+            url=f"{API_SERVER_URL}/manage/users",
+            headers=user_performing_action.headers
+            if user_performing_action
+            else GENERAL_HEADERS,
+        )
+        response.raise_for_status()
+
+        data = response.json()
+        return AllUsersResponse(
+            accepted=[FullUserSnapshot(**user) for user in data["accepted"]],
+            invited=[InvitedUserSnapshot(**user) for user in data["invited"]],
+            accepted_pages=data["accepted_pages"],
+            invited_pages=data["invited_pages"],
+        )
+
+    @staticmethod
+    def verify_user_in_tenant(
+        user: DATestUser, user_performing_action: DATestUser | None = None
+    ) -> None:
+        all_users = TenantManager.get_all_users(user_performing_action)
+        for accepted_user in all_users.accepted:
+            if accepted_user.email == user.email and accepted_user.id == user.id:
+                return
+        raise ValueError(f"User {user.email} not found in tenant")
diff --git a/backend/tests/integration/common_utils/managers/user.py b/backend/tests/integration/common_utils/managers/user.py
index 4dcafc23cf9..ecc6d3206b9 100644
--- a/backend/tests/integration/common_utils/managers/user.py
+++ b/backend/tests/integration/common_utils/managers/user.py
@@ -65,15 +65,23 @@ def login_as_user(test_user: DATestUser) -> DATestUser:
             data=data,
             headers=headers,
         )
+
         response.raise_for_status()
-        result_cookie = next(iter(response.cookies), None)
 
-        if not result_cookie:
+        cookies = response.cookies.get_dict()
+        session_cookie = cookies.get("fastapiusersauth")
+        tenant_details_cookie = cookies.get("tenant_details")
+
+        if not session_cookie:
             raise Exception("Failed to login")
 
         print(f"Logged in as {test_user.email}")
-        cookie = f"{result_cookie.name}={result_cookie.value}"
-        test_user.headers["Cookie"] = cookie
+
+        # Set both cookies in the headers
+        test_user.headers["Cookie"] = (
+            f"fastapiusersauth={session_cookie}; "
+            f"tenant_details={tenant_details_cookie}"
+        )
         return test_user
 
     @staticmethod
diff --git a/backend/tests/integration/common_utils/reset.py b/backend/tests/integration/common_utils/reset.py
index a532406c4cd..1792af9dbf9 100644
--- a/backend/tests/integration/common_utils/reset.py
+++ b/backend/tests/integration/common_utils/reset.py
@@ -1,5 +1,6 @@
 import logging
 import time
+from types import SimpleNamespace
 
 import psycopg2
 import requests
@@ -11,7 +12,9 @@
 from danswer.configs.app_configs import POSTGRES_PORT
 from danswer.configs.app_configs import POSTGRES_USER
 from danswer.db.engine import build_connection_string
+from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import get_session_context_manager
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import SYNC_DB_API
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.swap_index import check_index_swap
@@ -26,7 +29,11 @@
 
 
 def _run_migrations(
-    database_url: str, direction: str = "upgrade", revision: str = "head"
+    database_url: str,
+    config_name: str,
+    direction: str = "upgrade",
+    revision: str = "head",
+    schema: str = "public",
 ) -> None:
     # hide info logs emitted during migration
     logging.getLogger("alembic").setLevel(logging.CRITICAL)
@@ -35,6 +42,10 @@ def _run_migrations(
     alembic_cfg = Config("alembic.ini")
     alembic_cfg.set_section_option("logger_alembic", "level", "WARN")
     alembic_cfg.attributes["configure_logger"] = False
+    alembic_cfg.config_ini_section = config_name
+
+    alembic_cfg.cmd_opts = SimpleNamespace()  # type: ignore
+    alembic_cfg.cmd_opts.x = [f"schema={schema}"]  # type: ignore
 
     # Set the SQLAlchemy URL in the Alembic configuration
     alembic_cfg.set_main_option("sqlalchemy.url", database_url)
@@ -52,7 +63,9 @@ def _run_migrations(
     logging.getLogger("alembic").setLevel(logging.INFO)
 
 
-def reset_postgres(database: str = "postgres") -> None:
+def reset_postgres(
+    database: str = "postgres", config_name: str = "alembic", setup_danswer: bool = True
+) -> None:
     """Reset the Postgres database."""
 
     # NOTE: need to delete all rows to allow migrations to be rolled back
@@ -111,14 +124,18 @@ def reset_postgres(database: str = "postgres") -> None:
     )
     _run_migrations(
         conn_str,
+        config_name,
         direction="downgrade",
         revision="base",
     )
     _run_migrations(
         conn_str,
+        config_name,
         direction="upgrade",
         revision="head",
     )
+    if not setup_danswer:
+        return
 
     # do the same thing as we do on API server startup
     with get_session_context_manager() as db_session:
@@ -127,6 +144,7 @@ def reset_postgres(database: str = "postgres") -> None:
 
 def reset_vespa() -> None:
     """Wipe all data from the Vespa index."""
+
     with get_session_context_manager() as db_session:
         # swap to the correct default model
         check_index_swap(db_session)
@@ -166,10 +184,98 @@ def reset_vespa() -> None:
             time.sleep(5)
 
 
+def reset_postgres_multitenant() -> None:
+    """Reset the Postgres database for all tenants in a multitenant setup."""
+
+    conn = psycopg2.connect(
+        dbname="postgres",
+        user=POSTGRES_USER,
+        password=POSTGRES_PASSWORD,
+        host=POSTGRES_HOST,
+        port=POSTGRES_PORT,
+    )
+    conn.autocommit = True
+    cur = conn.cursor()
+
+    # Get all tenant schemas
+    cur.execute(
+        """
+        SELECT schema_name
+        FROM information_schema.schemata
+        WHERE schema_name LIKE 'tenant_%'
+        """
+    )
+    tenant_schemas = cur.fetchall()
+
+    # Drop all tenant schemas
+    for schema in tenant_schemas:
+        schema_name = schema[0]
+        cur.execute(f'DROP SCHEMA "{schema_name}" CASCADE')
+
+    cur.close()
+    conn.close()
+
+    reset_postgres(config_name="schema_private", setup_danswer=False)
+
+
+def reset_vespa_multitenant() -> None:
+    """Wipe all data from the Vespa index for all tenants."""
+
+    for tenant_id in get_all_tenant_ids():
+        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
+            # swap to the correct default model for each tenant
+            check_index_swap(db_session)
+
+            search_settings = get_current_search_settings(db_session)
+            index_name = search_settings.index_name
+
+        success = setup_vespa(
+            document_index=VespaIndex(index_name=index_name, secondary_index_name=None),
+            index_setting=IndexingSetting.from_db_model(search_settings),
+            secondary_index_setting=None,
+        )
+
+        if not success:
+            raise RuntimeError(
+                f"Could not connect to Vespa for tenant {tenant_id} within the specified timeout."
+            )
+
+        for _ in range(5):
+            try:
+                continuation = None
+                should_continue = True
+                while should_continue:
+                    params = {"selection": "true", "cluster": "danswer_index"}
+                    if continuation:
+                        params = {**params, "continuation": continuation}
+                    response = requests.delete(
+                        DOCUMENT_ID_ENDPOINT.format(index_name=index_name),
+                        params=params,
+                    )
+                    response.raise_for_status()
+
+                    response_json = response.json()
+
+                    continuation = response_json.get("continuation")
+                    should_continue = bool(continuation)
+
+                break
+            except Exception as e:
+                print(f"Error deleting documents for tenant {tenant_id}: {e}")
+                time.sleep(5)
+
+
 def reset_all() -> None:
-    """Reset both Postgres and Vespa."""
     logger.info("Resetting Postgres...")
     reset_postgres()
     logger.info("Resetting Vespa...")
     reset_vespa()
+
+
+def reset_all_multitenant() -> None:
+    """Reset both Postgres and Vespa for all tenants."""
+    logger.info("Resetting Postgres for all tenants...")
+    reset_postgres_multitenant()
+    logger.info("Resetting Vespa for all tenants...")
+    reset_vespa_multitenant()
     logger.info("Finished resetting all.")
diff --git a/backend/tests/integration/conftest.py b/backend/tests/integration/conftest.py
index f3d194e22b1..91e61966643 100644
--- a/backend/tests/integration/conftest.py
+++ b/backend/tests/integration/conftest.py
@@ -8,6 +8,7 @@
 from danswer.db.search_settings import get_current_search_settings
 from tests.integration.common_utils.managers.user import UserManager
 from tests.integration.common_utils.reset import reset_all
+from tests.integration.common_utils.reset import reset_all_multitenant
 from tests.integration.common_utils.test_models import DATestUser
 from tests.integration.common_utils.vespa import vespa_fixture
 
@@ -54,3 +55,8 @@ def new_admin_user(reset: None) -> DATestUser | None:
         return UserManager.create(name="admin_user")
     except Exception:
         return None
+
+
+@pytest.fixture
+def reset_multitenant() -> None:
+    reset_all_multitenant()
diff --git a/backend/tests/integration/multitenant_tests/cc_Pair b/backend/tests/integration/multitenant_tests/cc_Pair
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py b/backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py
new file mode 100644
index 00000000000..454b02412d4
--- /dev/null
+++ b/backend/tests/integration/multitenant_tests/syncing/test_search_permissions.py
@@ -0,0 +1,150 @@
+from danswer.db.models import UserRole
+from tests.integration.common_utils.managers.api_key import APIKeyManager
+from tests.integration.common_utils.managers.cc_pair import CCPairManager
+from tests.integration.common_utils.managers.chat import ChatSessionManager
+from tests.integration.common_utils.managers.document import DocumentManager
+from tests.integration.common_utils.managers.llm_provider import LLMProviderManager
+from tests.integration.common_utils.managers.tenant import TenantManager
+from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.test_models import DATestAPIKey
+from tests.integration.common_utils.test_models import DATestCCPair
+from tests.integration.common_utils.test_models import DATestChatSession
+from tests.integration.common_utils.test_models import DATestUser
+
+
+def test_multi_tenant_access_control(reset_multitenant: None) -> None:
+    # Create Tenant 1 and its Admin User
+    TenantManager.create("tenant_dev1", "test1@test.com")
+    test_user1: DATestUser = UserManager.create(name="test1", email="test1@test.com")
+    assert UserManager.verify_role(test_user1, UserRole.ADMIN)
+
+    # Create Tenant 2 and its Admin User
+    TenantManager.create("tenant_dev2", "test2@test.com")
+    test_user2: DATestUser = UserManager.create(name="test2", email="test2@test.com")
+    assert UserManager.verify_role(test_user2, UserRole.ADMIN)
+
+    # Create connectors for Tenant 1
+    cc_pair_1: DATestCCPair = CCPairManager.create_from_scratch(
+        user_performing_action=test_user1,
+    )
+    api_key_1: DATestAPIKey = APIKeyManager.create(
+        user_performing_action=test_user1,
+    )
+    api_key_1.headers.update(test_user1.headers)
+    LLMProviderManager.create(user_performing_action=test_user1)
+
+    # Seed documents for Tenant 1
+    cc_pair_1.documents = []
+    doc1_tenant1 = DocumentManager.seed_doc_with_content(
+        cc_pair=cc_pair_1,
+        content="Tenant 1 Document Content",
+        api_key=api_key_1,
+    )
+    doc2_tenant1 = DocumentManager.seed_doc_with_content(
+        cc_pair=cc_pair_1,
+        content="Tenant 1 Document Content",
+        api_key=api_key_1,
+    )
+    cc_pair_1.documents.extend([doc1_tenant1, doc2_tenant1])
+
+    # Create connectors for Tenant 2
+    cc_pair_2: DATestCCPair = CCPairManager.create_from_scratch(
+        user_performing_action=test_user2,
+    )
+    api_key_2: DATestAPIKey = APIKeyManager.create(
+        user_performing_action=test_user2,
+    )
+    api_key_2.headers.update(test_user2.headers)
+    LLMProviderManager.create(user_performing_action=test_user2)
+
+    # Seed documents for Tenant 2
+    cc_pair_2.documents = []
+    doc1_tenant2 = DocumentManager.seed_doc_with_content(
+        cc_pair=cc_pair_2,
+        content="Tenant 2 Document Content",
+        api_key=api_key_2,
+    )
+    doc2_tenant2 = DocumentManager.seed_doc_with_content(
+        cc_pair=cc_pair_2,
+        content="Tenant 2 Document Content",
+        api_key=api_key_2,
+    )
+    cc_pair_2.documents.extend([doc1_tenant2, doc2_tenant2])
+
+    tenant1_doc_ids = {doc1_tenant1.id, doc2_tenant1.id}
+    tenant2_doc_ids = {doc1_tenant2.id, doc2_tenant2.id}
+
+    # Create chat sessions for each user
+    chat_session1: DATestChatSession = ChatSessionManager.create(
+        user_performing_action=test_user1
+    )
+    chat_session2: DATestChatSession = ChatSessionManager.create(
+        user_performing_action=test_user2
+    )
+
+    # User 1 sends a message and gets a response
+    response1 = ChatSessionManager.send_message(
+        chat_session_id=chat_session1.id,
+        message="What is in Tenant 1's documents?",
+        user_performing_action=test_user1,
+    )
+    # Assert that the search tool was used
+    assert response1.tool_name == "run_search"
+
+    response_doc_ids = {doc["document_id"] for doc in response1.tool_result or []}
+    assert tenant1_doc_ids.issubset(
+        response_doc_ids
+    ), "Not all Tenant 1 document IDs are in the response"
+    assert not response_doc_ids.intersection(
+        tenant2_doc_ids
+    ), "Tenant 2 document IDs should not be in the response"
+
+    # Assert that the contents are correct
+    for doc in response1.tool_result or []:
+        assert doc["content"] == "Tenant 1 Document Content"
+
+    # User 2 sends a message and gets a response
+    response2 = ChatSessionManager.send_message(
+        chat_session_id=chat_session2.id,
+        message="What is in Tenant 2's documents?",
+        user_performing_action=test_user2,
+    )
+    # Assert that the search tool was used
+    assert response2.tool_name == "run_search"
+    # Assert that the tool_result contains Tenant 2's documents
+    response_doc_ids = {doc["document_id"] for doc in response2.tool_result or []}
+    assert tenant2_doc_ids.issubset(
+        response_doc_ids
+    ), "Not all Tenant 2 document IDs are in the response"
+    assert not response_doc_ids.intersection(
+        tenant1_doc_ids
+    ), "Tenant 1 document IDs should not be in the response"
+    # Assert that the contents are correct
+    for doc in response2.tool_result or []:
+        assert doc["content"] == "Tenant 2 Document Content"
+
+    # User 1 tries to access Tenant 2's documents
+    response_cross = ChatSessionManager.send_message(
+        chat_session_id=chat_session1.id,
+        message="What is in Tenant 2's documents?",
+        user_performing_action=test_user1,
+    )
+    # Assert that the search tool was used
+    assert response_cross.tool_name == "run_search"
+    # Assert that the tool_result is empty or does not contain Tenant 2's documents
+    response_doc_ids = {doc["document_id"] for doc in response_cross.tool_result or []}
+    # Ensure none of Tenant 2's document IDs are in the response
+    assert not response_doc_ids.intersection(tenant2_doc_ids)
+
+    # User 2 tries to access Tenant 1's documents
+    response_cross2 = ChatSessionManager.send_message(
+        chat_session_id=chat_session2.id,
+        message="What is in Tenant 1's documents?",
+        user_performing_action=test_user2,
+    )
+    # Assert that the search tool was used
+    assert response_cross2.tool_name == "run_search"
+    # Assert that the tool_result is empty or does not contain Tenant 1's documents
+    response_doc_ids = {doc["document_id"] for doc in response_cross2.tool_result or []}
+    # Ensure none of Tenant 1's document IDs are in the response
+    assert not response_doc_ids.intersection(tenant1_doc_ids)
diff --git a/backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py b/backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py
new file mode 100644
index 00000000000..6088743e317
--- /dev/null
+++ b/backend/tests/integration/multitenant_tests/tenants/test_tenant_creation.py
@@ -0,0 +1,41 @@
+from danswer.configs.constants import DocumentSource
+from danswer.db.enums import AccessType
+from danswer.db.models import UserRole
+from tests.integration.common_utils.managers.cc_pair import CCPairManager
+from tests.integration.common_utils.managers.connector import ConnectorManager
+from tests.integration.common_utils.managers.credential import CredentialManager
+from tests.integration.common_utils.managers.tenant import TenantManager
+from tests.integration.common_utils.managers.user import UserManager
+from tests.integration.common_utils.test_models import DATestUser
+
+
+# Test flow from creating tenant to registering as a user
+def test_tenant_creation(reset_multitenant: None) -> None:
+    TenantManager.create("tenant_dev", "test@test.com")
+    test_user: DATestUser = UserManager.create(name="test", email="test@test.com")
+
+    assert UserManager.verify_role(test_user, UserRole.ADMIN)
+
+    test_credential = CredentialManager.create(
+        name="admin_test_credential",
+        source=DocumentSource.FILE,
+        curator_public=False,
+        user_performing_action=test_user,
+    )
+
+    test_connector = ConnectorManager.create(
+        name="admin_test_connector",
+        source=DocumentSource.FILE,
+        is_public=False,
+        user_performing_action=test_user,
+    )
+
+    test_cc_pair = CCPairManager.create(
+        connector_id=test_connector.id,
+        credential_id=test_credential.id,
+        name="admin_test_cc_pair",
+        access_type=AccessType.PRIVATE,
+        user_performing_action=test_user,
+    )
+
+    CCPairManager.verify(cc_pair=test_cc_pair, user_performing_action=test_user)
diff --git a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
index c37d1a6235d..10d1950ae03 100644
--- a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
+++ b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
@@ -119,6 +119,7 @@ def test_using_reference_docs_with_simple_with_history_api_flow(reset: None) ->
     )
     assert response.status_code == 200
     response_json = response.json()
+
     # get the db_doc_id of the top document to use as a search doc id for second message
     first_db_doc_id = response_json["top_documents"][0]["db_doc_id"]
 

From 94edcac36ee2db602a47a263e5abfb69fb2dea02 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Fri, 25 Oct 2024 17:11:52 -0700
Subject: [PATCH 220/376] upgraded claude model strings (#2876)

* upgraded model strings

* trolled

* we do a little trolling

* reeeeeee

* alembic upgrade

* added ignore

* bump litellm

* k

* nit

---------

Co-authored-by: pablodanswer <pablo@danswer.ai>
---
 backend/danswer/llm/llm_provider_options.py | 9 +++++----
 backend/requirements/default.txt            | 4 ++--
 backend/requirements/model_server.txt       | 4 ++--
 web/src/lib/hooks.ts                        | 6 ++++--
 web/src/lib/llm/utils.ts                    | 3 +++
 5 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/backend/danswer/llm/llm_provider_options.py b/backend/danswer/llm/llm_provider_options.py
index 3cb6157d6da..cf562ee5a27 100644
--- a/backend/danswer/llm/llm_provider_options.py
+++ b/backend/danswer/llm/llm_provider_options.py
@@ -61,6 +61,7 @@ class WellKnownLLMProviderDescriptor(BaseModel):
 IGNORABLE_ANTHROPIC_MODELS = [
     "claude-2",
     "claude-instant-1",
+    "anthropic/claude-3-5-sonnet-20241022",
 ]
 ANTHROPIC_PROVIDER_NAME = "anthropic"
 ANTHROPIC_MODEL_NAMES = [
@@ -100,8 +101,8 @@ def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
             api_version_required=False,
             custom_config_keys=[],
             llm_names=fetch_models_for_provider(ANTHROPIC_PROVIDER_NAME),
-            default_model="claude-3-5-sonnet-20240620",
-            default_fast_model="claude-3-5-sonnet-20240620",
+            default_model="claude-3-5-sonnet-20241022",
+            default_fast_model="claude-3-5-sonnet-20241022",
         ),
         WellKnownLLMProviderDescriptor(
             name=AZURE_PROVIDER_NAME,
@@ -135,8 +136,8 @@ def fetch_available_well_known_llms() -> list[WellKnownLLMProviderDescriptor]:
                 ),
             ],
             llm_names=fetch_models_for_provider(BEDROCK_PROVIDER_NAME),
-            default_model="anthropic.claude-3-5-sonnet-20240620-v1:0",
-            default_fast_model="anthropic.claude-3-5-sonnet-20240620-v1:0",
+            default_model="anthropic.claude-3-5-sonnet-20241022-v2:0",
+            default_fast_model="anthropic.claude-3-5-sonnet-20241022-v2:0",
         ),
     ]
 
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 354103f200a..2240180e355 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -29,7 +29,7 @@ trafilatura==1.12.2
 langchain==0.1.17
 langchain-core==0.1.50
 langchain-text-splitters==0.0.1
-litellm==1.49.5
+litellm==1.50.2
 lxml==5.3.0
 lxml_html_clean==0.2.2
 llama-index==0.9.45
@@ -38,7 +38,7 @@ msal==1.28.0
 nltk==3.8.1
 Office365-REST-Python-Client==2.5.9
 oauthlib==3.2.2
-openai==1.51.2
+openai==1.52.2
 openpyxl==3.1.2
 playwright==1.41.2
 psutil==5.9.5
diff --git a/backend/requirements/model_server.txt b/backend/requirements/model_server.txt
index 6160555f7b1..3bc32a8f6d7 100644
--- a/backend/requirements/model_server.txt
+++ b/backend/requirements/model_server.txt
@@ -3,7 +3,7 @@ cohere==5.6.1
 fastapi==0.109.2
 google-cloud-aiplatform==1.58.0
 numpy==1.26.4
-openai==1.51.2
+openai==1.52.2
 pydantic==2.8.2
 retry==0.9.2
 safetensors==0.4.2
@@ -12,5 +12,5 @@ torch==2.2.0
 transformers==4.39.2
 uvicorn==0.21.1
 voyageai==0.2.3
-litellm==1.49.5
+litellm==1.50.2
 sentry-sdk[fastapi,celery,starlette]==2.14.0
\ No newline at end of file
diff --git a/web/src/lib/hooks.ts b/web/src/lib/hooks.ts
index 1f03decd44a..c5a11b320d4 100644
--- a/web/src/lib/hooks.ts
+++ b/web/src/lib/hooks.ts
@@ -278,6 +278,7 @@ const MODEL_DISPLAY_NAMES: { [key: string]: string } = {
   "claude-2.0": "Claude 2.0",
   "claude-instant-1.2": "Claude Instant 1.2",
   "claude-3-5-sonnet-20240620": "Claude 3.5 Sonnet",
+  "claude-3-5-sonnet-20241022": "Claude 3.5 Sonnet (New)",
 
   // Bedrock models
   "meta.llama3-1-70b-instruct-v1:0": "Llama 3.1 70B",
@@ -301,6 +302,7 @@ const MODEL_DISPLAY_NAMES: { [key: string]: string } = {
   "anthropic.claude-3-opus-20240229-v1:0": "Claude 3 Opus",
   "anthropic.claude-3-haiku-20240307-v1:0": "Claude 3 Haiku",
   "anthropic.claude-3-5-sonnet-20240620-v1:0": "Claude 3.5 Sonnet",
+  "anthropic.claude-3-5-sonnet-20241022-v2:0": "Claude 3.5 Sonnet (New)",
   "anthropic.claude-3-sonnet-20240229-v1:0": "Claude 3 Sonnet",
   "mistral.mistral-large-2402-v1:0": "Mistral Large",
   "mistral.mixtral-8x7b-instruct-v0:1": "Mixtral 8x7B Instruct",
@@ -323,7 +325,7 @@ export const defaultModelsByProvider: { [name: string]: string[] } = {
     "meta.llama3-1-8b-instruct-v1:0",
     "anthropic.claude-3-opus-20240229-v1:0",
     "mistral.mistral-large-2402-v1:0",
-    "anthropic.claude-3-5-sonnet-20240620-v1:0",
+    "anthropic.claude-3-5-sonnet-20241022-v2:0",
   ],
-  anthropic: ["claude-3-opus-20240229", "claude-3-5-sonnet-20240620"],
+  anthropic: ["claude-3-opus-20240229", "claude-3-5-sonnet-20241022"],
 };
diff --git a/web/src/lib/llm/utils.ts b/web/src/lib/llm/utils.ts
index 2adfbfc0543..9a47dbc8071 100644
--- a/web/src/lib/llm/utils.ts
+++ b/web/src/lib/llm/utils.ts
@@ -70,6 +70,7 @@ const MODEL_NAMES_SUPPORTING_IMAGE_INPUT = [
   "gpt-4-1106-vision-preview",
   // standard claude names
   "claude-3-5-sonnet-20240620",
+  "claude-3-5-sonnet-20241022",
   "claude-3-opus-20240229",
   "claude-3-sonnet-20240229",
   "claude-3-haiku-20240307",
@@ -78,11 +79,13 @@ const MODEL_NAMES_SUPPORTING_IMAGE_INPUT = [
   "claude-3-sonnet-20240229-v1:0",
   "claude-3-haiku-20240307-v1:0",
   "claude-3-5-sonnet-20240620-v1:0",
+  "claude-3-5-sonnet-20241022-v2:0",
   // claude names with full AWS Bedrock names
   "anthropic.claude-3-opus-20240229-v1:0",
   "anthropic.claude-3-sonnet-20240229-v1:0",
   "anthropic.claude-3-haiku-20240307-v1:0",
   "anthropic.claude-3-5-sonnet-20240620-v1:0",
+  "anthropic.claude-3-5-sonnet-20241022-v2:0",
 ];
 
 export function checkLLMSupportsImageInput(model: string) {

From 5e01d6befb76f9e5bad506820fed78c6eba4c183 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 25 Oct 2024 17:26:02 -0700
Subject: [PATCH 221/376] check for index swap (#2922)

* check for index swap

* k

* minor

* k

* nit
---
 .../background/celery/tasks/indexing/tasks.py | 21 +++++++++++++++++++
 backend/danswer/db/swap_index.py              |  3 +--
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 980266ec878..2b5b987672a 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -24,6 +24,7 @@
 from danswer.background.indexing.run_indexing import run_indexing_entrypoint
 from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
+from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import CELERY_INDEXING_LOCK_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
@@ -47,9 +48,14 @@
 from danswer.db.models import SearchSettings
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.search_settings import get_secondary_search_settings
+from danswer.db.swap_index import check_index_swap
+from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
+from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import global_version
+from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
+from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
 
 logger = setup_logger()
 
@@ -98,6 +104,21 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
         if not lock_beat.acquire(blocking=False):
             return None
 
+        with get_session_with_tenant(tenant_id=tenant_id) as db_session:
+            check_index_swap(db_session=db_session)
+            current_search_settings = get_current_search_settings(db_session)
+            # So that the first time users aren't surprised by really slow speed of first
+            # batch of documents indexed
+            if current_search_settings.provider_type is None and not MULTI_TENANT:
+                embedding_model = EmbeddingModel.from_db_model(
+                    search_settings=current_search_settings,
+                    server_host=INDEXING_MODEL_SERVER_HOST,
+                    server_port=INDEXING_MODEL_SERVER_PORT,
+                )
+                warm_up_bi_encoder(
+                    embedding_model=embedding_model,
+                )
+
         cc_pair_ids: list[int] = []
         with get_session_with_tenant(tenant_id) as db_session:
             cc_pairs = fetch_connector_credential_pairs(db_session)
diff --git a/backend/danswer/db/swap_index.py b/backend/danswer/db/swap_index.py
index a52b2c37d35..415ade5df00 100644
--- a/backend/danswer/db/swap_index.py
+++ b/backend/danswer/db/swap_index.py
@@ -42,6 +42,7 @@ def check_index_swap(db_session: Session) -> SearchSettings | None:
         logger.error("More unique indexings than cc pairs, should not occur")
 
     if cc_pair_count == 0 or cc_pair_count == unique_cc_indexings:
+        # Swap indices
         now_old_search_settings = get_current_search_settings(db_session)
         update_search_settings_status(
             search_settings=now_old_search_settings,
@@ -68,6 +69,4 @@ def check_index_swap(db_session: Session) -> SearchSettings | None:
 
             if MULTI_TENANT:
                 return now_old_search_settings
-    else:
-        logger.warning("No need to swap indices")
     return None

From 8023cafb2b3cb8b4279c0fa5577d231111e213a8 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Fri, 25 Oct 2024 23:46:47 -0400
Subject: [PATCH 222/376] Fixed polling issue with timezone

---
 .../danswer/connectors/freshdesk/connector.py | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 89a47b4714d..34651b43441 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -1,6 +1,6 @@
 import requests
 import json
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Any, List, Optional
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
@@ -75,24 +75,28 @@ def _fetch_tickets(self, start: datetime, end: datetime) -> List[dict]:
         return all_tickets
 
     def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
+        # Ensure start and end are in UTC
+        start = start.astimezone(timezone.utc)
+        end = end.astimezone(timezone.utc)
+        
         tickets = self._fetch_tickets(start, end)
         doc_batch: List[Document] = []
 
         for ticket in tickets:
-            #convert to iso format
+            # Convert date fields to UTC
             for date_field in ["created_at", "updated_at", "due_by"]:
                 if ticket[date_field].endswith('Z'):
                     ticket[date_field] = ticket[date_field][:-1] + '+00:00'
-                ticket[date_field] = datetime.fromisoformat(ticket[date_field]).strftime("%Y-%m-%d %H:%M:%S")
+                ticket[date_field] = datetime.fromisoformat(ticket[date_field]).replace(tzinfo=timezone.utc)
 
-            #convert all other values to strings
+            # Convert all other values to strings
             ticket = {
-                key: str(value) if not isinstance(value, str) else value
+                key: str(value) if not isinstance(value, (str, datetime)) else value
                 for key, value in ticket.items()
             }
 
             # Checking for overdue tickets
-            today = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+            today = datetime.now(timezone.utc)
             ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
 
             # Mapping the status field values
@@ -108,7 +112,7 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
             # Use list comprehension for building sections
             sections = self.build_doc_sections_from_ticket(ticket)
 
-            created_at = datetime.fromisoformat(ticket["created_at"])
+            created_at = ticket["created_at"]
             if start <= created_at <= end:
                 doc = Document(
                     id=ticket["id"],
@@ -116,9 +120,9 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
                     source=DocumentSource.FRESHDESK,
                     semantic_identifier=ticket["subject"],
                     metadata={
-                        key: value
+                        key: value.isoformat() if isinstance(value, datetime) else str(value)
                         for key, value in ticket.items()
-                        if isinstance(value, str) and key not in ["description", "description_text"]
+                        if isinstance(value, (str, datetime)) and key not in ["description", "description_text"]
                     },
                 )
                 doc_batch.append(doc)
@@ -134,7 +138,7 @@ def load_from_state(self) -> GenerateDocumentsOutput:
         return self._fetch_tickets()
 
     def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:
-        start_datetime = datetime.fromtimestamp(start)
-        end_datetime = datetime.fromtimestamp(end)
+        start_datetime = datetime.fromtimestamp(start, tz=timezone.utc)
+        end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
 
         yield from self._process_tickets(start_datetime, end_datetime)

From 9def9f0dba344be3e0066be41ac32d2afa1a5fcf Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 26 Oct 2024 11:15:01 -0700
Subject: [PATCH 223/376] add posthog + layout rework (#2926)

* add posthog + layout rework

* remove posthog node

* nit
---
 web/Dockerfile                  |  11 ++
 web/package-lock.json           |  41 ++++++
 web/package.json                |   1 +
 web/src/app/PostHogPageView.tsx |  30 +++++
 web/src/app/layout.tsx          | 231 +++++++++++++++-----------------
 web/src/app/providers.tsx       |  26 ++++
 6 files changed, 218 insertions(+), 122 deletions(-)
 create mode 100644 web/src/app/PostHogPageView.tsx
 create mode 100644 web/src/app/providers.tsx

diff --git a/web/Dockerfile b/web/Dockerfile
index 3cfd1a0f3e4..46ec203250e 100644
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -61,6 +61,10 @@ ENV NEXT_PUBLIC_DISABLE_LOGOUT=${NEXT_PUBLIC_DISABLE_LOGOUT}
 ARG NEXT_PUBLIC_CUSTOM_REFRESH_URL
 ENV NEXT_PUBLIC_CUSTOM_REFRESH_URL=${NEXT_PUBLIC_CUSTOM_REFRESH_URL}
 
+ARG NEXT_PUBLIC_POSTHOG_KEY
+ARG NEXT_PUBLIC_POSTHOG_HOST
+ENV NEXT_PUBLIC_POSTHOG_KEY=${NEXT_PUBLIC_POSTHOG_KEY}
+ENV NEXT_PUBLIC_POSTHOG_HOST=${NEXT_PUBLIC_POSTHOG_HOST}
 
 RUN npx next build
 
@@ -122,6 +126,13 @@ ENV NEXT_PUBLIC_DISABLE_LOGOUT=${NEXT_PUBLIC_DISABLE_LOGOUT}
 ARG NEXT_PUBLIC_CUSTOM_REFRESH_URL
 ENV NEXT_PUBLIC_CUSTOM_REFRESH_URL=${NEXT_PUBLIC_CUSTOM_REFRESH_URL}
 
+
+ARG NEXT_PUBLIC_POSTHOG_KEY
+ARG NEXT_PUBLIC_POSTHOG_HOST
+ENV NEXT_PUBLIC_POSTHOG_KEY=${NEXT_PUBLIC_POSTHOG_KEY}
+ENV NEXT_PUBLIC_POSTHOG_HOST=${NEXT_PUBLIC_POSTHOG_HOST}
+
+
 # Note: Don't expose ports here, Compose will handle that for us if necessary. 
 # If you want to run this without compose, specify the ports to 
 # expose via cli
diff --git a/web/package-lock.json b/web/package-lock.json
index ffb68fc16bd..8d914d5b661 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -33,6 +33,7 @@
         "next": "^14.2.3",
         "npm": "^10.8.0",
         "postcss": "^8.4.31",
+        "posthog-js": "^1.176.0",
         "prismjs": "^1.29.0",
         "react": "^18.3.1",
         "react-dom": "^18.3.1",
@@ -4493,6 +4494,16 @@
       "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
       "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg=="
     },
+    "node_modules/core-js": {
+      "version": "3.38.1",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.38.1.tgz",
+      "integrity": "sha512-OP35aUorbU3Zvlx7pjsFdu1rGNnD4pgw/CWoYzRY3t2EzoVT7shKHY1dlAy3f41cGIO7ZDPQimhGFTlEYkG/Hw==",
+      "hasInstallScript": true,
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/core-js"
+      }
+    },
     "node_modules/cosmiconfig": {
       "version": "7.1.0",
       "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz",
@@ -5639,6 +5650,11 @@
         "reusify": "^1.0.4"
       }
     },
+    "node_modules/fflate": {
+      "version": "0.4.8",
+      "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.4.8.tgz",
+      "integrity": "sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA=="
+    },
     "node_modules/file-entry-cache": {
       "version": "6.0.1",
       "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz",
@@ -11136,6 +11152,26 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/posthog-js": {
+      "version": "1.176.0",
+      "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.176.0.tgz",
+      "integrity": "sha512-T5XKNtRzp7q6CGb7Vc7wAI76rWap9fiuDUPxPsyPBPDkreKya91x9RIsSapAVFafwD1AEin1QMczCmt9Le9BWw==",
+      "dependencies": {
+        "core-js": "^3.38.1",
+        "fflate": "^0.4.8",
+        "preact": "^10.19.3",
+        "web-vitals": "^4.2.0"
+      }
+    },
+    "node_modules/preact": {
+      "version": "10.24.3",
+      "resolved": "https://registry.npmjs.org/preact/-/preact-10.24.3.tgz",
+      "integrity": "sha512-Z2dPnBnMUfyQfSQ+GBdsGa16hz35YmLmtTLhM169uW944hYL6xzTYkJjC07j+Wosz733pMWx0fgON3JNw1jJQA==",
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/preact"
+      }
+    },
     "node_modules/prelude-ls": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz",
@@ -13192,6 +13228,11 @@
         "url": "https://github.com/sponsors/wooorm"
       }
     },
+    "node_modules/web-vitals": {
+      "version": "4.2.4",
+      "resolved": "https://registry.npmjs.org/web-vitals/-/web-vitals-4.2.4.tgz",
+      "integrity": "sha512-r4DIlprAGwJ7YM11VZp4R884m0Vmgr6EAKe3P+kO0PPj3Unqyvv59rczf6UiGcb9Z8QxZVcqKNwv/g0WNdWwsw=="
+    },
     "node_modules/webidl-conversions": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
diff --git a/web/package.json b/web/package.json
index e7e2d4098ff..418eaf4f9ac 100644
--- a/web/package.json
+++ b/web/package.json
@@ -34,6 +34,7 @@
     "next": "^14.2.3",
     "npm": "^10.8.0",
     "postcss": "^8.4.31",
+    "posthog-js": "^1.176.0",
     "prismjs": "^1.29.0",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
diff --git a/web/src/app/PostHogPageView.tsx b/web/src/app/PostHogPageView.tsx
new file mode 100644
index 00000000000..e75c09731d1
--- /dev/null
+++ b/web/src/app/PostHogPageView.tsx
@@ -0,0 +1,30 @@
+"use client";
+
+import { usePathname, useSearchParams } from "next/navigation";
+import { useEffect } from "react";
+import { usePostHog } from "posthog-js/react";
+
+export default function PostHogPageView(): null {
+  const pathname = usePathname();
+  const searchParams = useSearchParams();
+  const posthog = usePostHog();
+
+  useEffect(() => {
+    if (!posthog) {
+      return;
+    }
+
+    // Track pageviews
+    if (pathname) {
+      let url = window.origin + pathname;
+      if (searchParams.toString()) {
+        url = url + `?${searchParams.toString()}`;
+      }
+      posthog.capture("$pageview", {
+        $current_url: url,
+      });
+    }
+  }, [pathname, searchParams, posthog]);
+
+  return null;
+}
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index e6339ade9fb..e7e3c2c416d 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -8,19 +8,21 @@ import {
   CUSTOM_ANALYTICS_ENABLED,
   SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED,
 } from "@/lib/constants";
-import { SettingsProvider } from "@/components/settings/SettingsProvider";
 import { Metadata } from "next";
 import { buildClientUrl } from "@/lib/utilsSS";
 import { Inter } from "next/font/google";
-import Head from "next/head";
 import { EnterpriseSettings, GatingType } from "./admin/settings/interfaces";
 import { Card } from "@tremor/react";
 import { HeaderTitle } from "@/components/header/HeaderTitle";
 import { Logo } from "@/components/Logo";
-import { UserProvider } from "@/components/user/UserProvider";
-import { ProviderContextProvider } from "@/components/chat_search/ProviderContext";
 import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
 import { AppProvider } from "@/components/context/AppProvider";
+import { PHProvider } from "./providers";
+import { default as dynamicImport } from "next/dynamic";
+
+const PostHogPageView = dynamicImport(() => import("./PostHogPageView"), {
+  ssr: false,
+});
 
 const inter = Inter({
   subsets: ["latin"],
@@ -57,139 +59,124 @@ export default async function RootLayout({
 }) {
   const combinedSettings = await fetchSettingsSS();
 
-  const data = await fetchAssistantData();
-
-  const { assistants, hasAnyConnectors, hasImageCompatibleModel } = data;
-
   const productGating =
     combinedSettings?.settings.product_gating ?? GatingType.NONE;
 
-  if (!combinedSettings) {
-    return (
-      <html lang="en" className={`${inter.variable} font-sans`}>
-        <Head>
-          <title>Settings Unavailable | Danswer</title>
-        </Head>
-        <body className="bg-background text-default">
-          <div className="flex flex-col items-center justify-center min-h-screen">
-            <div className="mb-2 flex items-center max-w-[175px]">
-              <HeaderTitle>Danswer</HeaderTitle>
-              <Logo height={40} width={40} />
-            </div>
-
-            <Card className="p-8 max-w-md">
-              <h1 className="text-2xl font-bold mb-4 text-error">Error</h1>
-              <p className="text-text-500">
-                Your Danswer instance was not configured properly and your
-                settings could not be loaded. This could be due to an admin
-                configuration issue or an incomplete setup.
-              </p>
-              <p className="mt-4">
-                If you&apos;re an admin, please check{" "}
-                <a
-                  className="text-link"
-                  href="https://docs.danswer.dev/introduction?utm_source=app&utm_medium=error_page&utm_campaign=config_error"
-                  target="_blank"
-                  rel="noopener noreferrer"
-                >
-                  our docs
-                </a>{" "}
-                to see how to configure Danswer properly. If you&apos;re a user,
-                please contact your admin to fix this error.
-              </p>
-              <p className="mt-4">
-                For additional support and guidance, you can reach out to our
-                community on{" "}
-                <a
-                  className="text-link"
-                  href="https://danswer.ai?utm_source=app&utm_medium=error_page&utm_campaign=config_error"
-                  target="_blank"
-                  rel="noopener noreferrer"
-                >
-                  Slack
-                </a>
-                .
-              </p>
-            </Card>
-          </div>
-        </body>
-      </html>
-    );
-  }
-  if (productGating === GatingType.FULL) {
-    return (
-      <html lang="en" className={`${inter.variable} font-sans`}>
-        <Head>
-          <title>Access Restricted | Danswer</title>
-        </Head>
-        <body className="bg-background text-default">
-          <div className="flex flex-col items-center justify-center min-h-screen">
-            <div className="mb-2 flex items-center max-w-[175px]">
-              <HeaderTitle>Danswer</HeaderTitle>
-              <Logo height={40} width={40} />
-            </div>
-            <Card className="p-8 max-w-md">
-              <h1 className="text-2xl font-bold mb-4 text-error">
-                Access Restricted
-              </h1>
-              <p className="text-text-500 mb-4">
-                We regret to inform you that your access to Danswer has been
-                temporarily suspended due to a lapse in your subscription.
-              </p>
-              <p className="text-text-500 mb-4">
-                To reinstate your access and continue benefiting from
-                Danswer&apos;s powerful features, please update your payment
-                information.
-              </p>
-              <p className="text-text-500">
-                If you&apos;re an admin, you can resolve this by visiting the
-                billing section. For other users, please reach out to your
-                administrator to address this matter.
-              </p>
-            </Card>
-          </div>
-        </body>
-      </html>
-    );
-  }
-
-  return (
-    <html lang="en">
-      <Head>
+  const getPageContent = (content: React.ReactNode) => (
+    <html lang="en" className={`${inter.variable} font-sans`}>
+      <head>
         <meta
           name="viewport"
           content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0, interactive-widget=resizes-content"
         />
-      </Head>
-
-      {CUSTOM_ANALYTICS_ENABLED && combinedSettings.customAnalyticsScript && (
-        <head>
-          <script
-            type="text/javascript"
-            dangerouslySetInnerHTML={{
-              __html: combinedSettings.customAnalyticsScript,
-            }}
-          />
-        </head>
-      )}
-
+        {CUSTOM_ANALYTICS_ENABLED &&
+          combinedSettings?.customAnalyticsScript && (
+            <script
+              type="text/javascript"
+              dangerouslySetInnerHTML={{
+                __html: combinedSettings.customAnalyticsScript,
+              }}
+            />
+          )}
+      </head>
       <body className={`relative ${inter.variable} font-sans`}>
         <div
           className={`text-default min-h-screen bg-background ${
-            // TODO: remove this once proper dark mode exists
             process.env.THEME_IS_DARK?.toLowerCase() === "true" ? "dark" : ""
           }`}
         >
-          <AppProvider
-            settings={combinedSettings}
-            assistants={assistants}
-            hasAnyConnectors={hasAnyConnectors}
-            hasImageCompatibleModel={hasImageCompatibleModel}
-          >
-            {children}
-          </AppProvider>
+          <PHProvider>{content}</PHProvider>
         </div>
       </body>
     </html>
   );
+
+  if (!combinedSettings) {
+    return getPageContent(
+      <div className="flex flex-col items-center justify-center min-h-screen">
+        <div className="mb-2 flex items-center max-w-[175px]">
+          <HeaderTitle>Danswer</HeaderTitle>
+          <Logo height={40} width={40} />
+        </div>
+
+        <Card className="p-8 max-w-md">
+          <h1 className="text-2xl font-bold mb-4 text-error">Error</h1>
+          <p className="text-text-500">
+            Your Danswer instance was not configured properly and your settings
+            could not be loaded. This could be due to an admin configuration
+            issue or an incomplete setup.
+          </p>
+          <p className="mt-4">
+            If you&apos;re an admin, please check{" "}
+            <a
+              className="text-link"
+              href="https://docs.danswer.dev/introduction?utm_source=app&utm_medium=error_page&utm_campaign=config_error"
+              target="_blank"
+              rel="noopener noreferrer"
+            >
+              our docs
+            </a>{" "}
+            to see how to configure Danswer properly. If you&apos;re a user,
+            please contact your admin to fix this error.
+          </p>
+          <p className="mt-4">
+            For additional support and guidance, you can reach out to our
+            community on{" "}
+            <a
+              className="text-link"
+              href="https://danswer.ai?utm_source=app&utm_medium=error_page&utm_campaign=config_error"
+              target="_blank"
+              rel="noopener noreferrer"
+            >
+              Slack
+            </a>
+            .
+          </p>
+        </Card>
+      </div>
+    );
+  }
+  if (productGating === GatingType.FULL) {
+    return getPageContent(
+      <div className="flex flex-col items-center justify-center min-h-screen">
+        <div className="mb-2 flex items-center max-w-[175px]">
+          <HeaderTitle>Danswer</HeaderTitle>
+          <Logo height={40} width={40} />
+        </div>
+        <Card className="p-8 max-w-md">
+          <h1 className="text-2xl font-bold mb-4 text-error">
+            Access Restricted
+          </h1>
+          <p className="text-text-500 mb-4">
+            We regret to inform you that your access to Danswer has been
+            temporarily suspended due to a lapse in your subscription.
+          </p>
+          <p className="text-text-500 mb-4">
+            To reinstate your access and continue benefiting from Danswer&apos;s
+            powerful features, please update your payment information.
+          </p>
+          <p className="text-text-500">
+            If you&apos;re an admin, you can resolve this by visiting the
+            billing section. For other users, please reach out to your
+            administrator to address this matter.
+          </p>
+        </Card>
+      </div>
+    );
+  }
+
+  const data = await fetchAssistantData();
+  const { assistants, hasAnyConnectors, hasImageCompatibleModel } = data;
+
+  return getPageContent(
+    <AppProvider
+      settings={combinedSettings}
+      assistants={assistants}
+      hasAnyConnectors={hasAnyConnectors}
+      hasImageCompatibleModel={hasImageCompatibleModel}
+    >
+      <PostHogPageView />
+      {children}
+    </AppProvider>
+  );
 }
diff --git a/web/src/app/providers.tsx b/web/src/app/providers.tsx
new file mode 100644
index 00000000000..bf5a296abae
--- /dev/null
+++ b/web/src/app/providers.tsx
@@ -0,0 +1,26 @@
+"use client";
+import posthog from "posthog-js";
+import { PostHogProvider } from "posthog-js/react";
+import { useEffect } from "react";
+
+const isPostHogEnabled = !!(
+  process.env.NEXT_PUBLIC_POSTHOG_KEY && process.env.NEXT_PUBLIC_POSTHOG_HOST
+);
+
+export function PHProvider({ children }: { children: React.ReactNode }) {
+  useEffect(() => {
+    if (isPostHogEnabled) {
+      posthog.init(process.env.NEXT_PUBLIC_POSTHOG_KEY!, {
+        api_host: process.env.NEXT_PUBLIC_POSTHOG_HOST!,
+        person_profiles: "identified_only",
+        capture_pageview: false,
+      });
+    }
+  }, []);
+
+  if (!isPostHogEnabled) {
+    return <>{children}</>;
+  }
+
+  return <PostHogProvider client={posthog}>{children}</PostHogProvider>;
+}

From 8c2138a6ef302bf04c0f6cc4cc1fde2898d60710 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 11:51:03 -0700
Subject: [PATCH 224/376] Backport Test (#2930)

---
 .github/workflows/pr-backport-autotrigger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 2d49c39402e..4d035a4da83 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -65,7 +65,7 @@ jobs:
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
           # Fetch all history for all branches and tags
-          git fetch --prune --unshallow
+          git fetch --prune
           # Checkout the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
           # Cherry-pick the merge commit from the merged PR

From c0ab86bac218a8f2b5aab71895e6b767684ff6be Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 12:04:52 -0700
Subject: [PATCH 225/376] Backport Branch Fix (#2931)

---
 .github/workflows/pr-backport-autotrigger.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 4d035a4da83..7f4e9da8a2a 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -33,7 +33,7 @@ jobs:
         id: list-branches
         run: |
           git fetch --all --tags
-          BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/release/* | sed 's|origin/release/||' | sort -Vr)
+          BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/* | sed 's|origin/||' | sort -Vr)
           BETA=$(echo "$BRANCHES" | head -n 1)
           STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
           echo "beta=$BETA" >> $GITHUB_OUTPUT

From 848d86b886ff9fa71639024ab03b6bb35ef16ae1 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 26 Oct 2024 12:06:46 -0700
Subject: [PATCH 226/376] feat: sentry updates (#2929)

---
 .../background/celery/apps/app_base.py        |  2 +-
 backend/danswer/configs/app_configs.py        |  1 -
 backend/danswer/main.py                       |  2 +-
 backend/model_server/main.py                  |  2 +-
 web/next.config.js                            | 15 +++++++-----
 web/sentry.client.config.ts                   | 23 ++++++-------------
 web/sentry.edge.config.ts                     | 13 ++++-------
 web/sentry.server.config.ts                   | 10 ++++----
 web/src/app/chat/ChatPage.tsx                 |  2 +-
 web/src/app/global-error.tsx                  |  4 +++-
 10 files changed, 33 insertions(+), 41 deletions(-)

diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
index b74cba1dfe6..add04c91ade 100644
--- a/backend/danswer/background/celery/apps/app_base.py
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -36,7 +36,7 @@
     sentry_sdk.init(
         dsn=SENTRY_DSN,
         integrations=[CeleryIntegration()],
-        traces_sample_rate=0.5,
+        traces_sample_rate=0.1,
     )
     logger.info("Sentry initialized")
 else:
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 5be2b68d2a4..8710569ea39 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -134,7 +134,6 @@
 except ValueError:
     INDEX_BATCH_SIZE = 16
 
-
 # Below are intended to match the env variables names used by the official postgres docker image
 # https://hub.docker.com/_/postgres
 POSTGRES_USER = os.environ.get("POSTGRES_USER") or "postgres"
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index a6a338b4c4c..ef262b2fa7b 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -213,7 +213,7 @@ def get_application() -> FastAPI:
         sentry_sdk.init(
             dsn=SENTRY_DSN,
             integrations=[StarletteIntegration(), FastApiIntegration()],
-            traces_sample_rate=0.5,
+            traces_sample_rate=0.1,
         )
         logger.info("Sentry initialized")
     else:
diff --git a/backend/model_server/main.py b/backend/model_server/main.py
index 5505bbc8cdb..ce9cc724a98 100644
--- a/backend/model_server/main.py
+++ b/backend/model_server/main.py
@@ -89,7 +89,7 @@ def get_model_app() -> FastAPI:
         sentry_sdk.init(
             dsn=SENTRY_DSN,
             integrations=[StarletteIntegration(), FastApiIntegration()],
-            traces_sample_rate=0.5,
+            traces_sample_rate=0.1,
         )
         logger.info("Sentry initialized")
     else:
diff --git a/web/next.config.js b/web/next.config.js
index 684b43ffc16..950ab216776 100644
--- a/web/next.config.js
+++ b/web/next.config.js
@@ -15,12 +15,15 @@ const nextConfig = {
 
 const { withSentryConfig } = require("@sentry/nextjs");
 
+// Sentry configuration for error monitoring:
+// - Without SENTRY_AUTH_TOKEN and NEXT_PUBLIC_SENTRY_DSN: Sentry is completely disabled
+// - With both configured: Only unhandled errors are captured (no performance/session tracking)
 module.exports = withSentryConfig(nextConfig, {
-  org: "danswer",
-  project: "javascript-nextjs",
-
-  // An auth token is required for uploading source maps.
+  org: process.env.SENTRY_ORG || "danswer",
+  project: process.env.SENTRY_PROJECT || "data-plane-web",
   authToken: process.env.SENTRY_AUTH_TOKEN,
-
-  silent: false, // Can be used to suppress logs
+  silent: false,
+  sourceMaps: {
+    skipUpload: !process.env.SENTRY_AUTH_TOKEN,
+  },
 });
diff --git a/web/sentry.client.config.ts b/web/sentry.client.config.ts
index f6e0e7d1ad7..0a0fe92c4ce 100644
--- a/web/sentry.client.config.ts
+++ b/web/sentry.client.config.ts
@@ -3,21 +3,12 @@ import * as Sentry from "@sentry/nextjs";
 if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
   Sentry.init({
     dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
-    // Replay may only be enabled for the client-side
-    integrations: [Sentry.replayIntegration()],
-
-    // Set tracesSampleRate to 1.0 to capture 100%
-    // of transactions for tracing.
-    // We recommend adjusting this value in production
-    tracesSampleRate: 1.0,
-
-    // Capture Replay for 10% of all sessions,
-    // plus for 100% of sessions with an error
-    replaysSessionSampleRate: 0.1,
-    replaysOnErrorSampleRate: 1.0,
-
-    // Note: if you want to override the automatic release value, do not set a
-    // `release` value here - use the environment variable `SENTRY_RELEASE`, so
-    // that it will also get attached to your source maps
+    // Only capture unhandled exceptions
+    enableTracing: false,
+    integrations: [],
+    tracesSampleRate: 0,
+    replaysSessionSampleRate: 0,
+    replaysOnErrorSampleRate: 0,
+    autoSessionTracking: false,
   });
 }
diff --git a/web/sentry.edge.config.ts b/web/sentry.edge.config.ts
index 4c0eac9e310..6a412ea21e6 100644
--- a/web/sentry.edge.config.ts
+++ b/web/sentry.edge.config.ts
@@ -3,14 +3,9 @@ import * as Sentry from "@sentry/nextjs";
 if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
   Sentry.init({
     dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
-
-    // Set tracesSampleRate to 1.0 to capture 100%
-    // of transactions for tracing.
-    // We recommend adjusting this value in production
-    tracesSampleRate: 1.0,
-
-    // Note: if you want to override the automatic release value, do not set a
-    // `release` value here - use the environment variable `SENTRY_RELEASE`, so
-    // that it will also get attached to your source maps
+    // Only capture unhandled exceptions
+    tracesSampleRate: 0,
+    enableTracing: false,
+    autoSessionTracking: false,
   });
 }
diff --git a/web/sentry.server.config.ts b/web/sentry.server.config.ts
index 4c0eac9e310..3cc7749a7ed 100644
--- a/web/sentry.server.config.ts
+++ b/web/sentry.server.config.ts
@@ -4,10 +4,12 @@ if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
   Sentry.init({
     dsn: process.env.NEXT_PUBLIC_SENTRY_DSN,
 
-    // Set tracesSampleRate to 1.0 to capture 100%
-    // of transactions for tracing.
-    // We recommend adjusting this value in production
-    tracesSampleRate: 1.0,
+    // Only capture unhandled exceptions and errors
+    tracesSampleRate: 0,
+    enableTracing: false,
+
+    // Disable performance monitoring and only capture errors
+    profilesSampleRate: 0,
 
     // Note: if you want to override the automatic release value, do not set a
     // `release` value here - use the environment variable `SENTRY_RELEASE`, so
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 2ce657b2caf..467a03dee76 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -50,7 +50,6 @@ import {
   useContext,
   useEffect,
   useLayoutEffect,
-  useMemo,
   useRef,
   useState,
 } from "react";
@@ -2392,6 +2391,7 @@ export function ChatPage({
 
                             {/* Some padding at the bottom so the search bar has space at the bottom to not cover the last message*/}
                             <div ref={endPaddingRef} className="h-[95px]" />
+
                             <div ref={endDivRef} />
                           </div>
                         </div>
diff --git a/web/src/app/global-error.tsx b/web/src/app/global-error.tsx
index 1ef2941b270..6fbb642c7c2 100644
--- a/web/src/app/global-error.tsx
+++ b/web/src/app/global-error.tsx
@@ -11,7 +11,9 @@ export default function GlobalError({
   error: Error & { digest?: string };
 }) {
   useEffect(() => {
-    Sentry.captureException(error);
+    if (process.env.NEXT_PUBLIC_SENTRY_DSN) {
+      Sentry.captureException(error);
+    }
   }, [error]);
 
   return (

From ccc68c5c3496deda5616d001b2b473c0680bd67d Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 13:09:07 -0700
Subject: [PATCH 227/376] Backport Test (#2935)

---
 .github/workflows/pr-backport-autotrigger.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 7f4e9da8a2a..50b4c971859 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -33,11 +33,11 @@ jobs:
         id: list-branches
         run: |
           git fetch --all --tags
-          BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/* | sed 's|origin/||' | sort -Vr)
+          BRANCHES=$(git for-each-ref --format='%(refname:short)' refs/remotes/origin/release/* | sed 's|origin/release/||' | sort -Vr)
           BETA=$(echo "$BRANCHES" | head -n 1)
           STABLE=$(echo "$BRANCHES" | head -n 2 | tail -n 1)
-          echo "beta=$BETA" >> $GITHUB_OUTPUT
-          echo "stable=$STABLE" >> $GITHUB_OUTPUT
+          echo "beta=release/$BETA" >> $GITHUB_OUTPUT
+          echo "stable=release/$STABLE" >> $GITHUB_OUTPUT
           # Fetch latest tags for beta and stable
           LATEST_BETA_TAG=$(git tag -l "v*.*.0-beta.*" | sort -Vr | head -n 1)
           LATEST_STABLE_TAG=$(git tag -l "v*.*.*" | grep -v -- "-beta" | sort -Vr | head -n 1)

From b4bc6d994d5ded984e5aa94cf0a45c9956b0efd1 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 13:20:02 -0700
Subject: [PATCH 228/376] Backport Auth (#2936)

---
 .github/workflows/pr-backport-autotrigger.yml | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 50b4c971859..b24eb8f0ff5 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -2,7 +2,10 @@ name: Backport on Merge
 
 on:
   pull_request:
-    types: [closed]
+    types: [closed]  # Later we check for merge so only PRs that go in can get backported
+
+permissions:
+  contents: write
 
 jobs:
   backport:
@@ -16,8 +19,9 @@ jobs:
 
       - name: Set up Git
         run: |
-          git config user.name "github-actions[bot]"
-          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
 
       - name: Check for Backport Checkbox
         id: checkbox-check

From e8c28e79c963114777a1c62a1de0aa2cd3ef7b79 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 26 Oct 2024 13:18:41 -0700
Subject: [PATCH 229/376] ensure proper sentry silencing (#2934)

* ensure proper sentry silencing

* add comments
---
 web/next.config.js | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/web/next.config.js b/web/next.config.js
index 950ab216776..3b4c7bbd379 100644
--- a/web/next.config.js
+++ b/web/next.config.js
@@ -4,6 +4,9 @@ const env_version = process.env.DANSWER_VERSION; // version from env variable
 // Use env version if set & valid, otherwise default to package version
 const version = env_version || package_version;
 
+// Always require withSentryConfig
+const { withSentryConfig } = require("@sentry/nextjs");
+
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: "standalone",
@@ -13,17 +16,29 @@ const nextConfig = {
   },
 };
 
-const { withSentryConfig } = require("@sentry/nextjs");
-
 // Sentry configuration for error monitoring:
 // - Without SENTRY_AUTH_TOKEN and NEXT_PUBLIC_SENTRY_DSN: Sentry is completely disabled
 // - With both configured: Only unhandled errors are captured (no performance/session tracking)
-module.exports = withSentryConfig(nextConfig, {
+
+// Determine if Sentry should be enabled
+const sentryEnabled = Boolean(
+  process.env.SENTRY_AUTH_TOKEN && process.env.NEXT_PUBLIC_SENTRY_DSN
+);
+
+// Sentry webpack plugin options
+const sentryWebpackPluginOptions = {
   org: process.env.SENTRY_ORG || "danswer",
   project: process.env.SENTRY_PROJECT || "data-plane-web",
   authToken: process.env.SENTRY_AUTH_TOKEN,
-  silent: false,
+  silent: !sentryEnabled, // Silence output when Sentry is disabled
+  dryRun: !sentryEnabled, // Don't upload source maps when Sentry is disabled
   sourceMaps: {
-    skipUpload: !process.env.SENTRY_AUTH_TOKEN,
+    include: ["./.next"],
+    validate: false,
+    urlPrefix: "~/_next",
+    skip: !sentryEnabled,
   },
-});
+};
+
+// Export the module with conditional Sentry configuration
+module.exports = withSentryConfig(nextConfig, sentryWebpackPluginOptions);

From 80f8d7a486352c1fc63e28c632544489e475b74c Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 13:42:09 -0700
Subject: [PATCH 230/376] Backport Permissions (#2937)

---
 .github/workflows/pr-backport-autotrigger.yml | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index b24eb8f0ff5..f60c7bb6c59 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -6,6 +6,7 @@ on:
 
 permissions:
   contents: write
+  actions: write  # Allows the workflow to trigger other workflows on push
 
 jobs:
   backport:
@@ -19,9 +20,10 @@ jobs:
 
       - name: Set up Git
         run: |
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git
+          git config --global user.name "yuhongsun96"
+          git config --global user.email "yuhongsun96@gmail.com"
+          # Configure Git to use the PAT for authentication
+          git remote set-url origin https://yuhongsun96:${{ secrets.YUHONG_GH_ACTIONS }}@github.com/${{ github.repository }}.git
 
       - name: Check for Backport Checkbox
         id: checkbox-check
@@ -80,7 +82,7 @@ jobs:
           # Create new beta tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch
-          git push origin ${{ steps.list-branches.outputs.beta }}
+          git push origin ${{ steps.list-branches.outputs.beta }} --force
           git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
           # Checkout the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
@@ -92,5 +94,5 @@ jobs:
           # Create new stable tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch
-          git push origin ${{ steps.list-branches.outputs.stable }}
-          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
+          git push origin ${{ steps.list-branches.outputs.stable }} --force
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
\ No newline at end of file

From 4bcd65ed9200883af8e8ad2693b2454ba16917d3 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 13:47:09 -0700
Subject: [PATCH 231/376] Harmless Backport Test (#2938)

---
 backend/danswer/main.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index ef262b2fa7b..f99320aa457 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -96,6 +96,7 @@
 from shared_configs.configs import CORS_ALLOWED_ORIGIN
 from shared_configs.configs import SENTRY_DSN
 
+
 logger = setup_logger()
 
 

From accdd580d762af1112d187fb9b99e4426f7c2e5d Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 13:59:55 -0700
Subject: [PATCH 232/376] Backport Test (#2939)

---
 backend/danswer/setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index a27580a2b15..593ced1540d 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -55,6 +55,7 @@
 from shared_configs.configs import SUPPORTED_EMBEDDING_MODELS
 from shared_configs.model_server_models import SupportedEmbeddingModel
 
+
 logger = setup_logger()
 
 

From f6f9112b76297cd964601c8739c3d403cb74487d Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 14:23:43 -0700
Subject: [PATCH 233/376] Backport Test (#2940)

---
 backend/danswer/auth/users.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 49e9f5f8736..a540cbaa4cc 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -97,6 +97,7 @@
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
+
 logger = setup_logger()
 
 

From e6bef573ba1a1ef85cb4d8542123b4a928d73321 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 14:34:24 -0700
Subject: [PATCH 234/376] Backport Correct Branch (#2941)

---
 .github/workflows/pr-backport-autotrigger.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index f60c7bb6c59..2376365c41d 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -45,10 +45,10 @@ jobs:
           echo "beta=release/$BETA" >> $GITHUB_OUTPUT
           echo "stable=release/$STABLE" >> $GITHUB_OUTPUT
           # Fetch latest tags for beta and stable
-          LATEST_BETA_TAG=$(git tag -l "v*.*.0-beta.*" | sort -Vr | head -n 1)
-          LATEST_STABLE_TAG=$(git tag -l "v*.*.*" | grep -v -- "-beta" | sort -Vr | head -n 1)
+          LATEST_BETA_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*-beta.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$" | grep -v -- "-cloud" | sort -Vr | head -n 1)
+          LATEST_STABLE_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -Vr | head -n 1)
           # Increment latest beta tag
-          NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 ".0-beta." ($NF+1)}')
+          NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}')
           # Increment latest stable tag
           NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
           echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT

From aa0f307cc7c5da91f8b52e874a1ca3063fa9fa3a Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sat, 26 Oct 2024 14:52:59 -0700
Subject: [PATCH 235/376] Backport Test Final (#2942)

---
 backend/danswer/background/celery/apps/app_base.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
index add04c91ade..5105b755f57 100644
--- a/backend/danswer/background/celery/apps/app_base.py
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -28,6 +28,7 @@
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import SENTRY_DSN
 
+
 logger = setup_logger()
 
 task_logger = get_task_logger(__name__)

From 088551a4ef63d6d65b0e75cc2a9663906e9a92a5 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 26 Oct 2024 14:58:42 -0700
Subject: [PATCH 236/376] remove rt + home-grown sitemap parsing (#2933)

* remove rt

* nit

* add minor alembic revision

* functional migration

* replace usp

* k

* typing
---
 .../versions/949b4a92a401_remove_rt.py        |  74 +++++
 backend/danswer/connectors/factory.py         |   2 -
 .../connectors/requesttracker/connector.py    | 277 ++++++++----------
 backend/danswer/utils/sitemap.py              |  87 ++++--
 backend/requirements/default.txt              |   2 -
 web/public/RequestTracker.png                 | Bin 17302 -> 0 bytes
 web/src/components/icons/icons.tsx            |   8 -
 web/src/lib/connectors/connectors.tsx         |   7 -
 web/src/lib/connectors/credentials.ts         |  17 --
 web/src/lib/sources.ts                        |   7 -
 web/src/lib/types.ts                          |   1 -
 11 files changed, 261 insertions(+), 221 deletions(-)
 create mode 100644 backend/alembic/versions/949b4a92a401_remove_rt.py
 delete mode 100644 web/public/RequestTracker.png

diff --git a/backend/alembic/versions/949b4a92a401_remove_rt.py b/backend/alembic/versions/949b4a92a401_remove_rt.py
new file mode 100644
index 00000000000..6dbb7859cd7
--- /dev/null
+++ b/backend/alembic/versions/949b4a92a401_remove_rt.py
@@ -0,0 +1,74 @@
+"""remove rt
+
+Revision ID: 949b4a92a401
+Revises: 1b10e1fda030
+Create Date: 2024-10-26 13:06:06.937969
+
+"""
+from alembic import op
+from sqlalchemy.orm import Session
+
+# Import your models and constants
+from danswer.db.models import (
+    Connector,
+    ConnectorCredentialPair,
+    Credential,
+    IndexAttempt,
+)
+from danswer.configs.constants import DocumentSource
+
+
+# revision identifiers, used by Alembic.
+revision = "949b4a92a401"
+down_revision = "1b10e1fda030"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Deletes all RequestTracker connectors and associated data
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    connectors_to_delete = (
+        session.query(Connector)
+        .filter(Connector.source == DocumentSource.REQUESTTRACKER)
+        .all()
+    )
+
+    connector_ids = [connector.id for connector in connectors_to_delete]
+
+    if connector_ids:
+        cc_pairs_to_delete = (
+            session.query(ConnectorCredentialPair)
+            .filter(ConnectorCredentialPair.connector_id.in_(connector_ids))
+            .all()
+        )
+
+        cc_pair_ids = [cc_pair.id for cc_pair in cc_pairs_to_delete]
+
+        if cc_pair_ids:
+            session.query(IndexAttempt).filter(
+                IndexAttempt.connector_credential_pair_id.in_(cc_pair_ids)
+            ).delete(synchronize_session=False)
+
+            session.query(ConnectorCredentialPair).filter(
+                ConnectorCredentialPair.id.in_(cc_pair_ids)
+            ).delete(synchronize_session=False)
+
+        credential_ids = [cc_pair.credential_id for cc_pair in cc_pairs_to_delete]
+        if credential_ids:
+            session.query(Credential).filter(Credential.id.in_(credential_ids)).delete(
+                synchronize_session=False
+            )
+
+        session.query(Connector).filter(Connector.id.in_(connector_ids)).delete(
+            synchronize_session=False
+        )
+
+    session.commit()
+
+
+def downgrade() -> None:
+    # No-op downgrade as we cannot restore deleted data
+    pass
diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py
index 05f201a45c2..ce74997333e 100644
--- a/backend/danswer/connectors/factory.py
+++ b/backend/danswer/connectors/factory.py
@@ -34,7 +34,6 @@
 from danswer.connectors.models import InputType
 from danswer.connectors.notion.connector import NotionConnector
 from danswer.connectors.productboard.connector import ProductboardConnector
-from danswer.connectors.requesttracker.connector import RequestTrackerConnector
 from danswer.connectors.salesforce.connector import SalesforceConnector
 from danswer.connectors.sharepoint.connector import SharepointConnector
 from danswer.connectors.slab.connector import SlabConnector
@@ -77,7 +76,6 @@ def identify_connector_class(
         DocumentSource.SLAB: SlabConnector,
         DocumentSource.NOTION: NotionConnector,
         DocumentSource.ZULIP: ZulipConnector,
-        DocumentSource.REQUESTTRACKER: RequestTrackerConnector,
         DocumentSource.GURU: GuruConnector,
         DocumentSource.LINEAR: LinearConnector,
         DocumentSource.HUBSPOT: HubSpotConnector,
diff --git a/backend/danswer/connectors/requesttracker/connector.py b/backend/danswer/connectors/requesttracker/connector.py
index 9c4590fc2ef..b520d0d7acf 100644
--- a/backend/danswer/connectors/requesttracker/connector.py
+++ b/backend/danswer/connectors/requesttracker/connector.py
@@ -1,153 +1,124 @@
-from datetime import datetime
-from datetime import timezone
-from logging import DEBUG as LOG_LVL_DEBUG
-from typing import Any
-from typing import List
-from typing import Optional
-
-from rt.rest1 import ALL_QUEUES
-from rt.rest1 import Rt
-
-from danswer.configs.app_configs import INDEX_BATCH_SIZE
-from danswer.configs.constants import DocumentSource
-from danswer.connectors.interfaces import GenerateDocumentsOutput
-from danswer.connectors.interfaces import PollConnector
-from danswer.connectors.interfaces import SecondsSinceUnixEpoch
-from danswer.connectors.models import ConnectorMissingCredentialError
-from danswer.connectors.models import Document
-from danswer.connectors.models import Section
-from danswer.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-class RequestTrackerError(Exception):
-    pass
-
-
-class RequestTrackerConnector(PollConnector):
-    def __init__(
-        self,
-        batch_size: int = INDEX_BATCH_SIZE,
-    ) -> None:
-        self.batch_size = batch_size
-
-    def txn_link(self, tid: int, txn: int) -> str:
-        return f"{self.rt_base_url}/Ticket/Display.html?id={tid}&txn={txn}"
-
-    def build_doc_sections_from_txn(
-        self, connection: Rt, ticket_id: int
-    ) -> List[Section]:
-        Sections: List[Section] = []
-
-        get_history_resp = connection.get_history(ticket_id)
-
-        if get_history_resp is None:
-            raise RequestTrackerError(f"Ticket {ticket_id} cannot be found")
-
-        for tx in get_history_resp:
-            Sections.append(
-                Section(
-                    link=self.txn_link(ticket_id, int(tx["id"])),
-                    text="\n".join(
-                        [
-                            f"{k}:\n{v}\n" if k != "Attachments" else ""
-                            for (k, v) in tx.items()
-                        ]
-                    ),
-                )
-            )
-        return Sections
-
-    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
-        self.rt_username = credentials.get("requesttracker_username")
-        self.rt_password = credentials.get("requesttracker_password")
-        self.rt_base_url = credentials.get("requesttracker_base_url")
-        return None
-
-    # This does not include RT file attachments yet.
-    def _process_tickets(
-        self, start: datetime, end: datetime
-    ) -> GenerateDocumentsOutput:
-        if any([self.rt_username, self.rt_password, self.rt_base_url]) is None:
-            raise ConnectorMissingCredentialError("requesttracker")
-
-        Rt0 = Rt(
-            f"{self.rt_base_url}/REST/1.0/",
-            self.rt_username,
-            self.rt_password,
-        )
-
-        Rt0.login()
-
-        d0 = start.strftime("%Y-%m-%d %H:%M:%S")
-        d1 = end.strftime("%Y-%m-%d %H:%M:%S")
-
-        tickets = Rt0.search(
-            Queue=ALL_QUEUES,
-            raw_query=f"Updated > '{d0}' AND Updated < '{d1}'",
-        )
-
-        doc_batch: List[Document] = []
-
-        for ticket in tickets:
-            ticket_keys_to_omit = ["id", "Subject"]
-            tid: int = int(ticket["numerical_id"])
-            ticketLink: str = f"{self.rt_base_url}/Ticket/Display.html?id={tid}"
-            logger.info(f"Processing ticket {tid}")
-            doc = Document(
-                id=ticket["id"],
-                # Will add title to the first section later in processing
-                sections=[Section(link=ticketLink, text="")]
-                + self.build_doc_sections_from_txn(Rt0, tid),
-                source=DocumentSource.REQUESTTRACKER,
-                semantic_identifier=ticket["Subject"],
-                metadata={
-                    key: value
-                    for key, value in ticket.items()
-                    if key not in ticket_keys_to_omit
-                },
-            )
-
-            doc_batch.append(doc)
-
-            if len(doc_batch) >= self.batch_size:
-                yield doc_batch
-                doc_batch = []
-
-        if doc_batch:
-            yield doc_batch
-
-    def poll_source(
-        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
-    ) -> GenerateDocumentsOutput:
-        # Keep query short, only look behind 1 day at maximum
-        one_day_ago: float = end - (24 * 60 * 60)
-        _start: float = start if start > one_day_ago else one_day_ago
-        start_datetime = datetime.fromtimestamp(_start, tz=timezone.utc)
-        end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
-        yield from self._process_tickets(start_datetime, end_datetime)
-
-
-if __name__ == "__main__":
-    import time
-    import os
-    from dotenv import load_dotenv
-
-    load_dotenv()
-    logger.setLevel(LOG_LVL_DEBUG)
-    rt_connector = RequestTrackerConnector()
-    rt_connector.load_credentials(
-        {
-            "requesttracker_username": os.getenv("RT_USERNAME"),
-            "requesttracker_password": os.getenv("RT_PASSWORD"),
-            "requesttracker_base_url": os.getenv("RT_BASE_URL"),
-        }
-    )
-
-    current = time.time()
-    one_day_ago = current - (24 * 60 * 60)  # 1 days
-    latest_docs = rt_connector.poll_source(one_day_ago, current)
-
-    for doc in latest_docs:
-        print(doc)
+# from datetime import datetime
+# from datetime import timezone
+# from logging import DEBUG as LOG_LVL_DEBUG
+# from typing import Any
+# from typing import List
+# from typing import Optional
+# from rt.rest1 import ALL_QUEUES
+# from rt.rest1 import Rt
+# from danswer.configs.app_configs import INDEX_BATCH_SIZE
+# from danswer.configs.constants import DocumentSource
+# from danswer.connectors.interfaces import GenerateDocumentsOutput
+# from danswer.connectors.interfaces import PollConnector
+# from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+# from danswer.connectors.models import ConnectorMissingCredentialError
+# from danswer.connectors.models import Document
+# from danswer.connectors.models import Section
+# from danswer.utils.logger import setup_logger
+# logger = setup_logger()
+# class RequestTrackerError(Exception):
+#     pass
+# class RequestTrackerConnector(PollConnector):
+#     def __init__(
+#         self,
+#         batch_size: int = INDEX_BATCH_SIZE,
+#     ) -> None:
+#         self.batch_size = batch_size
+#     def txn_link(self, tid: int, txn: int) -> str:
+#         return f"{self.rt_base_url}/Ticket/Display.html?id={tid}&txn={txn}"
+#     def build_doc_sections_from_txn(
+#         self, connection: Rt, ticket_id: int
+#     ) -> List[Section]:
+#         Sections: List[Section] = []
+#         get_history_resp = connection.get_history(ticket_id)
+#         if get_history_resp is None:
+#             raise RequestTrackerError(f"Ticket {ticket_id} cannot be found")
+#         for tx in get_history_resp:
+#             Sections.append(
+#                 Section(
+#                     link=self.txn_link(ticket_id, int(tx["id"])),
+#                     text="\n".join(
+#                         [
+#                             f"{k}:\n{v}\n" if k != "Attachments" else ""
+#                             for (k, v) in tx.items()
+#                         ]
+#                     ),
+#                 )
+#             )
+#         return Sections
+#     def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
+#         self.rt_username = credentials.get("requesttracker_username")
+#         self.rt_password = credentials.get("requesttracker_password")
+#         self.rt_base_url = credentials.get("requesttracker_base_url")
+#         return None
+#     # This does not include RT file attachments yet.
+#     def _process_tickets(
+#         self, start: datetime, end: datetime
+#     ) -> GenerateDocumentsOutput:
+#         if any([self.rt_username, self.rt_password, self.rt_base_url]) is None:
+#             raise ConnectorMissingCredentialError("requesttracker")
+#         Rt0 = Rt(
+#             f"{self.rt_base_url}/REST/1.0/",
+#             self.rt_username,
+#             self.rt_password,
+#         )
+#         Rt0.login()
+#         d0 = start.strftime("%Y-%m-%d %H:%M:%S")
+#         d1 = end.strftime("%Y-%m-%d %H:%M:%S")
+#         tickets = Rt0.search(
+#             Queue=ALL_QUEUES,
+#             raw_query=f"Updated > '{d0}' AND Updated < '{d1}'",
+#         )
+#         doc_batch: List[Document] = []
+#         for ticket in tickets:
+#             ticket_keys_to_omit = ["id", "Subject"]
+#             tid: int = int(ticket["numerical_id"])
+#             ticketLink: str = f"{self.rt_base_url}/Ticket/Display.html?id={tid}"
+#             logger.info(f"Processing ticket {tid}")
+#             doc = Document(
+#                 id=ticket["id"],
+#                 # Will add title to the first section later in processing
+#                 sections=[Section(link=ticketLink, text="")]
+#                 + self.build_doc_sections_from_txn(Rt0, tid),
+#                 source=DocumentSource.REQUESTTRACKER,
+#                 semantic_identifier=ticket["Subject"],
+#                 metadata={
+#                     key: value
+#                     for key, value in ticket.items()
+#                     if key not in ticket_keys_to_omit
+#                 },
+#             )
+#             doc_batch.append(doc)
+#             if len(doc_batch) >= self.batch_size:
+#                 yield doc_batch
+#                 doc_batch = []
+#         if doc_batch:
+#             yield doc_batch
+#     def poll_source(
+#         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+#     ) -> GenerateDocumentsOutput:
+#         # Keep query short, only look behind 1 day at maximum
+#         one_day_ago: float = end - (24 * 60 * 60)
+#         _start: float = start if start > one_day_ago else one_day_ago
+#         start_datetime = datetime.fromtimestamp(_start, tz=timezone.utc)
+#         end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
+#         yield from self._process_tickets(start_datetime, end_datetime)
+# if __name__ == "__main__":
+#     import time
+#     import os
+#     from dotenv import load_dotenv
+#     load_dotenv()
+#     logger.setLevel(LOG_LVL_DEBUG)
+#     rt_connector = RequestTrackerConnector()
+#     rt_connector.load_credentials(
+#         {
+#             "requesttracker_username": os.getenv("RT_USERNAME"),
+#             "requesttracker_password": os.getenv("RT_PASSWORD"),
+#             "requesttracker_base_url": os.getenv("RT_BASE_URL"),
+#         }
+#     )
+#     current = time.time()
+#     one_day_ago = current - (24 * 60 * 60)  # 1 days
+#     latest_docs = rt_connector.poll_source(one_day_ago, current)
+#     for doc in latest_docs:
+#         print(doc)
diff --git a/backend/danswer/utils/sitemap.py b/backend/danswer/utils/sitemap.py
index ababbec4575..551b2bb3bf0 100644
--- a/backend/danswer/utils/sitemap.py
+++ b/backend/danswer/utils/sitemap.py
@@ -1,39 +1,78 @@
-from datetime import datetime
-from urllib import robotparser
+import re
+import xml.etree.ElementTree as ET
+from typing import Set
+from urllib.parse import urljoin
 
-from usp.tree import sitemap_tree_for_homepage  # type: ignore
+import requests
 
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
 
 
-def test_url(rp: robotparser.RobotFileParser | None, url: str) -> bool:
-    if not rp:
-        return True
-    else:
-        return rp.can_fetch("*", url)
+def _get_sitemap_locations_from_robots(base_url: str) -> Set[str]:
+    """Extract sitemap URLs from robots.txt"""
+    sitemap_urls: set = set()
+    try:
+        robots_url = urljoin(base_url, "/robots.txt")
+        resp = requests.get(robots_url, timeout=10)
+        if resp.status_code == 200:
+            for line in resp.text.splitlines():
+                if line.lower().startswith("sitemap:"):
+                    sitemap_url = line.split(":", 1)[1].strip()
+                    sitemap_urls.add(sitemap_url)
+    except Exception as e:
+        logger.warning(f"Error fetching robots.txt: {e}")
+    return sitemap_urls
+
+
+def _extract_urls_from_sitemap(sitemap_url: str) -> Set[str]:
+    """Extract URLs from a sitemap XML file"""
+    urls: set[str] = set()
+    try:
+        resp = requests.get(sitemap_url, timeout=10)
+        if resp.status_code != 200:
+            return urls
 
+        root = ET.fromstring(resp.content)
 
-def init_robots_txt(site: str) -> robotparser.RobotFileParser:
-    ts = datetime.now().timestamp()
-    robots_url = f"{site}/robots.txt?ts={ts}"
-    rp = robotparser.RobotFileParser()
-    rp.set_url(robots_url)
-    rp.read()
-    return rp
+        # Handle both regular sitemaps and sitemap indexes
+        # Remove namespace for easier parsing
+        namespace = re.match(r"\{.*\}", root.tag)
+        ns = namespace.group(0) if namespace else ""
+
+        if root.tag == f"{ns}sitemapindex":
+            # This is a sitemap index
+            for sitemap in root.findall(f".//{ns}loc"):
+                if sitemap.text:
+                    sub_urls = _extract_urls_from_sitemap(sitemap.text)
+                    urls.update(sub_urls)
+        else:
+            # This is a regular sitemap
+            for url in root.findall(f".//{ns}loc"):
+                if url.text:
+                    urls.add(url.text)
+
+    except Exception as e:
+        logger.warning(f"Error processing sitemap {sitemap_url}: {e}")
+
+    return urls
 
 
 def list_pages_for_site(site: str) -> list[str]:
-    rp: robotparser.RobotFileParser | None = None
-    try:
-        rp = init_robots_txt(site)
-    except Exception:
-        logger.warning("Failed to load robots.txt")
+    """Get list of pages from a site's sitemaps"""
+    site = site.rstrip("/")
+    all_urls = set()
 
-    tree = sitemap_tree_for_homepage(site)
+    # Try both common sitemap locations
+    sitemap_paths = ["/sitemap.xml", "/sitemap_index.xml"]
+    for path in sitemap_paths:
+        sitemap_url = urljoin(site, path)
+        all_urls.update(_extract_urls_from_sitemap(sitemap_url))
 
-    pages = [page.url for page in tree.all_pages() if test_url(rp, page.url)]
-    pages = list(dict.fromkeys(pages))
+    # Check robots.txt for additional sitemaps
+    sitemap_locations = _get_sitemap_locations_from_robots(site)
+    for sitemap_url in sitemap_locations:
+        all_urls.update(_extract_urls_from_sitemap(sitemap_url))
 
-    return pages
+    return list(all_urls)
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index 2240180e355..fe0cdd01547 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -61,7 +61,6 @@ requests==2.32.2
 requests-oauthlib==1.3.1
 retry==0.9.2  # This pulls in py which is in CVE-2022-42969, must remove py from image
 rfc3986==1.5.0
-rt==3.1.2
 simple-salesforce==1.12.6
 slack-sdk==3.20.2
 SQLAlchemy[mypy]==2.0.15
@@ -79,7 +78,6 @@ asana==5.0.8
 zenpy==2.0.41
 dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
-ultimate_sitemap_parser==0.5
 stripe==10.12.0
 urllib3==2.2.3
 mistune==0.8.4
diff --git a/web/public/RequestTracker.png b/web/public/RequestTracker.png
deleted file mode 100644
index 95d6680e95cbc007dbc3609eb752d3ef9fd4febd..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 17302
zcmbSy<y%|b6D{smT#9>(yGw9)DDDufcq#7g1b5d01%kD>JHg$pxVv56-+k^saPuM0
z`H*vF?U}XL%$eDVR8^KiM<GUmf`USqla*A5f`ZlixK2Pq`1t<BPdo($C1EHhDX#f*
z`D9fIXrM(naM`R~q5VbEFrpk4uL7xXKIkD*9Zno;b_qDg)=JKaQ>qFF&x`Tr5MZF0
zWx;_^*P@mPz%z{4?1}&KqY9M~44J2DDv?SqwHY{AjlnYGION-DKTv!V{h2<@lypBP
ze7}A@(T)TAVgLX4Q>9R$sVB(8^CJa*WNrh<#li#C&*0lKpR{jUVgnVXGQDbTb1f6x
zxq2s;Nxs5nx^7BHBX^bR!WC7S#VmGxtJ6_2XS+Zc@Y1irorAR^F#}8~;{_@<GsfE}
zme7KfMXCp%Q%GVkz1YUKE4;ZUt;21*kMEX_9tJ|E><Dy+_DQN6mW}l73DI`idXGS*
z6;H|FwPEt7M`YB9qiUw|_Xxj^m4wcpN6yU;Obqcy=Z;!r_2rFAQne21H9U5;by)vs
zL<8;%BHHp5zaI41gO}~N9M4o|G}TT)Bh4d>ys2cIRiD-G4)5D*ge=V+5xvvIN;#a3
zRa{L6v(j{@DtCDAG+lF_HWsc_1X`6`J<D?Qeq0N+z}MPk7tWtnz7p+m#~#4$2?~pD
z-z{0*?TMyrBnJ~eFPydH59<B!Bl10vzs_h<u|_M?Iu9ApTd<RJlFMZM;+@PXKJmA!
z^>3fs<#sGzO)|6C`)sYB$!t_96`7_hM-iC7LK&Wi3F-)o+^F!aldHxd%!hWoNPeX^
z`#hV=e>(DjTDYSS=e&BKggYFTunP0CbE@)OuMB<&l!<v>v2KjWw*P&6!EbNw*Cl3b
z=0NU%es0HnRH>An3im<9`=h20;M>I!N<)1+r@zXb@V@^+!t_WS<*lJkx(e7AflcHc
zyQhr=d(PIq$SZ(V;P^0up;E{UT37SEl`pPYPpy#QkFwNfkA9{cd4<?9+I_~kAE2Wb
z-;Ae&ZN9Z7q*tDR1j%YdH?h@~duZ;?&qFn4F>T%zk6p}fI~CT35_|!5ctrjr)sHGo
zgDE($ga`h^ezdpSBa@-!ryWMOJjoe&1GgFLE8cA_r$?>AX?C4{M%<*w+OG2!I&^1X
zp+g?cxw}26eKqGx_wFj+O4sKr%n^f)owhEwyVzQ=q45N9Tj3P2pws=9ZZA#h_z<V4
zs4<HfYIN3LeSDrdwb`q^W5Ty=7_SZ>!kjirGREmey-?Kz`*p7NtNwX>Ow2=Ff)Vo#
zquB=HoR2G;KGY*%1Kp+B7tO>#s=rhEtIc}J$+#!<Jo8{O>4+DqxIpXQ(QkftSI;(2
zeuinMudR4=f3<F(p~ni$wpW65&{L1R<oMU`7oR47-KD6kI@~OYBI@<Er}|=D_!mw+
zWwpD;)|tojK5QV0Xz8g}%0dbwi&+W)4`PzCQYjV+S^0kt#<G_hzDr3^eHWKiO*5P1
zud4o{M2)l;-1J4wS&lj=pI0)JN9tJJu)67QuE!4^qB+mWY|e9sf%i0_r5~)E@BG$Q
z6VGGM9h0MA(VkNnQ~A4%+`G;o>7?GFqwx8tTow-GAAgXD73uw9cgKDcOeySnrYua)
zWBNutor>B#bq170WjI#ub3={yBW{WB!u+FDby95FjhpJhv6{M2b5i^R*1hTA6OD(|
zs-%4Geck=y_j(<Y`PBkvY)ixGY%x&*eO=yx!}f{pn*DG}`k(Tbds1Qh`il<Mf&2N;
z_jz~{=2Bsh3;Obi;GV<bCM}iARl^shS4b8qDd|&9yRTR9_fLWjYAYf7G>8Gie`;f&
zc;UsJ=-8jSTW-gAwY<G%A`3!NmSNxr%@n>9R?Am&_P)g2K0NDO-ZK9Cet7})&bqZh
z3V3)216d&GYDJz#7IFTzuKwGOgGp5;-|{Vxj8Y6r)V=NA-Fx|#`)cnQS|(10LJ?|>
zMX`?Tt^=1nASnNc{&fR~j8)u(jS<%Z&xJob_=g7N)<XliZ?xRn4U_~-i~7O)h!~R3
zEx8@cq(WjCQ(muc2Gi(c(-tT8tXzv>KX%S*>7z~FXVeD8mNWTjdSOzQONMzzm)#O;
z$_(bp;=2}{dr>rYU!ER&Zp0U43J{G;+bUS=)_b8Ew)=Kp-R!Jm`;P_TjXQtz03!<|
zt4Zd#A?6U;3=mBCL}yi9HGw{tQ5NGr;dlc{{8=jE@|q;B7~Rz(du;e0e7Y+gaP8qm
zHLhw4U+9V3sC;_tm{dHKPJ`C_&o^k}3Qog;E%XtSI6$5o3~-fF??vRDXUz6%<Q=>c
zdT^eq|Ni8n?BafzT(2{yqoBIxc?+8o;ow%Er;Zp~wN>(gR%-KNob>Zbo;U`?>NR0Q
z=}*8gA0>|;d-M?xfJ109E5!G{tQsW2!4w!4)DVl*De@^Z%M4>y$pX|ySp7N{S!W9#
z@ih+<P1pFxJNrU2F$|I=Xql8ec47{_D=xR;Ek2hBh)pHAFAyTCZs`_3kDEhajls74
zDLNMnl0V%a!&KV629^P?3rH(sQx_n4emoZ7CY$I2l{0pIc>%1!t%c@q!sjSMsz6Cg
z-@iHdO)?Rdzx<n1KY5v9lkjv%2M(jq8WcN$ERfnNWVQ(XZt!MBgywNqMh7Kt+P3Pa
zqL941F5r}}3R;8}4H~4#VpF^9glwy~nD+q>w~Y`JoLA_z*ny;_1oB$v!f(CwmK1}h
zyV&;N*(8}FJakXAh1vR^P3WmxPKGJ#hn(oxIN9`86I>$@iG(Y`oGEvJRGVl@W=En-
zX)HdLgurQzN54d0V>C^7%;`U+LPKSIRn2$psYl5#j457m-C+xsybR8dd6B0{(7$UV
zZDX~|?*#+}F{!1QJ=yfcXiDUE{<RM@SVXrGGwY)f9XMviU)Q#a<y@Z{T^pIJGZl-M
z3J&8f0xM{1rAO)f+NXEDGR%3$7`TE6#J=Lp&t%C;b%jltC2u`<yBEsHh(CK`N$>6G
zHdqlER!UmFLiF}Xr9aXG+}Ox5N>sF6RYm`p>ND`<+z_67aQ8g|w1;Lkn@CUuUr0$o
zf^)P+#IxzpC0#YQB}vX@6XMX{>Os^BLmXm8<p)}?4s<f)CNL=~!ROJdR`c}=N$*Y1
zA~Olu%P)QV=6hYv@Ox^6R!#3-VrgaUeOmd(Hb(*hqWMnt6Y92GmTFm5qV8|#C=IOS
zvA=Lx{(j3_$kLx6LhjvRAoQ#|GuJ;&G%k{AZDMZJtQye#>lmENr%n+G!l&Kk%Le&V
z`+9T|=EmPsw&=e=w5`(nplFJ5wn|4t_;&5EZ#IFk+GkkpJir4M4TyG}UD1JOSrRnY
zTqpYNaSrF9r#Qh6u!4A<^V~T**ok&*J9cry=wtawnnB(MhSU@~yW>FgTiaXu*%y?h
zfLQHi*V7R?y5ndz?s3bo>9MC0wd;m`LekuBX$4MuupVIA{5AFtK47Ka&W*ow;xkEg
z&&=AIuy}|x1CV+U_t)9ckw6k!e%aedrh0{!P==V>qiMK4E-vW;$DTRap4x5a*K+<E
z+S0*n{J1tEW{Qle#z6Zn>8}nwo0HhV_1%Y!726aUzvV1$s4Z}K^lVLw=0?2v^pX~J
z>2$g?V<4$))pfETu11dnP~YXG-NGzCNY4|`sfph3jIB^wV4paB{_zqTw7sJje@aYL
zslnorck=w^fnpn%Crhqfr6uU<8YLzYt33@#7S_|bv@S~7<JO~Lr}Gax9^6Hyv6G9l
zRw2xDME#%X6tY@pwQX*8*TBT^Kc^U^A@Z7Bi@-e?aI`mWm^9L<)XD>oMo|aXEGE24
zI<fmbIRySEThzDASBh<HmRZp}GztKR6U;AlRaXPiuD%@QvG-n}XKWQs>}sL~F^`_j
zyu`{^{z(Bw4>U50LtQ+t4Eql+GuwlwgZn#i!qGYQO^f!pdSoMPgiw~doKM9B{|2Mw
zSG;v@eLC@1K13b9@E(GqqJh0-)%>uq(nynpo8v=VJqogI!bY5su3CD&&Ah#}bz|fD
z?4oamVhZsl{)lj@-XgHkrFtW?#R}h?j7M*xkdbTqIXm3w1qI|z*$NjUl_9}#hm+Ke
zy8<jgq^Yp_YS-|4DvA3k9@JNOA1*q{nW9IbyW&9k|DJV4ww`;T=Rh|gWDm}kQ~|<;
z8v)H3Xs93Ly2S?$gxle9tSyjIBWwv0PdkGCgu}x&FV4bst%78VLbVQXY~O-E;;-<-
zZ|y68Cs|orFn=JG4civsd*O6Lc#_S>-VcpU%?yodABgNr|G0FGun{<tpbjEXYV>rU
z#U%e-ulXh8{g?e^*I>~it1mp_=NW;?>zGwS_YjNCKJQKgmUzn2pVFhXwQ}Vic>iHO
zX?zrLcJYCDGvPtks->XCp3u$wo>E68slG8G<sWW@$i(O_0{KJ3AQ6Et;YJUxO0k8b
zF-XH*sBhAD!Eb4P?1V||_|MO>;^WS~s0~2Y*RPl#n}~8;>8kHWtsa`By4dwN889)v
zOdttRn4i}ZFE#z2Y#eqIyO>RF6q6YNh1mB^iuT|#0n|T!8nMNGMAeTMz-oXz+{II6
z0re>fy7#s%D064|%yUg6f-!9B>ex5(HuZUeisbi{yqBRB!B-nquV1zF;}V=50qp-1
zho|s%&e-Zgl-`Rn1YRepSZ8tn2rS%gNeM+H6ANcb>Ch;%N{E2#pmxy1;3+2Y4Q3=V
zbg}CvO*jHHqyjN4sk(F|JlH}Q6l{ZX)W3QZ%(M5aaTYN-5~HsZ9Ilg~Hk+q|30~{{
zb^i0QjQit%&e^Q3-Sz83F3aluM9e!85Tr{1j9vqt8icWw=R^eDxlYVm%n~lxv}5Gv
zAvQvGcny<Xw;KilDIpX~jA0;to?M(8NwvW^mYm|lf+VvZS)RTP_~kc_rZipyxD9P(
zV&_(Ex}~_3-`TqxTYR|Vs6C&rs6U!n`fEBn(%}3X51rp4xosk_sxHf{B#cu}IubzL
z#4%>NQ;8Y0=`tNq_|$A7vYt~9wQA;nmVgFqtw}&j@Bg$t#UL?$nGmC6iho-+y`-oY
zlbY5^#C@8`MADX~Xc|8CrQtXo?@5tVmN1Punfv8;+3u=OnA>&?J}VC)vk-h#YFSt^
z!pm=b+r*L4+U^b8?AkL!TP?mKOEkQKPmrLitE-7(XuR7ps+(;K07t%~Zy0ox4Gw?~
zEXlExf75;UgJbqHufE+t9BEbOLQ?hOckmah92{nGyJQ0}0f~2}W$;vGrXOQdV{fK8
zf~o@V@TFIH{Xk!2S(Jw=_MHdDVIL8*ks=ARQ1!r%{ixHDA)tHo8jwFRvtY6=-t)!x
zGlD(g+g;J1(V=*Ej6w|fWC&#fNF*)}lq_&p`y;{XGNX>;<;+1zwAif|CLU$yfdv;V
z3;UrW`#@%iac76%$YUuC!C3q=C;X<<J!@R5l|6#P(}9H-g&2G;;|jgB15ysZu{VFC
zHAW&30b}&T7BBL%zKVdebTmQ6RF6hRVP~()9N=$$swNRL)++*+I5XN4>J!m2+}u_}
zn&b{AB1!>cXcIY*9$Gl`Jug5ydN1{%GwsqYgfV++I;s#7Aw8xe^B;jWl1Ct@3IL8c
zlM8Sggyjj=#M|-1c?;r`s9_ov!E!O(ADxQD391)EQbS4q!>;~&`me<6+ki>7#LdJ*
zIUB)Dc?tdzDp_^K#SPM89Bc5;?sUA@_FWCHnAEP(c(L(VwCC}HUdTLv`qDrN>e|xU
zRa5x`L)g!1=?{0oE|c4M=BcT@KC?}dNtc8;rF;Q!&_!8t9ywDIed(eJ+kz=IHu7^Y
z^^8L2E1H{!vui7!m`av%;XbE3IX9Y)<kV~9E7Ce~6gQr24>a9qbyxg_6d<jpoyNsq
zR02QcnkvWz<2u9Y7HU*pZfwnJbleq(IRJHHN!XVjWclYKb6mylVhjfWW;Y_`6K{yB
zdm*U*4BZfxf;jaYVuxLm1kMYXw!<56?wTY2$Il5Mo{HTUTa4vlHiR$WiUiv%?(8Xb
zWnALAEK@RKUR_k~aHHSznAMdvHD~fAik6mh+$aE!j-cm`OfIN~#WX=0SSx4RLMGOH
zb!uVhV@C$COKbT5Y&9ei%c$dD_IgI1r~Uhnf7w&pu-*7nl=ylUQ%OB7`;hd7iQ-?I
zF{$d3@w)pEr0N0!lp_K<XD=%WR)HusJi=s5->|?~8(u|-*fbPCgryg`HLhSk{pGwM
zS5tn`0gM3{9U9f42ploOvHyZrbQ^l%t_tD?O(V^6h;z`ztKxTG$}Vnf$kI;$FR4^W
z4b*S$`E*h91%N76G1{L+g|gaSoMN_ifubJbGDI~bpGT(~rVb{Fm}wbMVu3epo6!FW
zLYC?30H~<C8>5sB&B-9#xju#xJt6_E<)9Ufd$5rp&yyqt#igh*$+RA#EpHSpsnL&v
zS&n(;cLYo@6e`_W`-Bwa;Lu>E{={nZMmfx$E2c}Mb8TbB>5%iEGW7v%Y_N<($~Zez
z94#k+jqrZgsOC|wEhaq|f-qX9oq6~bK@a+{?WCa$%&|3U?uiA9NDlFOhA+CsTY4%u
zMk3MEah!T5AAle5@;p&(#6FmJnb8dD4YhD%i*xy$0>e>6%kpD&Gdp4$O&=S)3{?5W
zMa&$S*0~ZTX9IUcNWnd$OY`^&>i-|t*&V=eS~GtmXv;Hr^|HGdOjm1gJEhe4CXXRC
z7)-sn3O0j8%-WKex>E|Nrg20IUOs1d#n4i>?^tl%x4Xc!C1R>VLL!7$QA(nnfmp#b
zE~v6^`v4Ag6EK|7n}E+-;#i;miV67Q47N$9)M49$YG9({n~RT*>ik7-G!0JE@c&TM
zbpxU8&$TT(hf82`(<9gQrk;~!R!p0~LYAr@qHQ0sUWWppUazgJJOU7_s&yXhiLg9E
zqCu#`Lk@RD7y*$9X!Zxv_UAHTnLB!cv{45d+U%xaC=vgpT^J9jm3f!&aCnlK)Y$xP
zU{A}5%6L8GZhH?=Vf~R%{C|q1ACnrKHc#T`PS2*_m~53jW!8Nigfb<VYAWJs;3dJT
zIh&UiOV(Zn+OZhZ-V6QM4{u#K4ba7b^9eBq!XJ9*8-9^MmMo=6#N5|lPLf6IflZ*U
z=OkU_aKRFq=sF75nwge#r}Z3t`$5d@80{b3?y^jul5dx%-(L6nh^h+%68V(G<GLxw
z!pUjS87enIjhrCFNn0wGd=$>41H-p$Hew%j%5LW1M+N6C*$F*DT>76nVd4^SL@xb_
zpAO*ElRMBZv4G)>te3=<o(|(l`2Ml``iZLTsn<#FdbN`1r}vVzv!}j%;exsD*h6!b
zG5>G#c@z+=77Noz2xa#3#N!!XJ99^Nz$Viw{zn*p2@WF1-vaY%M9lniGk@gQ`&V-R
zKtQ|a5T&Fn_({-r8fnx@I(=v9QBxGr6XwT>v10IcOsZ1BC*x9~8#6>SaYvVn;6Tz5
z(j@4HdQBbKumkve^Q>4Ai8bej2$}k&pM!niDchWw_uM0doC)A?%#C-Fxi^6&w5X4U
z8Q3fFJ6VoraB(jEuy)da={H#M!*e#9;kU>K8;Jj{rJk0{Of6G_mF?iQFVu(uL8a-B
z`R`_2|KHnJyxQ8@#!|#xO;4s-==}?y(|*z%=gHU%f~Q~#1FP+0g>1opYcn|YhPxEy
z;!drTf3&?cF2?WoX~WY!gtyzp*Htm><oV;GNt)vtLno?|zlf_ULSP)&CHqabQRU%(
z-PIkeHdIH~B1NdN%lGMgS>y3OFKH5sMM%DaJc!9d<0(+1C@9RT+iLMZ=JlSI9h(0x
zY<Qf@PAuoP9-L)xUXQOm(srISt)2ONbi=&&d9Qst<q(u-R<BtTHTxQkt@=)QECrFO
zD*{`7XzXX`bx<KyD<Awp9m|ql<ZNf4_IsI_TO{Pqg&9H2ve*o(jYfAfh>~Cm8iL8c
zQjvb~=fZN$RJp=uendkCpsFIO@NhivPKC=L6r`Zfde=YaTZLP)Jfba1+Sv-SP!n{q
z)X|mJf;2aM+G8dQh<35Xfl4Ur{LurAtQI%dJXd()z*eqUzP|7N7SOc)qWFrepYtt~
z)O5e+!{(%0jAikjOM)Bc#l~#Tq6+`PT;ZvQ%onCO0<sXf&5jY0qA0*O<(kiJwY`FO
z?!L0FhAN3Udb8%SgEpBhsi2S5cE_ZbffV2hxRJSxV7A)G?*DmR!7l;q5`3hv^lB8T
za|hclgz5v5w*8?m|0*rz9;wn16(I;*`+p91>6kJH#UwJw?}OqKsK8LlcGHSjGK@rg
zSu|8g%)<M%evGv!c0Y8_?j{J$FQXQm9SRn6{<PeYpVNJFc35CH&-GY|{NH-_+pmys
zKl^0+K;u&aV&`W8a;2ZL>@5XSaQ)fZ6JU^{qZP9(@?HI-4C%kJYDM(7TBcik!;apk
zkOPjl?(l*C%27A*L@CN31FD1zpAaAYfIpcZVesQpj(Ul=TGm^gBz?b0`=}&HG8tWl
z#rzB(;tqlw-8}w;KO1*7fwlRSiwoKDjJ5M0|C!q<^AT?5(b7+s!hTssnq)oP*^<yM
zSW;8^V_6oOWHK^&^7xRM8Npt#T$qxK6m<d(7z+H%;-tt)`OYLKfu-l!KOeuGo|~>h
zxzF{-V$+9Pw}bdasJ7nR-k7f<2OIy%`BZt_J%-Pjc(!(;%v05(-Q3{WNxBU)!49rq
z&c^tzOmakO7xv5C(&yTt9rz#!M;wPDHQ(3KR1$Zh%Ry%s1-5o96rN!*dSTTz5WN$U
z?Q4z&^j^BfqN^qonhhZO(;Olf&Jt|-A%6}HB~rD|Km9wGRS@%^8&yuTDqV7yK#t1A
zGMn@Kdfa!K^V0oQ=XQ8|;7>fc0OilW%M-&|e#i{a+zaj!iv#x%%RE%vwpQuix;Exr
zUnQl<&!xyaWBy|Xk0LdODMo7qQ-1M7@L?#X2?z}hjE5lb@~*`3YuDpin|1l`=zpZ_
zi(6t!!iWS!)9-qf{c)erXb6I2p0s-*@=Z(s@&U!tpuD_o1jS;sqMtyQSt?Q(7by|I
zU7p+Z$BfF~j%R)&|Aa%IKaCLHpWBEZqAjwSu~(PET}R+u`x~~*v$Ru1gc_;3UWqza
zC27~^|I+mOB$}~{+$~a0x|d>!M&iykSF!*;|CRrbTy?qDGzEA}$v^{v2vm6<Vb_U(
z=;_lnd=+~FjBxfrS%|2Z?VsXtcrVGYO^11V`Tp{H$Mb$J?#$?^i}*WeTm&m{|7^#X
z$ocKBF6P014)RT7vb~NuL39MpJIRR~JXi8~n@!1!a%coWTuh1UTAP~}<bNpfUk>^Y
zT6O8-ZjvX<4diYIn*h#^RoEDmDNw~;$T{tkOf{H(#6w`PdWAF7Hf#1-g5lr_ESF8M
z`bEAgyIh{Q{S2GsQu;fiVPmWN88$75UL208cj1ypP}jeU<yYIEv2wo>y0fj)05{~>
zzgz3o6LFt>DWE^#@#pTOw2%wBo{^o~pwz%u87b9ZEhu*_Rvqtzto=Pl0z^MN<X|gO
zpBc6|6{I9jb8HWm4)u25^tjT<xzGt*XDoIF$X-r1DRQtOO-gxB5=D3h6%{hqElL57
zHJ$RTf}SOkpo~Mc4W4mZ{@Q4r!K55<Gkz-Ck$ySpm!2%C)MQoB&9}akjawwc*6*DX
zVDeJMc{aT`vWksz*j$n(f05@UB4|k1u7Z60bnk$~cthHfx<;#8I$S9K$a#_?3&@}U
zeQF?#Yis{{N3gU7vTg36_@TmF?F@`>`oV`%X(q!-w6%cnns{`y7~Jsuvq?(SF6~oB
z;5j=iyP&h?2};3S)Vb^DXrK%Oc_{wp`v+CS1CSI?r!2W<^?|l~d3i<(-^q8I5LvRf
zZw7xhr$8gF$@#Xx2d<Vrd3l{GvSX6Xb)HFgfcj<;#4HhO&fj{^wT*jVAi2cbUpgl>
zZf6q#|8+e0coYgB2kU^A-`soP@Oze=qFO;ektA)pwAe+T@c(o>Oew@JLRQ)LyO^`s
zEqVBap`}ScF~9u`4XoMa_YbU}q75m*2}Z@9x@^sY8)^_FW~VrYZ3@GtAw+TlPTjsJ
z?*SB7{CHJO|77};6P6+OZy<Tno`-=2qjpZy0p*XDZXtt2WE((y()8kh$>m|4up(X_
zL0KCkVd3i0>_H$?p-VpK>}_iUMHE!3TXDB_d|vGvw<C)Ier&K}E{3`1fA~j3z=qK6
z7)TBgUGZ}A#WPE<wdK6uwf7cW_WA0LTAB~B&5)+)WMk7&U|$N{5Ck#hXFq&>E@Eg6
zf%${amJ!i8)+mpF5N-CtqzfbXrLg^Nn_VS88?&|i?bCSWRVxiNGkvQ>X4+2kFzxp8
zm)~2=hN#S9{7QFrld3bW^bbe_*VXb5EdBSqK{hoOl+fRqAs8eMZ8<){rF6=@YED%g
zRS(b$RhPcI$tu6&r4TeuIgBv<bk?5#cF}VSrU<OtNp5CHY8o3n5V;Q5ho%nvlVm;-
zGQ1$p)iS@}ajGC5&GU4Ap*$AP(7V}rUP<X%5sv)9!|P%27x@srk&EA<88wk+*e2Z=
zojARyq?O3E>+rwOb&_V`?3H06wjc^Tz3wY=oFG?gzNB4$*SlQoN3ISfK4>15MJasJ
zal-+4fRLd;_dOuWv}6h}E+n24YsNgZwnFF(YC}e(kTxm|eKnYauO>#OG?7b8lDI|0
z!|idY94>7n`$1{TjxoHq?I(rHiP~1(bG)M3QfQZuLv`SzZq#tF0fI!TiHM)~Is!Bx
zUy-8;k6h?3G}XnQe=<QX>~leAgFhGGSfuz&m+@&#cV!CV6pf=1-g4)45fGTxE+~(!
zhZ12&_tSsN?rZqYrjn8JnAdla=y55uRanq!$YjBNlB9`8tj7VGL6M*UfgYn~WkbsG
z!B)?LWzf%ecdg1zHO8f7yF4XBgj{{1^gLbO|EeG-CARkT!p%8oBiai!V2L8f+ipAL
zsMQm{>|vL05-99<2XKj;xBZuy{TF>9kW(QKu4q7}_3pLdVsxFG7KIsJWJX>$;GESb
zC&$?%PHtI1h@_Twqw6e|&swo-VoN$sV$5t}YVrXO=1k4`za_k66xmAg2JD2^)^WxK
z*s<O1&dO)O{kn5@$eS>2?j&!+kTLtVQpZRV7BaHMAW!82C2@&36AeYBy+nvq<%_Eq
zrpU31%Dmg|abf1-)hQK)sAz=Nc47JZ#M=_4d@JbpBg3{KhKSZfX8gX<g`YBIP>W#J
zOwPptnN#ujJBj*%T1o#_hIIA6QMjY=mi^F4nBE0;*n71>>P-ajEjR1YYLsy;nmn$a
z*z!@H#^PfOhR=I#&qCI%&kmLwxUMVRu0qoyy?IEqt&J}VHN^rE<1$AoPF3h_1bVER
zwIl#4C0ZaYr8yoD9Y|Pnr3bu>)7JD?A=|dj@x5S6Sa$X*WAaUDb6mPyjq`Y6^*$i`
zB84}2X?!DdH$jGvOB|YU*MnUVL&1(NKMKLRa(2HToAAQ<BLx>oE2(+&P&BqiI<A<^
zvW|v=2G*4K&M4PSengR~PEf$SwPpOi(I7a4yWmiJoyBQw<wyQdAGENxF_zu>*5N(j
z6VLN0^y;;xW)#~J{AYv0o#rMv@5>$XjYQF+lX1=)Q@{GlU!4m9_yh9<9%=MkqLrDD
zz@J2a#Iuk#ehbOV!6wdob#mljA+^Iggx%%zxElH=kQYLyyf`kLcTKOmGRWZ)o$=08
zvZA-^7<cv&R-ak@y%0ocoD9Jiv&%PB3afGx7ksrGbaexwa?#*<o82-Ux=PMWi{@eX
z5aAtoYZMSn&9W8{z#P9pT@qD$S+rkUZKq+NER}XlPUu8(>x~m_t?z!FRabvK$?K$7
z)99HTs7H26HmUgaeTbOnk*zZ_V|WxM!a${X$`)?yNp3JN_ZFW+iLj_JSuoX%u2{{U
zR=g{z`}*e{V#MG_n{oy5)*rvak6kYap5ffIeNx_*=O6}qEa3=vZN(#~G{cDr2<nch
z?dUWGBP3B+1SD_*25;nPqY@Kt$rqr>8g@r(PI~@|26z^gB>%z4Yv=SiH3P#@)-gB&
zf=%KiB7fe=V!=^{vI*@JYl+7bIizcc-1geL=f5?`4&i^}^`b}U@}qx2$BLd(ujE6>
z^QQ1w`|tkCmXd-sk>--NZ%LIO=YQbHing;-@cPZ_c!|QQ{D9PemFdRECJ4kLc1Df?
z4VJ$A2M64dhx;nM#5gp+N&*Gau$1S*zu0<M=_)`F1rorXjTmoe3BgbS!L-J|R{cwk
zJ|R+e0m0=dX1)aLMfHGCSSGEP2G(?h#^W%rZeAJ73*ZB`Y5rj5B%}=fw(9s#?-16f
z+4nN_GhdowOj0jsP<@PPLZ6ws+dudt<55*t)xRm~S1(}{#n7Zgr{_f}BeT-tSRo<o
zs5XD%cG@N&weaq^?FVQaVv@HD01ib(Riv<?J-uhse4k!6zMy5xk>ZB}0zpDsj3Gl_
zhh;9C0XRx#zLbN4PFc@p5Cu|@2|Al}9FOdlck@pGM`6W=Ar&PNqu6=*IEk0(e6fjB
z{Jci>mL0~3;p+7CbTpi}ET)$+<KA!*+K2a011kt4$xfOtgjYKyI3SaTVh9n1e!$=m
z3mU@o<l&YG8A5~}<Sk)>3=YPprP^f?6g2l|0SpkgLa816$Nl11V?iA?qF!~(Xs-sU
z`iZi90ccoCuM<XX=UCWD!5Hn3W6{PsBvGhs<n{o{%>TS$DBp^ih|ZUCU7<=QDM!Uy
zOMaBf`2FKqfoXt8y2p1~!CjYD)G1K4*}vnUqp^dvEej|^(Wk8xG;Tz9tn)_-<U3BA
z>Eed1xDCsef8wM)-f{G>ia7$e*gosTVVS{CL<$U!wE7ImG7DzqYadMsxa^TR$fe%e
zN3?^wr2NjrNX>#5^ks+81T0_%w$a1hSQvyP_P|NdZm$n*OK9Aql~pu>(DMWYwm7~0
z7`DyFe}adhS`HDmuA4Y#KN}pikln6W0*|DtP#STp$cL>4Sow337YdH?ifEC>#(tin
zt?KhiXyg+V{P?E;l;K@QVjlEuLAu|adANR9UN;R&(9~N$3Uz*|Bdk8bRvMBlD2&A-
zq9}Y?Sy^75py_5Yp$psIJ~40U68ClZasxR3+C}OWNU|v`+$+<xMZ#<>3TTk?Wc@}z
z3-1tIIZsp#c$Aw3_a(Sp>vvmsge+L+4_+QP@1`P9OTQS5d>J8emOtP;ZYdGr^Oz~H
z<Rdp!<u1OHAQy`_F}N$^Qq7+EOK*-M>W}BF!JsnW?LSL>=x<KaqC(_{C|!VApj>WS
zJ=N1<#y61*?2hykdnXT970yoE@;dCeH695&ewQw=eqJ;E_Y9+#yPJS^Q)^5?v%`UX
z0!3g!<?#<5Jew>RNpVwXdwcU04OPO`(ujwMx$Nq*8{ZB4C6EBz^1~0!H`)sR7U8=g
z=EMv;5YI9N@c8}nyZS%JF!lx6tP9=v0`b4eN%CtzhwoQEn<dS_$-I?_+-~BicK0C;
zEhVPqr-EV5c-1G|dioONB=H#LKGprlish^&XbbMOGUrX4KIdtr2{AonLU(2xzKOI{
zDD7-b(wgV~0o{k}vo9NvXk-gcGw@GHtVKb+$c^)*t>DCrWTEZueLOeA`4ZjWq;RrS
z_6RI{><l#o@Rek7e{i1gRp@v)j{S;=<|XcN<NB91-P&}D1^3*{O_~Op`4@pDA>|x-
zk(<lQBZh&s6+UYn)hM8YLpzf2sco1DTe<R0rbi+)70M%DMD7nJ<o&6vY*VoxYzBSb
zk{&-)Lvg<BgX_1b>(@N>r0jY=CdkJBcA@~GSMqH}in{r#6Ws2~%SPk#nb+Id(8X<d
z1%ue}>oOWYs2~4XhCzCkIn$eh52lsPoU}D&H<wxw4sOIw&p|L}qO;x+rc6@QS+V$H
zu^V9`(te~7FQW*(vl?Zn*F;a^$irB_P}|v3$I>QM$cH|Z%wg!@dideT0r_2x@IXx=
z#`{w^(1${V%|w!hxEkL_Ga)=GUK&ophl@gF`uExz|NRLINzptZH@jU>H*i)nWZiq2
zd76+>)@%xY76HCGUEaPi1s|ebsh{u%KdRqSXZ4u(J~w^lG-Jm5{>AESEc-EumA4)B
zU~DYId*WR1I_uoy7bFv!zN`Iumoki`sS0q6Uh-TLe;-_ybY4CO!Z?@=)x@KCICOcR
z^WAJ&{Tn)2vbn9jtS;~KZUt9(p6m~@34E{*@CcUq!#C*PO}Z(3*vXrS8bQ1<agN!s
zfSh<U>ZY^js816;#9cnUIeic>t{lLrj9jhvY87T3gfY={hMX%Nv0P@MOBuGvmTm#(
zhDh1RzgkK_m-EPWow}VJP965c`b)55UupLN=&NifKA(87-N=G(1hM&Jpab=ny)Yi1
zH5(f2Ky%V2ZQyNur$MU<{Ik%v2r$KG#{x9CX<xbE&AG$MIj%*fP34#(LDvR-CgXRZ
zo6E?F5PTTCjX<~WH||>D(9`XNAT^(k`q^__?@o30`p)Q0`(>gXw7FL$zgpODd%bm?
z7!zM6yuS{-O1gZKHEp!zuFy6Px+ec;Ln7ay+H+6N`Wgd$naxLgt-jZ$-4rIP9b_l?
z;4mrB_-EO(%^SH{4q<yOj|#(=k|==d5s8i{j*`1QJ&KS;n>G7!mn2F)!|1aLa5x+d
z(G97ZNwR*Gzkn7N1qX;4OX*Z7X0j^QL6t+|`cQ6WJRbAvtt*fC?^s}Qrr}fG0XDCb
z-&mt1-<=^Tc*|}VMG>VM?^rqY#}53nnYKXt`!r7hEoG8|_j=gG$4P^-3^FW}O-xiH
zD8)e3VNNmJoX~HM#*g1F8O{I*vqTn9fQSVI5E|m<M@z<_-V*4VIs@o?-KE+!iY&im
zZbEGOUNBeV?gC1|>@h;E#;xXEZG5Utb|l%EbEGDxOmg549JIb@EeI-fJ#~hgFuI`P
zfKD_BaCOiD?*wRAA}b<S-`U=Z_qUP_tO{otgvSED+6tkzt$We88=*4GQzpgl^QQ@<
zV66J0J*xLrd9e4z>M9R7zkKI_QlcYz?`iQ#hZ&K`G%^WV>v2-6f8o`xuc3eqXxxv6
zyfLFWOkA<Qh^;F_TZN>K|CRzmA0EvccUfT!4)ao+a+l?S4*8f^e*g66ehmBJ+h_&t
z2p?NqHO9yzBE;L+uorcR?S@+Y<Ih35izq&tQX6|yof@zeMn42Uz;$56H@$v8HJp%?
z)R)I>_?*2jURRkK-$)!Ge~E_Lq(h8s--d9A6pf&WzLWqf617{D6U;yQOtJo8SK1Oo
z0Y1Uf6SI|B8{0BHH@GhxAmx8;qvOQy)QstG+{H1`H%^$K_C#Mfx_w#pcza)=PpTGn
zKs|Vkl>pwZIVCaO8JLi9?1ylgW9dDaf*`h?pIda;9Xln-B;w_y+fcrTOZy#oDRLBM
z1<4JxDX|v9?y+{My|gpVpemNq@@-It@mN3ZyuJB(s2eF#A~Ac1X#X`D7x|shTe6G#
z+n8d(1bU$ecIq*i_bc4D5}0>Aiuw*&9w?jW3a4+1HR5Ry0E$N->c);Ra=tq?60>MX
zYKZI;uIs(WE<cvBU%-kcdRwn02<?LGM9{Taw%uma%s*)52IeJ^(OTpMNxBv9?`o0N
zQEad)5@3|7K+8miIf*raRjh6-`)w_qTHu~xV0&~IR`O~bCUK5}gyJ;D6xLD6yXDOB
zp`+OSIeCO}mPlscqwChqQvhEE=$C)ewCf*rmHn`|6+zi?GeooFNQiT9R7x}egI-E7
zkp0M&VKlbc5{!!#k}3$Ong{(6Ee%jQ-c*chztH7}KK*URgdvb8A9;ls+WiM?)C}#w
z+c(+t`PKlg4D#nC6^8@bm281ulr<PK4ZHH;E2a3|Cb`l+Z72s&%<>wVvsEex5b=>>
z{&5U<nGJ?Bbfq8mCu3a>4naLVJt+stzVJ(AamHYp8LW$cw9r~<z^d<(pZcn)$W^QS
zsmmPoab2%lH;c`L!iFp(b_FZjg(;mZ;Xb@<1DyhaD*Ni(i3S+uEol7OtmM*IMU_BD
z6d~-$MD9qoW4k7UfThTR_8}dM6KpNIZ={uXJ{$o}hf=5FA2B98MR;|kp9%b+8@ArJ
zsjD9#mldJS;cSzg?V!tt%8|2G-6CkDp&-CG2}1zpXMh6Dd0Nnh-SOQ=hrdU;;1+1q
z4mdz_B+MVBF&|SUF2w+$#SCNqM!Nc3DZtiZeazM_mV5Yh2nqpl=YHGg-Qq(WvVsob
zB`B~%v>gLpWukoLf(&G49}JW?-XPuFnc8rSX^GM0bF~SlnBrB?$)EI)k}RC}VRVL=
z*9jZo3Q7o?kgI>lU--Km?`q*jl%-Mn3H4wFTM=%=X*J*sL#cye=tr9QM;~}oc!nYD
zjQJ+KnH=q1@NH50fmz_C8w7FS5-qrJpcb!t?0DD`eBZ3N3rGA_fPL~}gtAKey#t`=
zSOjG>rUOL(<;xyv(NG!D;xT_J4m=EX090uk7Xl~8H9uYirbxKcEly%~9<(f=D&z)W
z|I-o5&mNz&&`>SjimOTqILb7`58*?~-h0FlgCO-Xzyjf@o-)L2m4gUko&B5R#5hHA
zlVs?LFEGKV!;G5(g&QaZjY%*+*~trNlcXM=U!&d~QT*R&v_8>=rna%78J??*E3Aa1
z%4+M}wfeuDGL@mItqfqLSv_L*l}qPpQBza1PMIcA_CXLUP!gp2AzSEZ!8{&2Mxr0d
zqrL0VZve8-L_P@TM-G31K&92PwM+wEmZEXIo|$O-7#^oAduUf}R3K^k0a#3(#)<0J
zqK<tq;4cdG3qqfb0!fN@nf6yXDycp)MY%KHB8{;L2tsV&YBcc3Ju0E){Q0Dh1~$k}
zSi&cBZ-#F_#RAGj9QNaM`+sgB`h0!8%b76kfg<pzAJ#o`#ZK7=ZviDCAyK=-+g^T3
zL`dj|4u642B@WE!?OL93URR3h`i|}B5xe`-E?Aaj_;BG<M($icJBt=V2bD8@HS_>V
zl%LRAr7m7qG)Ablt!h1){f{pC9F@IB-Tx9g)Sxc~67zR1f+0-@UDcH%n8Q1hKwSK8
z8grH+XZtDsKp)sE>5V81<z({wq6d17{y8Vp^<D?BKxDVFU({mI#k5$+00r{pgLYYb
z%7FvcE}N9O<|V{By8L*4-{&(S?fxHMj|p0XZj^j6JT#z01wQG?s`Yj@2amI~3_cQ2
z7<?osgV;`~62_!0@Jp}AZ=w+V6qU{xoz76n`p*Y)P93lCRfqc36>HW+SawEX>DLtz
zl;6i6DD8hN(ZHT=5D#9C4iT6ni1avkL2tQc@y|p9C4&n)elWGY?uldRjejOXX#TBE
zqks9RYNV&$9rmmOI*uSd*uOa?$^Z-keMM)!K%KzY-qtd}<c#BX#lsY&u*36}KRD8L
zd38Iw32Px&Kse-$^I6>5i*6eBzMYMY9V$Oiz-5SvHz0`Fw<Fgbyj|7VrB0C}B1kLK
z!v-)2Xk_XoJ6@};9tX<}ye|qa5DJ&F#fgd_mEcxH{0-rXpr9(RRDegv4skbLKFRg8
zZq_|P<kF(N309Z^&Me=P+I%g<E2#!fB~ZZ0t#Ow~YAmFt7TUAgxxY?!oQK<R9Ju{;
zo!~qe%Qe303!fk3Jjj_4dgniQA^%!RpjsI3#CNUB(f+`jZbV3jRQ=K)@VFSz5-|ex
z1hDk!H{@ths^kpYdfzA}Y>*j7EL?VzdYe@160$A0g?#_DVk{XW`_~A=6Ba}PwnwTi
z+^Kjwur+DUcYu5<AJ!o*H=n6P{yIN^4LkjDV)9)d!9{rxe|GoI%F;odw>=`4lMU%`
zF%a5@5+UIkDcdeIFni__G0vM2e(XcT@+9ie*>RltQUc3=_>@MQt#Z>1vXf7b0pchY
z)LMKg_tpOgqs+FIWm{bSydXgVj>J2RqY%-V4E<xu_5}oaN|tm%hrdHw>LO@7Z9q6$
za<d;~ccezxUH{TLI3hOTmu0cY(H^n$6OG+uc<|lM(*Px%9;vz`$Q&_p*Zt)_ZW`$q
zLQ9P<QG+hID%tNoDHL@_Yl6aMw0@&lYeo~Y639qQUR|;KuWchT+Wxlb-*jhYhae3G
zao{5hxTDwA3S%4ld?ge&`ksympm~-I!{|S87?F3?Ljx;+jfPrEP_^|iI2{jTb_U2k
z=Y{1jOB3jzEfZoi38w&akZxkxxej<_60B!M(6wDCe?6^_E$6|W%5gqbl5C9^R%t9l
zP_kI_gH)lu@8PmdiiK*I{0X9)5KMN9*!3D`7hyc@v}9T6zT@vL$trpg;|vLzM}R#i
zo<j^UlB6GS`@GG7LI{8B1+oF5^=<F(@4rSuNAzFvD00v#Xa8hqMezeyS!(<q>gq4u
zw*2)~IDjyry=C`G@&l%ME&<At^@sEx2rwk2X^yVSzi(G*mRT9xH)8=F@52WS%Yb+D
zzi$|a*;(qS&bdF(orP+%`hmO=UqU<8eF4<-*k3rCLg#CCUS1Ng^ymx_r~Fl>{uvFp
zp?s4-auzt6B}be6W63f6fKPR1;ewv0#JbDBn`~`CaiDoKQlW&hNdHkc0Z^5H)*r-e
z!?&c5$ghF77h>!>f)xYzQlp#=_ipM5PgFF{&~5#~_7wILU?zEsN;%aW9B~KFfn)4&
zDR{~ST8@=}BSwOh3QFFgAklomF!SXf6JZf9)%#L&VYr2*z5r$r8s+xaVR)ttqnpk2
z>c0QUy%A@EtH4{bMkem&f7KhH>J8=ug6pNDLDbz2YK-qMPXKa>PoO<o*TD2*^!aN?
z*K~>HI+73L!}1fyyqC7TH{IQ{e4)+;RM{`$>)nQ}*x}vphD&Q>!-46D)NonZRNqU@
z1^eVV*;RRm%f~nq8uaOifcQRy*q$3*{l!%4NSD3a0W?{pJK0rN2W<+!KpGjin;(_n
zUjS;BK8Y~+iNma&K-trZ7}b&cHvuK-@y95^-6f;^u7m2njL`&Nf0B1B#3t45Rdu98
z@}na93Wkc~Ho8_jXF*vUkX9(oadFR_q$9%8M|K-t?^Ei%*&ZC~k}5Xij`J2`H|f2B
z;r+MLeNzZ!k4GiaMF|!&SDC`-M|Gl0K^~hXavJ3L#5Br+z%LC-X`lpIg<I##*^6ux
zF5^~jDJ^9lAr|a4L6RaY3by@D&hom^zsW>Ac+G<KuNV3bBa9?ZWY8AbhQ-=OZIc3{
zTSmZ#@Num%Q#WQoo-@%8hjDRnz6@RQ<A6T*mXK6Q8qBZv`<j(S6-<(i_h=EtFO<A9
ze^H*ID^fA3&gcj-*!+#JK2=RNceGnvJB$lI@TEjjfL96h+?y9-qr9)QP<~U~>l8Gt
zC3Nzv(~9&u$5SHMXMFrKq$y9bsgA9B$jXbdjD<Pf=!HO$6=lGda+~iXXy)@GQMB$H
z9by+M)mhRs2mAQ4`-KGFHp&z_k+!vrSL-2dn3QP|w)eLOqhOzYhNe76oPBbdw0hgV
zd2~(uk2$h2P)?FV+*Q;_1{$A2JFbsrNCx3(*X`G!^CZ=O0K}M##CZ}(_7W}tuyUnE
zG7;(hTjk%qK6Tns9JsxNW?i_T;6|SU`YicEQ;XGwjm;JOFdyS&jfQNKBEMPid*Mp;
z<LrJR=ERgtF(_0QZcSX^x<ZdTAbHvvw#<v#xFTmFWf|<)p#Z;N>3QkQqtD*$cH^3e
z-}M1<{`QHrXU944?SF7x=R>q2)WXtG>!Wb9?ZCJ{5U#9r_Mj0_KIv?I@V1ai?Q~N7
z_emWXhB#LKlWs&q;h#WfUNf53sAW1I`ko1$&Tf)yH|ihtFS&GEUL8`6+ObY=rT+V)
z9zl>R8DPKRpj=`t8|qOZ6n!@hwV>^Q5=Te`D<=qeW2vAZ_?$$TeSi1TdVfyujX3Di
z{6J*ugh$1q9s;Rc@qRD<d00bXJj_C)-EaervZc4Z6A~7y2{IDTB$|kwVDkN?=_t>;
zaz^fGyy^2{xiynkf3(fOizp<Os3jCXA+|;9Ip`D9*=(R8R{)!kX<!5k6`fNJU3P<@
z|3rkeiQ;$48Hy=JekJO4pq<@U`(*<Du?bHNA)Jr1I>8NC&aqS>_zB0m+{T@bbQ=-2
z;nYpm2B-?6ozfVipM2+^JwB$Y4P5s<0=RWDYKalMnK2u8Ur+dj5_|%#9}#-7K0lj}
zpL>cvDq`vRAv3oa*`5t@Kk$CGWw+APUzuRHPWq%mZ#9-_(T6&O?Ty|c?koiAy#Wjc
zKZHjctbdL<lrmQ;eXU)A_CWo#z#tEt8}zLhLF+!;`@Xcm!Mr2jXXTq1y$hohxFTJn
z?#T7jNOym(Hk2#By4eh(QsvxfO15lDa*1a>zeaCJ;Cmw{4p(cUeIGyujItKMJH2pf
zy{=zaYcO`jA%Do<ms%%gk};0qgwu4Fs9z};HNOumR+g6d?J*Hs5&btN6igw(2D*O{
zA3wktD?syZu}B&e$`*vW+M(39?xLQ$hEZ>f6D_c1v{Yz{s>U4X+3IQjlc|Je@_jL+
zZfc^)jEMg8`Z5B|C%7(CqtSP*M5AL1cVy*wZ-!8?mj|iL^A6iKT0NWN*u<V%Ul;k@
zcsVVOlHwR;uq2HRLi?BW7(Dh)VDLU=k(HV%H@I!li9t~?-jc2`#GWu?Gjy5<z5Cw&
z@)y@Bjp~E94C)`w@=F>)gYF;YyMj&xTkpPmP~z`VwI&kt<C={mI$l4ehi!$_@GkmP
zHRR-QhH+<J@O-{WMxc(T#OJjF4uD2+Z+&ovsWaJMRuA`nbs$&E-<s{Rr5coeD`?Cs
zh3{T58dH3qTwe<rQj>Is@+S~Rio$vkgOkAUlMMa&2^SM3j3NSnF(si|q(9S+oVY3(
zgh*+NtBFR(xq^5}jpT);)@BoQB0fgRfDk-Hmk+(Li!ISr&h+HJ;+vjOkxAD4_jvB)
z9WlNuiH$pFJ=aS^>s+a<UUtAj4uu%`evD>ND>0@7dBDG$-=o$Ip5VBHJ=Q;dkim;x
zyuOj&K#Ep0h;D=zX$jws%1r!35>?^-XS?Sl;-<=eR$r<$lyaucY?Rj5>Wlr>T+wt^
z=>XWHh=u}-D!s>LX2EqMY9+?W#Z&1Qk^RPas_&$ko8NJC-0ll9rmxZO3p@LGdhp2Q
zZJFrwS|+rd3;64ptO`pE>KL6^MQCwjT_1P!>f4PrmX>7t2oF{zGNI;~{H9*p!z}E=
zt;LK7jEH~nzX|7#1R?6L`OA$*a(}9$_)!S|&&DOXEFr`Xsgof!mgT4}WbRiMO{)EG
z&fipE-CwT#2&ck4LT6%v0hpD@vv}{v>mH#JKJ0E`y#((RM*YQiE^<m^2m0%s+Ms|g
zINoZj*LiH_>P<a7kZGaT@zPJo^Iq(F6|T}8z=l1${ALV)rhD<Mk5DBH^oz+zKd?Ua
zMJjlyOp2Zc+KUj={4GX`6+0E(=uXHdFU5w}+a%|p{hh(Ehi0BLPP8A7H(IK_brfq*
zI_5E<FN9fF&jxb;o+(+Mk1o~FO;0^=9*ycHg04o9i=!h{jl<U`EPy;fbL978ADx>e
z(RQqRd)@!(<@qm-R(q?C(79cj8?^pc*n`G5J(kNn6tLYzTf9a@XxvV+X6y(^7Hn{t
zOF!kM_^00Y{q~Pp?n~<?BR40fEA>7^MmU;)TsKcAJj#NimVfo<<S)5@(Z`#=@Y>5!
zS72-V7UNTHZ@t$@7fY=kygS<{XC{m*&V0&N?wro(oJ0;1hJ|mazoJtaT>j|`tBw{(
zrO`oVWv+BG7$l`s-qaH|{Sz@X($OzUVTk-O#On#LI2NeGFjoCX(;oO-S^YZ)xW)gy
z@QQpw$7>S)n1BN}1%Fsf{pzqkQvTfgwUtI}g*TQV$r(oOo2&ksj{4bD>x-!zF3XA6
z4dQQK!W18z7tUP7^Fn*?1L6$rKeh%b-*~6~;eSOBWR~_Gr_uk`Kkzdb)1dkBpLIcf
zH)r7GYx^q67rYjgTQvBAS65ij)c={)$GLf{y_25i%dBC`P82);+vaxUQPU|m_~MGg
zYqITb9uhml_HM1&;k-ptzrW|pIsbR*<+#eq`dp2d=9g7!OQp3YO`PxcaZg3`$M86n
z$9szNng!NcJ>9nc=0Yd7eLqAuYM<naeAsz@@zsO>>YFF>?#uccZ5L8?*=5zS?MFX%
zsBgSfw)My)-fh?R<lWy=clXcLtwu*4%`ntcym!2`J|L*E#k<-dRNLga)QlN?$MbiH
z{avzcWAE>|hgPNa-d|Z<b&NgQ^tLsB@7g)aTKjta{(Pu!*uOtc$C2N>=XC+|_lVnx
z`WG*qT75I2e#NW|(ey9I2|xGc3eP)Pv3C9OyX_s<r%iV5;yQV%?D;3(Ng6%1++vB_
zxGt^=k-HSPLbmch-%Pc=JI%i)l+VrB8!iyWo88YUxV6WyOMk}Byu}af=Dyf>t;N9g
zso6Sq0sYKWYyYhW0_Gg6{x4I%Z$T*Y;`I(@xu5Q@yuCy>Rj{T1b=%G-@~ih(_4>H}
zOg+!^i2rh@*t^x9_rhOWdwlR&XDe2FO;}J+Ffi2Nu;_N>##<}S_Ud!mzF*q;;#_Sw
z=S35Z@D2BkW@iAE0D-FJRJ$k_Fz|GgZqC@dr-K~uFPoo_1$3wd%g^Zb1`e!1)&b#X
W=l5kWRnPkj68CiVb6Mw<&;$U_ig}0t

diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 8a7179a0640..b74eaffecc5 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -62,7 +62,6 @@ import OCIStorageSVG from "../../../public/OCI.svg";
 import googleCloudStorageIcon from "../../../public/GoogleCloudStorage.png";
 import guruIcon from "../../../public/Guru.svg";
 import gongIcon from "../../../public/Gong.png";
-import requestTrackerIcon from "../../../public/RequestTracker.png";
 import zulipIcon from "../../../public/Zulip.png";
 import linearIcon from "../../../public/Linear.png";
 import hubSpotIcon from "../../../public/HubSpot.png";
@@ -1178,13 +1177,6 @@ export const GuruIcon = ({
   className = defaultTailwindCSS,
 }: IconProps) => <LogoIcon size={size} className={className} src={guruIcon} />;
 
-export const RequestTrackerIcon = ({
-  size = 16,
-  className = defaultTailwindCSS,
-}: IconProps) => (
-  <LogoIcon size={size} className={className} src={requestTrackerIcon} />
-);
-
 export const SalesforceIcon = ({
   size = 16,
   className = defaultTailwindCSS,
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index d722fcf9848..4e7df383599 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -552,11 +552,6 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
     ],
     advanced_values: [],
   },
-  requesttracker: {
-    description: "Configure HubSpot connector",
-    values: [],
-    advanced_values: [],
-  },
   hubspot: {
     description: "Configure HubSpot connector",
     values: [],
@@ -1116,8 +1111,6 @@ export interface NotionConfig {
 
 export interface HubSpotConfig {}
 
-export interface RequestTrackerConfig {}
-
 export interface Document360Config {
   workspace: string;
   categories?: string[];
diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts
index d7bcef0adaf..532f8f6de76 100644
--- a/web/src/lib/connectors/credentials.ts
+++ b/web/src/lib/connectors/credentials.ts
@@ -106,12 +106,6 @@ export interface HubSpotCredentialJson {
   hubspot_access_token: string;
 }
 
-export interface RequestTrackerCredentialJson {
-  requesttracker_username: string;
-  requesttracker_password: string;
-  requesttracker_base_url: string;
-}
-
 export interface Document360CredentialJson {
   portal_id: string;
   document360_api_token: string;
@@ -224,11 +218,6 @@ export const credentialTemplates: Record<ValidSources, any> = {
     portal_id: "",
     document360_api_token: "",
   } as Document360CredentialJson,
-  requesttracker: {
-    requesttracker_username: "",
-    requesttracker_password: "",
-    requesttracker_base_url: "",
-  } as RequestTrackerCredentialJson,
   loopio: {
     loopio_subdomain: "",
     loopio_client_id: "",
@@ -371,12 +360,6 @@ export const credentialDisplayNames: Record<string, string> = {
 
   // HubSpot
   hubspot_access_token: "HubSpot Access Token",
-
-  // Request Tracker
-  requesttracker_username: "Request Tracker Username",
-  requesttracker_password: "Request Tracker Password",
-  requesttracker_base_url: "Request Tracker Base URL",
-
   // Document360
   portal_id: "Document360 Portal ID",
   document360_api_token: "Document360 API Token",
diff --git a/web/src/lib/sources.ts b/web/src/lib/sources.ts
index 18bc3336ae7..aa053912947 100644
--- a/web/src/lib/sources.ts
+++ b/web/src/lib/sources.ts
@@ -21,7 +21,6 @@ import {
   LoopioIcon,
   NotionIcon,
   ProductboardIcon,
-  RequestTrackerIcon,
   R2Icon,
   SalesforceIcon,
   SharepointIcon,
@@ -243,12 +242,6 @@ const SOURCE_METADATA_MAP: SourceMap = {
     category: SourceCategory.Wiki,
     docs: "https://docs.danswer.dev/connectors/mediawiki",
   },
-  requesttracker: {
-    icon: RequestTrackerIcon,
-    displayName: "Request Tracker",
-    category: SourceCategory.CustomerSupport,
-    docs: "https://docs.danswer.dev/connectors/requesttracker",
-  },
   clickup: {
     icon: ClickupIcon,
     displayName: "Clickup",
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index d4f43a024d6..46d5efd8444 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -241,7 +241,6 @@ const validSources = [
   "linear",
   "hubspot",
   "document360",
-  "requesttracker",
   "file",
   "google_sites",
   "loopio",

From 1261d859acb5fb2492a57cea7d3bbcb1bbcdb623 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 26 Oct 2024 16:27:40 -0700
Subject: [PATCH 237/376] Tenant aware JWT strategy (#2943)

* add tenantJWTSrategy

* nit
---
 backend/danswer/auth/users.py                 | 40 +++++++------------
 backend/danswer/configs/app_configs.py        |  3 --
 backend/danswer/db/engine.py                  | 15 ++++---
 .../server/middleware/tenant_tracking.py      | 10 +++--
 .../integration/common_utils/managers/user.py |  8 +---
 web/src/app/auth/logout/route.ts              |  2 +-
 6 files changed, 34 insertions(+), 44 deletions(-)

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index a540cbaa4cc..66659d7c6ef 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -62,7 +62,6 @@
 from danswer.configs.app_configs import EMAIL_FROM
 from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
-from danswer.configs.app_configs import SECRET_JWT_KEY
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import SMTP_PASS
 from danswer.configs.app_configs import SMTP_PORT
@@ -295,29 +294,6 @@ async def create(
             CURRENT_TENANT_ID_CONTEXTVAR.reset(token)
             return user
 
-    async def on_after_login(
-        self,
-        user: User,
-        request: Request | None = None,
-        response: Response | None = None,
-    ) -> None:
-        if response is None or not MULTI_TENANT:
-            return
-
-        tenant_id = get_tenant_id_for_email(user.email)
-
-        tenant_token = jwt.encode(
-            {"tenant_id": tenant_id}, SECRET_JWT_KEY, algorithm="HS256"
-        )
-
-        response.set_cookie(
-            key="tenant_details",
-            value=tenant_token,
-            httponly=True,
-            secure=WEB_DOMAIN.startswith("https"),
-            samesite="lax",
-        )
-
     async def oauth_callback(
         self: "BaseUserManager[models.UOAP, models.ID]",
         oauth_name: str,
@@ -527,8 +503,22 @@ async def get_user_manager(
 )
 
 
+# This strategy is used to add tenant_id to the JWT token
+class TenantAwareJWTStrategy(JWTStrategy):
+    async def write_token(self, user: User) -> str:
+        tenant_id = get_tenant_id_for_email(user.email)
+        data = {
+            "sub": str(user.id),
+            "aud": self.token_audience,
+            "tenant_id": tenant_id,
+        }
+        return generate_jwt(
+            data, self.encode_key, self.lifetime_seconds, algorithm=self.algorithm
+        )
+
+
 def get_jwt_strategy() -> JWTStrategy:
-    return JWTStrategy(
+    return TenantAwareJWTStrategy(
         secret=USER_AUTH_SECRET,
         lifetime_seconds=SESSION_EXPIRE_TIME_SECONDS,
     )
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 8710569ea39..453a18cd594 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -472,9 +472,6 @@
 ENABLE_EMAIL_INVITES = os.environ.get("ENABLE_EMAIL_INVITES", "").lower() == "true"
 
 # Security and authentication
-SECRET_JWT_KEY = os.environ.get(
-    "SECRET_JWT_KEY", ""
-)  # Used for encryption of the JWT token for user's tenant context
 DATA_PLANE_SECRET = os.environ.get(
     "DATA_PLANE_SECRET", ""
 )  # Used for secure communication between the control and data plane
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index b071a41e8c3..d133675f91b 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -35,7 +35,7 @@
 from danswer.configs.app_configs import POSTGRES_POOL_RECYCLE
 from danswer.configs.app_configs import POSTGRES_PORT
 from danswer.configs.app_configs import POSTGRES_USER
-from danswer.configs.app_configs import SECRET_JWT_KEY
+from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
 from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.utils.logger import setup_logger
@@ -263,17 +263,20 @@ def get_current_tenant_id(request: Request) -> str:
         CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
         return tenant_id
 
-    token = request.cookies.get("tenant_details")
+    token = request.cookies.get("fastapiusersauth")
     if not token:
         current_value = CURRENT_TENANT_ID_CONTEXTVAR.get()
         # If no token is present, use the default schema or handle accordingly
         return current_value
 
     try:
-        payload = jwt.decode(token, SECRET_JWT_KEY, algorithms=["HS256"])
-        tenant_id = payload.get("tenant_id")
-        if not tenant_id:
-            return CURRENT_TENANT_ID_CONTEXTVAR.get()
+        payload = jwt.decode(
+            token,
+            USER_AUTH_SECRET,
+            audience=["fastapi-users:auth"],
+            algorithms=["HS256"],
+        )
+        tenant_id = payload.get("tenant_id", POSTGRES_DEFAULT_SCHEMA)
         if not is_valid_schema_name(tenant_id):
             raise HTTPException(status_code=400, detail="Invalid tenant ID format")
         CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index f7a4ab0b6a3..42291712a45 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -9,7 +9,7 @@
 from fastapi import Response
 
 from danswer.configs.app_configs import MULTI_TENANT
-from danswer.configs.app_configs import SECRET_JWT_KEY
+from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.db.engine import is_valid_schema_name
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
@@ -25,11 +25,15 @@ async def set_tenant_id(
             if not MULTI_TENANT:
                 tenant_id = POSTGRES_DEFAULT_SCHEMA
             else:
-                token = request.cookies.get("tenant_details")
+                token = request.cookies.get("fastapiusersauth")
+
                 if token:
                     try:
                         payload = jwt.decode(
-                            token, SECRET_JWT_KEY, algorithms=["HS256"]
+                            token,
+                            USER_AUTH_SECRET,
+                            audience=["fastapi-users:auth"],
+                            algorithms=["HS256"],
                         )
                         tenant_id = payload.get("tenant_id", POSTGRES_DEFAULT_SCHEMA)
                         if not is_valid_schema_name(tenant_id):
diff --git a/backend/tests/integration/common_utils/managers/user.py b/backend/tests/integration/common_utils/managers/user.py
index ecc6d3206b9..2b9aa6e189d 100644
--- a/backend/tests/integration/common_utils/managers/user.py
+++ b/backend/tests/integration/common_utils/managers/user.py
@@ -70,18 +70,14 @@ def login_as_user(test_user: DATestUser) -> DATestUser:
 
         cookies = response.cookies.get_dict()
         session_cookie = cookies.get("fastapiusersauth")
-        tenant_details_cookie = cookies.get("tenant_details")
 
         if not session_cookie:
             raise Exception("Failed to login")
 
         print(f"Logged in as {test_user.email}")
 
-        # Set both cookies in the headers
-        test_user.headers["Cookie"] = (
-            f"fastapiusersauth={session_cookie}; "
-            f"tenant_details={tenant_details_cookie}"
-        )
+        # Set cookies in the headers
+        test_user.headers["Cookie"] = f"fastapiusersauth={session_cookie}; "
         return test_user
 
     @staticmethod
diff --git a/web/src/app/auth/logout/route.ts b/web/src/app/auth/logout/route.ts
index cd731810ca3..9a7d22ae0dc 100644
--- a/web/src/app/auth/logout/route.ts
+++ b/web/src/app/auth/logout/route.ts
@@ -14,7 +14,7 @@ export const POST = async (request: NextRequest) => {
 
   // Delete cookies only if cloud is enabled (jwt auth)
   if (NEXT_PUBLIC_CLOUD_ENABLED) {
-    const cookiesToDelete = ["fastapiusersauth", "tenant_details"];
+    const cookiesToDelete = ["fastapiusersauth"];
     const cookieOptions = {
       path: "/",
       secure: process.env.NODE_ENV === "production",

From 0c2cc7499f3be257cf08e973d60c3558ce7514a7 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Sat, 26 Oct 2024 19:54:22 -0700
Subject: [PATCH 238/376] Move user fetching to SS + parallelize some
 server-side calls (#2932)

* Move user fetching to SS

* Cleanup

* Add more logging

* Small cleanup
---
 .../admin/connectors/[connector]/Sidebar.tsx  | 49 ++++++++++++++-----
 web/src/app/chat/page.tsx                     |  1 -
 web/src/app/layout.tsx                        | 12 +++--
 web/src/components/context/AppProvider.tsx    |  5 +-
 web/src/components/user/UserProvider.tsx      | 35 +++++++------
 5 files changed, 69 insertions(+), 33 deletions(-)

diff --git a/web/src/app/admin/connectors/[connector]/Sidebar.tsx b/web/src/app/admin/connectors/[connector]/Sidebar.tsx
index c31e3930c24..e227159ed9d 100644
--- a/web/src/app/admin/connectors/[connector]/Sidebar.tsx
+++ b/web/src/app/admin/connectors/[connector]/Sidebar.tsx
@@ -8,12 +8,47 @@ import { credentialTemplates } from "@/lib/connectors/credentials";
 import Link from "next/link";
 import { useUser } from "@/components/user/UserProvider";
 import { useContext } from "react";
+import { User } from "@/lib/types";
+
+function BackButton({
+  isAdmin,
+  isCurator,
+  user,
+}: {
+  isAdmin: boolean;
+  isCurator: boolean;
+  user: User | null;
+}) {
+  const buttonText = isAdmin ? "Admin Page" : "Curator Page";
+
+  if (!isAdmin && !isCurator) {
+    console.error(
+      `User is neither admin nor curator, defaulting to curator view. Found user:\n ${JSON.stringify(
+        user,
+        null,
+        2
+      )}`
+    );
+  }
+
+  return (
+    <div className="mx-3 mt-6 flex-col flex items-center">
+      <Link
+        href={"/admin/add-connector"}
+        className="w-full p-2 bg-white border-border border rounded items-center hover:bg-background-200 cursor-pointer transition-all duration-150 flex gap-x-2"
+      >
+        <SettingsIcon className="flex-none " />
+        <p className="my-auto flex items-center text-sm">{buttonText}</p>
+      </Link>
+    </div>
+  );
+}
 
 export default function Sidebar() {
   const { formStep, setFormStep, connector, allowAdvanced, allowCreate } =
     useFormContext();
   const combinedSettings = useContext(SettingsContext);
-  const { isLoadingUser, isAdmin } = useUser();
+  const { isCurator, isAdmin, user } = useUser();
   if (!combinedSettings) {
     return null;
   }
@@ -55,17 +90,7 @@ export default function Sidebar() {
             </div>
           </div>
 
-          <div className="mx-3 mt-6 gap-y-1 flex-col flex gap-x-1.5 items-center items-center">
-            <Link
-              href={"/admin/add-connector"}
-              className="w-full p-2 bg-white border-border border rounded items-center hover:bg-background-200 cursor-pointer transition-all duration-150 flex gap-x-2"
-            >
-              <SettingsIcon className="flex-none " />
-              <p className="my-auto flex items-center text-sm">
-                {isAdmin ? "Admin Page" : "Curator Page"}
-              </p>
-            </Link>
-          </div>
+          <BackButton isAdmin={isAdmin} isCurator={isCurator} user={user} />
 
           <div className="h-full flex">
             <div className="mx-auto w-full max-w-2xl px-4 py-8">
diff --git a/web/src/app/chat/page.tsx b/web/src/app/chat/page.tsx
index 4f91699b48c..05207ea9282 100644
--- a/web/src/app/chat/page.tsx
+++ b/web/src/app/chat/page.tsx
@@ -5,7 +5,6 @@ import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrap
 import { ChatProvider } from "@/components/context/ChatContext";
 import { fetchChatData } from "@/lib/chat/fetchChatData";
 import WrappedChat from "./WrappedChat";
-import { AssistantsProvider } from "@/components/context/AssistantsContext";
 
 export default async function Page({
   searchParams,
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index e7e3c2c416d..6fd2799ab85 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -19,6 +19,7 @@ import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
 import { AppProvider } from "@/components/context/AppProvider";
 import { PHProvider } from "./providers";
 import { default as dynamicImport } from "next/dynamic";
+import { getCurrentUserSS } from "@/lib/userSS";
 
 const PostHogPageView = dynamicImport(() => import("./PostHogPageView"), {
   ssr: false,
@@ -57,7 +58,11 @@ export default async function RootLayout({
 }: {
   children: React.ReactNode;
 }) {
-  const combinedSettings = await fetchSettingsSS();
+  const [combinedSettings, assistantsData, user] = await Promise.all([
+    fetchSettingsSS(),
+    fetchAssistantData(),
+    getCurrentUserSS(),
+  ]);
 
   const productGating =
     combinedSettings?.settings.product_gating ?? GatingType.NONE;
@@ -165,11 +170,12 @@ export default async function RootLayout({
     );
   }
 
-  const data = await fetchAssistantData();
-  const { assistants, hasAnyConnectors, hasImageCompatibleModel } = data;
+  const { assistants, hasAnyConnectors, hasImageCompatibleModel } =
+    assistantsData;
 
   return getPageContent(
     <AppProvider
+      user={user}
       settings={combinedSettings}
       assistants={assistants}
       hasAnyConnectors={hasAnyConnectors}
diff --git a/web/src/components/context/AppProvider.tsx b/web/src/components/context/AppProvider.tsx
index 9ce4d864fbf..66d8a0eff15 100644
--- a/web/src/components/context/AppProvider.tsx
+++ b/web/src/components/context/AppProvider.tsx
@@ -5,9 +5,11 @@ import { ProviderContextProvider } from "../chat_search/ProviderContext";
 import { SettingsProvider } from "../settings/SettingsProvider";
 import { AssistantsProvider } from "./AssistantsContext";
 import { Persona } from "@/app/admin/assistants/interfaces";
+import { User } from "@/lib/types";
 
 interface AppProviderProps {
   children: React.ReactNode;
+  user: User | null;
   settings: CombinedSettings;
   assistants: Persona[];
   hasAnyConnectors: boolean;
@@ -16,13 +18,14 @@ interface AppProviderProps {
 
 export const AppProvider = ({
   children,
+  user,
   settings,
   assistants,
   hasAnyConnectors,
   hasImageCompatibleModel,
 }: AppProviderProps) => {
   return (
-    <UserProvider>
+    <UserProvider user={user}>
       <ProviderContextProvider>
         <SettingsProvider settings={settings}>
           <AssistantsProvider
diff --git a/web/src/components/user/UserProvider.tsx b/web/src/components/user/UserProvider.tsx
index 5e7bb753520..42086d6350f 100644
--- a/web/src/components/user/UserProvider.tsx
+++ b/web/src/components/user/UserProvider.tsx
@@ -14,20 +14,21 @@ interface UserContextType {
 
 const UserContext = createContext<UserContextType | undefined>(undefined);
 
-export function UserProvider({ children }: { children: React.ReactNode }) {
-  const [user, setUser] = useState<User | null>(null);
-  const [isLoadingUser, setIsLoadingUser] = useState(true);
-  const [isAdmin, setIsAdmin] = useState(false);
-  const [isCurator, setIsCurator] = useState(false);
+export function UserProvider({
+  children,
+  user,
+}: {
+  children: React.ReactNode;
+  user: User | null;
+}) {
+  const [upToDateUser, setUpToDateUser] = useState<User | null>(user);
+  const [isLoadingUser, setIsLoadingUser] = useState(false);
 
   const fetchUser = async () => {
     try {
-      const user = await getCurrentUser();
-      setUser(user);
-      setIsAdmin(user?.role === UserRole.ADMIN);
-      setIsCurator(
-        user?.role === UserRole.CURATOR || user?.role == UserRole.GLOBAL_CURATOR
-      );
+      setIsLoadingUser(true);
+      const currentUser = await getCurrentUser();
+      setUpToDateUser(currentUser);
     } catch (error) {
       console.error("Error fetching current user:", error);
     } finally {
@@ -35,17 +36,19 @@ export function UserProvider({ children }: { children: React.ReactNode }) {
     }
   };
 
-  useEffect(() => {
-    fetchUser();
-  }, []);
-
   const refreshUser = async () => {
     await fetchUser();
   };
 
   return (
     <UserContext.Provider
-      value={{ user, isLoadingUser, isAdmin, refreshUser, isCurator }}
+      value={{
+        user: upToDateUser,
+        isLoadingUser,
+        refreshUser,
+        isAdmin: upToDateUser?.role === UserRole.ADMIN,
+        isCurator: upToDateUser?.role === UserRole.CURATOR,
+      }}
     >
       {children}
     </UserContext.Provider>

From e5d84cae1b2c422ae5abe957311bb5cb65a1e4cd Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Sat, 26 Oct 2024 23:06:24 -0400
Subject: [PATCH 239/376] Clean up code

---
 .../danswer/connectors/freshdesk/connector.py | 80 +++++++------------
 1 file changed, 29 insertions(+), 51 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 34651b43441..e0690e34c5b 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -11,41 +11,41 @@
 
 logger = setup_logger()
 
+def create_doc_from_ticket(ticket: dict, domain: str) -> Document:
+    return Document(
+        id=ticket["id"],
+        sections=Section(
+            link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
+            text=json.dumps({
+                key: value
+                for key, value in ticket.items()
+                if isinstance(value, str)
+            }, default=str),
+        ),
+        source=DocumentSource.FRESHDESK,
+        semantic_identifier=ticket["subject"],
+        metadata={
+            key: value.isoformat() if isinstance(value, datetime) else str(value)
+            for key, value in ticket.items()
+            if isinstance(value, (str, datetime)) and key not in ["description", "description_text"]
+        },
+    )
 
 class FreshdeskConnector(PollConnector, LoadConnector):
     def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
 
-    def ticket_link(self, tid: int) -> str:
-        return f"https://{self.domain}.freshdesk.com/helpdesk/tickets/{tid}"
-
-    def build_doc_sections_from_ticket(self, ticket: dict) -> List[Section]:
-        # Use list comprehension for building sections
-        return [
-            Section(
-                link=self.ticket_link(int(ticket["id"])),
-                text=json.dumps({
-                    key: value
-                    for key, value in ticket.items()
-                    if isinstance(value, str)
-                }, default=str),
-            )
-        ]
-
-    def strip_html_tags(self, html: str) -> str:
-        return parse_html_page_basic(html)
-
-    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
+    def _load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
         self.api_key = credentials.get("freshdesk_api_key")
         self.domain = credentials.get("freshdesk_domain")
         self.password = credentials.get("freshdesk_password")
         return None
     
-    def _fetch_tickets(self, start: datetime, end: datetime) -> List[dict]:
+    def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = None) -> List[dict]:
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
         
-        start_time = start.strftime("%Y-%m-%dT%H:%M:%SZ")
+        start_time = start.isoformat() if start else None
 
         all_tickets = []
         page = 1
@@ -54,7 +54,7 @@ def _fetch_tickets(self, start: datetime, end: datetime) -> List[dict]:
         while True:
             freshdesk_url = (
                 f"https://{self.domain}.freshdesk.com/api/v2/tickets"
-                f"?include=description&updated_since={start_time}"
+                f"?include=description{f"&updated_since={start_time}" if start_time else ""}"
                 f"&per_page={per_page}&page={page}"
             )
             response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
@@ -74,20 +74,14 @@ def _fetch_tickets(self, start: datetime, end: datetime) -> List[dict]:
 
         return all_tickets
 
-    def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsOutput:
-        # Ensure start and end are in UTC
-        start = start.astimezone(timezone.utc)
-        end = end.astimezone(timezone.utc)
-        
+    def _process_tickets(self, start: datetime | None = None, end: datetime | None = None) -> GenerateDocumentsOutput:
         tickets = self._fetch_tickets(start, end)
+
         doc_batch: List[Document] = []
 
         for ticket in tickets:
-            # Convert date fields to UTC
             for date_field in ["created_at", "updated_at", "due_by"]:
-                if ticket[date_field].endswith('Z'):
-                    ticket[date_field] = ticket[date_field][:-1] + '+00:00'
-                ticket[date_field] = datetime.fromisoformat(ticket[date_field]).replace(tzinfo=timezone.utc)
+                ticket[date_field] = datetime.fromisoformat(ticket[date_field].rstrip('Z'))
 
             # Convert all other values to strings
             ticket = {
@@ -104,28 +98,12 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
             ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
 
             # Stripping HTML tags from the description field
-            ticket["description"] = self.strip_html_tags(ticket["description"])
+            ticket["description"] = parse_html_page_basic(ticket["description"])
 
             # Remove extra white spaces from the description field
             ticket["description"] = " ".join(ticket["description"].split())
 
-            # Use list comprehension for building sections
-            sections = self.build_doc_sections_from_ticket(ticket)
-
-            created_at = ticket["created_at"]
-            if start <= created_at <= end:
-                doc = Document(
-                    id=ticket["id"],
-                    sections=sections,
-                    source=DocumentSource.FRESHDESK,
-                    semantic_identifier=ticket["subject"],
-                    metadata={
-                        key: value.isoformat() if isinstance(value, datetime) else str(value)
-                        for key, value in ticket.items()
-                        if isinstance(value, (str, datetime)) and key not in ["description", "description_text"]
-                    },
-                )
-                doc_batch.append(doc)
+            doc_batch.append(create_doc_from_ticket(ticket, self.domain))
 
             if len(doc_batch) >= self.batch_size:
                 yield doc_batch
@@ -135,7 +113,7 @@ def _process_tickets(self, start: datetime, end: datetime) -> GenerateDocumentsO
             yield doc_batch
 
     def load_from_state(self) -> GenerateDocumentsOutput:
-        return self._fetch_tickets()
+        return self._process_tickets()
 
     def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:
         start_datetime = datetime.fromtimestamp(start, tz=timezone.utc)

From da3c5e3711c5b7bf33129afd0b8a0b8776411e23 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 26 Oct 2024 22:15:41 -0700
Subject: [PATCH 240/376] Feat: add clean logging for api routes (#2928)

* feat: add clean logging for api routes

* nit

* `MULTI_TENANT` must be shared config

* nit
---
 backend/alembic/env.py                           |  2 +-
 backend/danswer/auth/users.py                    |  2 +-
 .../background/celery/tasks/indexing/tasks.py    |  2 +-
 backend/danswer/configs/app_configs.py           |  5 -----
 backend/danswer/configs/constants.py             |  3 ---
 backend/danswer/db/engine.py                     |  4 ++--
 backend/danswer/db/swap_index.py                 |  2 +-
 backend/danswer/document_index/factory.py        |  2 +-
 backend/danswer/document_index/vespa/index.py    |  2 +-
 backend/danswer/key_value_store/store.py         |  2 +-
 backend/danswer/main.py                          |  2 +-
 .../search/preprocessing/preprocessing.py        |  2 +-
 backend/danswer/server/manage/users.py           |  2 +-
 backend/danswer/setup.py                         |  2 +-
 backend/danswer/utils/logger.py                  | 16 ++++++++++++++++
 .../ee/danswer/background/celery/apps/primary.py |  2 +-
 backend/ee/danswer/main.py                       |  2 +-
 .../danswer/server/middleware/tenant_tracking.py |  5 +----
 backend/ee/danswer/server/tenants/api.py         |  2 +-
 backend/shared_configs/configs.py                |  5 +++++
 20 files changed, 38 insertions(+), 28 deletions(-)

diff --git a/backend/alembic/env.py b/backend/alembic/env.py
index 7ccd04cf16a..019ea94b836 100644
--- a/backend/alembic/env.py
+++ b/backend/alembic/env.py
@@ -9,7 +9,7 @@
 from sqlalchemy.ext.asyncio import create_async_engine
 from sqlalchemy.sql import text
 
-from danswer.configs.app_configs import MULTI_TENANT
+from shared_configs.configs import MULTI_TENANT
 from danswer.db.engine import build_connection_string
 from danswer.db.models import Base
 from celery.backends.database.session import ResultModelBase  # type: ignore
diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 66659d7c6ef..13a0902b8cc 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -60,7 +60,6 @@
 from danswer.configs.app_configs import DISABLE_AUTH
 from danswer.configs.app_configs import DISABLE_VERIFICATION
 from danswer.configs.app_configs import EMAIL_FROM
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import REQUIRE_EMAIL_VERIFICATION
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import SMTP_PASS
@@ -94,6 +93,7 @@
 from danswer.utils.telemetry import RecordType
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 2b5b987672a..1d8723f1121 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -24,7 +24,6 @@
 from danswer.background.indexing.run_indexing import run_indexing_entrypoint
 from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import CELERY_INDEXING_LOCK_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DANSWER_REDIS_FUNCTION_LOCK_PREFIX
@@ -56,6 +55,7 @@
 from danswer.utils.variable_functionality import global_version
 from shared_configs.configs import INDEXING_MODEL_SERVER_HOST
 from shared_configs.configs import INDEXING_MODEL_SERVER_PORT
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 453a18cd594..1e9e0099d8b 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -461,11 +461,6 @@
 AZURE_DALLE_DEPLOYMENT_NAME = os.environ.get("AZURE_DALLE_DEPLOYMENT_NAME")
 
 
-# Cloud configuration
-
-# Multi-tenancy configuration
-MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
-
 # Use managed Vespa (Vespa Cloud). If set, must also set VESPA_CLOUD_URL, VESPA_CLOUD_CERT_PATH and VESPA_CLOUD_KEY_PATH
 MANAGED_VESPA = os.environ.get("MANAGED_VESPA", "").lower() == "true"
 
diff --git a/backend/danswer/configs/constants.py b/backend/danswer/configs/constants.py
index caa903af71d..ac09f2fa90b 100644
--- a/backend/danswer/configs/constants.py
+++ b/backend/danswer/configs/constants.py
@@ -31,9 +31,6 @@
     "You can still use Danswer as a search engine."
 )
 
-# Prefix used for all tenant ids
-TENANT_ID_PREFIX = "tenant_"
-
 # Postgres connection constants for application_name
 POSTGRES_WEB_APP_NAME = "web"
 POSTGRES_INDEXER_APP_NAME = "indexer"
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index d133675f91b..7336493e0a0 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -25,7 +25,6 @@
 
 from danswer.configs.app_configs import LOG_POSTGRES_CONN_COUNTS
 from danswer.configs.app_configs import LOG_POSTGRES_LATENCY
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_SIZE
 from danswer.configs.app_configs import POSTGRES_DB
@@ -37,10 +36,11 @@
 from danswer.configs.app_configs import POSTGRES_USER
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
-from danswer.configs.constants import TENANT_ID_PREFIX
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.configs import TENANT_ID_PREFIX
 
 logger = setup_logger()
 
diff --git a/backend/danswer/db/swap_index.py b/backend/danswer/db/swap_index.py
index 415ade5df00..4961f05bbd4 100644
--- a/backend/danswer/db/swap_index.py
+++ b/backend/danswer/db/swap_index.py
@@ -1,6 +1,5 @@
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.db.connector_credential_pair import get_connector_credential_pairs
 from danswer.db.connector_credential_pair import resync_cc_pair
@@ -15,6 +14,7 @@
 from danswer.db.search_settings import update_search_settings_status
 from danswer.key_value_store.factory import get_kv_store
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
 
 
 logger = setup_logger()
diff --git a/backend/danswer/document_index/factory.py b/backend/danswer/document_index/factory.py
index 4f188ebc0ba..92dde3dda43 100644
--- a/backend/danswer/document_index/factory.py
+++ b/backend/danswer/document_index/factory.py
@@ -1,9 +1,9 @@
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.search_settings import get_current_search_settings
 from danswer.document_index.interfaces import DocumentIndex
 from danswer.document_index.vespa.index import VespaIndex
+from shared_configs.configs import MULTI_TENANT
 
 
 def get_default_document_index(
diff --git a/backend/danswer/document_index/vespa/index.py b/backend/danswer/document_index/vespa/index.py
index 86bc481e573..31d765b3195 100644
--- a/backend/danswer/document_index/vespa/index.py
+++ b/backend/danswer/document_index/vespa/index.py
@@ -17,7 +17,6 @@
 import requests  # type: ignore
 
 from danswer.configs.app_configs import DOCUMENT_INDEX_NAME
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.chat_configs import DOC_TIME_DECAY
 from danswer.configs.chat_configs import NUM_RETURNED_HITS
 from danswer.configs.chat_configs import TITLE_CONTENT_RATIO
@@ -73,6 +72,7 @@
 from danswer.search.models import InferenceChunkUncleaned
 from danswer.utils.batching import batch_generator
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.model_server_models import Embedding
 
 
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index d0a17b26565..5fec7dea40c 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -8,7 +8,6 @@
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.engine import get_sqlalchemy_engine
 from danswer.db.engine import is_valid_schema_name
 from danswer.db.models import KVStore
@@ -18,6 +17,7 @@
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 logger = setup_logger()
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index f99320aa457..65036c43ea4 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -32,7 +32,6 @@
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import DISABLE_GENERATIVE_AI
 from danswer.configs.app_configs import LOG_ENDPOINT_LATENCY
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_OVERFLOW
@@ -94,6 +93,7 @@
 from danswer.utils.variable_functionality import global_version
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 from shared_configs.configs import CORS_ALLOWED_ORIGIN
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import SENTRY_DSN
 
 
diff --git a/backend/danswer/search/preprocessing/preprocessing.py b/backend/danswer/search/preprocessing/preprocessing.py
index 10832ae7cbf..77098dac053 100644
--- a/backend/danswer/search/preprocessing/preprocessing.py
+++ b/backend/danswer/search/preprocessing/preprocessing.py
@@ -1,6 +1,5 @@
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.chat_configs import BASE_RECENCY_DECAY
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_ABOVE
 from danswer.configs.chat_configs import CONTEXT_CHUNKS_BELOW
@@ -31,6 +30,7 @@
 from danswer.utils.threadpool_concurrency import FunctionCall
 from danswer.utils.threadpool_concurrency import run_functions_in_parallel
 from danswer.utils.timing import log_function_time
+from shared_configs.configs import MULTI_TENANT
 
 
 logger = setup_logger()
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index cd2ffe08422..e8127ce0f79 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -33,7 +33,6 @@
 from danswer.auth.users import optional_user
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import ENABLE_EMAIL_INVITES
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
@@ -66,6 +65,7 @@
 from ee.danswer.server.tenants.billing import register_tenant_users
 from ee.danswer.server.tenants.provisioning import add_users_to_tenant
 from ee.danswer.server.tenants.provisioning import remove_users_from_tenant
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index 593ced1540d..8b6bede7563 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -5,7 +5,6 @@
 from danswer.chat.load_yamls import load_chat_yamls
 from danswer.configs.app_configs import DISABLE_INDEX_UPDATE_ON_SWAP
 from danswer.configs.app_configs import MANAGED_VESPA
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.constants import KV_REINDEX_KEY
 from danswer.configs.constants import KV_SEARCH_SETTINGS
 from danswer.configs.model_configs import FAST_GEN_AI_MODEL_VERSION
@@ -52,6 +51,7 @@
 from shared_configs.configs import ALT_INDEX_SUFFIX
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import SUPPORTED_EMBEDDING_MODELS
 from shared_configs.model_server_models import SupportedEmbeddingModel
 
diff --git a/backend/danswer/utils/logger.py b/backend/danswer/utils/logger.py
index 2aadbe379d6..344e3ea8ef1 100644
--- a/backend/danswer/utils/logger.py
+++ b/backend/danswer/utils/logger.py
@@ -4,10 +4,14 @@
 from logging.handlers import RotatingFileHandler
 from typing import Any
 
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import DEV_LOGGING_ENABLED
 from shared_configs.configs import LOG_FILE_NAME
 from shared_configs.configs import LOG_LEVEL
+from shared_configs.configs import MULTI_TENANT
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import SLACK_CHANNEL_ID
+from shared_configs.configs import TENANT_ID_PREFIX
 
 
 logging.addLevelName(logging.INFO + 5, "NOTICE")
@@ -66,6 +70,18 @@ def process(
         if cc_pair_id is not None:
             msg = f"[CC Pair: {cc_pair_id}] {msg}"
 
+        # Add tenant information if it differs from default
+        # This will always be the case for authenticated API requests
+        if MULTI_TENANT:
+            tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+            if tenant_id != POSTGRES_DEFAULT_SCHEMA:
+                # Strip tenant_ prefix and take first 8 chars for cleaner logs
+                tenant_display = tenant_id.removeprefix(TENANT_ID_PREFIX)
+                short_tenant = (
+                    tenant_display[:8] if len(tenant_display) > 8 else tenant_display
+                )
+                msg = f"[t:{short_tenant}] {msg}"
+
         # For Slack Bot, logs the channel relevant to the request
         channel_id = self.extra.get(SLACK_CHANNEL_ID) if self.extra else None
         if channel_id:
diff --git a/backend/ee/danswer/background/celery/apps/primary.py b/backend/ee/danswer/background/celery/apps/primary.py
index be27d22868e..b9929068688 100644
--- a/backend/ee/danswer/background/celery/apps/primary.py
+++ b/backend/ee/danswer/background/celery/apps/primary.py
@@ -1,7 +1,6 @@
 from danswer.background.celery.apps.primary import celery_app
 from danswer.background.task_utils import build_celery_task_wrapper
 from danswer.configs.app_configs import JOB_TIMEOUT
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.db.chat import delete_chat_sessions_older_than
 from danswer.db.engine import get_session_with_tenant
 from danswer.server.settings.store import load_settings
@@ -30,6 +29,7 @@
 )
 from ee.danswer.server.reporting.usage_export_generation import create_new_usage_report
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
diff --git a/backend/ee/danswer/main.py b/backend/ee/danswer/main.py
index 2d1793b8b27..affa5fd1cde 100644
--- a/backend/ee/danswer/main.py
+++ b/backend/ee/danswer/main.py
@@ -5,7 +5,6 @@
 from danswer.auth.users import create_danswer_oauth_router
 from danswer.auth.users import fastapi_users
 from danswer.configs.app_configs import AUTH_TYPE
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import OAUTH_CLIENT_ID
 from danswer.configs.app_configs import OAUTH_CLIENT_SECRET
 from danswer.configs.app_configs import USER_AUTH_SECRET
@@ -43,6 +42,7 @@
 )
 from ee.danswer.server.user_group.api import router as user_group_router
 from ee.danswer.utils.encryption import test_encryption
+from shared_configs.configs import MULTI_TENANT
 
 logger = setup_logger()
 
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index 42291712a45..8c076e57d5c 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -8,10 +8,10 @@
 from fastapi import Request
 from fastapi import Response
 
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.db.engine import is_valid_schema_name
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 
 
@@ -21,7 +21,6 @@ async def set_tenant_id(
         request: Request, call_next: Callable[[Request], Awaitable[Response]]
     ) -> Response:
         try:
-            logger.info(f"Request route: {request.url.path}")
             if not MULTI_TENANT:
                 tenant_id = POSTGRES_DEFAULT_SCHEMA
             else:
@@ -53,8 +52,6 @@ async def set_tenant_id(
                     tenant_id = POSTGRES_DEFAULT_SCHEMA
 
             CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-            logger.info(f"Middleware set current_tenant_id to: {tenant_id}")
-
             response = await call_next(request)
             return response
 
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index 342554c1c43..792d486947d 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -5,7 +5,6 @@
 
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import User
-from danswer.configs.app_configs import MULTI_TENANT
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.notification import create_notification
@@ -25,6 +24,7 @@
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
 from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 
 
 stripe.api_key = STRIPE_SECRET_KEY
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index 5c24aebe749..5f25deedfca 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -129,12 +129,17 @@ def validate_cors_origin(origin: str) -> None:
     CORS_ALLOWED_ORIGIN = ["*"]
 
 
+# Multi-tenancy configuration
+MULTI_TENANT = os.environ.get("MULTI_TENANT", "").lower() == "true"
+
 POSTGRES_DEFAULT_SCHEMA = os.environ.get("POSTGRES_DEFAULT_SCHEMA") or "public"
 
 CURRENT_TENANT_ID_CONTEXTVAR = contextvars.ContextVar(
     "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
 )
 
+# Prefix used for all tenant ids
+TENANT_ID_PREFIX = "tenant_"
 
 SUPPORTED_EMBEDDING_MODELS = [
     # Cloud-based models

From e5494f9742879eb5f047499fcbe249f10f61531e Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Sun, 27 Oct 2024 11:53:50 -0400
Subject: [PATCH 241/376] Refactor & cleanup code, process tickets in batches

---
 .../danswer/connectors/freshdesk/connector.py | 114 +++++++++---------
 1 file changed, 56 insertions(+), 58 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index e0690e34c5b..0b88e5b49dd 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -1,7 +1,7 @@
 import requests
 import json
 from datetime import datetime, timezone
-from typing import Any, List, Optional
+from typing import Any, List, Optional, Iterator
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
@@ -11,17 +11,41 @@
 
 logger = setup_logger()
 
-def create_doc_from_ticket(ticket: dict, domain: str) -> Document:
+def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
+    # Process date fields
+    for date_field in ["created_at", "updated_at", "due_by"]:
+        ticket[date_field] = datetime.fromisoformat(ticket[date_field].rstrip('Z')).replace(tzinfo=timezone.utc)
+
+    # Convert all other values to strings
+    ticket = {
+        key: str(value) if not isinstance(value, (str, datetime)) else value
+        for key, value in ticket.items()
+    }
+
+    # Checking for overdue tickets
+    today = datetime.now(timezone.utc)
+    ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
+
+    # Mapping the status field values
+    status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
+    ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
+
+    # Stripping HTML tags from the description field
+    ticket["description"] = parse_html_page_basic(ticket["description"])
+
+    # Remove extra white spaces from the description field
+    ticket["description"] = " ".join(ticket["description"].split())
+
     return Document(
         id=ticket["id"],
-        sections=Section(
+        sections=[Section(
             link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
             text=json.dumps({
                 key: value
                 for key, value in ticket.items()
                 if isinstance(value, str)
             }, default=str),
-        ),
+        )],
         source=DocumentSource.FRESHDESK,
         semantic_identifier=ticket["subject"],
         metadata={
@@ -35,79 +59,53 @@ class FreshdeskConnector(PollConnector, LoadConnector):
     def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
 
-    def _load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
+    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
         self.api_key = credentials.get("freshdesk_api_key")
         self.domain = credentials.get("freshdesk_domain")
         self.password = credentials.get("freshdesk_password")
         return None
     
-    def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = None) -> List[dict]:
+    def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = None) -> Iterator[List[dict]]:
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
         
-        start_time = start.isoformat() if start else None
-
-        all_tickets = []
-        page = 1
-        per_page = 50
+        base_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets"
+        params = {
+            "include": "description",
+            "per_page": 50,
+            "page": 1
+        }
+        
+        if start:
+            params["updated_since"] = start.isoformat()
 
         while True:
-            freshdesk_url = (
-                f"https://{self.domain}.freshdesk.com/api/v2/tickets"
-                f"?include=description{f"&updated_since={start_time}" if start_time else ""}"
-                f"&per_page={per_page}&page={page}"
-            )
-            response = requests.get(freshdesk_url, auth=(self.api_key, self.password))
+            response = requests.get(base_url, auth=(self.api_key, self.password), params=params)
             response.raise_for_status()
             
-            if response.status_code != 204:
-                tickets = json.loads(response.content)
-                all_tickets.extend(tickets)
-                logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API (Page {page})")
-
-                if len(tickets) < per_page:
-                    break
-                
-                page += 1
-            else:
+            if response.status_code == 204:
                 break
+            
+            tickets = json.loads(response.content)
+            logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API (Page {params['page']})")
 
-        return all_tickets
+            yield tickets
 
-    def _process_tickets(self, start: datetime | None = None, end: datetime | None = None) -> GenerateDocumentsOutput:
-        tickets = self._fetch_tickets(start, end)
+            if len(tickets) < params["per_page"]:
+                break
+            
+            params["page"] += 1
 
+    def _process_tickets(self, start: datetime | None = None, end: datetime | None = None) -> GenerateDocumentsOutput:
         doc_batch: List[Document] = []
 
-        for ticket in tickets:
-            for date_field in ["created_at", "updated_at", "due_by"]:
-                ticket[date_field] = datetime.fromisoformat(ticket[date_field].rstrip('Z'))
-
-            # Convert all other values to strings
-            ticket = {
-                key: str(value) if not isinstance(value, (str, datetime)) else value
-                for key, value in ticket.items()
-            }
-
-            # Checking for overdue tickets
-            today = datetime.now(timezone.utc)
-            ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
-
-            # Mapping the status field values
-            status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
-            ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
-
-            # Stripping HTML tags from the description field
-            ticket["description"] = parse_html_page_basic(ticket["description"])
-
-            # Remove extra white spaces from the description field
-            ticket["description"] = " ".join(ticket["description"].split())
-
-            doc_batch.append(create_doc_from_ticket(ticket, self.domain))
+        for ticket_batch in self._fetch_tickets(start, end):
+            for ticket in ticket_batch:
+                doc_batch.append(_create_doc_from_ticket(ticket, self.domain))
 
-            if len(doc_batch) >= self.batch_size:
-                yield doc_batch
-                doc_batch = []
+                if len(doc_batch) >= self.batch_size:
+                    yield doc_batch
+                    doc_batch = []
 
         if doc_batch:
             yield doc_batch

From a1bfa7847a0805cf3f11cd608b691ecfe231120c Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 27 Oct 2024 10:52:55 -0700
Subject: [PATCH 242/376] a (#2815)

---
 web/src/app/chat/ChatPage.tsx                 |  34 ++-
 .../app/chat/modal/ShareChatSessionModal.tsx  | 267 +++++++++++-------
 web/src/app/chat/searchParams.ts              |   2 +
 3 files changed, 207 insertions(+), 96 deletions(-)

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 467a03dee76..e4fcccd9047 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -140,7 +140,33 @@ export function ChatPage({
   const { user, isAdmin, isLoadingUser } = useUser();
 
   const existingChatIdRaw = searchParams.get("chatId");
+  const [sendOnLoad, setSendOnLoad] = useState<string | null>(
+    searchParams.get(SEARCH_PARAM_NAMES.SEND_ON_LOAD)
+  );
+
   const currentPersonaId = searchParams.get(SEARCH_PARAM_NAMES.PERSONA_ID);
+  const modelVersionFromSearchParams = searchParams.get(
+    SEARCH_PARAM_NAMES.STRUCTURED_MODEL
+  );
+
+  // Effect to handle sendOnLoad
+  useEffect(() => {
+    if (sendOnLoad) {
+      const newSearchParams = new URLSearchParams(searchParams.toString());
+      newSearchParams.delete(SEARCH_PARAM_NAMES.SEND_ON_LOAD);
+
+      // Update the URL without the send-on-load parameter
+      router.replace(`?${newSearchParams.toString()}`, { scroll: false });
+
+      // Update our local state to reflect the change
+      setSendOnLoad(null);
+
+      // If there's a message, submit it
+      if (message) {
+        onSubmit({ messageOverride: message });
+      }
+    }
+  }, [sendOnLoad, searchParams, router]);
 
   const existingChatSessionId = existingChatIdRaw ? existingChatIdRaw : null;
 
@@ -196,7 +222,7 @@ export function ChatPage({
   };
 
   const llmOverrideManager = useLlmOverride(
-    user?.preferences.default_model ?? null,
+    modelVersionFromSearchParams || (user?.preferences.default_model ?? null),
     selectedChatSession,
     defaultTemperature
   );
@@ -1853,6 +1879,9 @@ export function ChatPage({
 
       {sharedChatSession && (
         <ShareChatSessionModal
+          assistantId={liveAssistant?.id}
+          message={message}
+          modelOverride={llmOverrideManager.llmOverride}
           chatSessionId={sharedChatSession.id}
           existingSharedStatus={sharedChatSession.shared_status}
           onClose={() => setSharedChatSession(null)}
@@ -1867,6 +1896,9 @@ export function ChatPage({
       )}
       {sharingModalVisible && chatSessionIdRef.current !== null && (
         <ShareChatSessionModal
+          message={message}
+          assistantId={liveAssistant?.id}
+          modelOverride={llmOverrideManager.llmOverride}
           chatSessionId={chatSessionIdRef.current}
           existingSharedStatus={chatSessionSharedStatus}
           onClose={() => setSharingModalVisible(false)}
diff --git a/web/src/app/chat/modal/ShareChatSessionModal.tsx b/web/src/app/chat/modal/ShareChatSessionModal.tsx
index 1b797e77ab9..4743efdf0dc 100644
--- a/web/src/app/chat/modal/ShareChatSessionModal.tsx
+++ b/web/src/app/chat/modal/ShareChatSessionModal.tsx
@@ -5,6 +5,10 @@ import { Spinner } from "@/components/Spinner";
 import { ChatSessionSharedStatus } from "../interfaces";
 import { FiCopy } from "react-icons/fi";
 import { CopyButton } from "@/components/CopyButton";
+import { SEARCH_PARAM_NAMES } from "../searchParams";
+import { usePopup } from "@/components/admin/connectors/Popup";
+import { destructureValue, structureValue } from "@/lib/llm/utils";
+import { LlmOverride } from "@/lib/hooks";
 
 function buildShareLink(chatSessionId: string) {
   const baseUrl = `${window.location.protocol}//${window.location.host}`;
@@ -26,6 +30,34 @@ async function generateShareLink(chatSessionId: string) {
   return null;
 }
 
+async function generateCloneLink(
+  message?: string,
+  assistantId?: number,
+  modelOverride?: LlmOverride
+) {
+  const baseUrl = `${window.location.protocol}//${window.location.host}`;
+  const model = modelOverride
+    ? structureValue(
+        modelOverride.name,
+        modelOverride.provider,
+        modelOverride.modelName
+      )
+    : null;
+  return `${baseUrl}/chat${
+    message
+      ? `?${SEARCH_PARAM_NAMES.USER_PROMPT}=${encodeURIComponent(message)}`
+      : ""
+  }${
+    assistantId
+      ? `${message ? "&" : "?"}${SEARCH_PARAM_NAMES.PERSONA_ID}=${assistantId}`
+      : ""
+  }${
+    model
+      ? `${message || assistantId ? "&" : "?"}${SEARCH_PARAM_NAMES.STRUCTURED_MODEL}=${encodeURIComponent(model)}`
+      : ""
+  }${message ? `&${SEARCH_PARAM_NAMES.SEND_ON_LOAD}=true` : ""}`;
+}
+
 async function deleteShareLink(chatSessionId: string) {
   const response = await fetch(`/api/chat/chat-session/${chatSessionId}`, {
     method: "PATCH",
@@ -43,117 +75,162 @@ export function ShareChatSessionModal({
   existingSharedStatus,
   onShare,
   onClose,
+  message,
+  assistantId,
+  modelOverride,
 }: {
   chatSessionId: string;
   existingSharedStatus: ChatSessionSharedStatus;
   onShare?: (shared: boolean) => void;
   onClose: () => void;
+  message?: string;
+  assistantId?: number;
+  modelOverride?: LlmOverride;
 }) {
-  const [linkGenerating, setLinkGenerating] = useState(false);
   const [shareLink, setShareLink] = useState<string>(
     existingSharedStatus === ChatSessionSharedStatus.Public
       ? buildShareLink(chatSessionId)
       : ""
   );
+  const { popup, setPopup } = usePopup();
 
   return (
-    <Modal onOutsideClick={onClose} width="max-w-3xl">
-      <>
-        <div className="flex mb-4">
-          <h2 className="text-2xl text-emphasis font-bold flex my-auto">
-            Share link to Chat
-          </h2>
-        </div>
-
-        {linkGenerating && <Spinner />}
-
-        <div className="flex mt-2">
-          {shareLink ? (
-            <div>
-              <Text>
-                This chat session is currently shared. Anyone at your
-                organization can view the message history using the following
-                link:
-              </Text>
-
-              <div className="flex mt-2">
-                <CopyButton content={shareLink} />
-                <a
-                  href={shareLink}
-                  target="_blank"
-                  className="underline text-link mt-1 ml-1 text-sm my-auto"
-                  rel="noreferrer"
+    <>
+      {popup}
+      <Modal onOutsideClick={onClose} width="max-w-3xl">
+        <>
+          <div className="flex mb-4">
+            <h2 className="text-2xl text-emphasis font-bold flex my-auto">
+              Share link to Chat
+            </h2>
+          </div>
+
+          <div className="flex mt-2">
+            {shareLink ? (
+              <div>
+                <Text>
+                  This chat session is currently shared. Anyone at your
+                  organization can view the message history using the following
+                  link:
+                </Text>
+
+                <div className="flex mt-2">
+                  <CopyButton content={shareLink} />
+                  <a
+                    href={shareLink}
+                    target="_blank"
+                    className="underline text-link mt-1 ml-1 text-sm my-auto"
+                    rel="noreferrer"
+                  >
+                    {shareLink}
+                  </a>
+                </div>
+
+                <Divider />
+
+                <Text className="mb-4">
+                  Click the button below to make the chat private again.
+                </Text>
+
+                <Button
+                  onClick={async () => {
+                    const success = await deleteShareLink(chatSessionId);
+                    if (success) {
+                      setShareLink("");
+                      onShare && onShare(false);
+                    } else {
+                      alert("Failed to delete share link");
+                    }
+                  }}
+                  size="xs"
+                  color="red"
                 >
-                  {shareLink}
-                </a>
+                  Delete Share Link
+                </Button>
               </div>
-
-              <Divider />
-
-              <Text className="mb-4">
-                Click the button below to make the chat private again.
-              </Text>
-
-              <Button
-                onClick={async () => {
-                  setLinkGenerating(true);
-
-                  const success = await deleteShareLink(chatSessionId);
-                  if (success) {
-                    setShareLink("");
-                    onShare && onShare(false);
+            ) : (
+              <div>
+                <Callout title="Warning" color="yellow" className="mb-4">
+                  Ensure that all content in the chat is safe to share with the
+                  whole organization. The content of the retrieved documents
+                  will not be visible, but the names of cited documents as well
+                  as the AI and human messages will be visible.
+                </Callout>
+                <div className="flex w-full justify-between">
+                  <Button
+                    icon={FiCopy}
+                    onClick={async () => {
+                      // NOTE: for "insecure" non-https setup, the `navigator.clipboard.writeText` may fail
+                      // as the browser may not allow the clipboard to be accessed.
+                      try {
+                        const shareLink =
+                          await generateShareLink(chatSessionId);
+                        if (!shareLink) {
+                          alert("Failed to generate share link");
+                        } else {
+                          setShareLink(shareLink);
+                          onShare && onShare(true);
+                          navigator.clipboard.writeText(shareLink);
+                        }
+                      } catch (e) {
+                        console.error(e);
+                      }
+                    }}
+                    size="xs"
+                    color="green"
+                  >
+                    Generate and Copy Share Link
+                  </Button>
+                </div>
+              </div>
+            )}
+          </div>
+
+          <Divider className="my-4" />
+          <div className="mb-4">
+            <Callout title="Clone Chat" color="blue">
+              Generate a link to clone this chat session with the current query.
+              This allows others to start a new chat with the same initial
+              message and settings.
+            </Callout>
+          </div>
+          <div className="flex w-full justify-between">
+            <Button
+              icon={FiCopy}
+              onClick={async () => {
+                // NOTE: for "insecure" non-https setup, the `navigator.clipboard.writeText` may fail
+                // as the browser may not allow the clipboard to be accessed.
+                try {
+                  const cloneLink = await generateCloneLink(
+                    message,
+                    assistantId,
+                    modelOverride
+                  );
+                  if (!cloneLink) {
+                    setPopup({
+                      message: "Failed to generate clone link",
+                      type: "error",
+                    });
                   } else {
-                    alert("Failed to delete share link");
+                    navigator.clipboard.writeText(cloneLink);
+                    setPopup({
+                      message: "Link copied to clipboard!",
+                      type: "success",
+                    });
                   }
-
-                  setLinkGenerating(false);
-                }}
-                size="xs"
-                color="red"
-              >
-                Delete Share Link
-              </Button>
-            </div>
-          ) : (
-            <div>
-              <Callout title="Warning" color="yellow" className="mb-4">
-                Ensure that all content in the chat is safe to share with the
-                whole organization. The content of the retrieved documents will
-                not be visible, but the names of cited documents as well as the
-                AI and human messages will be visible.
-              </Callout>
-
-              <Button
-                icon={FiCopy}
-                onClick={async () => {
-                  setLinkGenerating(true);
-
-                  // NOTE: for "inescure" non-https setup, the `navigator.clipboard.writeText` may fail
-                  // as the browser may not allow the clipboard to be accessed.
-                  try {
-                    const shareLink = await generateShareLink(chatSessionId);
-                    if (!shareLink) {
-                      alert("Failed to generate share link");
-                    } else {
-                      setShareLink(shareLink);
-                      onShare && onShare(true);
-                      navigator.clipboard.writeText(shareLink);
-                    }
-                  } catch (e) {
-                    console.error(e);
-                  }
-
-                  setLinkGenerating(false);
-                }}
-                size="xs"
-                color="green"
-              >
-                Generate and Copy Share Link
-              </Button>
-            </div>
-          )}
-        </div>
-      </>
-    </Modal>
+                } catch (e) {
+                  console.error(e);
+                  alert("Failed to generate or copy link.");
+                }
+              }}
+              size="xs"
+              color="blue"
+            >
+              Generate and Copy Clone Link
+            </Button>
+          </div>
+        </>
+      </Modal>
+    </>
   );
 }
diff --git a/web/src/app/chat/searchParams.ts b/web/src/app/chat/searchParams.ts
index 56bcf4e33f4..7b3dc44fe97 100644
--- a/web/src/app/chat/searchParams.ts
+++ b/web/src/app/chat/searchParams.ts
@@ -9,6 +9,7 @@ export const SEARCH_PARAM_NAMES = {
   TEMPERATURE: "temperature",
   MODEL_VERSION: "model-version",
   SYSTEM_PROMPT: "system-prompt",
+  STRUCTURED_MODEL: "structured-model",
   // user message
   USER_PROMPT: "user-prompt",
   SUBMIT_ON_LOAD: "submit-on-load",
@@ -16,6 +17,7 @@ export const SEARCH_PARAM_NAMES = {
   TITLE: "title",
   // for seeding chats
   SEEDED: "seeded",
+  SEND_ON_LOAD: "send-on-load",
 };
 
 export function shouldSubmitOnLoad(searchParams: ReadonlyURLSearchParams) {

From 179dc418e0cf7d7e0bdabffd54acde3692a2188b Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 27 Oct 2024 11:48:30 -0700
Subject: [PATCH 243/376] Onboarding nits (#2907)

* temporary stash

* welcome flow

* minor update

* k

* minor updates to welcome flow
---
 .../one_shot_answer/answer_question.py        | 14 ++++-
 web/src/app/assistants/gallery/page.tsx       |  2 +-
 web/src/app/assistants/mine/page.tsx          |  6 +-
 web/src/app/chat/ChatPage.tsx                 |  4 +-
 web/src/app/search/page.tsx                   | 24 ++------
 .../chat_search/UnconfiguredProviderText.tsx  | 45 ++++++++++-----
 .../initialSetup/search/NoSourcesModal.tsx    | 55 -------------------
 .../initialSetup/welcome/WelcomeModal.tsx     |  4 ++
 web/src/components/llm/ApiKeyModal.tsx        |  9 ++-
 web/src/components/search/SearchSection.tsx   |  2 +
 10 files changed, 65 insertions(+), 100 deletions(-)
 delete mode 100644 web/src/components/initialSetup/search/NoSourcesModal.tsx

diff --git a/backend/danswer/one_shot_answer/answer_question.py b/backend/danswer/one_shot_answer/answer_question.py
index f051da82f14..1bfac570aee 100644
--- a/backend/danswer/one_shot_answer/answer_question.py
+++ b/backend/danswer/one_shot_answer/answer_question.py
@@ -129,7 +129,19 @@ def stream_answer_objects(
 
     persona = temporary_persona if temporary_persona else chat_session.persona
 
-    llm, fast_llm = get_llms_for_persona(persona=persona)
+    try:
+        llm, fast_llm = get_llms_for_persona(persona=persona)
+    except ValueError as e:
+        logger.error(
+            f"Failed to initialize LLMs for persona '{persona.name}': {str(e)}"
+        )
+        if "No LLM provider" in str(e):
+            raise ValueError(
+                "Please configure a Generative AI model to use this feature."
+            ) from e
+        raise ValueError(
+            "Failed to initialize the AI model. Please check your configuration and try again."
+        ) from e
 
     llm_tokenizer = get_tokenizer(
         model_name=llm.config.model_name,
diff --git a/web/src/app/assistants/gallery/page.tsx b/web/src/app/assistants/gallery/page.tsx
index 1ae928c598f..fd261fbcae8 100644
--- a/web/src/app/assistants/gallery/page.tsx
+++ b/web/src/app/assistants/gallery/page.tsx
@@ -24,8 +24,8 @@ export default async function GalleryPage({
     chatSessions,
     folders,
     openedFolders,
-    shouldShowWelcomeModal,
     toggleSidebar,
+    shouldShowWelcomeModal,
   } = data;
 
   return (
diff --git a/web/src/app/assistants/mine/page.tsx b/web/src/app/assistants/mine/page.tsx
index d14de250bae..5761a8c48e5 100644
--- a/web/src/app/assistants/mine/page.tsx
+++ b/web/src/app/assistants/mine/page.tsx
@@ -1,10 +1,10 @@
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
-import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
+
 import { fetchChatData } from "@/lib/chat/fetchChatData";
 import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
 import WrappedAssistantsMine from "./WrappedAssistantsMine";
-import { AssistantsProvider } from "@/components/context/AssistantsContext";
+import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 
 export default async function GalleryPage({
   searchParams,
@@ -24,8 +24,8 @@ export default async function GalleryPage({
     chatSessions,
     folders,
     openedFolders,
-    shouldShowWelcomeModal,
     toggleSidebar,
+    shouldShowWelcomeModal,
   } = data;
 
   return (
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index e4fcccd9047..056aa181548 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -135,7 +135,9 @@ export function ChatPage({
 
   const { assistants: availableAssistants, finalAssistants } = useAssistants();
 
-  const [showApiKeyModal, setShowApiKeyModal] = useState(true);
+  const [showApiKeyModal, setShowApiKeyModal] = useState(
+    !shouldShowWelcomeModal
+  );
 
   const { user, isAdmin, isLoadingUser } = useUser();
 
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index cfbc72e564b..8bd743fe763 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -10,16 +10,10 @@ import { CCPairBasicInfo, DocumentSet, Tag, User } from "@/lib/types";
 import { cookies } from "next/headers";
 import { SearchType } from "@/lib/search/interfaces";
 import { Persona } from "../admin/assistants/interfaces";
-import {
-  WelcomeModal,
-  hasCompletedWelcomeFlowSS,
-} from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 import { unstable_noStore as noStore } from "next/cache";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import { personaComparator } from "../admin/assistants/lib";
 import { FullEmbeddingModelResponse } from "@/components/embedding/interfaces";
-import { NoSourcesModal } from "@/components/initialSetup/search/NoSourcesModal";
-import { NoCompleteSourcesModal } from "@/components/initialSetup/search/NoCompleteSourceModal";
 import { ChatPopup } from "../chat/ChatPopup";
 import {
   FetchAssistantsResponse,
@@ -38,6 +32,10 @@ import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "../admin/configuration/llm/interfaces";
 import { AssistantsProvider } from "@/components/context/AssistantsContext";
 import { headers } from "next/headers";
+import {
+  hasCompletedWelcomeFlowSS,
+  WelcomeModal,
+} from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 
 export default async function Home({
   searchParams,
@@ -170,14 +168,6 @@ export default async function Home({
     ccPairs.length === 0 &&
     !shouldShowWelcomeModal;
 
-  const shouldDisplaySourcesIncompleteModal =
-    !ccPairs.some(
-      (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0
-    ) &&
-    !shouldDisplayNoSourcesModal &&
-    !shouldShowWelcomeModal &&
-    (!user || user.role == "admin");
-
   const sidebarToggled = cookies().get(SIDEBAR_TOGGLED_COOKIE_NAME);
   const agenticSearchToggle = cookies().get(AGENTIC_SEARCH_TYPE_COOKIE_NAME);
 
@@ -192,12 +182,8 @@ export default async function Home({
   return (
     <>
       <HealthCheckBanner />
-      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
       <InstantSSRAutoRefresh />
-      {shouldDisplayNoSourcesModal && <NoSourcesModal />}
-      {shouldDisplaySourcesIncompleteModal && (
-        <NoCompleteSourcesModal ccPairs={ccPairs} />
-      )}
+      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 
       Only used in the EE version of the app. */}
       <ChatPopup />
diff --git a/web/src/components/chat_search/UnconfiguredProviderText.tsx b/web/src/components/chat_search/UnconfiguredProviderText.tsx
index e990eecd610..53d65176aa2 100644
--- a/web/src/components/chat_search/UnconfiguredProviderText.tsx
+++ b/web/src/components/chat_search/UnconfiguredProviderText.tsx
@@ -2,26 +2,41 @@ import { useProviderStatus } from "./ProviderContext";
 
 export default function CredentialNotConfigured({
   showConfigureAPIKey,
+  noSources,
 }: {
   showConfigureAPIKey: () => void;
+  noSources?: boolean;
 }) {
   const { shouldShowConfigurationNeeded } = useProviderStatus();
 
-  if (!shouldShowConfigurationNeeded) {
-    return null;
-  }
-
   return (
-    <p className="text-base text-center w-full text-subtle">
-      Please note that you have not yet configured an LLM provider. You can
-      configure one{" "}
-      <button
-        onClick={showConfigureAPIKey}
-        className="text-link hover:underline cursor-pointer"
-      >
-        here
-      </button>
-      .
-    </p>
+    <>
+      {noSources ? (
+        <p className="text-base text-center w-full text-subtle">
+          You have not yet added any sources. Please add{" "}
+          <a
+            href="/admin/add-connector"
+            className="text-link hover:underline cursor-pointer"
+          >
+            a source
+          </a>{" "}
+          to continue.
+        </p>
+      ) : (
+        shouldShowConfigurationNeeded && (
+          <p className="text-base text-center w-full text-subtle">
+            Please note that you have not yet configured an LLM provider. You
+            can configure one{" "}
+            <button
+              onClick={showConfigureAPIKey}
+              className="text-link hover:underline cursor-pointer"
+            >
+              here
+            </button>
+            .
+          </p>
+        )
+      )}
+    </>
   );
 }
diff --git a/web/src/components/initialSetup/search/NoSourcesModal.tsx b/web/src/components/initialSetup/search/NoSourcesModal.tsx
deleted file mode 100644
index b4f6ec004ab..00000000000
--- a/web/src/components/initialSetup/search/NoSourcesModal.tsx
+++ /dev/null
@@ -1,55 +0,0 @@
-"use client";
-
-import { Button, Divider, Text } from "@tremor/react";
-import { Modal } from "../../Modal";
-import Link from "next/link";
-import { FiMessageSquare, FiShare2 } from "react-icons/fi";
-import { useContext, useState } from "react";
-import { SettingsContext } from "@/components/settings/SettingsProvider";
-
-export function NoSourcesModal() {
-  const settings = useContext(SettingsContext);
-  const [isHidden, setIsHidden] = useState(
-    !settings?.settings.search_page_enabled
-  );
-
-  if (isHidden) {
-    return null;
-  }
-
-  return (
-    <Modal
-      width="max-w-3xl w-full"
-      title="🧐 No sources connected"
-      onOutsideClick={() => setIsHidden(true)}
-    >
-      <div className="text-base">
-        <div>
-          <Text>
-            Before using Search you&apos;ll need to connect at least one source.
-            Without any connected knowledge sources, there isn&apos;t anything
-            to search over.
-          </Text>
-          <Link href="/admin/add-connector">
-            <Button className="mt-3" size="xs" icon={FiShare2}>
-              Connect a Source!
-            </Button>
-          </Link>
-          <Divider />
-          <div>
-            <Text>
-              Or, if you&apos;re looking for a pure ChatGPT-like experience
-              without any organization specific knowledge, then you can head
-              over to the Chat page and start chatting with Danswer right away!
-            </Text>
-            <Link href="/chat">
-              <Button className="mt-3" size="xs" icon={FiMessageSquare}>
-                Start Chatting!
-              </Button>
-            </Link>
-          </div>
-        </div>
-      </div>
-    </Modal>
-  );
-}
diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index 81496edb8f9..cd3dd8ff268 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -58,6 +58,10 @@ export function _WelcomeModal({ user }: { user: User | null }) {
       {popup}
 
       <Modal
+        onOutsideClick={() => {
+          setWelcomeFlowComplete();
+          router.refresh();
+        }}
         title={"Welcome to Danswer!"}
         width="w-full max-h-[900px] overflow-y-scroll max-w-3xl"
       >
diff --git a/web/src/components/llm/ApiKeyModal.tsx b/web/src/components/llm/ApiKeyModal.tsx
index d401036e493..775c6ac9cd5 100644
--- a/web/src/components/llm/ApiKeyModal.tsx
+++ b/web/src/components/llm/ApiKeyModal.tsx
@@ -26,18 +26,17 @@ export const ApiKeyModal = ({
   }
   return (
     <Modal
-      title="Set an API Key!"
+      title="Configure a Generative AI Model"
       width="max-w-3xl w-full"
       onOutsideClick={() => hide()}
     >
       <>
         <div className="mb-5 text-sm text-gray-700">
-          Please provide an API Key below in order to start using Danswer – you
-          can always change this later.
+          Please provide an API Key – you can always change this or switch
+          models later.
           <br />
-          If you&apos;d rather look around first, you can
+          If you would rather look around first, you can{" "}
           <strong onClick={() => hide()} className="text-link cursor-pointer">
-            {" "}
             skip this step
           </strong>
           .
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 9f49e4715cf..66b5cb481ed 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -44,6 +44,7 @@ import UnconfiguredProviderText from "../chat_search/UnconfiguredProviderText";
 import { DateRangePickerValue } from "@tremor/react";
 import { Tag } from "@/lib/types";
 import { isEqual } from "lodash";
+import { WelcomeModal } from "../initialSetup/welcome/WelcomeModalWrapper";
 
 export type searchState =
   | "input"
@@ -783,6 +784,7 @@ export const SearchSection = ({
                     </div>
 
                     <UnconfiguredProviderText
+                      noSources={shouldDisplayNoSources}
                       showConfigureAPIKey={() => setShowApiKeyModal(true)}
                     />
 

From 53e916552b05bc243152f3e7cabead0e9cd5c721 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 27 Oct 2024 11:48:47 -0700
Subject: [PATCH 244/376] tenant seeding docs (#2925)

* tenant seeding docs

* k
---
 backend/danswer/background/celery/apps/beat.py         | 1 +
 backend/danswer/document_index/vespa/indexing_utils.py | 1 -
 backend/danswer/document_index/vespa_constants.py      | 1 +
 backend/danswer/main.py                                | 2 +-
 backend/danswer/seeding/load_docs.py                   | 8 +++++---
 backend/danswer/setup.py                               | 4 ++--
 backend/ee/danswer/server/tenants/api.py               | 2 +-
 7 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/backend/danswer/background/celery/apps/beat.py b/backend/danswer/background/celery/apps/beat.py
index 8ddc17efc52..f88295fa139 100644
--- a/backend/danswer/background/celery/apps/beat.py
+++ b/backend/danswer/background/celery/apps/beat.py
@@ -78,6 +78,7 @@ def on_setup_logging(
     },
 ]
 
+
 # Build the celery beat schedule dynamically
 beat_schedule = {}
 
diff --git a/backend/danswer/document_index/vespa/indexing_utils.py b/backend/danswer/document_index/vespa/indexing_utils.py
index 8ecdc22672b..aafc6bf4efe 100644
--- a/backend/danswer/document_index/vespa/indexing_utils.py
+++ b/backend/danswer/document_index/vespa/indexing_utils.py
@@ -57,7 +57,6 @@ def _does_document_exist(
     chunk. This checks for whether the chunk exists already in the index"""
     doc_url = f"{DOCUMENT_ID_ENDPOINT.format(index_name=index_name)}/{doc_chunk_id}"
     doc_fetch_response = http_client.get(doc_url)
-
     if doc_fetch_response.status_code == 404:
         return False
 
diff --git a/backend/danswer/document_index/vespa_constants.py b/backend/danswer/document_index/vespa_constants.py
index d4a36ef9725..30039922f1a 100644
--- a/backend/danswer/document_index/vespa_constants.py
+++ b/backend/danswer/document_index/vespa_constants.py
@@ -29,6 +29,7 @@
 # main search application
 VESPA_APP_CONTAINER_URL = VESPA_CLOUD_URL or f"http://{VESPA_HOST}:{VESPA_PORT}"
 
+
 # danswer_chunk below is defined in vespa/app_configs/schemas/danswer_chunk.sd
 DOCUMENT_ID_ENDPOINT = (
     f"{VESPA_APP_CONTAINER_URL}/document/v1/default/{{index_name}}/docid"
diff --git a/backend/danswer/main.py b/backend/danswer/main.py
index 65036c43ea4..ae18ab3ccf2 100644
--- a/backend/danswer/main.py
+++ b/backend/danswer/main.py
@@ -184,7 +184,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator:
 
         # If we are multi-tenant, we need to only set up initial public tables
         with Session(engine) as db_session:
-            setup_danswer(db_session)
+            setup_danswer(db_session, None)
     else:
         setup_multitenant_danswer()
 
diff --git a/backend/danswer/seeding/load_docs.py b/backend/danswer/seeding/load_docs.py
index 2756e7ddf61..2e9c13b10ba 100644
--- a/backend/danswer/seeding/load_docs.py
+++ b/backend/danswer/seeding/load_docs.py
@@ -39,6 +39,7 @@
 
 def _create_indexable_chunks(
     preprocessed_docs: list[dict],
+    tenant_id: str | None,
 ) -> tuple[list[Document], list[DocMetadataAwareIndexChunk]]:
     ids_to_documents = {}
     chunks = []
@@ -80,7 +81,7 @@ def _create_indexable_chunks(
                 mini_chunk_embeddings=[],
             ),
             title_embedding=preprocessed_doc["title_embedding"],
-            tenant_id=None,
+            tenant_id=tenant_id,
             access=default_public_access,
             document_sets=set(),
             boost=DEFAULT_BOOST,
@@ -90,7 +91,7 @@ def _create_indexable_chunks(
     return list(ids_to_documents.values()), chunks
 
 
-def seed_initial_documents(db_session: Session) -> None:
+def seed_initial_documents(db_session: Session, tenant_id: str | None) -> None:
     """
     Seed initial documents so users don't have an empty index to start
 
@@ -177,7 +178,7 @@ def seed_initial_documents(db_session: Session) -> None:
     )
     processed_docs = json.load(open(initial_docs_path))
 
-    docs, chunks = _create_indexable_chunks(processed_docs)
+    docs, chunks = _create_indexable_chunks(processed_docs, tenant_id)
 
     index_doc_batch_prepare(
         document_batch=docs,
@@ -198,6 +199,7 @@ def seed_initial_documents(db_session: Session) -> None:
 
     # Retries here because the index may take a few seconds to become ready
     # as we just sent over the Vespa schema and there is a slight delay
+
     index_with_retries = retry_builder()(document_index.index)
     index_with_retries(chunks=chunks)
 
diff --git a/backend/danswer/setup.py b/backend/danswer/setup.py
index 8b6bede7563..7abd7482a78 100644
--- a/backend/danswer/setup.py
+++ b/backend/danswer/setup.py
@@ -59,7 +59,7 @@
 logger = setup_logger()
 
 
-def setup_danswer(db_session: Session) -> None:
+def setup_danswer(db_session: Session, tenant_id: str | None) -> None:
     """
     Setup Danswer for a particular tenant. In the Single Tenant case, it will set it up for the default schema
     on server startup. In the MT case, it will be called when the tenant is created.
@@ -148,7 +148,7 @@ def setup_danswer(db_session: Session) -> None:
     # update multipass indexing setting based on GPU availability
     update_default_multipass_indexing(db_session)
 
-    seed_initial_documents(db_session)
+    seed_initial_documents(db_session, tenant_id)
 
 
 def translate_saved_search_settings(db_session: Session) -> None:
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index 792d486947d..66485975f31 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -59,7 +59,7 @@ def create_tenant(
         run_alembic_migrations(tenant_id)
 
         with get_session_with_tenant(tenant_id) as db_session:
-            setup_danswer(db_session)
+            setup_danswer(db_session, tenant_id)
 
         add_users_to_tenant([email], tenant_id)
 

From 01463442ba3f3554a02b0066089493729bbb5dff Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 27 Oct 2024 13:08:18 -0700
Subject: [PATCH 245/376] avoid image generation tool confusion

---
 .../danswer/server/features/persona/api.py    | 11 +++
 .../danswer/server/features/persona/models.py |  4 +
 .../app/admin/assistants/AssistantEditor.tsx  | 75 +++++--------------
 .../components/context/AssistantsContext.tsx  | 55 +++++++-------
 4 files changed, 63 insertions(+), 82 deletions(-)

diff --git a/backend/danswer/server/features/persona/api.py b/backend/danswer/server/features/persona/api.py
index 8c0e1943861..1e0e84a5896 100644
--- a/backend/danswer/server/features/persona/api.py
+++ b/backend/danswer/server/features/persona/api.py
@@ -30,6 +30,7 @@
 from danswer.file_store.models import ChatFileType
 from danswer.llm.answering.prompts.utils import build_dummy_prompt
 from danswer.server.features.persona.models import CreatePersonaRequest
+from danswer.server.features.persona.models import ImageGenerationToolStatus
 from danswer.server.features.persona.models import PersonaSharedNotificationData
 from danswer.server.features.persona.models import PersonaSnapshot
 from danswer.server.features.persona.models import PromptTemplateResponse
@@ -227,6 +228,16 @@ def delete_persona(
     )
 
 
+@basic_router.get("/image-generation-tool")
+def get_image_generation_tool(
+    _: User
+    | None = Depends(current_user),  # User param not used but kept for consistency
+    db_session: Session = Depends(get_session),
+) -> ImageGenerationToolStatus:  # Use bool instead of str for boolean values
+    is_available = is_image_generation_available(db_session=db_session)
+    return ImageGenerationToolStatus(is_available=is_available)
+
+
 @basic_router.get("")
 def list_personas(
     user: User | None = Depends(current_user),
diff --git a/backend/danswer/server/features/persona/models.py b/backend/danswer/server/features/persona/models.py
index 866d70f11bb..0e69777a02c 100644
--- a/backend/danswer/server/features/persona/models.py
+++ b/backend/danswer/server/features/persona/models.py
@@ -124,3 +124,7 @@ class PromptTemplateResponse(BaseModel):
 
 class PersonaSharedNotificationData(BaseModel):
     persona_id: int
+
+
+class ImageGenerationToolStatus(BaseModel):
+    is_available: bool
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 2eec9632c66..131cbf5930d 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -45,12 +45,7 @@ import { FullLLMProvider } from "../configuration/llm/interfaces";
 import CollapsibleSection from "./CollapsibleSection";
 import { SuccessfulPersonaUpdateRedirectType } from "./enums";
 import { Persona, StarterMessage } from "./interfaces";
-import {
-  buildFinalPrompt,
-  createPersona,
-  providersContainImageGeneratingSupport,
-  updatePersona,
-} from "./lib";
+import { buildFinalPrompt, createPersona, updatePersona } from "./lib";
 import { Popover } from "@/components/popover/Popover";
 import {
   CameraIcon,
@@ -106,7 +101,7 @@ export function AssistantEditor({
   shouldAddAssistantToUserPreferences?: boolean;
   admin?: boolean;
 }) {
-  const { refreshAssistants } = useAssistants();
+  const { refreshAssistants, isImageGenerationAvailable } = useAssistants();
   const router = useRouter();
 
   const { popup, setPopup } = usePopup();
@@ -138,38 +133,10 @@ export function AssistantEditor({
 
   const [isIconDropdownOpen, setIsIconDropdownOpen] = useState(false);
 
-  const [finalPrompt, setFinalPrompt] = useState<string | null>("");
-  const [finalPromptError, setFinalPromptError] = useState<string>("");
   const [removePersonaImage, setRemovePersonaImage] = useState(false);
 
-  const triggerFinalPromptUpdate = async (
-    systemPrompt: string,
-    taskPrompt: string,
-    retrievalDisabled: boolean
-  ) => {
-    const response = await buildFinalPrompt(
-      systemPrompt,
-      taskPrompt,
-      retrievalDisabled
-    );
-    if (response.ok) {
-      setFinalPrompt((await response.json()).final_prompt_template);
-    }
-  };
-
   const isUpdate = existingPersona !== undefined && existingPersona !== null;
   const existingPrompt = existingPersona?.prompts[0] ?? null;
-
-  useEffect(() => {
-    if (isUpdate && existingPrompt) {
-      triggerFinalPromptUpdate(
-        existingPrompt.system_prompt,
-        existingPrompt.task_prompt,
-        existingPersona.num_chunks === 0
-      );
-    }
-  }, [isUpdate, existingPrompt, existingPersona?.num_chunks]);
-
   const defaultProvider = llmProviders.find(
     (llmProvider) => llmProvider.is_default_provider
   );
@@ -314,14 +281,6 @@ export function AssistantEditor({
             }
           )}
         onSubmit={async (values, formikHelpers) => {
-          if (finalPromptError) {
-            setPopup({
-              type: "error",
-              message: "Cannot submit while there are errors in the form",
-            });
-            return;
-          }
-
           if (
             values.llm_model_provider_override &&
             !values.llm_model_version_override
@@ -642,13 +601,7 @@ export function AssistantEditor({
                 placeholder="e.g. 'You are a professional email writing assistant that always uses a polite enthusiastic tone, emphasizes action items, and leaves blanks for the human to fill in when you have unknowns'"
                 onChange={(e) => {
                   setFieldValue("system_prompt", e.target.value);
-                  triggerFinalPromptUpdate(
-                    e.target.value,
-                    values.task_prompt,
-                    searchToolEnabled()
-                  );
                 }}
-                error={finalPromptError}
               />
 
               <div>
@@ -775,7 +728,8 @@ export function AssistantEditor({
                         <TooltipTrigger asChild>
                           <div
                             className={`w-fit ${
-                              !currentLLMSupportsImageOutput
+                              !currentLLMSupportsImageOutput ||
+                              !isImageGenerationAvailable
                                 ? "opacity-70 cursor-not-allowed"
                                 : ""
                             }`}
@@ -787,11 +741,14 @@ export function AssistantEditor({
                               onChange={() => {
                                 toggleToolInValues(imageGenerationTool.id);
                               }}
-                              disabled={!currentLLMSupportsImageOutput}
+                              disabled={
+                                !currentLLMSupportsImageOutput ||
+                                !isImageGenerationAvailable
+                              }
                             />
                           </div>
                         </TooltipTrigger>
-                        {!currentLLMSupportsImageOutput && (
+                        {!currentLLMSupportsImageOutput ? (
                           <TooltipContent side="top" align="center">
                             <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-white">
                               To use Image Generation, select GPT-4o or another
@@ -799,6 +756,15 @@ export function AssistantEditor({
                               this Assistant.
                             </p>
                           </TooltipContent>
+                        ) : (
+                          !isImageGenerationAvailable && (
+                            <TooltipContent side="top" align="center">
+                              <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-white">
+                                Image Generation requires an OpenAI or Azure
+                                Dalle configuration.
+                              </p>
+                            </TooltipContent>
+                          )
                         )}
                       </Tooltip>
                     </TooltipProvider>
@@ -1024,11 +990,6 @@ export function AssistantEditor({
                         placeholder="e.g. 'Remember to reference all of the points mentioned in my message to you and focus on identifying action items that can move things forward'"
                         onChange={(e) => {
                           setFieldValue("task_prompt", e.target.value);
-                          triggerFinalPromptUpdate(
-                            values.system_prompt,
-                            e.target.value,
-                            searchToolEnabled()
-                          );
                         }}
                         explanationText="Learn about prompting in our docs!"
                         explanationLink="https://docs.danswer.dev/guides/assistants"
diff --git a/web/src/components/context/AssistantsContext.tsx b/web/src/components/context/AssistantsContext.tsx
index 821429e1090..f6787f5641c 100644
--- a/web/src/components/context/AssistantsContext.tsx
+++ b/web/src/components/context/AssistantsContext.tsx
@@ -21,6 +21,7 @@ interface AssistantsContextProps {
   finalAssistants: Persona[];
   ownedButHiddenAssistants: Persona[];
   refreshAssistants: () => Promise<void>;
+  isImageGenerationAvailable: boolean;
 
   // Admin only
   editablePersonas: Persona[];
@@ -47,51 +48,54 @@ export const AssistantsProvider: React.FC<{
   );
   const { user, isLoadingUser, isAdmin } = useUser();
   const [editablePersonas, setEditablePersonas] = useState<Persona[]>([]);
+  const [allAssistants, setAllAssistants] = useState<Persona[]>([]);
 
-  useEffect(() => {
-    const fetchEditablePersonas = async () => {
-      if (!isAdmin) {
-        return;
-      }
+  const [isImageGenerationAvailable, setIsImageGenerationAvailable] =
+    useState<boolean>(false);
 
+  useEffect(() => {
+    const checkImageGenerationAvailability = async () => {
       try {
-        const response = await fetch("/api/admin/persona?get_editable=true");
-        if (!response.ok) {
-          console.error("Failed to fetch editable personas");
-          return;
+        const response = await fetch("/api/persona/image-generation-tool");
+        if (response.ok) {
+          const { is_available } = await response.json();
+          setIsImageGenerationAvailable(is_available);
         }
-        const personas = await response.json();
-        setEditablePersonas(personas);
       } catch (error) {
-        console.error("Error fetching editable personas:", error);
+        console.error("Error checking image generation availability:", error);
       }
     };
 
-    fetchEditablePersonas();
-  }, [isAdmin]);
-
-  const [allAssistants, setAllAssistants] = useState<Persona[]>([]);
+    checkImageGenerationAvailability();
+  }, []);
 
   useEffect(() => {
-    const fetchAllAssistants = async () => {
+    const fetchPersonas = async () => {
       if (!isAdmin) {
         return;
       }
 
       try {
-        const response = await fetch("/api/admin/persona");
-        if (!response.ok) {
-          console.error("Failed to fetch all personas");
-          return;
+        const [editableResponse, allResponse] = await Promise.all([
+          fetch("/api/admin/persona?get_editable=true"),
+          fetch("/api/admin/persona"),
+        ]);
+
+        if (editableResponse.ok) {
+          const editablePersonas = await editableResponse.json();
+          setEditablePersonas(editablePersonas);
+        }
+
+        if (allResponse.ok) {
+          const allPersonas = await allResponse.json();
+          setAllAssistants(allPersonas);
         }
-        const personas = await response.json();
-        setAllAssistants(personas);
       } catch (error) {
-        console.error("Error fetching all personas:", error);
+        console.error("Error fetching personas:", error);
       }
     };
 
-    fetchAllAssistants();
+    fetchPersonas();
   }, [isAdmin]);
 
   const refreshAssistants = async () => {
@@ -162,6 +166,7 @@ export const AssistantsProvider: React.FC<{
         refreshAssistants,
         editablePersonas,
         allAssistants,
+        isImageGenerationAvailable,
       }}
     >
       {children}

From 31a518a9d15978bdd7b555b4cc9ca9e46853b232 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 27 Oct 2024 13:09:13 -0700
Subject: [PATCH 246/376] nit

---
 web/src/app/admin/assistants/AssistantEditor.tsx | 1 -
 1 file changed, 1 deletion(-)

diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 131cbf5930d..9637e74026f 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -140,7 +140,6 @@ export function AssistantEditor({
   const defaultProvider = llmProviders.find(
     (llmProvider) => llmProvider.is_default_provider
   );
-  const defaultProviderName = defaultProvider?.provider;
   const defaultModelName = defaultProvider?.default_model_name;
   const providerDisplayNameToProviderName = new Map<string, string>();
   llmProviders.forEach((llmProvider) => {

From f27071cbc5b5253a0b70f955787f60d62e69ed52 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 14:49:10 -0700
Subject: [PATCH 247/376] Harmless Backport Test (#2950)

---
 backend/danswer/connectors/gitlab/connector.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/connectors/gitlab/connector.py b/backend/danswer/connectors/gitlab/connector.py
index f07baf3e141..805aff0073c 100644
--- a/backend/danswer/connectors/gitlab/connector.py
+++ b/backend/danswer/connectors/gitlab/connector.py
@@ -24,6 +24,10 @@
 from danswer.connectors.models import Section
 from danswer.utils.logger import setup_logger
 
+
+logger = setup_logger()
+
+
 # List of directories/Files to exclude
 exclude_patterns = [
     "logs",
@@ -31,7 +35,6 @@
     ".gitlab/",
     ".pre-commit-config.yaml",
 ]
-logger = setup_logger()
 
 
 def _batch_gitlab_objects(

From aea261d49e5ee35e210439293bac11a88a127296 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 27 Oct 2024 14:52:33 -0700
Subject: [PATCH 248/376] Ensure build args passed to cloud web images (#2947)

* ensure build args passed to cloud web images

* update web build workflow
---
 ...-build-push-cloud-web-container-on-tag.yml | 136 ++++++++++++++++++
 web/Dockerfile                                |   5 +
 2 files changed, 141 insertions(+)
 create mode 100644 .github/workflows/docker-build-push-cloud-web-container-on-tag.yml

diff --git a/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml b/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml
new file mode 100644
index 00000000000..871c96841ad
--- /dev/null
+++ b/.github/workflows/docker-build-push-cloud-web-container-on-tag.yml
@@ -0,0 +1,136 @@
+name: Build and Push Cloud Web Image on Tag
+# Identical to the web container build, but with correct image tag and build args
+
+on:
+  push:
+    tags:
+      - '*'
+
+env:
+  REGISTRY_IMAGE: danswer/danswer-cloud-web-server
+  LATEST_TAG: ${{ contains(github.ref_name, 'latest') }}
+  
+jobs:
+  build:
+    runs-on:
+      - runs-on
+      - runner=${{ matrix.platform == 'linux/amd64' && '8cpu-linux-x64' || '8cpu-linux-arm64' }}
+      - run-id=${{ github.run_id }}
+      - tag=platform-${{ matrix.platform }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux/amd64
+          - linux/arm64
+
+    steps:
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV          
+      
+      - name: Checkout
+        uses: actions/checkout@v4
+      
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+          tags: |
+            type=raw,value=${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+            type=raw,value=${{ env.LATEST_TAG == 'true' && format('{0}:latest', env.REGISTRY_IMAGE) || '' }}
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+    
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@v5
+        with:
+          context: ./web
+          file: ./web/Dockerfile
+          platforms: ${{ matrix.platform }}
+          push: true
+          build-args: |
+            DANSWER_VERSION=${{ github.ref_name }}
+            NEXT_PUBLIC_CLOUD_ENABLED=true
+            NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }}
+            NEXT_PUBLIC_POSTHOG_HOST=${{ secrets.POSTHOG_HOST }}
+            NEXT_PUBLIC_SENTRY_DSN=${{ secrets.SENTRY_DSN }}
+          # needed due to weird interactions with the builds for different platforms  
+          no-cache: true
+          labels: ${{ steps.meta.outputs.labels }}
+          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+      
+      - name: Export digest
+        run: |
+          mkdir -p /tmp/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "/tmp/digests/${digest#sha256:}"          
+      
+      - name: Upload digest
+        uses: actions/upload-artifact@v4
+        with:
+          name: digests-${{ env.PLATFORM_PAIR }}
+          path: /tmp/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    needs:
+      - build
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@v4
+        with:
+          path: /tmp/digests
+          pattern: digests-*
+          merge-multiple: true
+      
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY_IMAGE }}
+      
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      
+      - name: Create manifest list and push
+        working-directory: /tmp/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)          
+      
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}
+
+    # trivy has their own rate limiting issues causing this action to flake
+    # we worked around it by hardcoding to different db repos in env
+    # can re-enable when they figure it out
+    # https://github.com/aquasecurity/trivy/discussions/7538
+    # https://github.com/aquasecurity/trivy-action/issues/389
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        env:
+          TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+          TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+        with:
+          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
+          severity: 'CRITICAL,HIGH'
diff --git a/web/Dockerfile b/web/Dockerfile
index 46ec203250e..0edad2a9446 100644
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -66,6 +66,9 @@ ARG NEXT_PUBLIC_POSTHOG_HOST
 ENV NEXT_PUBLIC_POSTHOG_KEY=${NEXT_PUBLIC_POSTHOG_KEY}
 ENV NEXT_PUBLIC_POSTHOG_HOST=${NEXT_PUBLIC_POSTHOG_HOST}
 
+ARG NEXT_PUBLIC_SENTRY_DSN
+ENV NEXT_PUBLIC_SENTRY_DSN=${NEXT_PUBLIC_SENTRY_DSN} 
+
 RUN npx next build
 
 # Step 2. Production image, copy all the files and run next
@@ -131,6 +134,8 @@ ARG NEXT_PUBLIC_POSTHOG_KEY
 ARG NEXT_PUBLIC_POSTHOG_HOST
 ENV NEXT_PUBLIC_POSTHOG_KEY=${NEXT_PUBLIC_POSTHOG_KEY}
 ENV NEXT_PUBLIC_POSTHOG_HOST=${NEXT_PUBLIC_POSTHOG_HOST}
+ARG NEXT_PUBLIC_SENTRY_DSN
+ENV NEXT_PUBLIC_SENTRY_DSN=${NEXT_PUBLIC_SENTRY_DSN} 
 
 
 # Note: Don't expose ports here, Compose will handle that for us if necessary. 

From 7ea484aee27248bace1d72dc7908bc75db239ec1 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:18:46 -0700
Subject: [PATCH 249/376] Trigger from Workflow (#2951)

---
 .../docker-build-push-backend-container-on-tag.yml       | 2 +-
 .github/workflows/pr-backport-autotrigger.yml            | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/docker-build-push-backend-container-on-tag.yml b/.github/workflows/docker-build-push-backend-container-on-tag.yml
index ef07e051db3..7318fd64f13 100644
--- a/.github/workflows/docker-build-push-backend-container-on-tag.yml
+++ b/.github/workflows/docker-build-push-backend-container-on-tag.yml
@@ -1,7 +1,7 @@
 name: Build and Push Backend Image on Tag
 
 on:
-  push:
+  create:
     tags:
       - '*'
 
diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 2376365c41d..935ce04bf63 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -6,7 +6,8 @@ on:
 
 permissions:
   contents: write
-  actions: write  # Allows the workflow to trigger other workflows on push
+  actions: write
+  workflows: write
 
 jobs:
   backport:
@@ -81,7 +82,7 @@ jobs:
           }
           # Create new beta tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
-          # Push the changes and tag to the beta branch
+          # Push the changes and tag to the beta branch using PAT
           git push origin ${{ steps.list-branches.outputs.beta }} --force
           git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
           # Checkout the stable branch
@@ -93,6 +94,6 @@ jobs:
           }
           # Create new stable tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
-          # Push the changes and tag to the stable branch
+          # Push the changes and tag to the stable branch using PAT
           git push origin ${{ steps.list-branches.outputs.stable }} --force
-          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
\ No newline at end of file
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}

From fa88c1dba84f6204ef5ca6af0b81cf1e55730ec1 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:21:17 -0700
Subject: [PATCH 250/376] Test Workflow Trigger (#2952)

---
 backend/danswer/connectors/gitlab/connector.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/danswer/connectors/gitlab/connector.py b/backend/danswer/connectors/gitlab/connector.py
index 805aff0073c..39ec443e709 100644
--- a/backend/danswer/connectors/gitlab/connector.py
+++ b/backend/danswer/connectors/gitlab/connector.py
@@ -27,7 +27,6 @@
 
 logger = setup_logger()
 
-
 # List of directories/Files to exclude
 exclude_patterns = [
     "logs",

From 6568c7805a378c7c5387e13fb07c113f79a3422e Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:31:08 -0700
Subject: [PATCH 251/376] Update docker-build-push-backend-container-on-tag.yml

---
 .../workflows/docker-build-push-backend-container-on-tag.yml    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/docker-build-push-backend-container-on-tag.yml b/.github/workflows/docker-build-push-backend-container-on-tag.yml
index 7318fd64f13..ef07e051db3 100644
--- a/.github/workflows/docker-build-push-backend-container-on-tag.yml
+++ b/.github/workflows/docker-build-push-backend-container-on-tag.yml
@@ -1,7 +1,7 @@
 name: Build and Push Backend Image on Tag
 
 on:
-  create:
+  push:
     tags:
       - '*'
 

From ad3df42b52700df77d9055ef2f57a1c670258e4a Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:37:59 -0700
Subject: [PATCH 252/376] Backport Tag Test (#2954)

---
 backend/danswer/connectors/gitlab/connector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/gitlab/connector.py b/backend/danswer/connectors/gitlab/connector.py
index 39ec443e709..81aeae5191f 100644
--- a/backend/danswer/connectors/gitlab/connector.py
+++ b/backend/danswer/connectors/gitlab/connector.py
@@ -253,5 +253,6 @@ def poll_source(
             "gitlab_url": os.environ["GITLAB_URL"],
         }
     )
+
     document_batches = connector.load_from_state()
     print(next(document_batches))

From 18dcdd680d75cb9e45cbe4ea326851b0f6202b5c Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:44:41 -0700
Subject: [PATCH 253/376] GHA Trigger (#2955)

---
 .github/workflows/pr-backport-autotrigger.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 935ce04bf63..152db3bee07 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -7,7 +7,6 @@ on:
 permissions:
   contents: write
   actions: write
-  workflows: write
 
 jobs:
   backport:
@@ -97,3 +96,5 @@ jobs:
           # Push the changes and tag to the stable branch using PAT
           git push origin ${{ steps.list-branches.outputs.stable }} --force
           git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}

From 6f8e7abcbb5f92356128a9dc1b80305ff3f05683 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:45:57 -0700
Subject: [PATCH 254/376] Backport (#2956)

---
 backend/danswer/connectors/gitlab/connector.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/danswer/connectors/gitlab/connector.py b/backend/danswer/connectors/gitlab/connector.py
index 81aeae5191f..39ec443e709 100644
--- a/backend/danswer/connectors/gitlab/connector.py
+++ b/backend/danswer/connectors/gitlab/connector.py
@@ -253,6 +253,5 @@ def poll_source(
             "gitlab_url": os.environ["GITLAB_URL"],
         }
     )
-
     document_batches = connector.load_from_state()
     print(next(document_batches))

From 7dc06bfbe5ff9bb02b4b39a8f66604d703b3eb1b Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:55:07 -0700
Subject: [PATCH 255/376] Backport Test (#2957)

---
 backend/danswer/connectors/mediawiki/wiki.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/mediawiki/wiki.py b/backend/danswer/connectors/mediawiki/wiki.py
index a3de963982b..7ba63cc1925 100644
--- a/backend/danswer/connectors/mediawiki/wiki.py
+++ b/backend/danswer/connectors/mediawiki/wiki.py
@@ -22,6 +22,7 @@
 from danswer.connectors.models import Section
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 

From 1f2be542f0c5baa1d2df344e013400db95d6ff83 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 15:59:52 -0700
Subject: [PATCH 256/376] Backport Test

---
 .github/workflows/pr-backport-autotrigger.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 152db3bee07..f351a4f97cb 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -12,6 +12,8 @@ jobs:
   backport:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
@@ -96,5 +98,3 @@ jobs:
           # Push the changes and tag to the stable branch using PAT
           git push origin ${{ steps.list-branches.outputs.stable }} --force
           git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}

From e8dfed959e1f747f1737183bd8ce9c40814e0c83 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:06:36 -0700
Subject: [PATCH 257/376] Backport Test (#2958)

---
 backend/danswer/connectors/mediawiki/wiki.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/danswer/connectors/mediawiki/wiki.py b/backend/danswer/connectors/mediawiki/wiki.py
index 7ba63cc1925..8a09fd6d500 100644
--- a/backend/danswer/connectors/mediawiki/wiki.py
+++ b/backend/danswer/connectors/mediawiki/wiki.py
@@ -231,5 +231,7 @@ def poll_source(
     print("All docs", all_docs)
     current = datetime.datetime.now().timestamp()
     one_day_ago = current - 30 * 24 * 60 * 60  # 30 days
+
     latest_docs = list(test_connector.poll_source(one_day_ago, current))
+
     print("Latest docs", latest_docs)

From 2a6c0328834b052a1d8cecd6c073535bc46736cb Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:19:59 -0700
Subject: [PATCH 258/376] Backport No Tag

---
 .github/workflows/pr-backport-autotrigger.yml | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index f351a4f97cb..2215269388e 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -1,5 +1,7 @@
 name: Backport on Merge
 
+# Note this workflow does not trigger the builds, be sure to manually tag the branches to trigger the builds
+
 on:
   pull_request:
     types: [closed]  # Later we check for merge so only PRs that go in can get backported
@@ -72,29 +74,33 @@ jobs:
         run: |
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
+          
           # Fetch all history for all branches and tags
           git fetch --prune
-          # Checkout the beta branch
+
+          # Prepare the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
-          # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to beta failed due to conflicts."
             exit 1
           }
-          # Create new beta tag
+          
+          # Create new beta branch/tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
           git push origin ${{ steps.list-branches.outputs.beta }} --force
-          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
-          # Checkout the stable branch
+          #git push origin ${{ steps.list-branches.outputs.new_beta_tag }}  # This step does not trigger the workflow, better to do it manually for now
+
+          # Prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
           # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to stable failed due to conflicts."
             exit 1
           }
-          # Create new stable tag
+          
+          # Create new stable branch/tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT
           git push origin ${{ steps.list-branches.outputs.stable }} --force
-          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
+          #git push origin ${{ steps.list-branches.outputs.new_stable_tag }}  # This step does not trigger the workflow, better to do it manually for now

From 07d76b2954a101cad626f2a533fcf24d194145f2 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:25:43 -0700
Subject: [PATCH 259/376] Notion Child Block Fix (#2953)

---
 backend/danswer/connectors/notion/connector.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/connectors/notion/connector.py b/backend/danswer/connectors/notion/connector.py
index d4de5eaebd2..4680c3d044b 100644
--- a/backend/danswer/connectors/notion/connector.py
+++ b/backend/danswer/connectors/notion/connector.py
@@ -134,9 +134,14 @@ def _fetch_child_blocks(
                     f"This is likely due to the block not being shared "
                     f"with the Danswer integration. Exact exception:\n\n{e}"
                 )
-                return None
-            logger.exception(f"Error fetching blocks - {res.json()}")
-            raise e
+            else:
+                logger.exception(
+                    f"Error fetching blocks with status code {res.status_code}: {res.json()}"
+                )
+
+            # This can occasionally happen, the reason is unknown and cannot be reproduced on our internal Notion
+            # Assuming this will not be a critical loss of data
+            return None
         return res.json()
 
     @retry(tries=3, delay=1, backoff=2)

From 85c56f99424bcea1db50e0a138d4b0d669bcbc84 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:30:15 -0700
Subject: [PATCH 260/376] Backport Richard

---
 .github/workflows/pr-backport-autotrigger.yml | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 2215269388e..e72ce205ea2 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -14,20 +14,16 @@ jobs:
   backport:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
-    env:
-      GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Fetch all history for all branches and tags
+          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
 
-      - name: Set up Git
+      - name: Set up Git user
         run: |
-          git config --global user.name "yuhongsun96"
-          git config --global user.email "yuhongsun96@gmail.com"
-          # Configure Git to use the PAT for authentication
-          git remote set-url origin https://yuhongsun96:${{ secrets.YUHONG_GH_ACTIONS }}@github.com/${{ github.repository }}.git
+          git config user.name "Richard Kuo [bot]"
+          git config user.email "rkuo[bot]@danswer.ai"
 
       - name: Check for Backport Checkbox
         id: checkbox-check

From 4a9966148dd9fe9ed7d01a2e045d27895dee1047 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:33:04 -0700
Subject: [PATCH 261/376] Backport Test (#2959)

---
 backend/danswer/connectors/clickup/connector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/clickup/connector.py b/backend/danswer/connectors/clickup/connector.py
index 6ef60763baf..2ccc5ef4f65 100644
--- a/backend/danswer/connectors/clickup/connector.py
+++ b/backend/danswer/connectors/clickup/connector.py
@@ -210,6 +210,7 @@ def poll_source(
             "clickup_team_id": os.environ["clickup_team_id"],
         }
     )
+
     latest_docs = clickup_connector.load_from_state()
 
     for doc in latest_docs:

From 001801dee0f21ea973546e9cc5d3a60cd0e867bc Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:37:26 -0700
Subject: [PATCH 262/376] Add back Backport Tags

---
 .github/workflows/pr-backport-autotrigger.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index e72ce205ea2..47257862e85 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -85,7 +85,7 @@ jobs:
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
           git push origin ${{ steps.list-branches.outputs.beta }} --force
-          #git push origin ${{ steps.list-branches.outputs.new_beta_tag }}  # This step does not trigger the workflow, better to do it manually for now
+          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}  # This step does not trigger the workflow, better to do it manually for now
 
           # Prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
@@ -99,4 +99,4 @@ jobs:
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT
           git push origin ${{ steps.list-branches.outputs.stable }} --force
-          #git push origin ${{ steps.list-branches.outputs.new_stable_tag }}  # This step does not trigger the workflow, better to do it manually for now
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}  # This step does not trigger the workflow, better to do it manually for now

From 1187849afe6623abb9b79b74520c9486a43e7a90 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:42:08 -0700
Subject: [PATCH 263/376] Backport Touchup

---
 .github/workflows/pr-backport-autotrigger.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 47257862e85..a8e6b8a3d00 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -47,8 +47,14 @@ jobs:
           # Fetch latest tags for beta and stable
           LATEST_BETA_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*-beta.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$" | grep -v -- "-cloud" | sort -Vr | head -n 1)
           LATEST_STABLE_TAG=$(git tag -l "v[0-9]*.[0-9]*.[0-9]*" | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | sort -Vr | head -n 1)
-          # Increment latest beta tag
-          NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}')
+          
+          # Handle case where no beta tags exist
+          if [[ -z "$LATEST_BETA_TAG" ]]; then
+            NEW_BETA_TAG="v1.0.0-beta.1"
+          else
+            NEW_BETA_TAG=$(echo $LATEST_BETA_TAG | awk -F '[.-]' '{print $1 "." $2 "." $3 "-beta." ($NF+1)}')
+          fi
+          
           # Increment latest stable tag
           NEW_STABLE_TAG=$(echo $LATEST_STABLE_TAG | awk -F '.' '{print $1 "." $2 "." ($3+1)}')
           echo "latest_beta_tag=$LATEST_BETA_TAG" >> $GITHUB_OUTPUT
@@ -84,8 +90,8 @@ jobs:
           # Create new beta branch/tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
-          git push origin ${{ steps.list-branches.outputs.beta }} --force
-          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}  # This step does not trigger the workflow, better to do it manually for now
+          git push origin ${{ steps.list-branches.outputs.beta }}
+          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
 
           # Prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
@@ -98,5 +104,5 @@ jobs:
           # Create new stable branch/tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT
-          git push origin ${{ steps.list-branches.outputs.stable }} --force
-          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}  # This step does not trigger the workflow, better to do it manually for now
+          git push origin ${{ steps.list-branches.outputs.stable }}
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}

From 23988f8c494bcae63b0c33ef5e2e882f4bc000d9 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:45:11 -0700
Subject: [PATCH 264/376] Touchup (#2961)

---
 backend/danswer/connectors/bookstack/connector.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/bookstack/connector.py b/backend/danswer/connectors/bookstack/connector.py
index f2e692d2c5f..9255bc3b8e5 100644
--- a/backend/danswer/connectors/bookstack/connector.py
+++ b/backend/danswer/connectors/bookstack/connector.py
@@ -44,8 +44,6 @@ def _get_doc_batch(
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
     ) -> tuple[list[Document], int]:
-        doc_batch: list[Document] = []
-
         params = {
             "count": str(batch_size),
             "offset": str(start_ind),
@@ -63,8 +61,7 @@ def _get_doc_batch(
             )
 
         batch = bookstack_client.get(endpoint, params=params).get("data", [])
-        for item in batch:
-            doc_batch.append(transformer(bookstack_client, item))
+        doc_batch = [transformer(bookstack_client, item) for item in batch]
 
         return doc_batch, len(batch)
 

From 0fb6baef2b6e0dd99942c6ed537915ef178962f8 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:52:07 -0700
Subject: [PATCH 265/376] Echo Merge Commit (#2962)

---
 .github/workflows/pr-backport-autotrigger.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index a8e6b8a3d00..f73dafaff34 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -77,6 +77,9 @@ jobs:
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
           
+          # Echo the merge commit SHA
+          echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}"
+          
           # Fetch all history for all branches and tags
           git fetch --prune
 

From c7018f7a6cdedeb17a2fd813ca43a78f2b58c466 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 16:55:02 -0700
Subject: [PATCH 266/376] Backport Test (#2963)

---
 backend/danswer/connectors/guru/connector.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/connectors/guru/connector.py b/backend/danswer/connectors/guru/connector.py
index a27546425d3..510105b872f 100644
--- a/backend/danswer/connectors/guru/connector.py
+++ b/backend/danswer/connectors/guru/connector.py
@@ -19,13 +19,14 @@
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.utils.logger import setup_logger
 
+
+logger = setup_logger()
+
 # Potential Improvements
 # 1. Support fetching per collection via collection token (configured at connector creation)
-
 GURU_API_BASE = "https://api.getguru.com/api/v1/"
 GURU_QUERY_ENDPOINT = GURU_API_BASE + "search/query"
 GURU_CARDS_URL = "https://app.getguru.com/card/"
-logger = setup_logger()
 
 
 def unixtime_to_guru_time_str(unix_time: SecondsSinceUnixEpoch) -> str:

From 7ddfabed624c7381daa719453a802db263bdbd93 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 17:03:53 -0700
Subject: [PATCH 267/376] Backport Debugging

---
 .github/workflows/pr-backport-autotrigger.yml | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index f73dafaff34..af4fdd31c5e 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -83,8 +83,13 @@ jobs:
           # Fetch all history for all branches and tags
           git fetch --prune
 
-          # Prepare the beta branch
+          # Reset and prepare the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
+          git reset --hard origin/${{ steps.list-branches.outputs.beta }}
+          echo "Last 5 commits on beta branch:"
+          git log -n 5 --pretty=format:"%H"
+
+          # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to beta failed due to conflicts."
             exit 1
@@ -96,8 +101,12 @@ jobs:
           git push origin ${{ steps.list-branches.outputs.beta }}
           git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
 
-          # Prepare the stable branch
+          # Reset and prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
+          git reset --hard origin/${{ steps.list-branches.outputs.stable }}
+          echo "Last 5 commits on stable branch:"
+          git log -n 5 --pretty=format:"%H"
+
           # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to stable failed due to conflicts."

From e28ba4b55b132870dac5a0822b79b29f9d942971 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 17:10:31 -0700
Subject: [PATCH 268/376] Backport Test Conn (#2964)

---
 backend/danswer/connectors/linear/connector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/linear/connector.py b/backend/danswer/connectors/linear/connector.py
index 8455b20f5ba..22b769562d1 100644
--- a/backend/danswer/connectors/linear/connector.py
+++ b/backend/danswer/connectors/linear/connector.py
@@ -18,6 +18,7 @@
 from danswer.connectors.models import Section
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 _NUM_RETRIES = 5

From 1a31f1e773a33deef5aabdd3c1c70066f7cfdc09 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 18:58:26 -0700
Subject: [PATCH 269/376] New Credentials GH

---
 .github/workflows/pr-backport-autotrigger.yml | 24 +++++++------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index af4fdd31c5e..00c21a9c85f 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -14,26 +14,20 @@ jobs:
   backport:
     if: github.event.pull_request.merged == true
     runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
-          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
+          fetch-depth: 0  # Fetch all history for all branches and tags
 
-      - name: Set up Git user
+      - name: Set up Git
         run: |
-          git config user.name "Richard Kuo [bot]"
-          git config user.email "rkuo[bot]@danswer.ai"
-
-      - name: Check for Backport Checkbox
-        id: checkbox-check
-        run: |
-          PR_BODY="${{ github.event.pull_request.body }}"
-          if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
-            echo "backport=true" >> $GITHUB_OUTPUT
-          else
-            echo "backport=false" >> $GITHUB_OUTPUT
-          fi
+          git config --global user.name "yuhongsun96"
+          git config --global user.email "yuhongsun96@gmail.com"
+          # Configure Git to use the PAT for authentication
+          git remote set-url origin https://yuhongsun96:${{ secrets.YUHONG_GH_ACTIONS }}@github.com/${{ github.repository }}.git
 
       - name: List and sort release branches
         id: list-branches

From 9270782c49c940dca10023b5bf24f3b2dae5b2d6 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:00:37 -0700
Subject: [PATCH 270/376] Backport Test (#2966)

---
 .../cross_connector_utils/miscellaneous_utils.py          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py b/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py
index 897503dca99..1b79c4a4faf 100644
--- a/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py
+++ b/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py
@@ -11,6 +11,10 @@
 from danswer.utils.text_processing import is_valid_email
 
 
+T = TypeVar("T")
+U = TypeVar("U")
+
+
 def datetime_to_utc(dt: datetime) -> datetime:
     if dt.tzinfo is None or dt.tzinfo.utcoffset(dt) is None:
         dt = dt.replace(tzinfo=timezone.utc)
@@ -49,10 +53,6 @@ def get_experts_stores_representations(
     return [owner for owner in reps if owner is not None]
 
 
-T = TypeVar("T")
-U = TypeVar("U")
-
-
 def process_in_batches(
     objects: list[T], process_function: Callable[[T], U], batch_size: int
 ) -> Iterator[list[U]]:

From 4c417b5e3e0d5dad4b91166fd46d6522416882cc Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:12:31 -0700
Subject: [PATCH 271/376] Revert

---
 .github/workflows/pr-backport-autotrigger.yml | 24 +++++--------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 00c21a9c85f..05752b14701 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -71,19 +71,11 @@ jobs:
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
           
-          # Echo the merge commit SHA
-          echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}"
-          
           # Fetch all history for all branches and tags
           git fetch --prune
 
-          # Reset and prepare the beta branch
+          # Prepare the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
-          git reset --hard origin/${{ steps.list-branches.outputs.beta }}
-          echo "Last 5 commits on beta branch:"
-          git log -n 5 --pretty=format:"%H"
-
-          # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to beta failed due to conflicts."
             exit 1
@@ -92,15 +84,11 @@ jobs:
           # Create new beta branch/tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
-          git push origin ${{ steps.list-branches.outputs.beta }}
-          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
+          git push origin ${{ steps.list-branches.outputs.beta }} --force
+          #git push origin ${{ steps.list-branches.outputs.new_beta_tag }}  # This step does not trigger the workflow, better to do it manually for now
 
-          # Reset and prepare the stable branch
+          # Prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
-          git reset --hard origin/${{ steps.list-branches.outputs.stable }}
-          echo "Last 5 commits on stable branch:"
-          git log -n 5 --pretty=format:"%H"
-
           # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to stable failed due to conflicts."
@@ -110,5 +98,5 @@ jobs:
           # Create new stable branch/tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT
-          git push origin ${{ steps.list-branches.outputs.stable }}
-          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
+          git push origin ${{ steps.list-branches.outputs.stable }} --force
+          #git push origin ${{ steps.list-branches.outputs.new_stable_tag }}  # This step does not trigger the workflow, better to do it manually for now

From 57e5264df6d1eb27efb57b4167eff7dc6027fecd Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:31:29 -0700
Subject: [PATCH 272/376] Backport Test (#2967)

---
 backend/danswer/connectors/sharepoint/connector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/sharepoint/connector.py b/backend/danswer/connectors/sharepoint/connector.py
index a32b4ecb1ab..8d99baffe33 100644
--- a/backend/danswer/connectors/sharepoint/connector.py
+++ b/backend/danswer/connectors/sharepoint/connector.py
@@ -25,6 +25,7 @@
 from danswer.file_processing.extract_file_text import extract_file_text
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 

From 20e8c2287a1a58aaeef521fe805000163d73f154 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:39:18 -0700
Subject: [PATCH 273/376] Add Conditional

---
 .github/workflows/pr-backport-autotrigger.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 05752b14701..bf5bb6ba97e 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -28,6 +28,16 @@ jobs:
           git config --global user.email "yuhongsun96@gmail.com"
           # Configure Git to use the PAT for authentication
           git remote set-url origin https://yuhongsun96:${{ secrets.YUHONG_GH_ACTIONS }}@github.com/${{ github.repository }}.git
+      
+      - name: Check for Backport Checkbox
+        id: checkbox-check
+        run: |
+          PR_BODY="${{ github.event.pull_request.body }}"
+          if [[ "$PR_BODY" == *"[x] This PR should be backported"* ]]; then
+            echo "backport=true" >> $GITHUB_OUTPUT
+          else
+            echo "backport=false" >> $GITHUB_OUTPUT
+          fi
 
       - name: List and sort release branches
         id: list-branches

From 3d9cc769d92f0568b9f1b119a6cce1a19856dd4e Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:41:04 -0700
Subject: [PATCH 274/376] Backport Test 4 (#2968)

---
 backend/danswer/connectors/slab/connector.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/connectors/slab/connector.py b/backend/danswer/connectors/slab/connector.py
index 80380ff7c29..ae76332838b 100644
--- a/backend/danswer/connectors/slab/connector.py
+++ b/backend/danswer/connectors/slab/connector.py
@@ -20,10 +20,13 @@
 from danswer.connectors.models import Section
 from danswer.utils.logger import setup_logger
 
+
+logger = setup_logger()
+
+
 # Fairly generous retry because it's not understood why occasionally GraphQL requests fail even with timeout > 1 min
 SLAB_GRAPHQL_MAX_TRIES = 10
 SLAB_API_URL = "https://api.slab.com/v1/graphql"
-logger = setup_logger()
 
 
 def run_graphql_request(

From 7856718db80dcbb440f0e896d48a432b2a08a55f Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:54:53 -0700
Subject: [PATCH 275/376] k

---
 .github/workflows/pr-backport-autotrigger.yml | 24 ++++++++++++++-----
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index bf5bb6ba97e..1c3bd23df0f 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -81,11 +81,19 @@ jobs:
           set -e
           echo "Backporting to beta ${{ steps.list-branches.outputs.beta }} and stable ${{ steps.list-branches.outputs.stable }}"
           
+          # Echo the merge commit SHA
+          echo "Merge commit SHA: ${{ github.event.pull_request.merge_commit_sha }}"
+          
           # Fetch all history for all branches and tags
           git fetch --prune
 
-          # Prepare the beta branch
+          # Reset and prepare the beta branch
           git checkout ${{ steps.list-branches.outputs.beta }}
+          echo "Last 5 commits on beta branch:"
+          git log -n 5 --pretty=format:"%H"
+          echo ""  # Newline for formatting
+
+          # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to beta failed due to conflicts."
             exit 1
@@ -94,11 +102,15 @@ jobs:
           # Create new beta branch/tag
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
-          git push origin ${{ steps.list-branches.outputs.beta }} --force
-          #git push origin ${{ steps.list-branches.outputs.new_beta_tag }}  # This step does not trigger the workflow, better to do it manually for now
+          git push origin ${{ steps.list-branches.outputs.beta }}
+          #git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
 
-          # Prepare the stable branch
+          # Reset and prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
+          echo "Last 5 commits on stable branch:"
+          git log -n 5 --pretty=format:"%H"
+          echo ""  # Newline for formatting
+          
           # Cherry-pick the merge commit from the merged PR
           git cherry-pick -m 1 ${{ github.event.pull_request.merge_commit_sha }} || {
             echo "Cherry-pick to stable failed due to conflicts."
@@ -108,5 +120,5 @@ jobs:
           # Create new stable branch/tag
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT
-          git push origin ${{ steps.list-branches.outputs.stable }} --force
-          #git push origin ${{ steps.list-branches.outputs.new_stable_tag }}  # This step does not trigger the workflow, better to do it manually for now
+          git push origin ${{ steps.list-branches.outputs.stable }}
+          #git push origin ${{ steps.list-branches.outputs.new_stable_tag }}

From e2c4c07c348f79a7950feb570f304fa792ccf939 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 19:56:29 -0700
Subject: [PATCH 276/376] Push Tag

---
 .github/workflows/pr-backport-autotrigger.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 1c3bd23df0f..de5ccb3217b 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -103,7 +103,7 @@ jobs:
           git tag ${{ steps.list-branches.outputs.new_beta_tag }}
           # Push the changes and tag to the beta branch using PAT
           git push origin ${{ steps.list-branches.outputs.beta }}
-          #git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
+          git push origin ${{ steps.list-branches.outputs.new_beta_tag }}
 
           # Reset and prepare the stable branch
           git checkout ${{ steps.list-branches.outputs.stable }}
@@ -121,4 +121,4 @@ jobs:
           git tag ${{ steps.list-branches.outputs.new_stable_tag }}
           # Push the changes and tag to the stable branch using PAT
           git push origin ${{ steps.list-branches.outputs.stable }}
-          #git push origin ${{ steps.list-branches.outputs.new_stable_tag }}
+          git push origin ${{ steps.list-branches.outputs.new_stable_tag }}

From d8ad3e73bf504edd7478b5e7177e8e3f8e91d552 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 20:07:29 -0700
Subject: [PATCH 277/376] Backport Test 5 (#2969)

---
 backend/danswer/connectors/slack/connector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/slack/connector.py b/backend/danswer/connectors/slack/connector.py
index f5728950e4f..92b5a0b7558 100644
--- a/backend/danswer/connectors/slack/connector.py
+++ b/backend/danswer/connectors/slack/connector.py
@@ -441,6 +441,7 @@ def poll_source(
 
     current = time.time()
     one_day_ago = current - 24 * 60 * 60  # 1 day
+
     document_batches = connector.poll_source(one_day_ago, current)
 
     print(next(document_batches))

From b2d8e1033947f7cc56a7c1ecd1ba2fb2a2fd589f Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 20:09:42 -0700
Subject: [PATCH 278/376] Richard Key

---
 .github/workflows/pr-backport-autotrigger.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index de5ccb3217b..57f3006af74 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -18,16 +18,14 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.YUHONG_GH_ACTIONS }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Fetch all history for all branches and tags
+          ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
 
-      - name: Set up Git
+      - name: Set up Git user
         run: |
-          git config --global user.name "yuhongsun96"
-          git config --global user.email "yuhongsun96@gmail.com"
-          # Configure Git to use the PAT for authentication
-          git remote set-url origin https://yuhongsun96:${{ secrets.YUHONG_GH_ACTIONS }}@github.com/${{ github.repository }}.git
+          git config user.name "Richard Kuo [bot]"
+          git config user.email "rkuo[bot]@danswer.ai"
       
       - name: Check for Backport Checkbox
         id: checkbox-check

From f84e75cee7e1caa95962f222ebfc1083e018a784 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 22:45:22 -0700
Subject: [PATCH 279/376] Backport Test 6 (#2970)

---
 backend/danswer/connectors/slack/load_connector.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/connectors/slack/load_connector.py b/backend/danswer/connectors/slack/load_connector.py
index ebcfce5b845..7350ac6284d 100644
--- a/backend/danswer/connectors/slack/load_connector.py
+++ b/backend/danswer/connectors/slack/load_connector.py
@@ -16,6 +16,7 @@
 from danswer.connectors.slack.utils import get_message_link
 from danswer.utils.logger import setup_logger
 
+
 logger = setup_logger()
 
 

From e46facb76585115e4e1b54f9680474f9a73f2bb6 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 22:52:52 -0700
Subject: [PATCH 280/376] Backport Final

---
 .github/workflows/pr-backport-autotrigger.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/pr-backport-autotrigger.yml b/.github/workflows/pr-backport-autotrigger.yml
index 57f3006af74..273f00a5c5a 100644
--- a/.github/workflows/pr-backport-autotrigger.yml
+++ b/.github/workflows/pr-backport-autotrigger.yml
@@ -21,11 +21,13 @@ jobs:
         uses: actions/checkout@v4
         with:
           ssh-key: "${{ secrets.RKUO_DEPLOY_KEY }}"
+          fetch-depth: 0
 
       - name: Set up Git user
         run: |
           git config user.name "Richard Kuo [bot]"
           git config user.email "rkuo[bot]@danswer.ai"
+          git fetch --prune
       
       - name: Check for Backport Checkbox
         id: checkbox-check

From 5062075b8d5afdf2daf9ebcfd533a7d727e427f6 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 27 Oct 2024 22:55:35 -0700
Subject: [PATCH 281/376] Backport Test 7 (#2971)

---
 .../tests/unit/danswer/connectors/mediawiki/test_wiki.py    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py b/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
index 9aaacfc1eae..78601446a66 100644
--- a/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
+++ b/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
@@ -9,12 +9,15 @@
 
 from danswer.connectors.mediawiki import wiki
 
+# These tests are disabled for now
+
 
 @pytest.fixture
 def site() -> pywikibot.Site:
     return pywikibot.Site("en", "wikipedia")
 
 
+@pytest.mark.skip(reason="Test disabled")
 def test_pywikibot_timestamp_to_utc_datetime() -> None:
     timestamp_without_tzinfo = pywikibot.Timestamp(2023, 12, 27, 15, 38, 49)
     timestamp_min_timezone = timestamp_without_tzinfo.astimezone(datetime.timezone.min)
@@ -80,6 +83,7 @@ def latest_revision(self) -> pywikibot.page.Revision:
         )
 
 
+@pytest.mark.skip(reason="Test disabled")
 def test_get_doc_from_page(site: pywikibot.Site) -> None:
     test_page = MockPage(site, "Test Page", _has_categories=True)
     doc = wiki.get_doc_from_page(test_page, site, wiki.DocumentSource.MEDIAWIKI)
@@ -103,6 +107,7 @@ def test_get_doc_from_page(site: pywikibot.Site) -> None:
     assert doc.id == f"MEDIAWIKI_{test_page.pageid}_{test_page.full_url()}"
 
 
+@pytest.mark.skip(reason="Test disabled")
 def test_mediawiki_connector_recurse_depth() -> None:
     """Test that the recurse_depth parameter is parsed correctly.
 
@@ -132,6 +137,7 @@ def test_mediawiki_connector_recurse_depth() -> None:
     assert connector.recurse_depth == recurse_depth
 
 
+@pytest.mark.skip(reason="Test disabled")
 def test_load_from_state_calls_poll_source_with_nones(mocker: MockFixture) -> None:
     connector = wiki.MediaWikiConnector("wikipedia.org", [], [], 0, "test")
     poll_source = mocker.patch.object(connector, "poll_source")

From 075e4f18bccedea403f13ad258500df5d5b13aad Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Mon, 28 Oct 2024 11:26:37 -0400
Subject: [PATCH 282/376] Clean up & comment fetch_tickets

---
 .../danswer/connectors/freshdesk/connector.py | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 0b88e5b49dd..47b10c8a4c1 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -5,7 +5,8 @@
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
-from danswer.connectors.interfaces import GenerateDocumentsOutput, PollConnector, LoadConnector, SecondsSinceUnixEpoch
+from danswer.connectors.interfaces import GenerateDocumentsOutput, SecondsSinceUnixEpoch
+from danswer.connectors.interfaces import PollConnector, LoadConnector
 from danswer.connectors.models import ConnectorMissingCredentialError, Document, Section
 from danswer.utils.logger import setup_logger
 
@@ -23,19 +24,16 @@ def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     }
 
     # Checking for overdue tickets
-    today = datetime.now(timezone.utc)
-    ticket["overdue"] = "true" if today > ticket["due_by"] else "false"
+    ticket["overdue"] = datetime.now(timezone.utc) > ticket["due_by"]
 
-    # Mapping the status field values
+    # Map ticket status codes to human readable values
     status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
-    ticket["status"] = status_mapping.get(ticket["status"], str(ticket["status"]))
+    if status_string := status_mapping.get(ticket.get("status")):
+        ticket["status"] = status_string
 
-    # Stripping HTML tags from the description field
+    # Parse HTML from the description field
     ticket["description"] = parse_html_page_basic(ticket["description"])
 
-    # Remove extra white spaces from the description field
-    ticket["description"] = " ".join(ticket["description"].split())
-
     return Document(
         id=ticket["id"],
         sections=[Section(
@@ -66,6 +64,10 @@ def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, An
         return None
     
     def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = None) -> Iterator[List[dict]]:
+        #"end" is not currently used, so we may double fetch tickets created after the indexing starts but before the actual call is made. 
+        #To use "end" would require us to use the search endpoint but it has limitations, 
+        #namely having to fetch all IDs and then individually fetch each ticket because there is no "include" field available for this endpoint:
+        #https://developers.freshdesk.com/api/#filter_tickets
         if any([self.api_key, self.domain, self.password]) is None:
             raise ConnectorMissingCredentialError("freshdesk")
         

From 1d89fea73ed4aabe836f18f80c56f9df296f8a50 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 28 Oct 2024 09:14:51 -0700
Subject: [PATCH 283/376] Bugfix/celery light backoff (#2880)

* logging cleanup

* raise vespa_timeout to 15 by default

* implement backoff for document index methods specifically

* do not retry on 400 BAD_REQUEST

* handle RetryError

* actually check status code and fix type errors
---
 .../celery/tasks/connector_deletion/tasks.py  |  2 +-
 .../background/celery/tasks/indexing/tasks.py | 14 +++--
 .../background/celery/tasks/pruning/tasks.py  | 10 +--
 .../celery/tasks/shared/RetryDocumentIndex.py | 40 ++++++++++++
 .../background/celery/tasks/shared/tasks.py   | 62 ++++++++++++++-----
 .../background/celery/tasks/vespa/tasks.py    | 51 +++++++++++----
 backend/danswer/configs/app_configs.py        |  2 +-
 7 files changed, 143 insertions(+), 38 deletions(-)
 create mode 100644 backend/danswer/background/celery/tasks/shared/RetryDocumentIndex.py

diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index 59d236cde31..0b8c98c18c6 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -81,7 +81,7 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> N
             "Soft time limit exceeded, task is being terminated gracefully."
         )
     except Exception:
-        task_logger.exception("Unexpected exception")
+        task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 1d8723f1121..7c1460cbddf 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -175,7 +175,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     )
                     if attempt_id:
                         task_logger.info(
-                            f"Indexing queued: cc_pair_id={cc_pair.id} index_attempt_id={attempt_id}"
+                            f"Indexing queued: cc_pair={cc_pair.id} index_attempt={attempt_id}"
                         )
                         tasks_created += 1
     except SoftTimeLimitExceeded:
@@ -183,7 +183,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
             "Soft time limit exceeded, task is being terminated gracefully."
         )
     except Exception:
-        task_logger.exception("Unexpected exception")
+        task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
@@ -366,7 +366,12 @@ def try_creating_indexing_task(
         r.set(rci.fence_key, fence_value.model_dump_json())
     except Exception:
         r.delete(rci.fence_key)
-        task_logger.exception("Unexpected exception")
+        task_logger.exception(
+            f"Unexpected exception: "
+            f"tenant={tenant_id} "
+            f"cc_pair={cc_pair.id} "
+            f"search_settings={search_settings.id}"
+        )
         return None
     finally:
         if lock.owned():
@@ -470,10 +475,9 @@ def connector_indexing_task(
         # read related data and evaluate/print task progress
         fence_value = cast(bytes, r.get(rci.fence_key))
         if fence_value is None:
-            task_logger.info(
+            raise ValueError(
                 f"connector_indexing_task: fence_value not found: fence={rci.fence_key}"
             )
-            raise RuntimeError(f"Fence not found: fence={rci.fence_key}")
 
         try:
             fence_json = fence_value.decode("utf-8")
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index 2e68986e839..d9579ccf93c 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -79,13 +79,13 @@ def check_for_pruning(self: Task, *, tenant_id: str | None) -> None:
                 if not tasks_created:
                     continue
 
-                task_logger.info(f"Pruning queued: cc_pair_id={cc_pair.id}")
+                task_logger.info(f"Pruning queued: cc_pair={cc_pair.id}")
     except SoftTimeLimitExceeded:
         task_logger.info(
             "Soft time limit exceeded, task is being terminated gracefully."
         )
     except Exception:
-        task_logger.exception("Unexpected exception")
+        task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
@@ -201,7 +201,7 @@ def try_creating_prune_generator_task(
         # set this only after all tasks have been added
         r.set(rcp.fence_key, 1)
     except Exception:
-        task_logger.exception("Unexpected exception")
+        task_logger.exception(f"Unexpected exception: cc_pair={cc_pair.id}")
         return None
     finally:
         if lock.owned():
@@ -300,7 +300,7 @@ def connector_pruning_generator_task(
             rcp.documents_to_prune = set(doc_ids_to_remove)
 
             task_logger.info(
-                f"RedisConnectorPruning.generate_tasks starting. cc_pair_id={cc_pair_id}"
+                f"RedisConnectorPruning.generate_tasks starting. cc_pair={cc_pair.id}"
             )
             tasks_generated = rcp.generate_tasks(
                 self.app, db_session, r, None, tenant_id
@@ -310,7 +310,7 @@ def connector_pruning_generator_task(
 
             task_logger.info(
                 f"RedisConnectorPruning.generate_tasks finished. "
-                f"cc_pair={cc_pair_id} tasks_generated={tasks_generated}"
+                f"cc_pair={cc_pair.id} tasks_generated={tasks_generated}"
             )
 
             r.set(rcp.generator_complete_key, tasks_generated)
diff --git a/backend/danswer/background/celery/tasks/shared/RetryDocumentIndex.py b/backend/danswer/background/celery/tasks/shared/RetryDocumentIndex.py
new file mode 100644
index 00000000000..bdaca0d811e
--- /dev/null
+++ b/backend/danswer/background/celery/tasks/shared/RetryDocumentIndex.py
@@ -0,0 +1,40 @@
+import httpx
+from tenacity import retry
+from tenacity import retry_if_exception_type
+from tenacity import stop_after_delay
+from tenacity import wait_random_exponential
+
+from danswer.document_index.interfaces import DocumentIndex
+from danswer.document_index.interfaces import VespaDocumentFields
+
+
+class RetryDocumentIndex:
+    """A wrapper class to help with specific retries against Vespa involving
+    read timeouts.
+
+    wait_random_exponential implements full jitter as per this article:
+    https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/"""
+
+    MAX_WAIT = 30
+
+    # STOP_AFTER + MAX_WAIT should be slightly less (5?) than the celery soft_time_limit
+    STOP_AFTER = 70
+
+    def __init__(self, index: DocumentIndex):
+        self.index: DocumentIndex = index
+
+    @retry(
+        retry=retry_if_exception_type(httpx.ReadTimeout),
+        wait=wait_random_exponential(multiplier=1, max=MAX_WAIT),
+        stop=stop_after_delay(STOP_AFTER),
+    )
+    def delete_single(self, doc_id: str) -> int:
+        return self.index.delete_single(doc_id)
+
+    @retry(
+        retry=retry_if_exception_type(httpx.ReadTimeout),
+        wait=wait_random_exponential(multiplier=1, max=MAX_WAIT),
+        stop=stop_after_delay(STOP_AFTER),
+    )
+    def update_single(self, doc_id: str, fields: VespaDocumentFields) -> int:
+        return self.index.update_single(doc_id, fields)
diff --git a/backend/danswer/background/celery/tasks/shared/tasks.py b/backend/danswer/background/celery/tasks/shared/tasks.py
index 52a49d467e9..116e7e1ff7e 100644
--- a/backend/danswer/background/celery/tasks/shared/tasks.py
+++ b/backend/danswer/background/celery/tasks/shared/tasks.py
@@ -1,9 +1,14 @@
+from http import HTTPStatus
+
+import httpx
 from celery import shared_task
 from celery import Task
 from celery.exceptions import SoftTimeLimitExceeded
+from tenacity import RetryError
 
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.apps.app_base import task_logger
+from danswer.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
 from danswer.db.document import delete_document_by_connector_credential_pair__no_commit
 from danswer.db.document import delete_documents_complete__no_commit
 from danswer.db.document import get_document
@@ -20,12 +25,17 @@
 DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES = 3
 
 
+# 5 seconds more than RetryDocumentIndex STOP_AFTER+MAX_WAIT
+LIGHT_SOFT_TIME_LIMIT = 105
+LIGHT_TIME_LIMIT = LIGHT_SOFT_TIME_LIMIT + 15
+
+
 @shared_task(
     name="document_by_cc_pair_cleanup_task",
-    bind=True,
-    soft_time_limit=45,
-    time_limit=60,
+    soft_time_limit=LIGHT_SOFT_TIME_LIMIT,
+    time_limit=LIGHT_TIME_LIMIT,
     max_retries=DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES,
+    bind=True,
 )
 def document_by_cc_pair_cleanup_task(
     self: Task,
@@ -49,7 +59,7 @@ def document_by_cc_pair_cleanup_task(
     connector / credential pair from the access list
     (6) delete all relevant entries from postgres
     """
-    task_logger.info(f"tenant_id={tenant_id} document_id={document_id}")
+    task_logger.info(f"tenant={tenant_id} doc={document_id}")
 
     try:
         with get_session_with_tenant(tenant_id) as db_session:
@@ -57,17 +67,19 @@ def document_by_cc_pair_cleanup_task(
             chunks_affected = 0
 
             curr_ind_name, sec_ind_name = get_both_index_names(db_session)
-            document_index = get_default_document_index(
+            doc_index = get_default_document_index(
                 primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
             )
 
+            retry_index = RetryDocumentIndex(doc_index)
+
             count = get_document_connector_count(db_session, document_id)
             if count == 1:
                 # count == 1 means this is the only remaining cc_pair reference to the doc
                 # delete it from vespa and the db
                 action = "delete"
 
-                chunks_affected = document_index.delete_single(document_id)
+                chunks_affected = retry_index.delete_single(document_id)
                 delete_documents_complete__no_commit(
                     db_session=db_session,
                     document_ids=[document_id],
@@ -97,9 +109,7 @@ def document_by_cc_pair_cleanup_task(
                 )
 
                 # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-                chunks_affected = document_index.update_single(
-                    document_id, fields=fields
-                )
+                chunks_affected = retry_index.update_single(document_id, fields=fields)
 
                 # there are still other cc_pair references to the doc, so just resync to Vespa
                 delete_document_by_connector_credential_pair__no_commit(
@@ -118,19 +128,41 @@ def document_by_cc_pair_cleanup_task(
             db_session.commit()
 
             task_logger.info(
-                f"tenant_id={tenant_id} "
-                f"document_id={document_id} "
+                f"tenant={tenant_id} "
+                f"doc={document_id} "
                 f"action={action} "
                 f"refcount={count} "
                 f"chunks={chunks_affected}"
             )
     except SoftTimeLimitExceeded:
         task_logger.info(
-            f"SoftTimeLimitExceeded exception. tenant_id={tenant_id} doc_id={document_id}"
+            f"SoftTimeLimitExceeded exception. tenant={tenant_id} doc={document_id}"
         )
         return False
-    except Exception as e:
-        task_logger.exception("Unexpected exception")
+    except Exception as ex:
+        if isinstance(ex, RetryError):
+            task_logger.info(f"Retry failed: {ex.last_attempt.attempt_number}")
+
+            # only set the inner exception if it is of type Exception
+            e_temp = ex.last_attempt.exception()
+            if isinstance(e_temp, Exception):
+                e = e_temp
+        else:
+            e = ex
+
+        if isinstance(e, httpx.HTTPStatusError):
+            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+                task_logger.exception(
+                    f"Non-retryable HTTPStatusError: "
+                    f"tenant={tenant_id} "
+                    f"doc={document_id} "
+                    f"status={e.response.status_code}"
+                )
+            return False
+
+        task_logger.exception(
+            f"Unexpected exception: tenant={tenant_id} doc={document_id}"
+        )
 
         if self.request.retries < DOCUMENT_BY_CC_PAIR_CLEANUP_MAX_RETRIES:
             # Still retrying. Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
@@ -141,7 +173,7 @@ def document_by_cc_pair_cleanup_task(
             # eventually gets fixed out of band via stale document reconciliation
             task_logger.info(
                 f"Max retries reached. Marking doc as dirty for reconciliation: "
-                f"tenant_id={tenant_id} document_id={document_id}"
+                f"tenant={tenant_id} doc={document_id}"
             )
             with get_session_with_tenant(tenant_id):
                 mark_document_as_modified(document_id, db_session)
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index fcc4d2aa5bc..b058a97e1bc 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -4,6 +4,7 @@
 from http import HTTPStatus
 from typing import cast
 
+import httpx
 import redis
 from celery import Celery
 from celery import shared_task
@@ -13,6 +14,7 @@
 from celery.states import READY_STATES
 from redis import Redis
 from sqlalchemy.orm import Session
+from tenacity import RetryError
 
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.apps.app_base import task_logger
@@ -29,6 +31,9 @@
 from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
     RedisConnectorIndexingFenceData,
 )
+from danswer.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
+from danswer.background.celery.tasks.shared.tasks import LIGHT_SOFT_TIME_LIMIT
+from danswer.background.celery.tasks.shared.tasks import LIGHT_TIME_LIMIT
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryQueues
@@ -152,7 +157,7 @@ def check_for_vespa_sync_task(self: Task, *, tenant_id: str | None) -> None:
             "Soft time limit exceeded, task is being terminated gracefully."
         )
     except Exception:
-        task_logger.exception("Unexpected exception")
+        task_logger.exception(f"Unexpected exception: tenant={tenant_id}")
     finally:
         if lock_beat.owned():
             lock_beat.release()
@@ -809,22 +814,22 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
 @shared_task(
     name="vespa_metadata_sync_task",
     bind=True,
-    soft_time_limit=45,
-    time_limit=60,
+    soft_time_limit=LIGHT_SOFT_TIME_LIMIT,
+    time_limit=LIGHT_TIME_LIMIT,
     max_retries=3,
 )
 def vespa_metadata_sync_task(
     self: Task, document_id: str, tenant_id: str | None
 ) -> bool:
-    task_logger.info(f"document_id={document_id}")
-
     try:
         with get_session_with_tenant(tenant_id) as db_session:
             curr_ind_name, sec_ind_name = get_both_index_names(db_session)
-            document_index = get_default_document_index(
+            doc_index = get_default_document_index(
                 primary_index_name=curr_ind_name, secondary_index_name=sec_ind_name
             )
 
+            retry_index = RetryDocumentIndex(doc_index)
+
             doc = get_document(document_id, db_session)
             if not doc:
                 return False
@@ -846,19 +851,43 @@ def vespa_metadata_sync_task(
             )
 
             # update Vespa. OK if doc doesn't exist. Raises exception otherwise.
-            chunks_affected = document_index.update_single(document_id, fields=fields)
+            chunks_affected = retry_index.update_single(document_id, fields)
 
             # update db last. Worst case = we crash right before this and
             # the sync might repeat again later
             mark_document_as_synced(document_id, db_session)
 
             task_logger.info(
-                f"document_id={document_id} action=sync chunks={chunks_affected}"
+                f"tenant={tenant_id} doc={document_id} action=sync chunks={chunks_affected}"
             )
     except SoftTimeLimitExceeded:
-        task_logger.info(f"SoftTimeLimitExceeded exception. doc_id={document_id}")
-    except Exception as e:
-        task_logger.exception("Unexpected exception")
+        task_logger.info(
+            f"SoftTimeLimitExceeded exception. tenant={tenant_id} doc={document_id}"
+        )
+    except Exception as ex:
+        if isinstance(ex, RetryError):
+            task_logger.warning(f"Retry failed: {ex.last_attempt.attempt_number}")
+
+            # only set the inner exception if it is of type Exception
+            e_temp = ex.last_attempt.exception()
+            if isinstance(e_temp, Exception):
+                e = e_temp
+        else:
+            e = ex
+
+        if isinstance(e, httpx.HTTPStatusError):
+            if e.response.status_code == HTTPStatus.BAD_REQUEST:
+                task_logger.exception(
+                    f"Non-retryable HTTPStatusError: "
+                    f"tenant={tenant_id} "
+                    f"doc={document_id} "
+                    f"status={e.response.status_code}"
+                )
+            return False
+
+        task_logger.exception(
+            f"Unexpected exception: tenant={tenant_id} doc={document_id}"
+        )
 
         # Exponential backoff from 2^4 to 2^6 ... i.e. 16, 32, 64
         countdown = 2 ** (self.request.retries + 4)
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 1e9e0099d8b..6d6cc6b9639 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -437,7 +437,7 @@
     os.environ.get("CUSTOM_ANSWER_VALIDITY_CONDITIONS", "[]")
 )
 
-VESPA_REQUEST_TIMEOUT = int(os.environ.get("VESPA_REQUEST_TIMEOUT") or "5")
+VESPA_REQUEST_TIMEOUT = int(os.environ.get("VESPA_REQUEST_TIMEOUT") or "15")
 
 SYSTEM_RECURSION_LIMIT = int(os.environ.get("SYSTEM_RECURSION_LIMIT") or "1000")
 

From 1dec69bb8214720b41e9efccd12d6ea27f1d769d Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Mon, 28 Oct 2024 12:33:58 -0400
Subject: [PATCH 284/376] Fix document time parsing

---
 backend/danswer/connectors/freshdesk/connector.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 47b10c8a4c1..43d6339bd9a 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -15,7 +15,7 @@
 def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     # Process date fields
     for date_field in ["created_at", "updated_at", "due_by"]:
-        ticket[date_field] = datetime.fromisoformat(ticket[date_field].rstrip('Z')).replace(tzinfo=timezone.utc)
+        ticket[date_field] = datetime.fromisoformat(ticket[date_field].replace('Z', '+00:00'))
 
     # Convert all other values to strings
     ticket = {
@@ -74,7 +74,7 @@ def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = N
         base_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets"
         params = {
             "include": "description",
-            "per_page": 50,
+            "per_page": 3,
             "page": 1
         }
         

From 195e2c335dcd070c4470c6ed5150e1541288773e Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Mon, 28 Oct 2024 12:35:40 -0400
Subject: [PATCH 285/376] Fix per_page count

---
 backend/danswer/connectors/freshdesk/connector.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 43d6339bd9a..e25010efdb2 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -74,7 +74,7 @@ def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = N
         base_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets"
         params = {
             "include": "description",
-            "per_page": 3,
+            "per_page": 50,
             "page": 1
         }
         

From e05846db9f34d51909deab569e70e77a28285b8d Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 28 Oct 2024 11:33:32 -0700
Subject: [PATCH 286/376] change test port to 8889 (docker desktop is now using
 port 8888 which blocks the test from working on mac) (#2972)

---
 backend/tests/integration/tests/pruning/test_pruning.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/tests/integration/tests/pruning/test_pruning.py b/backend/tests/integration/tests/pruning/test_pruning.py
index 7498f760cef..cd8a7bde4d0 100644
--- a/backend/tests/integration/tests/pruning/test_pruning.py
+++ b/backend/tests/integration/tests/pruning/test_pruning.py
@@ -110,7 +110,7 @@ def test_web_pruning(reset: None, vespa_client: vespa_fixture) -> None:
     test_filename = os.path.realpath(__file__)
     test_directory = os.path.dirname(test_filename)
     with tempfile.TemporaryDirectory() as temp_dir:
-        port = 8888
+        port = 8889
 
         website_src = os.path.join(test_directory, "website")
         website_tgt = os.path.join(temp_dir, "website")

From e5af4681d307b6a9ac1c4f96233e00b86b1f64b4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 12:44:45 -0700
Subject: [PATCH 287/376] Fix nagging double auth issue (#2960)

* fix nagging double auth issue

* ports
---
 backend/danswer/auth/users.py | 5 +++++
 backend/danswer/db/auth.py    | 5 ++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index 13a0902b8cc..aa5054983fa 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -49,6 +49,7 @@
 from httpx_oauth.oauth2 import OAuth2Token
 from pydantic import BaseModel
 from sqlalchemy import select
+from sqlalchemy import text
 from sqlalchemy.orm import attributes
 from sqlalchemy.orm import Session
 
@@ -366,6 +367,10 @@ async def oauth_callback(
                     }
 
                     user = await self.user_db.create(user_dict)
+
+                    # Explicitly set the Postgres schema for this session to ensure
+                    # OAuth account creation happens in the correct tenant schema
+                    await db_session.execute(text(f'SET search_path = "{tenant_id}"'))
                     user = await self.user_db.add_oauth_account(
                         user, oauth_account_dict
                     )
diff --git a/backend/danswer/db/auth.py b/backend/danswer/db/auth.py
index 9eba3806df5..53fba1e52dc 100644
--- a/backend/danswer/db/auth.py
+++ b/backend/danswer/db/auth.py
@@ -57,7 +57,10 @@ async def get_user_count() -> int:
 
 # Need to override this because FastAPI Users doesn't give flexibility for backend field creation logic in OAuth flow
 class SQLAlchemyUserAdminDB(SQLAlchemyUserDatabase):
-    async def create(self, create_dict: Dict[str, Any]) -> UP:
+    async def create(
+        self,
+        create_dict: Dict[str, Any],
+    ) -> UP:
         user_count = await get_user_count()
         if user_count == 0 or create_dict["email"] in get_default_admin_user_emails():
             create_dict["role"] = UserRole.ADMIN

From a40082c5dad2abf62ea8edaee9acc6f2f526766b Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 12:47:26 -0700
Subject: [PATCH 288/376] Distinguish users in posthog (#2965)

* distinguish tenants in posthog

* nit
---
 backend/danswer/server/manage/models.py  |  3 +++
 backend/danswer/server/manage/users.py   |  4 ++++
 web/src/components/user/UserProvider.tsx | 19 +++++++++++++++++++
 web/src/lib/types.ts                     |  1 +
 4 files changed, 27 insertions(+)

diff --git a/backend/danswer/server/manage/models.py b/backend/danswer/server/manage/models.py
index e3be4c4891d..6e021b562ff 100644
--- a/backend/danswer/server/manage/models.py
+++ b/backend/danswer/server/manage/models.py
@@ -57,6 +57,7 @@ class UserInfo(BaseModel):
     oidc_expiry: datetime | None = None
     current_token_created_at: datetime | None = None
     current_token_expiry_length: int | None = None
+    organization_name: str | None = None
 
     @classmethod
     def from_model(
@@ -64,6 +65,7 @@ def from_model(
         user: User,
         current_token_created_at: datetime | None = None,
         expiry_length: int | None = None,
+        organization_name: str | None = None,
     ) -> "UserInfo":
         return cls(
             id=str(user.id),
@@ -80,6 +82,7 @@ def from_model(
                     visible_assistants=user.visible_assistants,
                 )
             ),
+            organization_name=organization_name,
             # set to None if TRACK_EXTERNAL_IDP_EXPIRY is False so that we avoid cases
             # where they previously had this set + used OIDC, and now they switched to
             # basic auth are now constantly getting redirected back to the login page
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index e8127ce0f79..bbaf08b7db9 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -30,6 +30,7 @@
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
+from danswer.auth.users import get_tenant_id_for_email
 from danswer.auth.users import optional_user
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import ENABLE_EMAIL_INVITES
@@ -493,10 +494,13 @@ def verify_user_logged_in(
     token_created_at = (
         None if MULTI_TENANT else get_current_token_creation(user, db_session)
     )
+    organization_name = get_tenant_id_for_email(user.email)
+
     user_info = UserInfo.from_model(
         user,
         current_token_created_at=token_created_at,
         expiry_length=SESSION_EXPIRE_TIME_SECONDS,
+        organization_name=organization_name,
     )
 
     return user_info
diff --git a/web/src/components/user/UserProvider.tsx b/web/src/components/user/UserProvider.tsx
index 42086d6350f..5a3e7cbc099 100644
--- a/web/src/components/user/UserProvider.tsx
+++ b/web/src/components/user/UserProvider.tsx
@@ -3,6 +3,7 @@
 import React, { createContext, useContext, useState, useEffect } from "react";
 import { User, UserRole } from "@/lib/types";
 import { getCurrentUser } from "@/lib/user";
+import { usePostHog } from "posthog-js/react";
 
 interface UserContextType {
   user: User | null;
@@ -24,6 +25,24 @@ export function UserProvider({
   const [upToDateUser, setUpToDateUser] = useState<User | null>(user);
   const [isLoadingUser, setIsLoadingUser] = useState(false);
 
+  const posthog = usePostHog();
+
+  useEffect(() => {
+    if (!posthog) return;
+
+    if (user?.id) {
+      const identifyData: Record<string, any> = {
+        email: user.email,
+      };
+      if (user.organization_name) {
+        identifyData.organization_name = user.organization_name;
+      }
+      posthog.identify(user.id, identifyData);
+    } else {
+      posthog.reset();
+    }
+  }, [posthog, user]);
+
   const fetchUser = async () => {
     try {
       setIsLoadingUser(true);
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index 46d5efd8444..f8b7fb57536 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -42,6 +42,7 @@ export interface User {
   current_token_created_at?: Date;
   current_token_expiry_length?: number;
   oidc_expiry?: Date;
+  organization_name: string | null;
 }
 
 export interface MinimalUserSnapshot {

From 678ba41321dbbcad0ef0db75e0a618716af3eb3d Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 14:39:34 -0700
Subject: [PATCH 289/376] Cleaner initial chat screen (#2528)

* cleaner initial chat screen

* slightly cleaner animation

* cleaner cards

* use display name + minor updates to models

* minor udpate to ui

* remove logs

* update based on feedback

* minor nits

* formatting
---
 backend/danswer/chat/personas.yaml            |  57 ++++++++-
 .../app/admin/assistants/AssistantEditor.tsx  |   4 +
 web/src/app/chat/ChatIntro.tsx                | 116 +++++-------------
 web/src/app/chat/ChatPage.tsx                 |  96 +++++++--------
 web/src/app/chat/StarterMessage.tsx           |  22 ++--
 .../components/assistants/AssistantCards.tsx  |  60 +++++++++
 .../components/assistants/AssistantIcon.tsx   |  46 +++++--
 web/src/components/tooltip/CustomTooltip.tsx  |   6 +-
 8 files changed, 253 insertions(+), 154 deletions(-)

diff --git a/backend/danswer/chat/personas.yaml b/backend/danswer/chat/personas.yaml
index 0aececcee6c..d7b16aad99e 100644
--- a/backend/danswer/chat/personas.yaml
+++ b/backend/danswer/chat/personas.yaml
@@ -41,6 +41,19 @@ personas:
     icon_color: "#6FB1FF"
     display_priority: 1
     is_visible: true
+    starter_messages:
+      - name: "General Information"
+        description: "Ask about available information"
+        message: "Hello! I'm interested in learning more about the information available here. Could you give me an overview of the types of data or documents that might be accessible?"
+      - name: "Specific Topic Search"
+        description: "Search for specific information"
+        message: "Hi! I'd like to learn more about a specific topic. Could you help me find relevant documents and information?"
+      - name: "Recent Updates"
+        description: "Inquire about latest additions"
+        message: "Hello! I'm curious about any recent updates or additions to the knowledge base. Can you tell me what new information has been added lately?"
+      - name: "Cross-referencing Information"
+        description: "Connect information from different sources"
+        message: "Hi! I'm working on a project that requires connecting information from multiple sources. How can I effectively cross-reference data across different documents or categories?"
 
   - id: 1
     name: "General"
@@ -57,6 +70,19 @@ personas:
     icon_color: "#FF6F6F"
     display_priority: 0
     is_visible: true
+    starter_messages:
+      - name: "Open Discussion"
+        description: "Start an open-ended conversation"
+        message: "Hi! Can you help me write a professional email?"
+      - name: "Problem Solving"
+        description: "Get help with a challenge"
+        message: "Hello! I need help managing my daily tasks better. Do you have any simple tips?"
+      - name: "Learn Something New"
+        description: "Explore a new topic"
+        message: "Hi! Could you explain what project management is in simple terms?"
+      - name: "Creative Brainstorming"
+        description: "Generate creative ideas"
+        message: "Hello! I need to brainstorm some team building activities. Do you have any fun suggestions?"
 
   - id: 2
     name: "Paraphrase"
@@ -73,7 +99,19 @@ personas:
     icon_color: "#6FFF8D"
     display_priority: 2
     is_visible: false
-
+    starter_messages:
+      - name: "Document Search"
+        description: "Find exact information"
+        message: "Hi! Could you help me find information about our team structure and reporting lines from our internal documents?"
+      - name: "Process Verification"
+        description: "Find exact quotes"
+        message: "Hello! I need to understand our project approval process. Could you find the exact steps from our documentation?"
+      - name: "Technical Documentation"
+        description: "Search technical details"
+        message: "Hi there! I'm looking for information about our deployment procedures. Can you find the specific steps from our technical guides?"
+      - name: "Policy Reference"
+        description: "Check official policies"
+        message: "Hello! Could you help me find our official guidelines about client communication? I need the exact wording from our documentation."
 
   - id: 3
     name: "Art"
@@ -86,8 +124,21 @@ personas:
     llm_filter_extraction: false
     recency_bias: "no_decay"
     document_sets: []
-    icon_shape: 234124 
+    icon_shape: 234124
     icon_color: "#9B59B6"
-    image_generation: true 
+    image_generation: true
     display_priority: 3
     is_visible: true
+    starter_messages:
+      - name: "Landscape"
+        description: "Generate a landscape image"
+        message: "Create an image of a serene mountain lake at sunset, with snow-capped peaks reflected in the calm water and a small wooden cabin on the shore."
+      - name: "Character"
+        description: "Generate a character image"
+        message: "Generate an image of a futuristic robot with glowing blue eyes, sleek metallic body, and intricate circuitry visible through transparent panels on its chest and arms."
+      - name: "Abstract"
+        description: "Create an abstract image"
+        message: "Create an abstract image representing the concept of time, using swirling clock hands, fragmented hourglasses, and streaks of light to convey the passage of moments and eras."
+      - name: "Urban Scene"
+        description: "Generate an urban landscape"
+        message: "Generate an image of a bustling futuristic cityscape at night, with towering skyscrapers, flying vehicles, holographic advertisements, and a mix of neon and bioluminescent lighting."
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 9637e74026f..69b2d9c1276 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -1002,6 +1002,10 @@ export function AssistantEditor({
                         Starter Messages (Optional){" "}
                       </div>
                     </div>
+                    <SubLabel>
+                      Add pre-defined messages to help users get started. Only
+                      the first 4 will be displayed.
+                    </SubLabel>
                     <FieldArray
                       name="starter_messages"
                       render={(
diff --git a/web/src/app/chat/ChatIntro.tsx b/web/src/app/chat/ChatIntro.tsx
index 3703655d7f9..a1c15bf6269 100644
--- a/web/src/app/chat/ChatIntro.tsx
+++ b/web/src/app/chat/ChatIntro.tsx
@@ -1,96 +1,44 @@
-import { getSourceMetadataForSources } from "@/lib/sources";
-import { ValidSources } from "@/lib/types";
 import { Persona } from "../admin/assistants/interfaces";
-import { Divider } from "@tremor/react";
-import { FiBookmark, FiInfo } from "react-icons/fi";
-import { HoverPopup } from "@/components/HoverPopup";
+import { AssistantIcon } from "@/components/assistants/AssistantIcon";
+import { useState } from "react";
+import { DisplayAssistantCard } from "@/components/assistants/AssistantCards";
 
-export function ChatIntro({
-  availableSources,
-  selectedPersona,
-}: {
-  availableSources: ValidSources[];
-  selectedPersona: Persona;
-}) {
-  const availableSourceMetadata = getSourceMetadataForSources(availableSources);
+export function ChatIntro({ selectedPersona }: { selectedPersona: Persona }) {
+  const [hoveredAssistant, setHoveredAssistant] = useState(false);
 
   return (
     <>
-      <div className="flex justify-center items-center h-full">
-        <div className="mobile:w-[90%] mobile:px-4 w-message-xs 2xl:w-message-sm 3xl:w-message">
-          <div className="flex">
-            <div className="mx-auto">
-              <div className="m-auto text-3xl font-strong font-bold text-strong w-fit">
-                {selectedPersona?.name || "How can I help you today?"}
+      <div className="mobile:w-[90%] mobile:px-4 w-message-xs 2xl:w-message-sm 3xl:w-message">
+        <div className="relative flex w-fit mx-auto justify-center">
+          <div className="absolute z-10 -left-20 top-1/2 -translate-y-1/2">
+            <div className="relative">
+              <div
+                onMouseEnter={() => setHoveredAssistant(true)}
+                onMouseLeave={() => setHoveredAssistant(false)}
+                className="p-4 scale-[.8] cursor-pointer border-dashed rounded-full flex border border-border border-2 border-dashed"
+                style={{
+                  borderStyle: "dashed",
+                  borderWidth: "1.5px",
+                  borderSpacing: "4px",
+                }}
+              >
+                <AssistantIcon
+                  disableToolip
+                  size={"large"}
+                  assistant={selectedPersona}
+                />
+              </div>
+              <div className="absolute right-full mr-2 w-[300px] top-0">
+                {hoveredAssistant && (
+                  <DisplayAssistantCard selectedPersona={selectedPersona} />
+                )}
               </div>
-              {selectedPersona && (
-                <div className="mt-1">{selectedPersona.description}</div>
-              )}
             </div>
           </div>
 
-          {selectedPersona && selectedPersona.num_chunks !== 0 && (
-            <>
-              <Divider />
-              <div>
-                {selectedPersona.document_sets.length > 0 && (
-                  <div className="mt-2">
-                    <p className="font-bold mb-1 mt-4 text-emphasis">
-                      Knowledge Sets:{" "}
-                    </p>
-                    <div className="flex flex-wrap gap-2">
-                      {selectedPersona.document_sets.map((documentSet) => (
-                        <div key={documentSet.id} className="w-fit">
-                          <HoverPopup
-                            mainContent={
-                              <span className="flex w-fit p-1 rounded border border-border text-xs font-medium cursor-default">
-                                <div className="mr-1 my-auto">
-                                  <FiBookmark />
-                                </div>
-                                {documentSet.name}
-                              </span>
-                            }
-                            popupContent={
-                              <div className="flex py-1 w-96">
-                                <FiInfo className="my-auto mr-2" />
-                                <div className="text-sm">
-                                  {documentSet.description}
-                                </div>
-                              </div>
-                            }
-                            direction="top"
-                          />
-                        </div>
-                      ))}
-                    </div>
-                  </div>
-                )}
-
-                {availableSources.length > 0 && (
-                  <div className="mt-1">
-                    <p className="font-bold mb-1 mt-4 text-emphasis">
-                      Connected Sources:{" "}
-                    </p>
-                    <div className={`flex flex-wrap gap-2`}>
-                      {availableSourceMetadata.map((sourceMetadata) => (
-                        <span
-                          key={sourceMetadata.internalName}
-                          className="flex w-fit p-1 rounded border border-border text-xs font-medium cursor-default"
-                        >
-                          <div className="mr-1 my-auto">
-                            {sourceMetadata.icon({})}
-                          </div>
-                          <div className="my-auto">
-                            {sourceMetadata.displayName}
-                          </div>
-                        </span>
-                      ))}
-                    </div>
-                  </div>
-                )}
-              </div>
-            </>
-          )}
+          <div className="text-3xl line-clamp-2 text-text-800 font-base font-semibold text-strong">
+            {selectedPersona?.name || "How can I help you today?"}
+          </div>
         </div>
       </div>
     </>
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 056aa181548..790ef2017e5 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -103,6 +103,7 @@ import { ApiKeyModal } from "@/components/llm/ApiKeyModal";
 import BlurBackground from "./shared_chat_search/BlurBackground";
 import { NoAssistantModal } from "@/components/modals/NoAssistantModal";
 import { useAssistants } from "@/components/context/AssistantsContext";
+import { Divider } from "@tremor/react";
 
 const TEMP_USER_MESSAGE_ID = -1;
 const TEMP_ASSISTANT_MESSAGE_ID = -2;
@@ -741,12 +742,6 @@ export function ChatPage({
   }, [liveAssistant]);
 
   const filterManager = useFilters();
-  const [finalAvailableSources, finalAvailableDocumentSets] =
-    computeAvailableFilters({
-      selectedPersona: selectedAssistant,
-      availableSources,
-      availableDocumentSets,
-    });
 
   const [currentFeedback, setCurrentFeedback] = useState<
     [FeedbackType, number] | null
@@ -2002,7 +1997,7 @@ export function ChatPage({
                         {...getRootProps()}
                       >
                         <div
-                          className={`w-full h-full  flex flex-col overflow-y-auto include-scrollbar overflow-x-hidden relative`}
+                          className={`w-full h-full flex flex-col overflow-y-auto include-scrollbar overflow-x-hidden relative`}
                           ref={scrollableDivRef}
                         >
                           {/* ChatBanner is a custom banner that displays a admin-specified message at 
@@ -2012,11 +2007,51 @@ export function ChatPage({
                             !isFetchingChatMessages &&
                             currentSessionChatState == "input" &&
                             !loadingError && (
-                              <ChatIntro
-                                availableSources={finalAvailableSources}
-                                selectedPersona={liveAssistant}
-                              />
+                              <div className="h-full flex flex-col justify-center items-center">
+                                <ChatIntro selectedPersona={liveAssistant} />
+
+                                <div
+                                  key={-4}
+                                  className={`
+                                      mx-auto 
+                                      px-4 
+                                      w-full
+                                      max-w-[750px]
+                                      flex 
+                                      flex-wrap
+                                      justify-center
+                                      mt-2
+                                      h-40
+                                      items-start
+                                      mb-6`}
+                                >
+                                  {currentPersona?.starter_messages &&
+                                    currentPersona.starter_messages.length >
+                                      0 && (
+                                      <>
+                                        <Divider className="mx-2" />
+
+                                        {currentPersona.starter_messages
+                                          .slice(0, 4)
+                                          .map((starterMessage, i) => (
+                                            <div key={i} className="w-1/2">
+                                              <StarterMessage
+                                                starterMessage={starterMessage}
+                                                onClick={() =>
+                                                  onSubmit({
+                                                    messageOverride:
+                                                      starterMessage.message,
+                                                  })
+                                                }
+                                              />
+                                            </div>
+                                          ))}
+                                      </>
+                                    )}
+                                </div>
+                              </div>
                             )}
+
                           <div
                             className={
                               "-ml-4 w-full mx-auto " +
@@ -2383,45 +2418,6 @@ export function ChatPage({
                                 />
                               </div>
                             )}
-                            {currentPersona &&
-                              currentPersona.starter_messages &&
-                              currentPersona.starter_messages.length > 0 &&
-                              selectedAssistant &&
-                              messageHistory.length === 0 &&
-                              !isFetchingChatMessages && (
-                                <div
-                                  key={-4}
-                                  className={`
-                                      mx-auto 
-                                      px-4 
-                                      w-searchbar-xs 
-                                      2xl:w-searchbar-sm 
-                                      3xl:w-searchbar 
-                                      grid 
-                                      gap-4 
-                                      grid-cols-1 
-                                      grid-rows-1 
-                                      mt-4 
-                                      md:grid-cols-2 
-                                      mb-6`}
-                                >
-                                  {currentPersona.starter_messages.map(
-                                    (starterMessage, i) => (
-                                      <div key={i} className="w-full">
-                                        <StarterMessage
-                                          starterMessage={starterMessage}
-                                          onClick={() =>
-                                            onSubmit({
-                                              messageOverride:
-                                                starterMessage.message,
-                                            })
-                                          }
-                                        />
-                                      </div>
-                                    )
-                                  )}
-                                </div>
-                              )}
 
                             {/* Some padding at the bottom so the search bar has space at the bottom to not cover the last message*/}
                             <div ref={endPaddingRef} className="h-[95px]" />
diff --git a/web/src/app/chat/StarterMessage.tsx b/web/src/app/chat/StarterMessage.tsx
index f344739148f..af65ddc9c81 100644
--- a/web/src/app/chat/StarterMessage.tsx
+++ b/web/src/app/chat/StarterMessage.tsx
@@ -1,21 +1,29 @@
-import { StarterMessage } from "../admin/assistants/interfaces";
+import { StarterMessage as StarterMessageType } from "../admin/assistants/interfaces";
 
 export function StarterMessage({
   starterMessage,
   onClick,
 }: {
-  starterMessage: StarterMessage;
+  starterMessage: StarterMessageType;
   onClick: () => void;
 }) {
   return (
     <div
-      className={
-        "py-2 px-3 rounded border border-border bg-white cursor-pointer hover:bg-hover-light h-full"
-      }
+      className="mb-4 mx-2 group relative overflow-hidden rounded-xl border border-border bg-gradient-to-br from-white to-background p-4 shadow-sm transition-all duration-300 hover:shadow-md hover:scale-[1.005] cursor-pointer"
       onClick={onClick}
     >
-      <p className="font-medium text-emphasis">{starterMessage.name}</p>
-      <p className="text-subtle text-sm">{starterMessage.description}</p>
+      <div className="absolute inset-0 bg-gradient-to-r from-blue-100 to-purple-100 opacity-0 group-hover:opacity-20 transition-opacity duration-300" />
+      <h3
+        className="text-base flex items-center font-medium text-text-800 group-hover:text-text-900 transition-colors duration-300
+                     line-clamp-2  gap-x-2 overflow-hidden"
+      >
+        {starterMessage.name}
+      </h3>
+      <div className={`overflow-hidden transition-all duration-300 max-h-20}`}>
+        <p className="text-sm text-text-600 mt-2">
+          {starterMessage.description}
+        </p>
+      </div>
     </div>
   );
 }
diff --git a/web/src/components/assistants/AssistantCards.tsx b/web/src/components/assistants/AssistantCards.tsx
index 60c6ee24896..075ec0ab06f 100644
--- a/web/src/components/assistants/AssistantCards.tsx
+++ b/web/src/components/assistants/AssistantCards.tsx
@@ -115,3 +115,63 @@ export function DraggableAssistantCard(props: {
     </div>
   );
 }
+
+export function DisplayAssistantCard({
+  selectedPersona,
+}: {
+  selectedPersona: Persona;
+}) {
+  return (
+    <div className="p-4 bg-white/90 backdrop-blur-sm rounded-lg shadow-md border border-border/50 max-w-md w-full mx-auto transition-all duration-300 ease-in-out hover:shadow-lg">
+      <div className="flex items-center mb-3">
+        <AssistantIcon
+          disableToolip
+          size="medium"
+          assistant={selectedPersona}
+        />
+        <h2 className="ml-3 text-xl font-semibold text-text-900">
+          {selectedPersona.name}
+        </h2>
+      </div>
+      <p className="text-sm text-text-600 mb-3 leading-relaxed">
+        {selectedPersona.description}
+      </p>
+      {selectedPersona.tools.length > 0 ||
+      selectedPersona.llm_relevance_filter ||
+      selectedPersona.llm_filter_extraction ? (
+        <div className="space-y-2">
+          <h3 className="text-base font-medium text-text-900">Capabilities:</h3>
+          <ul className="space-y-.5">
+            {/* display all tools */}
+            {selectedPersona.tools.map((tool, index) => (
+              <li
+                key={index}
+                className="flex items-center text-sm text-text-700"
+              >
+                <span className="mr-2 text-text-500 opacity-70">•</span>{" "}
+                {tool.display_name}
+              </li>
+            ))}
+            {/* Built in capabilities */}
+            {selectedPersona.llm_relevance_filter && (
+              <li className="flex items-center text-sm text-text-700">
+                <span className="mr-2 text-text-500 opacity-70">•</span>{" "}
+                Advanced Relevance Filtering
+              </li>
+            )}
+            {selectedPersona.llm_filter_extraction && (
+              <li className="flex items-center text-sm text-text-700">
+                <span className="mr-2 text-text-500 opacity-70">•</span> Smart
+                Information Extraction
+              </li>
+            )}
+          </ul>
+        </div>
+      ) : (
+        <p className="text-sm text-text-600 italic">
+          No specific capabilities listed for this assistant.
+        </p>
+      )}
+    </div>
+  );
+}
diff --git a/web/src/components/assistants/AssistantIcon.tsx b/web/src/components/assistants/AssistantIcon.tsx
index 58a5b11b237..4126fdfca2f 100644
--- a/web/src/components/assistants/AssistantIcon.tsx
+++ b/web/src/components/assistants/AssistantIcon.tsx
@@ -23,22 +23,38 @@ export function AssistantIcon({
   assistant,
   size,
   border,
+  disableToolip,
 }: {
   assistant: Persona;
-  size?: "small" | "medium" | "large";
+  size?: "small" | "medium" | "large" | "header";
   border?: boolean;
+  disableToolip?: boolean;
 }) {
   const color = darkerGenerateColorFromId(assistant.id.toString());
 
   return (
-    <CustomTooltip showTick line wrap content={assistant.description}>
+    <CustomTooltip
+      disabled={disableToolip}
+      showTick
+      line
+      wrap
+      content={assistant.description}
+    >
       {
         // Prioritization order: image, graph, defaults
         assistant.uploaded_image_id ? (
           <img
             alt={assistant.name}
             className={`object-cover object-center rounded-sm overflow-hidden transition-opacity duration-300 opacity-100
-              ${size === "large" ? "w-8 h-8" : "w-6 h-6"}`}
+              ${
+                size === "large"
+                  ? "w-10 h-10"
+                  : size === "header"
+                    ? "w-14 h-14"
+                    : size === "medium"
+                      ? "w-8 h-8"
+                      : "w-6 h-6"
+              }`}
             src={buildImgUrl(assistant.uploaded_image_id)}
             loading="lazy"
           />
@@ -46,20 +62,36 @@ export function AssistantIcon({
           <div
             className={`flex-none
                   ${border && "ring ring-[1px] ring-border-strong "}
-                  ${size === "large" ? "w-10 h-10" : "w-6 h-6"} `}
+                  ${
+                    size === "large"
+                      ? "w-10 h-10"
+                      : size === "header"
+                        ? "w-14 h-14"
+                        : size === "medium"
+                          ? "w-8 h-8"
+                          : "w-6 h-6"
+                  } `}
           >
             {createSVG(
               { encodedGrid: assistant.icon_shape, filledSquares: 0 },
               assistant.icon_color,
-              size == "large" ? 36 : 24
+              size === "large"
+                ? 40
+                : size === "header"
+                  ? 56
+                  : size === "medium"
+                    ? 32
+                    : 24
             )}
           </div>
         ) : (
           <div
             className={`flex-none rounded-sm overflow-hidden
                   ${border && "border border-.5 border-border-strong "}
-                  ${size === "large" && "w-12 h-12"}
-                  ${(!size || size === "small") && "w-6 h-6"} `}
+                  ${size === "large" ? "w-10 h-10" : ""}
+                  ${size === "header" ? "w-14 h-14" : ""}
+                  ${size === "medium" ? "w-8 h-8" : ""}
+                  ${!size || size === "small" ? "w-6 h-6" : ""} `}
             style={{ backgroundColor: color }}
           />
         )
diff --git a/web/src/components/tooltip/CustomTooltip.tsx b/web/src/components/tooltip/CustomTooltip.tsx
index 52a98d8aad6..42efdbe2e4a 100644
--- a/web/src/components/tooltip/CustomTooltip.tsx
+++ b/web/src/components/tooltip/CustomTooltip.tsx
@@ -124,9 +124,9 @@ export const CustomTooltip = ({
         !disabled &&
         createPortal(
           <div
-            className={`min-w-8 fixed z-[1000] ${citation ? "max-w-[350px]" : "w-40"} ${
-              large ? (medium ? "w-88" : "w-96") : line && "max-w-64 w-auto"
-            } 
+            className={`min-w-8 fixed z-[1000] ${
+              citation ? "max-w-[350px]" : "w-40"
+            } ${large ? (medium ? "w-88" : "w-96") : line && "max-w-64 w-auto"} 
             transform -translate-x-1/2 text-sm 
             ${
               light

From 2eefb3c15fe2ba22d57d2bae1c613ba4746a3df0 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 15:20:21 -0700
Subject: [PATCH 290/376] add srem and sadd to tenant wrapper (#2973)

---
 backend/danswer/redis/redis_pool.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py
index 3f2ec03d77f..9434875680c 100644
--- a/backend/danswer/redis/redis_pool.py
+++ b/backend/danswer/redis/redis_pool.py
@@ -102,6 +102,8 @@ def __getattribute__(self, item: str) -> Any:
             "reacquire",
             "create_lock",
             "startswith",
+            "sadd",
+            "srem",
         ]  # Regular methods that need simple prefixing
 
         if item == "scan_iter":

From 11d849b5538fb70e49d0b577be7efeeb32429703 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 16:08:47 -0700
Subject: [PATCH 291/376] add indent to scan_iter (#2948)

---
 backend/danswer/background/celery/apps/primary.py | 4 ++--
 background/celery/apps/primary.py                 | 0
 2 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 background/celery/apps/primary.py

diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 983b76773ed..8790dce5903 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -162,8 +162,8 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
         for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
             r.delete(key)
 
-    for key in r.scan_iter(RedisConnectorStop.FENCE_PREFIX + "*"):
-        r.delete(key)
+        for key in r.scan_iter(RedisConnectorStop.FENCE_PREFIX + "*"):
+            r.delete(key)
 
 
 # @worker_process_init.connect
diff --git a/background/celery/apps/primary.py b/background/celery/apps/primary.py
new file mode 100644
index 00000000000..e69de29bb2d

From 5640230f5b8f6b20404b7eadd6e7357cc8df8f62 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 16:11:00 -0700
Subject: [PATCH 292/376] remove empty directory (#2977)

---
 background/celery/apps/primary.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 background/celery/apps/primary.py

diff --git a/background/celery/apps/primary.py b/background/celery/apps/primary.py
deleted file mode 100644
index e69de29bb2d..00000000000

From a60613ec11fe8754909f59bf53608753e952399f Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Mon, 28 Oct 2024 16:25:08 -0700
Subject: [PATCH 293/376] missing scard

---
 backend/danswer/redis/redis_pool.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/redis/redis_pool.py b/backend/danswer/redis/redis_pool.py
index 9434875680c..0fe5c0ddd42 100644
--- a/backend/danswer/redis/redis_pool.py
+++ b/backend/danswer/redis/redis_pool.py
@@ -104,6 +104,7 @@ def __getattribute__(self, item: str) -> Any:
             "startswith",
             "sadd",
             "srem",
+            "scard",
         ]  # Regular methods that need simple prefixing
 
         if item == "scan_iter":

From aafcf7af55cf333d4a236867556092602444bc22 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 28 Oct 2024 21:17:53 -0700
Subject: [PATCH 294/376] fail gracefully on provider fetch (#2981)

---
 web/src/lib/chat/fetchAssistantdata.ts | 110 ++++++++++++++-----------
 1 file changed, 64 insertions(+), 46 deletions(-)

diff --git a/web/src/lib/chat/fetchAssistantdata.ts b/web/src/lib/chat/fetchAssistantdata.ts
index 08910320a83..8638036d91f 100644
--- a/web/src/lib/chat/fetchAssistantdata.ts
+++ b/web/src/lib/chat/fetchAssistantdata.ts
@@ -1,5 +1,4 @@
 import { fetchSS } from "@/lib/utilsSS";
-import { CCPairBasicInfo } from "@/lib/types";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { personaComparator } from "@/app/admin/assistants/lib";
@@ -12,58 +11,77 @@ interface AssistantData {
   hasImageCompatibleModel: boolean;
 }
 export async function fetchAssistantData(): Promise<AssistantData> {
-  const [assistants, assistantsFetchError] = await fetchAssistantsSS();
-  const ccPairsResponse = await fetchSS("/manage/indexing-status");
-
-  let ccPairs: CCPairBasicInfo[] = [];
-  if (ccPairsResponse?.ok) {
-    ccPairs = await ccPairsResponse.json();
-  } else {
-    console.log(`Failed to fetch connectors - ${ccPairsResponse?.status}`);
-  }
-
-  const hasAnyConnectors = ccPairs.length > 0;
+  // Default state if anything fails
+  const defaultState: AssistantData = {
+    assistants: [],
+    hasAnyConnectors: false,
+    hasImageCompatibleModel: false,
+  };
 
-  // if no connectors are setup, only show personas that are pure
-  // passthrough and don't do any retrieval
-  let filteredAssistants = assistants;
-  if (assistantsFetchError) {
-    console.log(`Failed to fetch assistants - ${assistantsFetchError}`);
-  }
+  try {
+    // Fetch core assistants data first
+    const [assistants, assistantsFetchError] = await fetchAssistantsSS();
+    if (assistantsFetchError) {
+      console.error(`Failed to fetch assistants - ${assistantsFetchError}`);
+      return defaultState;
+    }
 
-  // remove those marked as hidden by an admin
-  filteredAssistants = filteredAssistants.filter(
-    (assistant) => assistant.is_visible
-  );
+    // Parallel fetch of additional data
+    const [ccPairsResponse, llmProviders] = await Promise.all([
+      fetchSS("/manage/indexing-status").catch((error) => {
+        console.error("Failed to fetch connectors:", error);
+        return null;
+      }),
+      fetchLLMProvidersSS().catch((error) => {
+        console.error("Failed to fetch LLM providers:", error);
+        return [];
+      }),
+    ]);
 
-  if (!hasAnyConnectors) {
-    filteredAssistants = filteredAssistants.filter(
-      (assistant) => assistant.num_chunks === 0
+    // Process visible assistants
+    let filteredAssistants = assistants.filter(
+      (assistant) => assistant.is_visible
     );
-  }
 
-  // sort them in priority order
-  filteredAssistants.sort(personaComparator);
+    // Process connector status
+    const hasAnyConnectors = ccPairsResponse?.ok
+      ? (await ccPairsResponse.json()).length > 0
+      : false;
 
-  const llmProviders = await fetchLLMProvidersSS();
-  const hasImageCompatibleModel = llmProviders.some(
-    (provider) =>
-      provider.provider === "openai" ||
-      provider.model_names.some((model) => checkLLMSupportsImageInput(model))
-  );
+    // Filter assistants based on connector status
+    if (!hasAnyConnectors) {
+      filteredAssistants = filteredAssistants.filter(
+        (assistant) => assistant.num_chunks === 0
+      );
+    }
 
-  if (!hasImageCompatibleModel) {
-    filteredAssistants = filteredAssistants.filter(
-      (assistant) =>
-        !assistant.tools.some(
-          (tool) => tool.in_code_tool_id === "ImageGenerationTool"
-        )
+    // Sort assistants
+    filteredAssistants.sort(personaComparator);
+
+    // Check for image-compatible models
+    const hasImageCompatibleModel = llmProviders.some(
+      (provider) =>
+        provider.provider === "openai" ||
+        provider.model_names.some((model) => checkLLMSupportsImageInput(model))
     );
-  }
 
-  return {
-    assistants: filteredAssistants,
-    hasAnyConnectors,
-    hasImageCompatibleModel,
-  };
+    // Filter out image generation tools if no compatible model
+    if (!hasImageCompatibleModel) {
+      filteredAssistants = filteredAssistants.filter(
+        (assistant) =>
+          !assistant.tools.some(
+            (tool) => tool.in_code_tool_id === "ImageGenerationTool"
+          )
+      );
+    }
+
+    return {
+      assistants: filteredAssistants,
+      hasAnyConnectors,
+      hasImageCompatibleModel,
+    };
+  } catch (error) {
+    console.error("Unexpected error in fetchAssistantData:", error);
+    return defaultState;
+  }
 }

From 75dd10323818ce7a4a71c829bfacbcfa9cd07f39 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 29 Oct 2024 06:29:39 -0700
Subject: [PATCH 295/376] add additional configuration options (#2980)

---
 web/src/app/chat/ChatPage.tsx                  |  2 +-
 .../chat/sessionSidebar/ChatSessionDisplay.tsx | 12 +++++++++---
 .../app/chat/sessionSidebar/HistorySidebar.tsx |  2 +-
 .../app/chat/shared_chat_search/FixedLogo.tsx  |  8 +++++---
 .../shared_chat_search/FunctionalWrapper.tsx   |  6 ++++--
 web/src/components/BasicClickable.tsx          | 12 +++++++++++-
 web/src/components/header/HeaderTitle.tsx      | 12 ++++++++++--
 web/src/components/search/SearchAnswer.tsx     |  6 +++++-
 web/src/components/search/SearchBar.tsx        | 18 +++++++++++++-----
 web/src/components/search/SearchSection.tsx    |  6 ++++--
 10 files changed, 63 insertions(+), 21 deletions(-)

diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 790ef2017e5..4c4291405f9 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -2512,7 +2512,7 @@ export function ChatPage({
               )}
             </div>
           </div>
-          <FixedLogo />
+          <FixedLogo chat />
         </div>
       </div>
       <DocumentSidebar
diff --git a/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx b/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
index 8b690968c74..d06ee14be25 100644
--- a/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
+++ b/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
@@ -122,7 +122,7 @@ export function ChatSessionDisplay({
           );
         }}
       >
-        <BasicSelectable padding="extra" fullWidth selected={isSelected}>
+        <BasicSelectable chat padding="extra" fullWidth selected={isSelected}>
           <>
             <div className="flex relative">
               {isRenamingChat ? (
@@ -142,7 +142,11 @@ export function ChatSessionDisplay({
                   {chatName || `Chat ${chatSession.id}`}
                   <span
                     className={`absolute right-0 top-0 h-full w-8 bg-gradient-to-r from-transparent 
-                    ${isSelected ? "to-background-200" : " to-background-100 group-hover:to-background-200"} `}
+                    ${
+                      isSelected
+                        ? "to-background-chat-hover"
+                        : " to-background-chat-selected group-hover:to-background-chat-hover"
+                    } `}
                   />
                 </p>
               )}
@@ -176,7 +180,9 @@ export function ChatSessionDisplay({
                             This chat will expire{" "}
                             {daysUntilExpiration < 1
                               ? "today"
-                              : `in ${daysUntilExpiration} day${daysUntilExpiration !== 1 ? "s" : ""}`}
+                              : `in ${daysUntilExpiration} day${
+                                  daysUntilExpiration !== 1 ? "s" : ""
+                                }`}
                           </p>
                         }
                       >
diff --git a/web/src/app/chat/sessionSidebar/HistorySidebar.tsx b/web/src/app/chat/sessionSidebar/HistorySidebar.tsx
index 9e072050b5a..9dad5c05079 100644
--- a/web/src/app/chat/sessionSidebar/HistorySidebar.tsx
+++ b/web/src/app/chat/sessionSidebar/HistorySidebar.tsx
@@ -94,7 +94,7 @@ export const HistorySidebar = forwardRef<HTMLDivElement, HistorySidebarProps>(
             bg-background-sidebar
             w-full
             border-r 
-            border-border 
+            border-sidebar-border 
             flex 
             flex-col relative
             h-screen
diff --git a/web/src/app/chat/shared_chat_search/FixedLogo.tsx b/web/src/app/chat/shared_chat_search/FixedLogo.tsx
index 2961db9df5b..717b407c8a4 100644
--- a/web/src/app/chat/shared_chat_search/FixedLogo.tsx
+++ b/web/src/app/chat/shared_chat_search/FixedLogo.tsx
@@ -8,7 +8,7 @@ import Link from "next/link";
 import { useContext } from "react";
 import { FiSidebar } from "react-icons/fi";
 
-export default function FixedLogo() {
+export default function FixedLogo({ chat }: { chat?: boolean }) {
   const combinedSettings = useContext(SettingsContext);
   const settings = combinedSettings?.settings;
   const enterpriseSettings = combinedSettings?.enterpriseSettings;
@@ -28,13 +28,15 @@ export default function FixedLogo() {
           <div className="w-full">
             {enterpriseSettings && enterpriseSettings.application_name ? (
               <div>
-                <HeaderTitle>{enterpriseSettings.application_name}</HeaderTitle>
+                <HeaderTitle chat={chat}>
+                  {enterpriseSettings.application_name}
+                </HeaderTitle>
                 {!NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED && (
                   <p className="text-xs text-subtle">Powered by Danswer</p>
                 )}
               </div>
             ) : (
-              <HeaderTitle>Danswer</HeaderTitle>
+              <HeaderTitle chat={chat}>Danswer</HeaderTitle>
             )}
           </div>
         </div>
diff --git a/web/src/app/chat/shared_chat_search/FunctionalWrapper.tsx b/web/src/app/chat/shared_chat_search/FunctionalWrapper.tsx
index 4f8d31d39ee..bfa8ef934f3 100644
--- a/web/src/app/chat/shared_chat_search/FunctionalWrapper.tsx
+++ b/web/src/app/chat/shared_chat_search/FunctionalWrapper.tsx
@@ -36,7 +36,7 @@ const ToggleSwitch = () => {
   };
 
   return (
-    <div className="bg-gray-100 mobile:mt-8 flex rounded-full p-1">
+    <div className="bg-background-toggle mobile:mt-8 flex rounded-full p-1">
       <div
         className={`absolute mobile:mt-8 top-1 bottom-1 ${
           activeTab === "chat" ? "w-[45%]" : "w-[50%]"
@@ -148,7 +148,9 @@ export default function FunctionalWrapper({
       {(!settings ||
         (settings.search_page_enabled && settings.chat_page_enabled)) && (
         <div
-          className={`mobile:hidden z-30 flex fixed ${chatBannerPresent ? (twoLines ? "top-20" : "top-14") : "top-4"} left-1/2 transform -translate-x-1/2`}
+          className={`mobile:hidden z-30 flex fixed ${
+            chatBannerPresent ? (twoLines ? "top-20" : "top-14") : "top-4"
+          } left-1/2 transform -translate-x-1/2`}
         >
           <div
             style={{ transition: "width 0.30s ease-out" }}
diff --git a/web/src/components/BasicClickable.tsx b/web/src/components/BasicClickable.tsx
index 34132c7bce5..44386a9d177 100644
--- a/web/src/components/BasicClickable.tsx
+++ b/web/src/components/BasicClickable.tsx
@@ -81,12 +81,14 @@ export function BasicSelectable({
   children,
   selected,
   hasBorder,
+  chat,
   fullWidth = false,
   padding = "normal",
 }: {
   children: string | JSX.Element;
   selected: boolean;
   hasBorder?: boolean;
+  chat?: boolean;
   fullWidth?: boolean;
   padding?: "none" | "normal" | "extra";
 }) {
@@ -100,7 +102,15 @@ export function BasicSelectable({
         ${padding == "extra" && "p-1.5"}
         select-none
         ${hasBorder ? "border border-border" : ""}
-        ${selected ? "bg-hover" : "hover:bg-hover"}
+        ${
+          selected
+            ? chat
+              ? "bg-background-chat-selected"
+              : "bg-hover"
+            : chat
+              ? "bg-background-chat-hover"
+              : "hover:bg-hover"
+        }
         ${fullWidth ? "w-full" : ""}`}
     >
       {children}
diff --git a/web/src/components/header/HeaderTitle.tsx b/web/src/components/header/HeaderTitle.tsx
index f63a8769c44..f7c9d21c47d 100644
--- a/web/src/components/header/HeaderTitle.tsx
+++ b/web/src/components/header/HeaderTitle.tsx
@@ -2,13 +2,21 @@
 
 import React from "react";
 
-export function HeaderTitle({ children }: { children: JSX.Element | string }) {
+export function HeaderTitle({
+  children,
+  chat,
+}: {
+  children: JSX.Element | string;
+  chat?: boolean;
+}) {
   const isString = typeof children === "string";
   const textSize = isString && children.length > 10 ? "text-xl" : "text-2xl";
 
   return (
     <h1
-      className={`${textSize} break-words line-clamp-2 ellipsis text-strong leading-none font-bold`}
+      className={`${textSize} ${
+        chat ? "text-text-sidebar-header" : "text-text-header"
+      } break-words line-clamp-2 ellipsis text-strong leading-none font-bold`}
     >
       {children}
     </h1>
diff --git a/web/src/components/search/SearchAnswer.tsx b/web/src/components/search/SearchAnswer.tsx
index e8645dfcea0..a9fe728b1c2 100644
--- a/web/src/components/search/SearchAnswer.tsx
+++ b/web/src/components/search/SearchAnswer.tsx
@@ -70,7 +70,11 @@ export default function SearchAnswer({
   return (
     <div
       ref={answerContainerRef}
-      className={`my-4 ${searchAnswerExpanded ? "min-h-[16rem]" : "h-[16rem]"} ${!searchAnswerExpanded && searchAnswerOverflowing && "overflow-y-hidden"} p-4 border-2 border-border rounded-lg relative`}
+      className={`my-4 ${
+        searchAnswerExpanded ? "min-h-[16rem]" : "h-[16rem]"
+      } ${
+        !searchAnswerExpanded && searchAnswerOverflowing && "overflow-y-hidden"
+      } p-4 border-2 border-search-answer-border rounded-lg relative`}
     >
       <div>
         <div className="flex gap-x-2">
diff --git a/web/src/components/search/SearchBar.tsx b/web/src/components/search/SearchBar.tsx
index df6f45eacdb..30b61b20c7e 100644
--- a/web/src/components/search/SearchBar.tsx
+++ b/web/src/components/search/SearchBar.tsx
@@ -68,8 +68,8 @@ export const AnimatedToggle = ({
           {/* Toggle switch */}
           <div
             className={`
-            w-10 h-6 flex items-center rounded-full p-1 transition-all duration-300 ease-in-out
-            ${isOn ? "bg-background-400" : "bg-background-200"}
+            w-10 h-6 flex items-center rounded-full p-1 transition-all duration-300 ease-in-out 
+            ${isOn ? "bg-toggled-background" : "bg-untoggled-background"}
           `}
           >
             <div
@@ -178,10 +178,14 @@ export const FullSearchBar = ({
         suppressContentEditableWarning={true}
       />
       <div
-        className={`flex flex-nowrap ${showingSidebar ? " 2xl:justify-between" : "2xl:justify-end"} justify-between 4xl:justify-end w-full max-w-full items-center space-x-3 py-3 px-4`}
+        className={`flex flex-nowrap ${
+          showingSidebar ? " 2xl:justify-between" : "2xl:justify-end"
+        } justify-between 4xl:justify-end w-full max-w-full items-center space-x-3 py-3 px-4`}
       >
         <div
-          className={`-my-1 flex-grow 4xl:hidden ${!showingSidebar && "2xl:hidden"}`}
+          className={`-my-1 flex-grow 4xl:hidden ${
+            !showingSidebar && "2xl:hidden"
+          }`}
         >
           {(ccPairs.length > 0 || documentSets.length > 0) && (
             <HorizontalSourceSelector
@@ -208,7 +212,11 @@ export const FullSearchBar = ({
             >
               <SendIcon
                 size={28}
-                className={`text-emphasis ${disabled || !query ? "bg-disabled-submit-background" : "bg-submit-background"} text-white p-1 rounded-full`}
+                className={`text-emphasis ${
+                  disabled || !query
+                    ? "bg-disabled-submit-background"
+                    : "bg-submit-background"
+                } text-white p-1 rounded-full`}
               />
             </button>
           </div>
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 66b5cb481ed..454f180bf21 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -724,7 +724,9 @@ export const SearchSection = ({
 
             {
               <div
-                className={`desktop:px-24 w-full ${chatBannerPresent && "mt-10"} pt-10 relative max-w-[2000px] xl:max-w-[1430px] mx-auto`}
+                className={`desktop:px-24 w-full ${
+                  chatBannerPresent && "mt-10"
+                } pt-10 relative max-w-[2000px] xl:max-w-[1430px] mx-auto`}
               >
                 <div className="absolute z-10 mobile:px-4 mobile:max-w-searchbar-max mobile:w-[90%] top-12 desktop:left-4 hidden 2xl:block mobile:left-1/2 mobile:transform mobile:-translate-x-1/2 desktop:w-52 3xl:w-64">
                   {!settings?.isMobile &&
@@ -849,7 +851,7 @@ export const SearchSection = ({
           </div>
         </div>
       </div>
-      <FixedLogo />
+      <FixedLogo chat />
     </>
   );
 };

From cc40f0d27bdb6d75a05227a8360e9ac72c7e8f3d Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Tue, 29 Oct 2024 09:05:01 -0700
Subject: [PATCH 296/376] fixed label filter  (#2978)

* added old error handling to comment fetching

* Not

* properly escaped cql labels

* reverted changes
---
 backend/danswer/connectors/confluence/connector.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index fe52862982d..f0945547e57 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -91,12 +91,13 @@ def __init__(
                 cql_page_query += f" and id='{page_id}'"
 
         self.cql_page_query = cql_page_query
-        self.cql_label_filter = ""
         self.cql_time_filter = ""
+
+        self.cql_label_filter = ""
         if labels_to_skip:
             labels_to_skip = list(set(labels_to_skip))
-            comma_separated_labels = ",".join(labels_to_skip)
-            self.cql_label_filter = f"&label not in ({comma_separated_labels})"
+            comma_separated_labels = ",".join(f"'{label}'" for label in labels_to_skip)
+            self.cql_label_filter = f" and label not in ({comma_separated_labels})"
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         # see https://github.com/atlassian-api/atlassian-python-api/blob/master/atlassian/rest_client.py
@@ -125,7 +126,8 @@ def _get_comment_string_for_page_id(self, page_id: str) -> str:
             for comment in comments:
                 comment_string += "\nComment:\n"
                 comment_string += extract_text_from_confluence_html(
-                    confluence_client=self.confluence_client, confluence_object=comment
+                    confluence_client=self.confluence_client,
+                    confluence_object=comment,
                 )
 
         return comment_string

From fcb94f117330384f06f54a5de183a770965e2802 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Tue, 29 Oct 2024 09:44:02 -0700
Subject: [PATCH 297/376] Tiny logging clarity improvement (#2985)

---
 backend/danswer/utils/logger.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/utils/logger.py b/backend/danswer/utils/logger.py
index 344e3ea8ef1..98d86f799a3 100644
--- a/backend/danswer/utils/logger.py
+++ b/backend/danswer/utils/logger.py
@@ -61,11 +61,11 @@ def process(
     ) -> tuple[str, MutableMapping[str, Any]]:
         # If this is an indexing job, add the attempt ID to the log message
         # This helps filter the logs for this specific indexing
-        attempt_id = IndexAttemptSingleton.get_index_attempt_id()
+        index_attempt_id = IndexAttemptSingleton.get_index_attempt_id()
         cc_pair_id = IndexAttemptSingleton.get_connector_credential_pair_id()
 
-        if attempt_id is not None:
-            msg = f"[Attempt: {attempt_id}] {msg}"
+        if index_attempt_id is not None:
+            msg = f"[Index Attempt: {index_attempt_id}] {msg}"
 
         if cc_pair_id is not None:
             msg = f"[CC Pair: {cc_pair_id}] {msg}"

From 40a0f719608d7239ffa76db7faefb305c223e8bd Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Tue, 29 Oct 2024 10:33:54 -0700
Subject: [PATCH 298/376] Temp fix to add retries to
 get_all_vespa_ids_for_document_id

---
 backend/danswer/document_index/vespa/chunk_retrieval.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/danswer/document_index/vespa/chunk_retrieval.py b/backend/danswer/document_index/vespa/chunk_retrieval.py
index 3622b9a3e02..ce95ff6fecb 100644
--- a/backend/danswer/document_index/vespa/chunk_retrieval.py
+++ b/backend/danswer/document_index/vespa/chunk_retrieval.py
@@ -231,6 +231,7 @@ def _get_chunks_via_visit_api(
     return document_chunks
 
 
+@retry(tries=10, delay=1, backoff=2)
 def get_all_vespa_ids_for_document_id(
     document_id: str,
     index_name: str,

From ea80cdce0217b24b657f7583baf9312f38d14dc1 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Tue, 29 Oct 2024 11:01:34 -0700
Subject: [PATCH 299/376] init sqlalchemy in child process (#2987)

---
 backend/danswer/background/celery/apps/beat.py    |  2 ++
 backend/danswer/background/celery/apps/heavy.py   |  2 +-
 backend/danswer/background/celery/apps/primary.py | 13 -------------
 backend/danswer/background/indexing/job_client.py |  7 +++++--
 4 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/backend/danswer/background/celery/apps/beat.py b/backend/danswer/background/celery/apps/beat.py
index f88295fa139..097f8f5cfa3 100644
--- a/backend/danswer/background/celery/apps/beat.py
+++ b/backend/danswer/background/celery/apps/beat.py
@@ -21,6 +21,8 @@
 @beat_init.connect
 def on_beat_init(sender: Any, **kwargs: Any) -> None:
     logger.info("beat_init signal received.")
+
+    # celery beat shouldn't touch the db at all. But just setting a low minimum here.
     SqlEngine.set_app_name(POSTGRES_CELERY_BEAT_APP_NAME)
     SqlEngine.init_engine(pool_size=2, max_overflow=0)
     app_base.wait_for_redis(sender, **kwargs)
diff --git a/backend/danswer/background/celery/apps/heavy.py b/backend/danswer/background/celery/apps/heavy.py
index ba53776bedb..4bd95162b3a 100644
--- a/backend/danswer/background/celery/apps/heavy.py
+++ b/backend/danswer/background/celery/apps/heavy.py
@@ -58,7 +58,7 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     logger.info(f"Multiprocessing start method: {multiprocessing.get_start_method()}")
 
     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_HEAVY_APP_NAME)
-    SqlEngine.init_engine(pool_size=8, max_overflow=0)
+    SqlEngine.init_engine(pool_size=4, max_overflow=12)
 
     app_base.wait_for_redis(sender, **kwargs)
     app_base.on_secondary_worker_init(sender, **kwargs)
diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 8790dce5903..33719608b52 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -166,19 +166,6 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
             r.delete(key)
 
 
-# @worker_process_init.connect
-# def on_worker_process_init(sender: Any, **kwargs: Any) -> None:
-#     """This only runs inside child processes when the worker is in pool=prefork mode.
-#     This may be technically unnecessary since we're finding prefork pools to be
-#     unstable and currently aren't planning on using them."""
-#     logger.info("worker_process_init signal received.")
-#     SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME)
-#     SqlEngine.init_engine(pool_size=5, max_overflow=0)
-
-#     # https://stackoverflow.com/questions/43944787/sqlalchemy-celery-with-scoped-session-error
-#     SqlEngine.get_engine().dispose(close=False)
-
-
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:
     app_base.on_worker_ready(sender, **kwargs)
diff --git a/backend/danswer/background/indexing/job_client.py b/backend/danswer/background/indexing/job_client.py
index 68d706895fd..6808a52c5ca 100644
--- a/backend/danswer/background/indexing/job_client.py
+++ b/backend/danswer/background/indexing/job_client.py
@@ -11,7 +11,8 @@
 from typing import Literal
 from typing import Optional
 
-from danswer.db.engine import get_sqlalchemy_engine
+from danswer.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME
+from danswer.db.engine import SqlEngine
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -37,7 +38,9 @@ def _initializer(
     if kwargs is None:
         kwargs = {}
 
-    get_sqlalchemy_engine().dispose(close=False)
+    logger.info("Initializing spawned worker child process.")
+    SqlEngine.set_app_name(POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME)
+    SqlEngine.init_engine(pool_size=4, max_overflow=12, pool_recycle=60)
     return func(*args, **kwargs)
 
 

From ea406c55cdc2f54bf9fae09f6ff1c01491f21b73 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 30 Oct 2024 10:54:29 -0700
Subject: [PATCH 300/376] add extra tags to pruning logs (#2994)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 .../background/celery/tasks/pruning/tasks.py  |  8 ++++++-
 backend/danswer/utils/logger.py               | 21 +++++++++++++++----
 2 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index d9579ccf93c..d20430df1f8 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -34,6 +34,7 @@
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.models import ConnectorCredentialPair
 from danswer.redis.redis_pool import get_redis_client
+from danswer.utils.logger import pruning_ctx
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -229,10 +230,15 @@ def connector_pruning_generator_task(
     and compares those IDs to locally stored documents and deletes all locally stored IDs missing
     from the most recently pulled document ID list"""
 
-    r = get_redis_client(tenant_id=tenant_id)
+    pruning_ctx_dict = pruning_ctx.get()
+    pruning_ctx_dict["cc_pair_id"] = cc_pair_id
+    pruning_ctx_dict["request_id"] = self.request.id
+    pruning_ctx.set(pruning_ctx_dict)
 
     rcp = RedisConnectorPruning(cc_pair_id)
 
+    r = get_redis_client(tenant_id=tenant_id)
+
     lock = r.lock(
         DanswerRedisLocks.PRUNING_LOCK_PREFIX + f"_{rcp._id}",
         timeout=CELERY_PRUNING_LOCK_TIMEOUT,
diff --git a/backend/danswer/utils/logger.py b/backend/danswer/utils/logger.py
index 98d86f799a3..bc872b0c423 100644
--- a/backend/danswer/utils/logger.py
+++ b/backend/danswer/utils/logger.py
@@ -1,3 +1,4 @@
+import contextvars
 import logging
 import os
 from collections.abc import MutableMapping
@@ -16,6 +17,10 @@
 
 logging.addLevelName(logging.INFO + 5, "NOTICE")
 
+pruning_ctx: contextvars.ContextVar[dict[str, Any]] = contextvars.ContextVar(
+    "pruning_ctx", default=dict()
+)
+
 
 class IndexAttemptSingleton:
     """Used to tell if this process is an indexing job, and if so what is the
@@ -64,11 +69,19 @@ def process(
         index_attempt_id = IndexAttemptSingleton.get_index_attempt_id()
         cc_pair_id = IndexAttemptSingleton.get_connector_credential_pair_id()
 
-        if index_attempt_id is not None:
-            msg = f"[Index Attempt: {index_attempt_id}] {msg}"
+        pruning_ctx_dict = pruning_ctx.get()
+        if len(pruning_ctx_dict) > 0:
+            if "request_id" in pruning_ctx_dict:
+                msg = f"[Prune: {pruning_ctx_dict['request_id']}] {msg}"
+
+            if "cc_pair_id" in pruning_ctx_dict:
+                msg = f"[CC Pair: {pruning_ctx_dict['cc_pair_id']}] {msg}"
+        else:
+            if index_attempt_id is not None:
+                msg = f"[Index Attempt: {index_attempt_id}] {msg}"
 
-        if cc_pair_id is not None:
-            msg = f"[CC Pair: {cc_pair_id}] {msg}"
+            if cc_pair_id is not None:
+                msg = f"[CC Pair: {cc_pair_id}] {msg}"
 
         # Add tenant information if it differs from default
         # This will always be the case for authenticated API requests

From 8223dc763de99d22d65637652c3609133e6acc1b Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 30 Oct 2024 11:55:47 -0700
Subject: [PATCH 301/376] add regeneration clarity (#2986)

* add regeneration clarity

* minor udpate
---
 web/src/app/chat/RegenerateOption.tsx |  5 +++--
 web/src/app/chat/message/Messages.tsx | 30 ++++++++++++++++++++-------
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/web/src/app/chat/RegenerateOption.tsx b/web/src/app/chat/RegenerateOption.tsx
index 8a9234b0a23..606f5ed589d 100644
--- a/web/src/app/chat/RegenerateOption.tsx
+++ b/web/src/app/chat/RegenerateOption.tsx
@@ -16,6 +16,7 @@ import { Hoverable } from "@/components/Hoverable";
 import { Popover } from "@/components/popover/Popover";
 import { StarFeedback } from "@/components/icons/icons";
 import { IconType } from "react-icons";
+import { FiRefreshCw } from "react-icons/fi";
 
 export function RegenerateDropdown({
   options,
@@ -68,7 +69,7 @@ export function RegenerateDropdown({
                 py-1.5 
                 "
       >
-        Pick a model
+        Regenerate with
       </p>
       {options.map((option, ind) => {
         const isSelected = option.value === selected;
@@ -92,7 +93,7 @@ export function RegenerateDropdown({
       content={
         <div onClick={() => toggleDropdownVisible(!isOpen)}>
           {!alternate ? (
-            <Hoverable size={16} icon={StarFeedback as IconType} />
+            <Hoverable size={16} icon={FiRefreshCw as IconType} />
           ) : (
             <Hoverable
               size={16}
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 181e00845ee..8526be1b922 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -299,7 +299,9 @@ export const AIMessage = ({
       className={"py-5 ml-4 px-5 relative flex "}
     >
       <div
-        className={`mx-auto ${shared ? "w-full" : "w-[90%]"}  max-w-message-max`}
+        className={`mx-auto ${
+          shared ? "w-full" : "w-[90%]"
+        }  max-w-message-max`}
       >
         <div className={`desktop:mr-12 ${!shared && "mobile:ml-0 md:ml-8"}`}>
           <div className="flex">
@@ -545,9 +547,21 @@ export const AIMessage = ({
                         className={`
                         absolute -bottom-5
                         z-10
-                        invisible ${(isHovering || isRegenerateHovered || settings?.isMobile) && "!visible"}
-                        opacity-0 ${(isHovering || isRegenerateHovered || settings?.isMobile) && "!opacity-100"}
-                        translate-y-2 ${(isHovering || settings?.isMobile) && "!translate-y-0"}
+                        invisible ${
+                          (isHovering ||
+                            isRegenerateHovered ||
+                            settings?.isMobile) &&
+                          "!visible"
+                        }
+                        opacity-0 ${
+                          (isHovering ||
+                            isRegenerateHovered ||
+                            settings?.isMobile) &&
+                          "!opacity-100"
+                        }
+                        translate-y-2 ${
+                          (isHovering || settings?.isMobile) && "!translate-y-0"
+                        }
                         transition-transform duration-300 ease-in-out 
                         flex md:flex-row gap-x-0.5 bg-background-125/40 -mx-1.5 p-1.5 rounded-lg
                         `}
@@ -703,9 +717,7 @@ export const HumanMessage = ({
   }, [isEditing]);
 
   const handleEditSubmit = () => {
-    if (editedContent.trim() !== content.trim()) {
-      onEdit?.(editedContent);
-    }
+    onEdit?.(editedContent);
     setIsEditing(false);
   };
 
@@ -721,7 +733,9 @@ export const HumanMessage = ({
       onMouseLeave={() => setIsHovered(false)}
     >
       <div
-        className={`text-user-text mx-auto ${shared ? "w-full" : "w-[90%]"} max-w-[790px]`}
+        className={`text-user-text mx-auto ${
+          shared ? "w-full" : "w-[90%]"
+        } max-w-[790px]`}
       >
         <div className="xl:ml-8">
           <div className="flex flex-col mr-4">

From 5be457e321bb8f3b58978d8276b9ef449a338211 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Wed, 30 Oct 2024 12:10:03 -0700
Subject: [PATCH 302/376] Add alternative auth header (#2999)

---
 backend/ee/danswer/auth/api_key.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/backend/ee/danswer/auth/api_key.py b/backend/ee/danswer/auth/api_key.py
index d4f99d13891..74e391ad1da 100644
--- a/backend/ee/danswer/auth/api_key.py
+++ b/backend/ee/danswer/auth/api_key.py
@@ -10,6 +10,11 @@
 
 
 _API_KEY_HEADER_NAME = "Authorization"
+# NOTE for others who are curious: In the context of a header, "X-" often refers
+# to non-standard, experimental, or custom headers in HTTP or other protocols. It
+# indicates that the header is not part of the official standards defined by
+# organizations like the Internet Engineering Task Force (IETF).
+_API_KEY_HEADER_ALTERNATIVE_NAME = "X-Danswer-Authorization"
 _BEARER_PREFIX = "Bearer "
 _API_KEY_PREFIX = "dn_"
 _API_KEY_LEN = 192
@@ -43,7 +48,9 @@ def build_displayable_api_key(api_key: str) -> str:
 
 
 def get_hashed_api_key_from_request(request: Request) -> str | None:
-    raw_api_key_header = request.headers.get(_API_KEY_HEADER_NAME)
+    raw_api_key_header = request.headers.get(
+        _API_KEY_HEADER_ALTERNATIVE_NAME
+    ) or request.headers.get(_API_KEY_HEADER_NAME)
     if raw_api_key_header is None:
         return None
 

From 53d2d333ab349d6e1833aafa27700263c2fae475 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Wed, 30 Oct 2024 17:23:20 -0400
Subject: [PATCH 303/376] Refactor metadata

---
 .../danswer/connectors/freshdesk/connector.py | 181 ++++++++++++------
 1 file changed, 126 insertions(+), 55 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index e25010efdb2..bd5df1012a2 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -12,97 +12,166 @@
 
 logger = setup_logger()
 
-def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
-    # Process date fields
-    for date_field in ["created_at", "updated_at", "due_by"]:
-        ticket[date_field] = datetime.fromisoformat(ticket[date_field].replace('Z', '+00:00'))
-
-    # Convert all other values to strings
-    ticket = {
-        key: str(value) if not isinstance(value, (str, datetime)) else value
-        for key, value in ticket.items()
+
+def _create_metadata_from_ticket(ticket: dict) -> dict:
+    included_fields = {
+        "fr_escalated",
+        "spam",
+        "priority",
+        "source",
+        "status",
+        "type",
+        "is_escalated",
+        "tags",
+        "nr_due_by",
+        "nr_escalated",
+        "cc_emails",
+        "fwd_emails",
+        "reply_cc_emails",
+        "ticket_cc_emails",
+        "support_email",
+        "to_emails",
+    }
+
+    metadata = {}
+    email_data = {}
+
+    for key, value in ticket.items():
+        if (
+            key in included_fields
+            and value is not None
+            and value != []
+            and value != {}
+            and value != "[]"
+            and value != ""
+        ):
+            value_to_str = (
+                [str(item) for item in value] if isinstance(value, List) else str(value)
+            )
+            if "email" in key:
+                email_data[key] = value_to_str
+            else:
+                metadata[key] = value_to_str
+
+    if email_data:
+        metadata["email_data"] = str(email_data)
+
+    # Convert source to human-parsable string
+    source_types = {
+        "1": "Email",
+        "2": "Portal",
+        "3": "Phone",
+        "7": "Chat",
+        "9": "Feedback Widget",
+        "10": "Outbound Email",
     }
+    if ticket.get("source"):
+        metadata["source"] = source_types.get(
+            str(ticket.get("source")), "Unknown Source Type"
+        )
 
-    # Checking for overdue tickets
-    ticket["overdue"] = datetime.now(timezone.utc) > ticket["due_by"]
+    # Convert priority to human-parsable string
+    priority_types = {"1": "low", "2": "medium", "3": "high", "4": "urgent"}
+    if ticket.get("priority"):
+        metadata["priority"] = priority_types.get(
+            str(ticket.get("priority")), "Unknown Priority"
+        )
 
-    # Map ticket status codes to human readable values
-    status_mapping = {2: "open", 3: "pending", 4: "resolved", 5: "closed"}
-    if status_string := status_mapping.get(ticket.get("status")):
-        ticket["status"] = status_string
+    # Convert status to human-parsable string
+    status_types = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"}
+    if ticket.get("status"):
+        metadata["status"] = status_types.get(
+            str(ticket.get("status")), "Unknown Status"
+        )
 
-    # Parse HTML from the description field
-    ticket["description"] = parse_html_page_basic(ticket["description"])
+    metadata["overdue"] = datetime.now(timezone.utc) > ticket["due_by"]
 
+    return metadata
+
+
+def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     return Document(
         id=ticket["id"],
-        sections=[Section(
-            link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
-            text=json.dumps({
-                key: value
-                for key, value in ticket.items()
-                if isinstance(value, str)
-            }, default=str),
-        )],
+        sections=[
+            Section(
+                link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
+                text=f"description: {parse_html_page_basic(ticket.get('description_text', ''))}",
+            )
+        ],
         source=DocumentSource.FRESHDESK,
         semantic_identifier=ticket["subject"],
-        metadata={
-            key: value.isoformat() if isinstance(value, datetime) else str(value)
-            for key, value in ticket.items()
-            if isinstance(value, (str, datetime)) and key not in ["description", "description_text"]
-        },
+        metadata=_create_metadata_from_ticket(ticket),
+        doc_updated_at=ticket["updated_at"],
     )
 
+
 class FreshdeskConnector(PollConnector, LoadConnector):
     def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
 
-    def load_credentials(self, credentials: dict[str, Any]) -> Optional[dict[str, Any]]:
-        self.api_key = credentials.get("freshdesk_api_key")
-        self.domain = credentials.get("freshdesk_domain")
-        self.password = credentials.get("freshdesk_password")
-        return None
-    
-    def _fetch_tickets(self, start: datetime | None = None, end: datetime | None = None) -> Iterator[List[dict]]:
-        #"end" is not currently used, so we may double fetch tickets created after the indexing starts but before the actual call is made. 
-        #To use "end" would require us to use the search endpoint but it has limitations, 
-        #namely having to fetch all IDs and then individually fetch each ticket because there is no "include" field available for this endpoint:
-        #https://developers.freshdesk.com/api/#filter_tickets
-        if any([self.api_key, self.domain, self.password]) is None:
+    def load_credentials(self, credentials: dict[str, str | int]) -> None:
+        api_key = credentials.get("freshdesk_api_key")
+        domain = credentials.get("freshdesk_domain")
+        password = credentials.get("freshdesk_password")
+
+        if not all(isinstance(cred, str) for cred in [domain, api_key, password]):
+            raise ConnectorMissingCredentialError(
+                "All Freshdesk credentials must be strings"
+            )
+
+        self.api_key = str(api_key)
+        self.domain = str(domain)
+        self.password = str(password)
+
+    def _fetch_tickets(
+        self, start: datetime | None = None, end: datetime | None = None
+    ) -> Iterator[List[dict]]:
+        # "end" is not currently used, so we may double fetch tickets created after the indexing starts but before the actual call is made.
+        # To use "end" would require us to use the search endpoint but it has limitations,
+        # namely having to fetch all IDs and then individually fetch each ticket because there is no "include" field available for this endpoint:
+        # https://developers.freshdesk.com/api/#filter_tickets
+        if any(attr is None for attr in [self.api_key, self.domain, self.password]):
             raise ConnectorMissingCredentialError("freshdesk")
-        
+
         base_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets"
-        params = {
+        params: dict[str, int | str] = {
             "include": "description",
             "per_page": 50,
-            "page": 1
+            "page": 1,
         }
-        
+
         if start:
             params["updated_since"] = start.isoformat()
 
         while True:
-            response = requests.get(base_url, auth=(self.api_key, self.password), params=params)
+            response = requests.get(
+                base_url, auth=(self.api_key, self.password), params=params
+            )
             response.raise_for_status()
-            
+
             if response.status_code == 204:
                 break
-            
+
             tickets = json.loads(response.content)
-            logger.info(f"Fetched {len(tickets)} tickets from Freshdesk API (Page {params['page']})")
+            logger.info(
+                f"Fetched {len(tickets)} tickets from Freshdesk API (Page {params['page']})"
+            )
 
             yield tickets
 
-            if len(tickets) < params["per_page"]:
+            if len(tickets) < int(params["per_page"]):
                 break
-            
-            params["page"] += 1
 
-    def _process_tickets(self, start: datetime | None = None, end: datetime | None = None) -> GenerateDocumentsOutput:
+            params["page"] = int(params["page"]) + 1
+
+    def _process_tickets(
+        self, start: datetime | None = None, end: datetime | None = None
+    ) -> GenerateDocumentsOutput:
         doc_batch: List[Document] = []
 
         for ticket_batch in self._fetch_tickets(start, end):
             for ticket in ticket_batch:
+                logger.info(_create_doc_from_ticket(ticket, self.domain))
                 doc_batch.append(_create_doc_from_ticket(ticket, self.domain))
 
                 if len(doc_batch) >= self.batch_size:
@@ -115,7 +184,9 @@ def _process_tickets(self, start: datetime | None = None, end: datetime | None =
     def load_from_state(self) -> GenerateDocumentsOutput:
         return self._process_tickets()
 
-    def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch) -> GenerateDocumentsOutput:
+    def poll_source(
+        self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
+    ) -> GenerateDocumentsOutput:
         start_datetime = datetime.fromtimestamp(start, tz=timezone.utc)
         end_datetime = datetime.fromtimestamp(end, tz=timezone.utc)
 

From 73ee709801bc03c94cfc9a6dcd487768f33aa667 Mon Sep 17 00:00:00 2001
From: Skylar Kesselring <skylarkesselring@gmail.com>
Date: Wed, 30 Oct 2024 17:46:04 -0400
Subject: [PATCH 304/376] Fix typing errors

---
 backend/danswer/connectors/freshdesk/connector.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index bd5df1012a2..40b8cd7085b 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -1,7 +1,7 @@
 import requests
 import json
 from datetime import datetime, timezone
-from typing import Any, List, Optional, Iterator
+from typing import List, Iterator
 from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
@@ -84,14 +84,15 @@ def _create_metadata_from_ticket(ticket: dict) -> dict:
             str(ticket.get("status")), "Unknown Status"
         )
 
-    metadata["overdue"] = datetime.now(timezone.utc) > ticket["due_by"]
+    due_by = datetime.fromisoformat(ticket["due_by"].replace("Z", "+00:00"))
+    metadata["overdue"] = str(datetime.now(timezone.utc) > due_by)
 
     return metadata
 
 
 def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     return Document(
-        id=ticket["id"],
+        id=str(ticket["id"]),
         sections=[
             Section(
                 link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
@@ -101,7 +102,9 @@ def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
         source=DocumentSource.FRESHDESK,
         semantic_identifier=ticket["subject"],
         metadata=_create_metadata_from_ticket(ticket),
-        doc_updated_at=ticket["updated_at"],
+        doc_updated_at=datetime.fromisoformat(
+            ticket["updated_at"].replace("Z", "+00:00")
+        ),
     )
 
 

From 7384ca87686aded011a6ac99be07337fd0388025 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 30 Oct 2024 15:53:26 -0700
Subject: [PATCH 305/376] clarity (#3001)

---
 web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index c2e903e2776..a5b6b3bd1ee 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -253,9 +253,9 @@ export default function AddConnector({
 
         // Apply advanced configuration-specific transforms.
         const advancedConfiguration: any = {
-          pruneFreq: (pruneFreq || defaultPruneFreqDays) * 60 * 60 * 24,
+          pruneFreq: (pruneFreq ?? defaultPruneFreqDays) * 60 * 60 * 24,
           indexingStart: convertStringToDateTime(indexingStart),
-          refreshFreq: (refreshFreq || defaultRefreshFreqMinutes) * 60,
+          refreshFreq: (refreshFreq ?? defaultRefreshFreqMinutes) * 60,
         };
 
         // Google sites-specific handling

From f318e302c5effa46de6bac84380472112ccd0402 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Wed, 30 Oct 2024 16:05:32 -0700
Subject: [PATCH 306/376] Minor theming (#2993)

* ensure functionality

* naming

* ensure tailwind theme updated

* add comments

* nit

* remove pr

* enforce colors

* update our tailwind config
---
 web/src/app/assistants/SidebarWrapper.tsx           |  6 ++++--
 web/src/app/chat/ChatPage.tsx                       |  2 +-
 .../app/chat/sessionSidebar/ChatSessionDisplay.tsx  |  6 +++---
 .../app/chat/shared/[chatId]/SharedChatDisplay.tsx  |  1 -
 web/src/app/chat/shared_chat_search/FixedLogo.tsx   | 13 ++++++++++---
 web/src/components/BasicClickable.tsx               | 10 ++--------
 web/src/components/admin/ClientLayout.tsx           |  2 +-
 .../components/admin/connectors/AdminSidebar.tsx    |  6 +++---
 web/src/components/header/HeaderTitle.tsx           |  8 +++++---
 web/src/components/header/LogoType.tsx              | 12 ++++++++----
 web/src/components/search/SearchSection.tsx         |  2 +-
 web/tailwind-themes/tailwind.config.js              | 12 +++++++++++-
 12 files changed, 49 insertions(+), 31 deletions(-)

diff --git a/web/src/app/assistants/SidebarWrapper.tsx b/web/src/app/assistants/SidebarWrapper.tsx
index 58190228bbd..a5a1cdb4148 100644
--- a/web/src/app/assistants/SidebarWrapper.tsx
+++ b/web/src/app/assistants/SidebarWrapper.tsx
@@ -153,13 +153,15 @@ export default function SidebarWrapper<T extends object>({
           />
 
           <div
-            className={`mt-4 w-full ${size == "lg" ? "max-w-4xl" : "max-w-3xl"} mx-auto`}
+            className={`mt-4 w-full ${
+              size == "lg" ? "max-w-4xl" : "max-w-3xl"
+            } mx-auto`}
           >
             {children}
           </div>
         </div>
       </div>
-      <FixedLogo />
+      <FixedLogo backgroundToggled={toggledSidebar || showDocSidebar} />
     </div>
   );
 }
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 4c4291405f9..a4fcc00e1d9 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -2512,7 +2512,7 @@ export function ChatPage({
               )}
             </div>
           </div>
-          <FixedLogo chat />
+          <FixedLogo backgroundToggled={toggledSidebar || showDocSidebar} />
         </div>
       </div>
       <DocumentSidebar
diff --git a/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx b/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
index d06ee14be25..08e3ad7e3cc 100644
--- a/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
+++ b/web/src/app/chat/sessionSidebar/ChatSessionDisplay.tsx
@@ -122,7 +122,7 @@ export function ChatSessionDisplay({
           );
         }}
       >
-        <BasicSelectable chat padding="extra" fullWidth selected={isSelected}>
+        <BasicSelectable padding="extra" fullWidth selected={isSelected}>
           <>
             <div className="flex relative">
               {isRenamingChat ? (
@@ -144,8 +144,8 @@ export function ChatSessionDisplay({
                     className={`absolute right-0 top-0 h-full w-8 bg-gradient-to-r from-transparent 
                     ${
                       isSelected
-                        ? "to-background-chat-hover"
-                        : " to-background-chat-selected group-hover:to-background-chat-hover"
+                        ? "to-background-chat-selected"
+                        : "group-hover:to-background-chat-hover"
                     } `}
                   />
                 </p>
diff --git a/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx b/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
index de2d443963f..e633d01ba0b 100644
--- a/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
+++ b/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
@@ -27,7 +27,6 @@ function BackToDanswerButton() {
           Back to {enterpriseSettings?.application_name || "Danswer Chat"}
         </Button>
       </div>
-      pr
     </div>
   );
 }
diff --git a/web/src/app/chat/shared_chat_search/FixedLogo.tsx b/web/src/app/chat/shared_chat_search/FixedLogo.tsx
index 717b407c8a4..921220adbd7 100644
--- a/web/src/app/chat/shared_chat_search/FixedLogo.tsx
+++ b/web/src/app/chat/shared_chat_search/FixedLogo.tsx
@@ -8,7 +8,12 @@ import Link from "next/link";
 import { useContext } from "react";
 import { FiSidebar } from "react-icons/fi";
 
-export default function FixedLogo({ chat }: { chat?: boolean }) {
+export default function FixedLogo({
+  // Whether the sidebar is toggled or not
+  backgroundToggled,
+}: {
+  backgroundToggled?: boolean;
+}) {
   const combinedSettings = useContext(SettingsContext);
   const settings = combinedSettings?.settings;
   const enterpriseSettings = combinedSettings?.enterpriseSettings;
@@ -28,7 +33,7 @@ export default function FixedLogo({ chat }: { chat?: boolean }) {
           <div className="w-full">
             {enterpriseSettings && enterpriseSettings.application_name ? (
               <div>
-                <HeaderTitle chat={chat}>
+                <HeaderTitle backgroundToggled={backgroundToggled}>
                   {enterpriseSettings.application_name}
                 </HeaderTitle>
                 {!NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED && (
@@ -36,7 +41,9 @@ export default function FixedLogo({ chat }: { chat?: boolean }) {
                 )}
               </div>
             ) : (
-              <HeaderTitle chat={chat}>Danswer</HeaderTitle>
+              <HeaderTitle backgroundToggled={backgroundToggled}>
+                Danswer
+              </HeaderTitle>
             )}
           </div>
         </div>
diff --git a/web/src/components/BasicClickable.tsx b/web/src/components/BasicClickable.tsx
index 44386a9d177..c7a58d3cb03 100644
--- a/web/src/components/BasicClickable.tsx
+++ b/web/src/components/BasicClickable.tsx
@@ -81,14 +81,12 @@ export function BasicSelectable({
   children,
   selected,
   hasBorder,
-  chat,
   fullWidth = false,
   padding = "normal",
 }: {
   children: string | JSX.Element;
   selected: boolean;
   hasBorder?: boolean;
-  chat?: boolean;
   fullWidth?: boolean;
   padding?: "none" | "normal" | "extra";
 }) {
@@ -104,12 +102,8 @@ export function BasicSelectable({
         ${hasBorder ? "border border-border" : ""}
         ${
           selected
-            ? chat
-              ? "bg-background-chat-selected"
-              : "bg-hover"
-            : chat
-              ? "bg-background-chat-hover"
-              : "hover:bg-hover"
+            ? "bg-background-chat-selected"
+            : "hover:bg-background-chat-hover"
         }
         ${fullWidth ? "w-full" : ""}`}
     >
diff --git a/web/src/components/admin/ClientLayout.tsx b/web/src/components/admin/ClientLayout.tsx
index 16500b5e0eb..7ca8eb75283 100644
--- a/web/src/components/admin/ClientLayout.tsx
+++ b/web/src/components/admin/ClientLayout.tsx
@@ -58,7 +58,7 @@ export function ClientLayout({
   return (
     <div className="h-screen overflow-y-hidden">
       <div className="flex h-full">
-        <div className="flex-none text-text-settings-sidebar bg-background-settings-sidebar w-[250px] z-20 pt-4 pb-8 h-full border-r border-border miniscroll overflow-auto">
+        <div className="flex-none text-text-settings-sidebar bg-background-sidebar w-[250px] z-20 pt-4 pb-8 h-full border-r border-border miniscroll overflow-auto">
           <AdminSidebar
             collections={[
               {
diff --git a/web/src/components/admin/connectors/AdminSidebar.tsx b/web/src/components/admin/connectors/AdminSidebar.tsx
index 7ada0e52ed9..e07ca690fbc 100644
--- a/web/src/components/admin/connectors/AdminSidebar.tsx
+++ b/web/src/components/admin/connectors/AdminSidebar.tsx
@@ -55,7 +55,7 @@ export function AdminSidebar({ collections }: { collections: Collection[] }) {
                 <div className="flex-grow min-w-0 my-auto">
                   {enterpriseSettings && enterpriseSettings.application_name ? (
                     <div className="w-full">
-                      <HeaderTitle>
+                      <HeaderTitle backgroundToggled={true}>
                         {enterpriseSettings.application_name}
                       </HeaderTitle>
                       {!NEXT_PUBLIC_DO_NOT_USE_TOGGLE_OFF_DANSWER_POWERED && (
@@ -65,7 +65,7 @@ export function AdminSidebar({ collections }: { collections: Collection[] }) {
                       )}
                     </div>
                   ) : (
-                    <HeaderTitle>Danswer</HeaderTitle>
+                    <HeaderTitle backgroundToggled={true}>Danswer</HeaderTitle>
                   )}
                 </div>
               </div>
@@ -74,7 +74,7 @@ export function AdminSidebar({ collections }: { collections: Collection[] }) {
         </div>
         <div className="flex w-full justify-center">
           <Link href={"/chat"}>
-            <button className="text-sm flex items-center block w-52 py-2.5 flex px-2 text-left bg-background-200 hover:bg-background-200/80 cursor-pointer rounded">
+            <button className="text-sm flex items-center block w-52 py-2.5 flex px-2 text-left text-text-back-button bg-background-back-button hover:bg-opacity-80 cursor-pointer rounded">
               <BackIcon className="my-auto" size={18} />
               <p className="ml-1 break-words line-clamp-2 ellipsis leading-none">
                 Back to{" "}
diff --git a/web/src/components/header/HeaderTitle.tsx b/web/src/components/header/HeaderTitle.tsx
index f7c9d21c47d..36cc9f7c493 100644
--- a/web/src/components/header/HeaderTitle.tsx
+++ b/web/src/components/header/HeaderTitle.tsx
@@ -4,10 +4,10 @@ import React from "react";
 
 export function HeaderTitle({
   children,
-  chat,
+  backgroundToggled,
 }: {
   children: JSX.Element | string;
-  chat?: boolean;
+  backgroundToggled?: boolean;
 }) {
   const isString = typeof children === "string";
   const textSize = isString && children.length > 10 ? "text-xl" : "text-2xl";
@@ -15,7 +15,9 @@ export function HeaderTitle({
   return (
     <h1
       className={`${textSize} ${
-        chat ? "text-text-sidebar-header" : "text-text-header"
+        backgroundToggled
+          ? "text-text-sidebar-toggled-header"
+          : "text-text-sidebar-header"
       } break-words line-clamp-2 ellipsis text-strong leading-none font-bold`}
     >
       {children}
diff --git a/web/src/components/header/LogoType.tsx b/web/src/components/header/LogoType.tsx
index 522aace1861..32981bbe26c 100644
--- a/web/src/components/header/LogoType.tsx
+++ b/web/src/components/header/LogoType.tsx
@@ -37,7 +37,9 @@ export default function LogoType({
 
   return (
     <div
-      className={`${hideOnMobile && "mobile:hidden"} z-[100] mb-auto shrink-0 flex items-center text-xl font-bold`}
+      className={`${
+        hideOnMobile && "mobile:hidden"
+      } z-[100] mb-auto shrink-0 flex items-center text-xl font-bold`}
     >
       {toggleSidebar && page == "chat" ? (
         <button
@@ -55,7 +57,9 @@ export default function LogoType({
         </div>
       )}
       <div
-        className={`cursor-pointer ${showArrow ? "desktop:invisible" : "invisible"} break-words inline-block w-fit ml-2 text-text-700 text-xl`}
+        className={`cursor-pointer ${
+          showArrow ? "desktop:invisible" : "invisible"
+        } break-words inline-block w-fit ml-2 text-text-700 text-xl`}
       >
         <div className="max-w-[175px]">
           {enterpriseSettings && enterpriseSettings.application_name ? (
@@ -111,9 +115,9 @@ export default function LogoType({
             }}
           >
             {!toggled && !combinedSettings?.isMobile ? (
-              <RightToLineIcon />
+              <RightToLineIcon className="text-sidebar-toggle" />
             ) : (
-              <LeftToLineIcon />
+              <LeftToLineIcon className="text-sidebar-toggle" />
             )}
           </button>
         </Tooltip>
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 454f180bf21..656d57f2a1b 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -851,7 +851,7 @@ export const SearchSection = ({
           </div>
         </div>
       </div>
-      <FixedLogo chat />
+      <FixedLogo backgroundToggled={toggledSidebar || showDocSidebar} />
     </>
   );
 };
diff --git a/web/tailwind-themes/tailwind.config.js b/web/tailwind-themes/tailwind.config.js
index b427e19a3f0..c50d77f7073 100644
--- a/web/tailwind-themes/tailwind.config.js
+++ b/web/tailwind-themes/tailwind.config.js
@@ -103,10 +103,15 @@ module.exports = {
 
         // colors for sidebar in chat, search, and manage settings
         "background-sidebar": "var(--background-100)",
-        "background-settings-sidebar": "var(--background-100)",
         "background-chatbar": "var(--background-100)",
         "text-sidebar": "var(--text-500)",
 
+        "toggled-background": "var(--background-400)",
+        "untoggled-background": "var(--background-200)",
+
+        "background-back-button": "var(--background-200)",
+        "text-back-button": "var(--text-800)",
+
         // Settings
         "text-sidebar-subtle": "var(--text-500)",
         "icon-settings-sidebar": "var(--text-600)",
@@ -114,9 +119,14 @@ module.exports = {
         "text-settings-sidebar-strong": "var(--text-900)",
         "background-settings-hover": "var(--background-200)",
 
+        "background-chat-hover": "var(--background-200)",
+        "background-chat-selected": "var(--background-200)",
+
         // Background for chat messages (user bubbles)
         user: "var(--user-bubble)",
 
+        "background-toggle": "var(--background-100)",
+
         // Colors for the search toggle buttons
         "background-agentic-toggled": "var(--light-success)",
         "background-agentic-untoggled": "var(--undo)",

From 1a7d62794969c2eb78cd4b26faf4856e90d3f3a8 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Wed, 30 Oct 2024 17:27:58 -0700
Subject: [PATCH 307/376] Disable Mediawiki Tests (#3005)

---
 .../danswer/connectors/mediawiki/test_mediawiki_family.py  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py b/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py
index 3e62df7a0c9..95e8a0cd902 100644
--- a/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py
+++ b/backend/tests/unit/danswer/connectors/mediawiki/test_mediawiki_family.py
@@ -7,6 +7,10 @@
 
 from danswer.connectors.mediawiki import family
 
+
+# Disabling these tests as they are flaky and rely on external wikis that are maintained by just fan communities
+
+
 NON_BUILTIN_WIKIS: Final[list[tuple[str, str]]] = [
     ("https://fallout.fandom.com", "falloutwiki"),
     ("https://harrypotter.fandom.com/wiki/", "harrypotterwiki"),
@@ -19,6 +23,7 @@
 
 
 # TODO: Add support for more builtin family types from `pywikibot.families`.
+@pytest.mark.skip(reason="Temporarily skipped")
 @pytest.mark.parametrize(
     "url, name, expected",
     [
@@ -48,6 +53,7 @@ def test_family_class_dispatch_builtins(
     assert family.family_class_dispatch(url, name) == expected
 
 
+@pytest.mark.skip(reason="Temporarily skipped")
 @pytest.mark.parametrize("url, name", NON_BUILTIN_WIKIS)
 def test_family_class_dispatch_on_non_builtins_generates_new_class_fast(
     url: str, name: str, mocker: MockFixture
@@ -58,6 +64,7 @@ def test_family_class_dispatch_on_non_builtins_generates_new_class_fast(
     mock_generate_family_class.assert_called_once_with(url, name)
 
 
+@pytest.mark.skip(reason="Temporarily skipped")
 @pytest.mark.slow
 @pytest.mark.parametrize("url, name", NON_BUILTIN_WIKIS)
 def test_family_class_dispatch_on_non_builtins_generates_new_class_slow(

From dba2d67cdbbb432526932a8d8cd2a966ac3fce6f Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Wed, 30 Oct 2024 17:40:03 -0700
Subject: [PATCH 308/376] only warmup on index swap (#3003)

* only warmup on index swap

* move conditional
---
 .../background/celery/tasks/indexing/tasks.py | 21 +++++++++++--------
 backend/danswer/db/swap_index.py              | 20 +++++++++++-------
 2 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 7c1460cbddf..d275ffd0633 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -105,19 +105,22 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
             return None
 
         with get_session_with_tenant(tenant_id=tenant_id) as db_session:
-            check_index_swap(db_session=db_session)
+            old_search_settings = check_index_swap(db_session=db_session)
             current_search_settings = get_current_search_settings(db_session)
             # So that the first time users aren't surprised by really slow speed of first
             # batch of documents indexed
             if current_search_settings.provider_type is None and not MULTI_TENANT:
-                embedding_model = EmbeddingModel.from_db_model(
-                    search_settings=current_search_settings,
-                    server_host=INDEXING_MODEL_SERVER_HOST,
-                    server_port=INDEXING_MODEL_SERVER_PORT,
-                )
-                warm_up_bi_encoder(
-                    embedding_model=embedding_model,
-                )
+                if old_search_settings:
+                    embedding_model = EmbeddingModel.from_db_model(
+                        search_settings=current_search_settings,
+                        server_host=INDEXING_MODEL_SERVER_HOST,
+                        server_port=INDEXING_MODEL_SERVER_PORT,
+                    )
+
+                    # only warm up if search settings were changed
+                    warm_up_bi_encoder(
+                        embedding_model=embedding_model,
+                    )
 
         cc_pair_ids: list[int] = []
         with get_session_with_tenant(tenant_id) as db_session:
diff --git a/backend/danswer/db/swap_index.py b/backend/danswer/db/swap_index.py
index 4961f05bbd4..8b583bd2e4b 100644
--- a/backend/danswer/db/swap_index.py
+++ b/backend/danswer/db/swap_index.py
@@ -14,7 +14,6 @@
 from danswer.db.search_settings import update_search_settings_status
 from danswer.key_value_store.factory import get_kv_store
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import MULTI_TENANT
 
 
 logger = setup_logger()
@@ -23,7 +22,14 @@
 def check_index_swap(db_session: Session) -> SearchSettings | None:
     """Get count of cc-pairs and count of successful index_attempts for the
     new model grouped by connector + credential, if it's the same, then assume
-    new index is done building. If so, swap the indices and expire the old one."""
+    new index is done building. If so, swap the indices and expire the old one.
+
+    Returns None if search settings did not change, or the old search settings if they
+    did change.
+    """
+
+    old_search_settings = None
+
     # Default CC-pair created for Ingestion API unused here
     all_cc_pairs = get_connector_credential_pairs(db_session)
     cc_pair_count = max(len(all_cc_pairs) - 1, 0)
@@ -43,9 +49,9 @@ def check_index_swap(db_session: Session) -> SearchSettings | None:
 
     if cc_pair_count == 0 or cc_pair_count == unique_cc_indexings:
         # Swap indices
-        now_old_search_settings = get_current_search_settings(db_session)
+        current_search_settings = get_current_search_settings(db_session)
         update_search_settings_status(
-            search_settings=now_old_search_settings,
+            search_settings=current_search_settings,
             new_status=IndexModelStatus.PAST,
             db_session=db_session,
         )
@@ -67,6 +73,6 @@ def check_index_swap(db_session: Session) -> SearchSettings | None:
             for cc_pair in all_cc_pairs:
                 resync_cc_pair(cc_pair, db_session=db_session)
 
-            if MULTI_TENANT:
-                return now_old_search_settings
-    return None
+            old_search_settings = current_search_settings
+
+    return old_search_settings

From ff9d7141a9d8ecb15bfef3b7fe2dcacc884ba88b Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Wed, 30 Oct 2024 20:21:54 -0700
Subject: [PATCH 309/376] Gmail Connector Robustify (#3000)

---
 .../miscellaneous_utils.py                    | 11 ++-
 backend/danswer/connectors/gmail/connector.py | 70 +++++++++++++++++--
 2 files changed, 76 insertions(+), 5 deletions(-)

diff --git a/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py b/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py
index 1b79c4a4faf..8e8ea8d7d65 100644
--- a/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py
+++ b/backend/danswer/connectors/cross_connector_utils/miscellaneous_utils.py
@@ -23,7 +23,16 @@ def datetime_to_utc(dt: datetime) -> datetime:
 
 
 def time_str_to_utc(datetime_str: str) -> datetime:
-    dt = parse(datetime_str)
+    try:
+        dt = parse(datetime_str)
+    except ValueError:
+        # Handle malformed timezone by attempting to fix common format issues
+        if "0000" in datetime_str:
+            # Convert "0000" to "+0000" for proper timezone parsing
+            fixed_dt_str = datetime_str.replace(" 0000", " +0000")
+            dt = parse(fixed_dt_str)
+        else:
+            raise
     return datetime_to_utc(dt)
 
 
diff --git a/backend/danswer/connectors/gmail/connector.py b/backend/danswer/connectors/gmail/connector.py
index 42d2f305f73..376f03e71d0 100644
--- a/backend/danswer/connectors/gmail/connector.py
+++ b/backend/danswer/connectors/gmail/connector.py
@@ -1,4 +1,8 @@
+import re
+import time
 from base64 import urlsafe_b64decode
+from datetime import datetime
+from datetime import timezone
 from typing import Any
 from typing import cast
 from typing import Dict
@@ -6,6 +10,7 @@
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
 from googleapiclient import discovery  # type: ignore
+from googleapiclient.errors import HttpError  # type: ignore
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
@@ -34,6 +39,64 @@
 logger = setup_logger()
 
 
+def _execute_with_retry(request: Any) -> Any:
+    max_attempts = 10
+    attempt = 0
+
+    while attempt < max_attempts:
+        # Note for reasons unknown, the Google API will sometimes return a 429
+        # and even after waiting the retry period, it will return another 429.
+        # It could be due to a few possibilities:
+        # 1. Other things are also requesting from the Gmail API with the same key
+        # 2. It's a rolling rate limit so the moment we get some amount of requests cleared, we hit it again very quickly
+        # 3. The retry-after has a maximum and we've already hit the limit for the day
+        # or it's something else...
+        try:
+            return request.execute()
+        except HttpError as error:
+            attempt += 1
+
+            if error.resp.status == 429:
+                # Attempt to get 'Retry-After' from headers
+                retry_after = error.resp.get("Retry-After")
+                if retry_after:
+                    sleep_time = int(retry_after)
+                else:
+                    # Extract 'Retry after' timestamp from error message
+                    match = re.search(
+                        r"Retry after (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)",
+                        str(error),
+                    )
+                    if match:
+                        retry_after_timestamp = match.group(1)
+                        retry_after_dt = datetime.strptime(
+                            retry_after_timestamp, "%Y-%m-%dT%H:%M:%S.%fZ"
+                        ).replace(tzinfo=timezone.utc)
+                        current_time = datetime.now(timezone.utc)
+                        sleep_time = max(
+                            int((retry_after_dt - current_time).total_seconds()),
+                            0,
+                        )
+                    else:
+                        logger.error(
+                            f"No Retry-After header or timestamp found in error message: {error}"
+                        )
+                        sleep_time = 60
+
+                sleep_time += 3  # Add a buffer to be safe
+
+                logger.info(
+                    f"Rate limit exceeded. Attempt {attempt}/{max_attempts}. Sleeping for {sleep_time} seconds."
+                )
+                time.sleep(sleep_time)
+
+            else:
+                raise
+
+    # If we've exhausted all attempts
+    raise Exception(f"Failed to execute request after {max_attempts} attempts")
+
+
 class GmailConnector(LoadConnector, PollConnector):
     def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
@@ -156,7 +219,7 @@ def _fetch_mails_from_gmail(
         query = GmailConnector._build_time_range_query(time_range_start, time_range_end)
         service = discovery.build("gmail", "v1", credentials=self.creds)
         while page_token is not None:
-            result = (
+            result = _execute_with_retry(
                 service.users()
                 .messages()
                 .list(
@@ -165,18 +228,17 @@ def _fetch_mails_from_gmail(
                     q=query,
                     maxResults=self.batch_size,
                 )
-                .execute()
             )
+
             page_token = result.get("nextPageToken")
             messages = result.get("messages", [])
             doc_batch = []
             for message in messages:
                 message_id = message["id"]
-                msg = (
+                msg = _execute_with_retry(
                     service.users()
                     .messages()
                     .get(userId="me", id=message_id, format="full")
-                    .execute()
                 )
                 doc = self._email_to_document(msg)
                 doc_batch.append(doc)

From 231ab3fb5d23ad03ea8a0db48cd2771335612a33 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 31 Oct 2024 09:43:46 -0700
Subject: [PATCH 310/376] Feature/indexing logs (#3002)

* improve logging around indexing tasks

* task_logger doesn't work inside the spawned task
---
 .../background/celery/tasks/indexing/tasks.py | 92 +++++++++++++++----
 1 file changed, 76 insertions(+), 16 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index d275ffd0633..175aef54055 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -391,7 +391,12 @@ def connector_indexing_proxy_task(
     tenant_id: str | None,
 ) -> None:
     """celery tasks are forked, but forking is unstable.  This proxies work to a spawned task."""
-
+    task_logger.info(
+        f"Indexing proxy - starting: attempt={index_attempt_id} "
+        f"tenant={tenant_id} "
+        f"cc_pair={cc_pair_id} "
+        f"search_settings={search_settings_id}"
+    )
     client = SimpleJobClient()
 
     job = client.submit(
@@ -405,29 +410,56 @@ def connector_indexing_proxy_task(
     )
 
     if not job:
+        task_logger.info(
+            f"Indexing proxy - spawn failed: attempt={index_attempt_id} "
+            f"tenant={tenant_id} "
+            f"cc_pair={cc_pair_id} "
+            f"search_settings={search_settings_id}"
+        )
         return
 
+    task_logger.info(
+        f"Indexing proxy - spawn succeeded: attempt={index_attempt_id} "
+        f"tenant={tenant_id} "
+        f"cc_pair={cc_pair_id} "
+        f"search_settings={search_settings_id}"
+    )
+
     while True:
         sleep(10)
-        with get_session_with_tenant(tenant_id) as db_session:
-            index_attempt = get_index_attempt(
-                db_session=db_session, index_attempt_id=index_attempt_id
-            )
 
-            # do nothing for ongoing jobs that haven't been stopped
-            if not job.done():
+        # do nothing for ongoing jobs that haven't been stopped
+        if not job.done():
+            with get_session_with_tenant(tenant_id) as db_session:
+                index_attempt = get_index_attempt(
+                    db_session=db_session, index_attempt_id=index_attempt_id
+                )
+
                 if not index_attempt:
                     continue
 
                 if not index_attempt.is_finished():
                     continue
 
-            if job.status == "error":
-                logger.error(job.exception())
+        if job.status == "error":
+            task_logger.error(
+                f"Indexing proxy - spawned task exceptioned: "
+                f"attempt={index_attempt_id} "
+                f"tenant={tenant_id} "
+                f"cc_pair={cc_pair_id} "
+                f"search_settings={search_settings_id} "
+                f"error={job.exception()}"
+            )
 
-            job.release()
-            break
+        job.release()
+        break
 
+    task_logger.info(
+        f"Indexing proxy - finished: attempt={index_attempt_id} "
+        f"tenant={tenant_id} "
+        f"cc_pair={cc_pair_id} "
+        f"search_settings={search_settings_id}"
+    )
     return
 
 
@@ -449,7 +481,17 @@ def connector_indexing_task(
 
     Returns None if the task did not run (possibly due to a conflict).
     Otherwise, returns an int >= 0 representing the number of indexed docs.
+
+    NOTE: if an exception is raised out of this task, the primary worker will detect
+    that the task transitioned to a "READY" state but the generator_complete_key doesn't exist.
+    This will cause the primary worker to abort the indexing attempt and clean up.
     """
+    logger.info(
+        f"Indexing spawned task starting: attempt={index_attempt_id} "
+        f"tenant={tenant_id} "
+        f"cc_pair={cc_pair_id} "
+        f"search_settings={search_settings_id}"
+    )
 
     attempt = None
     n_final_progress = 0
@@ -488,19 +530,19 @@ def connector_indexing_task(
                 cast(str, fence_json)
             )
         except ValueError:
-            task_logger.exception(
+            logger.exception(
                 f"connector_indexing_task: fence_data not decodeable: fence={rci.fence_key}"
             )
             raise
 
         if fence_data.index_attempt_id is None or fence_data.celery_task_id is None:
-            task_logger.info(
+            logger.info(
                 f"connector_indexing_task - Waiting for fence: fence={rci.fence_key}"
             )
             sleep(1)
             continue
 
-        task_logger.info(
+        logger.info(
             f"connector_indexing_task - Fence found, continuing...: fence={rci.fence_key}"
         )
         break
@@ -512,7 +554,7 @@ def connector_indexing_task(
 
     acquired = lock.acquire(blocking=False)
     if not acquired:
-        task_logger.warning(
+        logger.warning(
             f"Indexing task already running, exiting...: "
             f"cc_pair={cc_pair_id} search_settings={search_settings_id}"
         )
@@ -555,6 +597,13 @@ def connector_indexing_task(
                 rcs.fence_key, rci.generator_progress_key, lock, r
             )
 
+            logger.info(
+                f"Indexing spawned task running entrypoint: attempt={index_attempt_id} "
+                f"tenant={tenant_id} "
+                f"cc_pair={cc_pair_id} "
+                f"search_settings={search_settings_id}"
+            )
+
             run_indexing_entrypoint(
                 index_attempt_id,
                 tenant_id,
@@ -573,7 +622,12 @@ def connector_indexing_task(
 
             r.set(rci.generator_complete_key, HTTPStatus.OK.value)
     except Exception as e:
-        task_logger.exception(f"Indexing failed: cc_pair={cc_pair_id}")
+        logger.exception(
+            f"Indexing spawned task failed: attempt={index_attempt_id} "
+            f"tenant={tenant_id} "
+            f"cc_pair={cc_pair_id} "
+            f"search_settings={search_settings_id}"
+        )
         if attempt:
             with get_session_with_tenant(tenant_id) as db_session:
                 mark_attempt_failed(attempt, db_session, failure_reason=str(e))
@@ -587,4 +641,10 @@ def connector_indexing_task(
         if lock.owned():
             lock.release()
 
+    logger.info(
+        f"Indexing spawned task finished: attempt={index_attempt_id} "
+        f"tenant={tenant_id} "
+        f"cc_pair={cc_pair_id} "
+        f"search_settings={search_settings_id}"
+    )
     return n_final_progress

From e3be318781fcac71e8f278f22f830883c700d70b Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 31 Oct 2024 09:50:48 -0700
Subject: [PATCH 311/376] Update connector.py

---
 backend/danswer/connectors/freshdesk/connector.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 40b8cd7085b..a98ca34bdd2 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -129,10 +129,15 @@ def load_credentials(self, credentials: dict[str, str | int]) -> None:
     def _fetch_tickets(
         self, start: datetime | None = None, end: datetime | None = None
     ) -> Iterator[List[dict]]:
-        # "end" is not currently used, so we may double fetch tickets created after the indexing starts but before the actual call is made.
-        # To use "end" would require us to use the search endpoint but it has limitations,
-        # namely having to fetch all IDs and then individually fetch each ticket because there is no "include" field available for this endpoint:
-        # https://developers.freshdesk.com/api/#filter_tickets
+        """ 
+        'end' is not currently used, so we may double fetch tickets created after the indexing
+        starts but before the actual call is made.
+        
+        To use 'end' would require us to use the search endpoint but it has limitations,
+        namely having to fetch all IDs and then individually fetch each ticket because there is no
+        'include' field available for this endpoint:
+        https://developers.freshdesk.com/api/#filter_tickets
+        """
         if any(attr is None for attr in [self.api_key, self.domain, self.password]):
             raise ConnectorMissingCredentialError("freshdesk")
 

From 4499c630b3060be07cde23b5249c1dd74e422d22 Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Thu, 31 Oct 2024 10:11:42 -0700
Subject: [PATCH 312/376] Fix model test action name

---
 .github/workflows/pr-python-model-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pr-python-model-tests.yml b/.github/workflows/pr-python-model-tests.yml
index 11d1d70d99d..a070eea27a7 100644
--- a/.github/workflows/pr-python-model-tests.yml
+++ b/.github/workflows/pr-python-model-tests.yml
@@ -15,7 +15,7 @@ env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
 jobs:
-  connectors-check:
+  model-check:
     # See https://runs-on.com/runners/linux/
     runs-on: [runs-on,runner=8cpu-linux-x64,"run-id=${{ github.run_id }}"]
 

From 787fdf2e38c50e46fe278b78a37f4f2af028f019 Mon Sep 17 00:00:00 2001
From: Samarth Mishra <139863510+samarth29jc@users.noreply.github.com>
Date: Thu, 31 Oct 2024 23:14:36 +0530
Subject: [PATCH 313/376] Update README.md (#3011)

---
 README.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/README.md b/README.md
index 1ff4338c530..0cc95ff4464 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 <!-- DANSWER_METADATA={"link": "https://github.com/danswer-ai/danswer/blob/main/README.md"} -->
+<a name="readme-top"></a>
 
 <h2 align="center">
 <a href="https://www.danswer.ai/"> <img width="50%" src="https://github.com/danswer-owners/danswer/blob/1fabd9372d66cd54238847197c33f091a724803b/DanswerWithName.png?raw=true)" /></a>
@@ -127,3 +128,19 @@ To try the Danswer Enterprise Edition:
 
 ## 💡 Contributing
 Looking to contribute? Please check out the [Contribution Guide](CONTRIBUTING.md) for more details.
+
+## ⭐Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=danswer-ai/danswer&type=Date)](https://star-history.com/#danswer-ai/danswer&Date)
+
+## ✨Contributors
+
+<a href="https://github.com/aryn-ai/sycamore/graphs/contributors">
+  <img alt="contributors" src="https://contrib.rocks/image?repo=danswer-ai/danswer"/>
+</a>
+
+<p align="right" style="font-size: 14px; color: #555; margin-top: 20px;">
+    <a href="#readme-top" style="text-decoration: none; color: #007bff; font-weight: bold;">
+        ↑ Back to Top ↑
+    </a>
+</p>

From add87fa1b44e6557fcae1c8f952b8886e4af9e23 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 31 Oct 2024 12:43:15 -0700
Subject: [PATCH 314/376] remove endpoint (#3014)

---
 backend/danswer/server/documents/credential.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/backend/danswer/server/documents/credential.py b/backend/danswer/server/documents/credential.py
index 2c6e41bf968..42c72b8f34f 100644
--- a/backend/danswer/server/documents/credential.py
+++ b/backend/danswer/server/documents/credential.py
@@ -81,18 +81,6 @@ def get_cc_source_full_info(
     ]
 
 
-@router.get("/credential/{id}")
-def list_credentials_by_id(
-    user: User | None = Depends(current_user),
-    db_session: Session = Depends(get_session),
-) -> list[CredentialSnapshot]:
-    credentials = fetch_credentials(db_session=db_session, user=user)
-    return [
-        CredentialSnapshot.from_credential_db_model(credential)
-        for credential in credentials
-    ]
-
-
 @router.delete("/admin/credential/{credential_id}")
 def delete_credential_by_id_admin(
     credential_id: int,

From 0b08bf4e3f0a8133ca8fc21d734d4693a6fb58ae Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Thu, 31 Oct 2024 12:45:35 -0700
Subject: [PATCH 315/376] Proper tenant reset (#3015)

* add proper tenant reset

* clear comment

* minor formatting
---
 backend/danswer/db/engine.py           | 58 ++++++++++++++++----------
 backend/danswer/server/manage/users.py |  4 +-
 2 files changed, 38 insertions(+), 24 deletions(-)

diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 7336493e0a0..a1fbbddc65f 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -322,11 +322,18 @@ async def get_async_session_with_tenant(
 def get_session_with_tenant(
     tenant_id: str | None = None,
 ) -> Generator[Session, None, None]:
-    """Generate a database session bound to a connection with the appropriate tenant schema set."""
+    """
+    Generate a database session bound to a connection with the appropriate tenant schema set.
+    This preserves the tenant ID across the session and reverts to the previous tenant ID
+    after the session is closed.
+    """
     engine = get_sqlalchemy_engine()
 
+    # Store the previous tenant ID
+    previous_tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+
     if tenant_id is None:
-        tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+        tenant_id = previous_tenant_id
     else:
         CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
 
@@ -335,30 +342,35 @@ def get_session_with_tenant(
     if not is_valid_schema_name(tenant_id):
         raise HTTPException(status_code=400, detail="Invalid tenant ID")
 
-    # Establish a raw connection
-    with engine.connect() as connection:
-        # Access the raw DBAPI connection and set the search_path
-        dbapi_connection = connection.connection
-
-        # Set the search_path outside of any transaction
-        cursor = dbapi_connection.cursor()
-        try:
-            cursor.execute(f'SET search_path = "{tenant_id}"')
-        finally:
-            cursor.close()
+    try:
+        # Establish a raw connection
+        with engine.connect() as connection:
+            # Access the raw DBAPI connection and set the search_path
+            dbapi_connection = connection.connection
 
-        # Bind the session to the connection
-        with Session(bind=connection, expire_on_commit=False) as session:
+            # Set the search_path outside of any transaction
+            cursor = dbapi_connection.cursor()
             try:
-                yield session
+                cursor.execute(f'SET search_path = "{tenant_id}"')
             finally:
-                # Reset search_path to default after the session is used
-                if MULTI_TENANT:
-                    cursor = dbapi_connection.cursor()
-                    try:
-                        cursor.execute('SET search_path TO "$user", public')
-                    finally:
-                        cursor.close()
+                cursor.close()
+
+            # Bind the session to the connection
+            with Session(bind=connection, expire_on_commit=False) as session:
+                try:
+                    yield session
+                finally:
+                    # Reset search_path to default after the session is used
+                    if MULTI_TENANT:
+                        cursor = dbapi_connection.cursor()
+                        try:
+                            cursor.execute('SET search_path TO "$user", public')
+                        finally:
+                            cursor.close()
+
+    finally:
+        # Restore the previous tenant ID
+        CURRENT_TENANT_ID_CONTEXTVAR.set(previous_tenant_id)
 
 
 def set_search_path_on_checkout(
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index bbaf08b7db9..9cb1cec1c1a 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -190,7 +190,6 @@ def bulk_invite_users(
         )
 
     tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
-
     normalized_emails = []
     try:
         for email in emails:
@@ -206,6 +205,7 @@ def bulk_invite_users(
     if MULTI_TENANT:
         try:
             add_users_to_tenant(normalized_emails, tenant_id)
+
         except IntegrityError as e:
             if isinstance(e.orig, UniqueViolation):
                 raise HTTPException(
@@ -213,6 +213,8 @@ def bulk_invite_users(
                     detail="User has already been invited to a Danswer organization",
                 )
             raise
+        except Exception as e:
+            logger.error(f"Failed to add users to tenant {tenant_id}: {str(e)}")
 
     initial_invited_users = get_invited_users()
 

From b34f5862d71b716227eb20e88284e66141954c6b Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Thu, 31 Oct 2024 17:31:19 -0700
Subject: [PATCH 316/376] Remove License Issues (#3013)

* k

* k

* k

* k

* k
---
 .../danswer/connectors/zendesk/connector.py   | 407 +++++++++++-------
 backend/requirements/default.txt              |   1 -
 .../tests/pruning/website/css/flexslider.css  | 226 ----------
 3 files changed, 262 insertions(+), 372 deletions(-)
 delete mode 100644 backend/tests/integration/tests/pruning/website/css/flexslider.css

diff --git a/backend/danswer/connectors/zendesk/connector.py b/backend/danswer/connectors/zendesk/connector.py
index f85f2efff57..195fe45af0f 100644
--- a/backend/danswer/connectors/zendesk/connector.py
+++ b/backend/danswer/connectors/zendesk/connector.py
@@ -1,10 +1,7 @@
+from collections.abc import Iterator
 from typing import Any
 
 import requests
-from retry import retry
-from zenpy import Zenpy  # type: ignore
-from zenpy.lib.api_objects import Ticket  # type: ignore
-from zenpy.lib.api_objects.help_centre_objects import Article  # type: ignore
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.app_configs import ZENDESK_CONNECTOR_SKIP_ARTICLE_LABELS
@@ -20,43 +17,244 @@
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
 from danswer.file_processing.html_utils import parse_html_page_basic
+from danswer.utils.retry_wrapper import retry_builder
 
 
-def _article_to_document(article: Article, content_tags: dict[str, str]) -> Document:
-    author = BasicExpertInfo(
-        display_name=article.author.name, email=article.author.email
+MAX_PAGE_SIZE = 30  # Zendesk API maximum
+
+
+class ZendeskCredentialsNotSetUpError(PermissionError):
+    def __init__(self) -> None:
+        super().__init__(
+            "Zendesk Credentials are not set up, was load_credentials called?"
+        )
+
+
+class ZendeskClient:
+    def __init__(self, subdomain: str, email: str, token: str):
+        self.base_url = f"https://{subdomain}.zendesk.com/api/v2"
+        self.auth = (f"{email}/token", token)
+
+    @retry_builder()
+    def make_request(self, endpoint: str, params: dict[str, Any]) -> dict[str, Any]:
+        response = requests.get(
+            f"{self.base_url}/{endpoint}", auth=self.auth, params=params
+        )
+        response.raise_for_status()
+        return response.json()
+
+
+def _get_content_tag_mapping(client: ZendeskClient) -> dict[str, str]:
+    content_tags: dict[str, str] = {}
+    params = {"page[size]": MAX_PAGE_SIZE}
+
+    try:
+        while True:
+            data = client.make_request("guide/content_tags", params)
+
+            for tag in data.get("records", []):
+                content_tags[tag["id"]] = tag["name"]
+
+            # Check if there are more pages
+            if data.get("meta", {}).get("has_more", False):
+                params["page[after]"] = data["meta"]["after_cursor"]
+            else:
+                break
+
+        return content_tags
+    except Exception as e:
+        raise Exception(f"Error fetching content tags: {str(e)}")
+
+
+def _get_articles(
+    client: ZendeskClient, start_time: int | None = None, page_size: int = MAX_PAGE_SIZE
+) -> Iterator[dict[str, Any]]:
+    params = (
+        {"start_time": start_time, "page[size]": page_size}
+        if start_time
+        else {"page[size]": page_size}
+    )
+
+    while True:
+        data = client.make_request("help_center/articles", params)
+        for article in data["articles"]:
+            yield article
+
+        if not data.get("meta", {}).get("has_more"):
+            break
+        params["page[after]"] = data["meta"]["after_cursor"]
+
+
+def _get_tickets(
+    client: ZendeskClient, start_time: int | None = None
+) -> Iterator[dict[str, Any]]:
+    params = {"start_time": start_time} if start_time else {"start_time": 0}
+
+    while True:
+        data = client.make_request("incremental/tickets.json", params)
+        for ticket in data["tickets"]:
+            yield ticket
+
+        if not data.get("end_of_stream", False):
+            params["start_time"] = data["end_time"]
+        else:
+            break
+
+
+def _fetch_author(client: ZendeskClient, author_id: str) -> BasicExpertInfo | None:
+    author_data = client.make_request(f"users/{author_id}", {})
+    user = author_data.get("user")
+    return (
+        BasicExpertInfo(display_name=user.get("name"), email=user.get("email"))
+        if user and user.get("name") and user.get("email")
+        else None
     )
-    update_time = time_str_to_utc(article.updated_at)
 
-    # build metadata
+
+def _article_to_document(
+    article: dict[str, Any],
+    content_tags: dict[str, str],
+    author_map: dict[str, BasicExpertInfo],
+    client: ZendeskClient,
+) -> tuple[dict[str, BasicExpertInfo] | None, Document]:
+    author_id = article.get("author_id")
+    if not author_id:
+        author = None
+    else:
+        author = (
+            author_map.get(author_id)
+            if author_id in author_map
+            else _fetch_author(client, author_id)
+        )
+
+    new_author_mapping = {author_id: author} if author_id and author else None
+
+    updated_at = article.get("updated_at")
+    update_time = time_str_to_utc(updated_at) if updated_at else None
+
+    # Build metadata
     metadata: dict[str, str | list[str]] = {
-        "labels": [str(label) for label in article.label_names if label],
+        "labels": [str(label) for label in article.get("label_names", []) if label],
         "content_tags": [
             content_tags[tag_id]
-            for tag_id in article.content_tag_ids
+            for tag_id in article.get("content_tag_ids", [])
             if tag_id in content_tags
         ],
     }
 
-    # remove empty values
+    # Remove empty values
     metadata = {k: v for k, v in metadata.items() if v}
 
-    return Document(
-        id=f"article:{article.id}",
+    return new_author_mapping, Document(
+        id=f"article:{article['id']}",
         sections=[
-            Section(link=article.html_url, text=parse_html_page_basic(article.body))
+            Section(
+                link=article.get("html_url"),
+                text=parse_html_page_basic(article["body"]),
+            )
         ],
         source=DocumentSource.ZENDESK,
-        semantic_identifier=article.title,
+        semantic_identifier=article["title"],
         doc_updated_at=update_time,
-        primary_owners=[author],
+        primary_owners=[author] if author else None,
         metadata=metadata,
     )
 
 
-class ZendeskClientNotSetUpError(PermissionError):
-    def __init__(self) -> None:
-        super().__init__("Zendesk Client is not set up, was load_credentials called?")
+def _get_comment_text(
+    comment: dict[str, Any],
+    author_map: dict[str, BasicExpertInfo],
+    client: ZendeskClient,
+) -> tuple[dict[str, BasicExpertInfo] | None, str]:
+    author_id = comment.get("author_id")
+    if not author_id:
+        author = None
+    else:
+        author = (
+            author_map.get(author_id)
+            if author_id in author_map
+            else _fetch_author(client, author_id)
+        )
+
+    new_author_mapping = {author_id: author} if author_id and author else None
+
+    comment_text = f"Comment{' by ' + author.display_name if author and author.display_name else ''}"
+    comment_text += f"{' at ' + comment['created_at'] if comment.get('created_at') else ''}:\n{comment['body']}"
+
+    return new_author_mapping, comment_text
+
+
+def _ticket_to_document(
+    ticket: dict[str, Any],
+    author_map: dict[str, BasicExpertInfo],
+    client: ZendeskClient,
+    default_subdomain: str,
+) -> tuple[dict[str, BasicExpertInfo] | None, Document]:
+    submitter_id = ticket.get("submitter")
+    if not submitter_id:
+        submitter = None
+    else:
+        submitter = (
+            author_map.get(submitter_id)
+            if submitter_id in author_map
+            else _fetch_author(client, submitter_id)
+        )
+
+    new_author_mapping = (
+        {submitter_id: submitter} if submitter_id and submitter else None
+    )
+
+    updated_at = ticket.get("updated_at")
+    update_time = time_str_to_utc(updated_at) if updated_at else None
+
+    metadata: dict[str, str | list[str]] = {}
+    if status := ticket.get("status"):
+        metadata["status"] = status
+    if priority := ticket.get("priority"):
+        metadata["priority"] = priority
+    if tags := ticket.get("tags"):
+        metadata["tags"] = tags
+    if ticket_type := ticket.get("type"):
+        metadata["ticket_type"] = ticket_type
+
+    # Fetch comments for the ticket
+    comments_data = client.make_request(f"tickets/{ticket.get('id')}/comments", {})
+    comments = comments_data.get("comments", [])
+
+    comment_texts = []
+    for comment in comments:
+        new_author_mapping, comment_text = _get_comment_text(
+            comment, author_map, client
+        )
+        if new_author_mapping:
+            author_map.update(new_author_mapping)
+        comment_texts.append(comment_text)
+
+    comments_text = "\n\n".join(comment_texts)
+
+    subject = ticket.get("subject")
+    full_text = f"Ticket Subject:\n{subject}\n\nComments:\n{comments_text}"
+
+    ticket_url = ticket.get("url")
+    subdomain = (
+        ticket_url.split("//")[1].split(".zendesk.com")[0]
+        if ticket_url
+        else default_subdomain
+    )
+
+    ticket_display_url = (
+        f"https://{subdomain}.zendesk.com/agent/tickets/{ticket.get('id')}"
+    )
+
+    return new_author_mapping, Document(
+        id=f"zendesk_ticket_{ticket['id']}",
+        sections=[Section(link=ticket_display_url, text=full_text)],
+        source=DocumentSource.ZENDESK,
+        semantic_identifier=f"Ticket #{ticket['id']}: {subject or 'No Subject'}",
+        doc_updated_at=update_time,
+        primary_owners=[submitter] if submitter else None,
+        metadata=metadata,
+    )
 
 
 class ZendeskConnector(LoadConnector, PollConnector):
@@ -66,44 +264,10 @@ def __init__(
         content_type: str = "articles",
     ) -> None:
         self.batch_size = batch_size
-        self.zendesk_client: Zenpy | None = None
-        self.content_tags: dict[str, str] = {}
         self.content_type = content_type
-
-    @retry(tries=3, delay=2, backoff=2)
-    def _set_content_tags(
-        self, subdomain: str, email: str, token: str, page_size: int = 30
-    ) -> None:
-        # Construct the base URL
-        base_url = f"https://{subdomain}.zendesk.com/api/v2/guide/content_tags"
-
-        # Set up authentication
-        auth = (f"{email}/token", token)
-
-        # Set up pagination parameters
-        params = {"page[size]": page_size}
-
-        try:
-            while True:
-                # Make the GET request
-                response = requests.get(base_url, auth=auth, params=params)
-
-                # Check if the request was successful
-                if response.status_code == 200:
-                    data = response.json()
-                    content_tag_list = data.get("records", [])
-                    for tag in content_tag_list:
-                        self.content_tags[tag["id"]] = tag["name"]
-
-                    # Check if there are more pages
-                    if data.get("meta", {}).get("has_more", False):
-                        params["page[after]"] = data["meta"]["after_cursor"]
-                    else:
-                        break
-                else:
-                    raise Exception(f"Error: {response.status_code}\n{response.text}")
-        except Exception as e:
-            raise Exception(f"Error fetching content tags: {str(e)}")
+        self.subdomain = ""
+        # Fetch all tags ahead of time
+        self.content_tags: dict[str, str] = {}
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:
         # Subdomain is actually the whole URL
@@ -112,87 +276,23 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
             .replace("https://", "")
             .split(".zendesk.com")[0]
         )
+        self.subdomain = subdomain
 
-        self.zendesk_client = Zenpy(
-            subdomain=subdomain,
-            email=credentials["zendesk_email"],
-            token=credentials["zendesk_token"],
-        )
-        self._set_content_tags(
-            subdomain,
-            credentials["zendesk_email"],
-            credentials["zendesk_token"],
+        self.client = ZendeskClient(
+            subdomain, credentials["zendesk_email"], credentials["zendesk_token"]
         )
         return None
 
     def load_from_state(self) -> GenerateDocumentsOutput:
         return self.poll_source(None, None)
 
-    def _ticket_to_document(self, ticket: Ticket) -> Document:
-        if self.zendesk_client is None:
-            raise ZendeskClientNotSetUpError()
-
-        owner = None
-        if ticket.requester and ticket.requester.name and ticket.requester.email:
-            owner = [
-                BasicExpertInfo(
-                    display_name=ticket.requester.name, email=ticket.requester.email
-                )
-            ]
-        update_time = time_str_to_utc(ticket.updated_at) if ticket.updated_at else None
-
-        metadata: dict[str, str | list[str]] = {}
-        if ticket.status is not None:
-            metadata["status"] = ticket.status
-        if ticket.priority is not None:
-            metadata["priority"] = ticket.priority
-        if ticket.tags:
-            metadata["tags"] = ticket.tags
-        if ticket.type is not None:
-            metadata["ticket_type"] = ticket.type
-
-        # Fetch comments for the ticket
-        comments = self.zendesk_client.tickets.comments(ticket=ticket)
-
-        # Combine all comments into a single text
-        comments_text = "\n\n".join(
-            [
-                f"Comment{f' by {comment.author.name}' if comment.author and comment.author.name else ''}"
-                f"{f' at {comment.created_at}' if comment.created_at else ''}:\n{comment.body}"
-                for comment in comments
-                if comment.body
-            ]
-        )
-
-        # Combine ticket description and comments
-        description = (
-            ticket.description
-            if hasattr(ticket, "description") and ticket.description
-            else ""
-        )
-        full_text = f"Ticket Description:\n{description}\n\nComments:\n{comments_text}"
-
-        # Extract subdomain from ticket.url
-        subdomain = ticket.url.split("//")[1].split(".zendesk.com")[0]
-
-        # Build the html url for the ticket
-        ticket_url = f"https://{subdomain}.zendesk.com/agent/tickets/{ticket.id}"
-
-        return Document(
-            id=f"zendesk_ticket_{ticket.id}",
-            sections=[Section(link=ticket_url, text=full_text)],
-            source=DocumentSource.ZENDESK,
-            semantic_identifier=f"Ticket #{ticket.id}: {ticket.subject or 'No Subject'}",
-            doc_updated_at=update_time,
-            primary_owners=owner,
-            metadata=metadata,
-        )
-
     def poll_source(
         self, start: SecondsSinceUnixEpoch | None, end: SecondsSinceUnixEpoch | None
     ) -> GenerateDocumentsOutput:
-        if self.zendesk_client is None:
-            raise ZendeskClientNotSetUpError()
+        if self.client is None:
+            raise ZendeskCredentialsNotSetUpError()
+
+        self.content_tags = _get_content_tag_mapping(self.client)
 
         if self.content_type == "articles":
             yield from self._poll_articles(start)
@@ -204,26 +304,30 @@ def poll_source(
     def _poll_articles(
         self, start: SecondsSinceUnixEpoch | None
     ) -> GenerateDocumentsOutput:
-        articles = (
-            self.zendesk_client.help_center.articles(cursor_pagination=True)  # type: ignore
-            if start is None
-            else self.zendesk_client.help_center.articles.incremental(  # type: ignore
-                start_time=int(start)
-            )
-        )
+        articles = _get_articles(self.client, start_time=int(start) if start else None)
+
+        # This one is built on the fly as there may be more many more authors than tags
+        author_map: dict[str, BasicExpertInfo] = {}
+
         doc_batch = []
         for article in articles:
             if (
-                article.body is None
-                or article.draft
+                article.get("body") is None
+                or article.get("draft")
                 or any(
                     label in ZENDESK_CONNECTOR_SKIP_ARTICLE_LABELS
-                    for label in article.label_names
+                    for label in article.get("label_names", [])
                 )
             ):
                 continue
 
-            doc_batch.append(_article_to_document(article, self.content_tags))
+            new_author_map, documents = _article_to_document(
+                article, self.content_tags, author_map, self.client
+            )
+            if new_author_map:
+                author_map.update(new_author_map)
+
+            doc_batch.append(documents)
             if len(doc_batch) >= self.batch_size:
                 yield doc_batch
                 doc_batch.clear()
@@ -234,10 +338,14 @@ def _poll_articles(
     def _poll_tickets(
         self, start: SecondsSinceUnixEpoch | None
     ) -> GenerateDocumentsOutput:
-        if self.zendesk_client is None:
-            raise ZendeskClientNotSetUpError()
+        if self.client is None:
+            raise ZendeskCredentialsNotSetUpError()
 
-        ticket_generator = self.zendesk_client.tickets.incremental(start_time=start)
+        author_map: dict[str, BasicExpertInfo] = {}
+
+        ticket_generator = _get_tickets(
+            self.client, start_time=int(start) if start else None
+        )
 
         while True:
             doc_batch = []
@@ -246,10 +354,20 @@ def _poll_tickets(
                     ticket = next(ticket_generator)
 
                     # Check if the ticket status is deleted and skip it if so
-                    if ticket.status == "deleted":
+                    if ticket.get("status") == "deleted":
                         continue
 
-                    doc_batch.append(self._ticket_to_document(ticket))
+                    new_author_map, documents = _ticket_to_document(
+                        ticket=ticket,
+                        author_map=author_map,
+                        client=self.client,
+                        default_subdomain=self.subdomain,
+                    )
+
+                    if new_author_map:
+                        author_map.update(new_author_map)
+
+                    doc_batch.append(documents)
 
                     if len(doc_batch) >= self.batch_size:
                         yield doc_batch
@@ -267,7 +385,6 @@ def _poll_tickets(
 
 if __name__ == "__main__":
     import os
-
     import time
 
     connector = ZendeskConnector()
diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt
index fe0cdd01547..c06ab2ca7cf 100644
--- a/backend/requirements/default.txt
+++ b/backend/requirements/default.txt
@@ -75,7 +75,6 @@ uvicorn==0.21.1
 zulip==0.8.2
 hubspot-api-client==8.1.0
 asana==5.0.8
-zenpy==2.0.41
 dropbox==11.36.2
 boto3-stubs[s3]==1.34.133
 stripe==10.12.0
diff --git a/backend/tests/integration/tests/pruning/website/css/flexslider.css b/backend/tests/integration/tests/pruning/website/css/flexslider.css
deleted file mode 100644
index 6088235631c..00000000000
--- a/backend/tests/integration/tests/pruning/website/css/flexslider.css
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * jQuery FlexSlider v2.0
- * http://www.woothemes.com/flexslider/
- *
- * Copyright 2012 WooThemes
- * Free to use under the GPLv2 license.
- * http://www.gnu.org/licenses/gpl-2.0.html
- *
- * Contributing author: Tyler Smith (@mbmufffin)
- */
-
-/* Browser Resets */
-.flex-container a:active,
-.flexslider a:active,
-.flex-container a:focus,
-.flexslider a:focus {
-  outline: none;
-}
-.slides,
-.flex-control-nav,
-.flex-direction-nav {
-  margin: 0;
-  padding: 0;
-  list-style: none;
-}
-
-/* FlexSlider Necessary Styles
-*********************************/
-.flexslider {
-  margin: 0;
-  padding: 0;
-}
-.flexslider .slides > li {
-  display: none;
-  -webkit-backface-visibility: hidden;
-} /* Hide the slides before the JS is loaded. Avoids image jumping */
-.flexslider .slides img {
-  width: 100%;
-  display: block;
-}
-.flex-pauseplay span {
-  text-transform: capitalize;
-}
-
-/* Clearfix for the .slides element */
-.slides:after {
-  content: ".";
-  display: block;
-  clear: both;
-  visibility: hidden;
-  line-height: 0;
-  height: 0;
-}
-html[xmlns] .slides {
-  display: block;
-}
-* html .slides {
-  height: 1%;
-}
-
-/* No JavaScript Fallback */
-/* If you are not using another script, such as Modernizr, make sure you
- * include js that eliminates this class on page load */
-.no-js .slides > li:first-child {
-  display: block;
-}
-
-/* FlexSlider Default Theme
-*********************************/
-.flexslider {
-  background: none;
-  position: relative;
-  zoom: 1;
-}
-.flex-viewport {
-  max-height: 2000px;
-  -webkit-transition: all 1s ease;
-  -moz-transition: all 1s ease;
-  transition: all 1s ease;
-}
-.loading .flex-viewport {
-  max-height: 300px;
-}
-.flexslider .slides {
-  zoom: 1;
-}
-
-.carousel li {
-  margin-right: 5px;
-}
-
-/* Caption style */
-
-.flex-caption {
-  background: rgba(0, 0, 0, 0.8);
-  margin-left: 5px;
-  bottom: 5px;
-  position: absolute;
-  padding: 20px;
-  z-index: 99;
-}
-.flex-caption p {
-  font-size: 14px !important;
-  line-height: 22px;
-  font-weight: 300;
-  color: #fff;
-}
-.flex-caption h2,
-.flex-caption h4 {
-  color: #fff;
-}
-
-/* Direction Nav */
-.flex-direction-nav {
-  *height: 0;
-}
-.flex-direction-nav a {
-  width: 30px;
-  height: 40px;
-  margin: 0;
-  display: block;
-  background: url(../img/bg_direction_nav.png) no-repeat 0 0;
-  position: absolute;
-  top: 45%;
-  z-index: 10;
-  cursor: pointer;
-  text-indent: -9999px;
-  opacity: 0;
-  -webkit-transition: all 0.3s ease;
-}
-.flex-direction-nav .flex-next {
-  background-position: 100% 0;
-  right: -36px;
-}
-.flex-direction-nav .flex-prev {
-  left: -36px;
-}
-.flexslider:hover .flex-next {
-  opacity: 0.8;
-  right: 5px;
-}
-.flexslider:hover .flex-prev {
-  opacity: 0.8;
-  left: 5px;
-}
-.flexslider:hover .flex-next:hover,
-.flexslider:hover .flex-prev:hover {
-  opacity: 1;
-}
-.flex-direction-nav .flex-disabled {
-  opacity: 0.3 !important;
-  filter: alpha(opacity=30);
-  cursor: default;
-}
-
-/* Control Nav */
-.flex-control-nav {
-  width: 100%;
-  position: absolute;
-  bottom: 0;
-  text-align: center;
-}
-.flex-control-nav li {
-  margin: 0 6px;
-  display: inline-block;
-  zoom: 1;
-  *display: inline;
-}
-.flex-control-paging li a {
-  width: 11px;
-  height: 11px;
-  display: block;
-  background: #666;
-  background: rgba(0, 0, 0, 0.5);
-  cursor: pointer;
-  text-indent: -9999px;
-  -webkit-border-radius: 20px;
-  -moz-border-radius: 20px;
-  -o-border-radius: 20px;
-  border-radius: 20px;
-  box-shadow: inset 0 0 3px rgba(0, 0, 0, 0.3);
-}
-.flex-control-paging li a:hover {
-  background: #333;
-  background: rgba(0, 0, 0, 0.7);
-}
-.flex-control-paging li a.flex-active {
-  background: #000;
-  background: rgba(0, 0, 0, 0.9);
-  cursor: default;
-}
-
-.flex-control-thumbs {
-  margin: 5px 0 0;
-  position: static;
-  overflow: hidden;
-}
-.flex-control-thumbs li {
-  width: 25%;
-  float: left;
-  margin: 0;
-}
-.flex-control-thumbs img {
-  width: 100%;
-  display: block;
-  opacity: 0.7;
-  cursor: pointer;
-}
-.flex-control-thumbs img:hover {
-  opacity: 1;
-}
-.flex-control-thumbs .flex-active {
-  opacity: 1;
-  cursor: default;
-}
-
-@media screen and (max-width: 860px) {
-  .flex-direction-nav .flex-prev {
-    opacity: 1;
-    left: 0;
-  }
-  .flex-direction-nav .flex-next {
-    opacity: 1;
-    right: 0;
-  }
-}

From 71d4fb98d3031173a17e19531aba34501cce734e Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Thu, 31 Oct 2024 19:25:00 -0700
Subject: [PATCH 317/376] Refactored Google Drive Connector + Permission
 Syncing (#2945)

* refactoring changes

* everything working for service account

* works with service account

* combined scopes

* copy change

* oauth prep

* Works for oauth and service account credentials

* mypy

* merge fixes

* Refactor Google Drive connector

* finished backend

* auth changes

* if its stupid but it works, its not stupid

* npm run dev fixes

* addressed change requests

* string fix

* minor fixes and cleanup

* spacing cleanup

* Update connector.py

* everything done

* testing!

* Delete backend/tests/daily/connectors/google_drive/file_generator.py

* cleaned up

---------

Co-authored-by: Chris Weaver <25087905+Weves@users.noreply.github.com>
---
 .../workflows/pr-python-connector-tests.yml   |   3 +
 backend/danswer/configs/app_configs.py        |   3 -
 .../connectors/confluence/connector.py        |   7 +-
 .../connectors/google_drive/connector.py      | 775 ++++++------------
 .../connectors/google_drive/connector_auth.py | 125 +--
 .../connectors/google_drive/constants.py      |  41 +-
 .../connectors/google_drive/doc_conversion.py | 115 +++
 .../connectors/google_drive/file_retrieval.py | 192 +++++
 .../connectors/google_drive/google_utils.py   |  35 +
 .../danswer/connectors/google_drive/models.py |  18 +
 backend/danswer/connectors/interfaces.py      |   6 +-
 .../connectors/salesforce/connector.py        |   6 +-
 backend/danswer/connectors/slack/connector.py |   6 +-
 backend/danswer/db/credentials.py             |   6 +-
 backend/danswer/server/documents/connector.py |  34 +-
 backend/danswer/server/documents/models.py    |   6 +-
 .../ee/danswer/background/celery/apps/beat.py |   4 +-
 .../google_drive/doc_sync.py                  | 225 +++--
 .../google_drive/group_sync.py                | 134 +--
 .../external_permissions/permission_sync.py   |   2 +
 .../daily/connectors/google_drive/conftest.py |  98 +++
 .../daily/connectors/google_drive/helpers.py  | 164 ++++
 .../google_drive/test_google_drive_oauth.py   | 246 ++++++
 .../test_google_drive_service_acct.py         | 257 ++++++
 .../test_google_drive_slim_docs.py            | 174 ++++
 .../[connector]/AddConnectorPage.tsx          |   6 +
 .../pages/ConnectorInput/ListInput.tsx        |  16 +-
 .../pages/DynamicConnectorCreationForm.tsx    | 116 ++-
 .../[connector]/pages/gdrive/Credential.tsx   |  34 +-
 .../pages/gdrive/GoogleDrivePage.tsx          |   7 +-
 .../admin/connectors/AccessTypeForm.tsx       |  37 +-
 .../admin/connectors/ConnectorTitle.tsx       |  15 -
 .../lib/connectors/AutoSyncOptionFields.tsx   |  33 +-
 web/src/lib/connectors/connectors.tsx         |  78 +-
 web/src/lib/connectors/credentials.ts         |   5 +-
 35 files changed, 2045 insertions(+), 984 deletions(-)
 create mode 100644 backend/danswer/connectors/google_drive/doc_conversion.py
 create mode 100644 backend/danswer/connectors/google_drive/file_retrieval.py
 create mode 100644 backend/danswer/connectors/google_drive/google_utils.py
 create mode 100644 backend/danswer/connectors/google_drive/models.py
 create mode 100644 backend/tests/daily/connectors/google_drive/conftest.py
 create mode 100644 backend/tests/daily/connectors/google_drive/helpers.py
 create mode 100644 backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
 create mode 100644 backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
 create mode 100644 backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py

diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml
index 108012100b3..fa7df201b5e 100644
--- a/.github/workflows/pr-python-connector-tests.yml
+++ b/.github/workflows/pr-python-connector-tests.yml
@@ -18,6 +18,9 @@ env:
   # Jira
   JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
   JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
+  # Google
+  GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR }}
+  GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
 
 jobs:
   connectors-check:
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index 6d6cc6b9639..fb6b4996363 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -251,9 +251,6 @@
 # for some connectors
 ENABLE_EXPENSIVE_EXPERT_CALLS = False
 
-GOOGLE_DRIVE_INCLUDE_SHARED = False
-GOOGLE_DRIVE_FOLLOW_SHORTCUTS = False
-GOOGLE_DRIVE_ONLY_ORG_PUBLIC = False
 
 # TODO these should be available for frontend configuration, via advanced options expandable
 WEB_CONNECTOR_IGNORED_CLASSES = os.environ.get(
diff --git a/backend/danswer/connectors/confluence/connector.py b/backend/danswer/connectors/confluence/connector.py
index f0945547e57..9c93f93f99b 100644
--- a/backend/danswer/connectors/confluence/connector.py
+++ b/backend/danswer/connectors/confluence/connector.py
@@ -17,6 +17,7 @@
 from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
 from danswer.connectors.interfaces import SlimConnector
 from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import ConnectorMissingCredentialError
@@ -249,7 +250,11 @@ def poll_source(self, start: float, end: float) -> GenerateDocumentsOutput:
         self.cql_time_filter += f" and lastmodified <= '{formatted_end_time}'"
         return self._fetch_document_batches()
 
-    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
+    def retrieve_all_slim_documents(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
         if self.confluence_client is None:
             raise ConnectorMissingCredentialError("Confluence")
 
diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index c48df5bb741..4ddd51f749f 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -1,556 +1,305 @@
-import io
 from collections.abc import Iterator
-from collections.abc import Sequence
-from datetime import datetime
-from datetime import timezone
-from enum import Enum
-from itertools import chain
 from typing import Any
 
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
-from googleapiclient import discovery  # type: ignore
-from googleapiclient.errors import HttpError  # type: ignore
+from googleapiclient.discovery import build  # type: ignore
+from googleapiclient.discovery import Resource  # type: ignore
 
-from danswer.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
-from danswer.configs.app_configs import GOOGLE_DRIVE_FOLLOW_SHORTCUTS
-from danswer.configs.app_configs import GOOGLE_DRIVE_INCLUDE_SHARED
-from danswer.configs.app_configs import GOOGLE_DRIVE_ONLY_ORG_PUBLIC
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
-from danswer.configs.constants import DocumentSource
-from danswer.configs.constants import IGNORE_FOR_QA
-from danswer.connectors.google_drive.connector_auth import get_google_drive_creds
-from danswer.connectors.google_drive.constants import (
-    DB_CREDENTIALS_DICT_DELEGATED_USER_KEY,
+from danswer.connectors.google_drive.connector_auth import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
 )
-from danswer.connectors.google_drive.constants import (
-    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+from danswer.connectors.google_drive.connector_auth import get_google_drive_creds
+from danswer.connectors.google_drive.constants import MISSING_SCOPES_ERROR_STR
+from danswer.connectors.google_drive.constants import ONYX_SCOPE_INSTRUCTIONS
+from danswer.connectors.google_drive.constants import SCOPE_DOC_URL
+from danswer.connectors.google_drive.constants import SLIM_BATCH_SIZE
+from danswer.connectors.google_drive.constants import USER_FIELDS
+from danswer.connectors.google_drive.doc_conversion import (
+    convert_drive_item_to_document,
 )
+from danswer.connectors.google_drive.file_retrieval import crawl_folders_for_files
+from danswer.connectors.google_drive.file_retrieval import get_files_in_my_drive
+from danswer.connectors.google_drive.file_retrieval import get_files_in_shared_drive
+from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_drive.models import GoogleDriveFileType
 from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
-from danswer.connectors.models import Document
-from danswer.connectors.models import Section
-from danswer.file_processing.extract_file_text import docx_to_text
-from danswer.file_processing.extract_file_text import pptx_to_text
-from danswer.file_processing.extract_file_text import read_pdf_file
-from danswer.file_processing.unstructured import get_unstructured_api_key
-from danswer.file_processing.unstructured import unstructured_to_text
-from danswer.utils.batching import batch_generator
+from danswer.connectors.interfaces import SlimConnector
+from danswer.connectors.models import SlimDocument
 from danswer.utils.logger import setup_logger
-from danswer.utils.retry_wrapper import retry_builder
 
 logger = setup_logger()
 
-DRIVE_FOLDER_TYPE = "application/vnd.google-apps.folder"
-DRIVE_SHORTCUT_TYPE = "application/vnd.google-apps.shortcut"
-UNSUPPORTED_FILE_TYPE_CONTENT = ""  # keep empty for now
-
-
-class GDriveMimeType(str, Enum):
-    DOC = "application/vnd.google-apps.document"
-    SPREADSHEET = "application/vnd.google-apps.spreadsheet"
-    PDF = "application/pdf"
-    WORD_DOC = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-    PPT = "application/vnd.google-apps.presentation"
-    POWERPOINT = (
-        "application/vnd.openxmlformats-officedocument.presentationml.presentation"
-    )
-    PLAIN_TEXT = "text/plain"
-    MARKDOWN = "text/markdown"
-
-
-GoogleDriveFileType = dict[str, Any]
-
-# Google Drive APIs are quite flakey and may 500 for an
-# extended period of time. Trying to combat here by adding a very
-# long retry period (~20 minutes of trying every minute)
-add_retries = retry_builder(tries=50, max_delay=30)
-
-
-def _run_drive_file_query(
-    service: discovery.Resource,
-    query: str,
-    continue_on_failure: bool,
-    include_shared: bool = GOOGLE_DRIVE_INCLUDE_SHARED,
-    follow_shortcuts: bool = GOOGLE_DRIVE_FOLLOW_SHORTCUTS,
-    batch_size: int = INDEX_BATCH_SIZE,
-) -> Iterator[GoogleDriveFileType]:
-    next_page_token = ""
-    while next_page_token is not None:
-        logger.debug(f"Running Google Drive fetch with query: {query}")
-        results = add_retries(
-            lambda: (
-                service.files()
-                .list(
-                    corpora="allDrives"
-                    if include_shared
-                    else "user",  # needed to search through shared drives
-                    pageSize=batch_size,
-                    supportsAllDrives=include_shared,
-                    includeItemsFromAllDrives=include_shared,
-                    fields=(
-                        "nextPageToken, files(mimeType, id, name, permissions, "
-                        "modifiedTime, webViewLink, shortcutDetails)"
-                    ),
-                    pageToken=next_page_token,
-                    q=query,
-                )
-                .execute()
-            )
-        )()
-        next_page_token = results.get("nextPageToken")
-        files = results["files"]
-        for file in files:
-            if follow_shortcuts and "shortcutDetails" in file:
-                try:
-                    file_shortcut_points_to = add_retries(
-                        lambda: (
-                            service.files()
-                            .get(
-                                fileId=file["shortcutDetails"]["targetId"],
-                                supportsAllDrives=include_shared,
-                                fields="mimeType, id, name, modifiedTime, webViewLink, permissions, shortcutDetails",
-                            )
-                            .execute()
-                        )
-                    )()
-                    yield file_shortcut_points_to
-                except HttpError:
-                    logger.error(
-                        f"Failed to follow shortcut with details: {file['shortcutDetails']}"
-                    )
-                    if continue_on_failure:
-                        continue
-                    raise
-            else:
-                yield file
-
-
-def _get_folder_id(
-    service: discovery.Resource,
-    parent_id: str,
-    folder_name: str,
-    include_shared: bool,
-    follow_shortcuts: bool,
-) -> str | None:
-    """
-    Get the ID of a folder given its name and the ID of its parent folder.
-    """
-    query = f"'{parent_id}' in parents and name='{folder_name}' and "
-    if follow_shortcuts:
-        query += f"(mimeType='{DRIVE_FOLDER_TYPE}' or mimeType='{DRIVE_SHORTCUT_TYPE}')"
-    else:
-        query += f"mimeType='{DRIVE_FOLDER_TYPE}'"
-
-    # TODO: support specifying folder path in shared drive rather than just `My Drive`
-    results = add_retries(
-        lambda: (
-            service.files()
-            .list(
-                q=query,
-                spaces="drive",
-                fields="nextPageToken, files(id, name, shortcutDetails)",
-                supportsAllDrives=include_shared,
-                includeItemsFromAllDrives=include_shared,
-            )
-            .execute()
-        )
-    )()
-    items = results.get("files", [])
-
-    folder_id = None
-    if items:
-        if follow_shortcuts and "shortcutDetails" in items[0]:
-            folder_id = items[0]["shortcutDetails"]["targetId"]
-        else:
-            folder_id = items[0]["id"]
-    return folder_id
-
-
-def _get_folders(
-    service: discovery.Resource,
-    continue_on_failure: bool,
-    folder_id: str | None = None,  # if specified, only fetches files within this folder
-    include_shared: bool = GOOGLE_DRIVE_INCLUDE_SHARED,
-    follow_shortcuts: bool = GOOGLE_DRIVE_FOLLOW_SHORTCUTS,
-    batch_size: int = INDEX_BATCH_SIZE,
-) -> Iterator[GoogleDriveFileType]:
-    query = f"mimeType = '{DRIVE_FOLDER_TYPE}' "
-    if follow_shortcuts:
-        query = "(" + query + f" or mimeType = '{DRIVE_SHORTCUT_TYPE}'" + ") "
-
-    if folder_id:
-        query += f"and '{folder_id}' in parents "
-    query = query.rstrip()  # remove the trailing space(s)
-
-    for file in _run_drive_file_query(
-        service=service,
-        query=query,
-        continue_on_failure=continue_on_failure,
-        include_shared=include_shared,
-        follow_shortcuts=follow_shortcuts,
-        batch_size=batch_size,
-    ):
-        # Need to check this since file may have been a target of a shortcut
-        # and not necessarily a folder
-        if file["mimeType"] == DRIVE_FOLDER_TYPE:
-            yield file
-        else:
-            pass
-
-
-def _get_files(
-    service: discovery.Resource,
-    continue_on_failure: bool,
-    time_range_start: SecondsSinceUnixEpoch | None = None,
-    time_range_end: SecondsSinceUnixEpoch | None = None,
-    folder_id: str | None = None,  # if specified, only fetches files within this folder
-    include_shared: bool = GOOGLE_DRIVE_INCLUDE_SHARED,
-    follow_shortcuts: bool = GOOGLE_DRIVE_FOLLOW_SHORTCUTS,
-    batch_size: int = INDEX_BATCH_SIZE,
-) -> Iterator[GoogleDriveFileType]:
-    query = f"mimeType != '{DRIVE_FOLDER_TYPE}' "
-    if time_range_start is not None:
-        time_start = datetime.utcfromtimestamp(time_range_start).isoformat() + "Z"
-        query += f"and modifiedTime >= '{time_start}' "
-    if time_range_end is not None:
-        time_stop = datetime.utcfromtimestamp(time_range_end).isoformat() + "Z"
-        query += f"and modifiedTime <= '{time_stop}' "
-    if folder_id:
-        query += f"and '{folder_id}' in parents "
-    query = query.rstrip()  # remove the trailing space(s)
-
-    files = _run_drive_file_query(
-        service=service,
-        query=query,
-        continue_on_failure=continue_on_failure,
-        include_shared=include_shared,
-        follow_shortcuts=follow_shortcuts,
-        batch_size=batch_size,
-    )
-
-    return files
-
-
-def get_all_files_batched(
-    service: discovery.Resource,
-    continue_on_failure: bool,
-    include_shared: bool = GOOGLE_DRIVE_INCLUDE_SHARED,
-    follow_shortcuts: bool = GOOGLE_DRIVE_FOLLOW_SHORTCUTS,
-    batch_size: int = INDEX_BATCH_SIZE,
-    time_range_start: SecondsSinceUnixEpoch | None = None,
-    time_range_end: SecondsSinceUnixEpoch | None = None,
-    folder_id: str | None = None,  # if specified, only fetches files within this folder
-    # if True, will fetch files in sub-folders of the specified folder ID.
-    # Only applies if folder_id is specified.
-    traverse_subfolders: bool = True,
-    folder_ids_traversed: list[str] | None = None,
-) -> Iterator[list[GoogleDriveFileType]]:
-    """Gets all files matching the criteria specified by the args from Google Drive
-    in batches of size `batch_size`.
-    """
-    found_files = _get_files(
-        service=service,
-        continue_on_failure=continue_on_failure,
-        time_range_start=time_range_start,
-        time_range_end=time_range_end,
-        folder_id=folder_id,
-        include_shared=include_shared,
-        follow_shortcuts=follow_shortcuts,
-        batch_size=batch_size,
-    )
-    yield from batch_generator(
-        items=found_files,
-        batch_size=batch_size,
-        pre_batch_yield=lambda batch_files: logger.debug(
-            f"Parseable Documents in batch: {[file['name'] for file in batch_files]}"
-        ),
-    )
-
-    if traverse_subfolders and folder_id is not None:
-        folder_ids_traversed = folder_ids_traversed or []
-        subfolders = _get_folders(
-            service=service,
-            folder_id=folder_id,
-            continue_on_failure=continue_on_failure,
-            include_shared=include_shared,
-            follow_shortcuts=follow_shortcuts,
-            batch_size=batch_size,
-        )
-        for subfolder in subfolders:
-            if subfolder["id"] not in folder_ids_traversed:
-                logger.info("Fetching all files in subfolder: " + subfolder["name"])
-                folder_ids_traversed.append(subfolder["id"])
-                yield from get_all_files_batched(
-                    service=service,
-                    continue_on_failure=continue_on_failure,
-                    include_shared=include_shared,
-                    follow_shortcuts=follow_shortcuts,
-                    batch_size=batch_size,
-                    time_range_start=time_range_start,
-                    time_range_end=time_range_end,
-                    folder_id=subfolder["id"],
-                    traverse_subfolders=traverse_subfolders,
-                    folder_ids_traversed=folder_ids_traversed,
-                )
-            else:
-                logger.debug(
-                    "Skipping subfolder since already traversed: " + subfolder["name"]
-                )
-
 
-def extract_text(file: dict[str, str], service: discovery.Resource) -> str:
-    mime_type = file["mimeType"]
-
-    if mime_type not in set(item.value for item in GDriveMimeType):
-        # Unsupported file types can still have a title, finding this way is still useful
-        return UNSUPPORTED_FILE_TYPE_CONTENT
-
-    if mime_type in [
-        GDriveMimeType.DOC.value,
-        GDriveMimeType.PPT.value,
-        GDriveMimeType.SPREADSHEET.value,
-    ]:
-        export_mime_type = (
-            "text/plain"
-            if mime_type != GDriveMimeType.SPREADSHEET.value
-            else "text/csv"
-        )
-        return (
-            service.files()
-            .export(fileId=file["id"], mimeType=export_mime_type)
-            .execute()
-            .decode("utf-8")
-        )
-    elif mime_type in [
-        GDriveMimeType.PLAIN_TEXT.value,
-        GDriveMimeType.MARKDOWN.value,
-    ]:
-        return service.files().get_media(fileId=file["id"]).execute().decode("utf-8")
-    if mime_type in [
-        GDriveMimeType.WORD_DOC.value,
-        GDriveMimeType.POWERPOINT.value,
-        GDriveMimeType.PDF.value,
-    ]:
-        response = service.files().get_media(fileId=file["id"]).execute()
-        if get_unstructured_api_key():
-            return unstructured_to_text(
-                file=io.BytesIO(response), file_name=file.get("name", file["id"])
-            )
+def _extract_str_list_from_comma_str(string: str | None) -> list[str]:
+    if not string:
+        return []
+    return [s.strip() for s in string.split(",") if s.strip()]
 
-        if mime_type == GDriveMimeType.WORD_DOC.value:
-            return docx_to_text(file=io.BytesIO(response))
-        elif mime_type == GDriveMimeType.PDF.value:
-            text, _ = read_pdf_file(file=io.BytesIO(response))
-            return text
-        elif mime_type == GDriveMimeType.POWERPOINT.value:
-            return pptx_to_text(file=io.BytesIO(response))
 
-    return UNSUPPORTED_FILE_TYPE_CONTENT
+def _extract_ids_from_urls(urls: list[str]) -> list[str]:
+    return [url.split("/")[-1] for url in urls]
 
 
-class GoogleDriveConnector(LoadConnector, PollConnector):
+class GoogleDriveConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(
         self,
-        # optional list of folder paths e.g. "[My Folder/My Subfolder]"
-        # if specified, will only index files in these folders
-        folder_paths: list[str] | None = None,
+        include_shared_drives: bool = True,
+        shared_drive_urls: str | None = None,
+        include_my_drives: bool = True,
+        my_drive_emails: str | None = None,
+        shared_folder_urls: str | None = None,
         batch_size: int = INDEX_BATCH_SIZE,
-        include_shared: bool = GOOGLE_DRIVE_INCLUDE_SHARED,
-        follow_shortcuts: bool = GOOGLE_DRIVE_FOLLOW_SHORTCUTS,
-        only_org_public: bool = GOOGLE_DRIVE_ONLY_ORG_PUBLIC,
-        continue_on_failure: bool = CONTINUE_ON_CONNECTOR_FAILURE,
+        # OLD PARAMETERS
+        folder_paths: list[str] | None = None,
+        include_shared: bool | None = None,
+        follow_shortcuts: bool | None = None,
+        only_org_public: bool | None = None,
+        continue_on_failure: bool | None = None,
     ) -> None:
-        self.folder_paths = folder_paths or []
+        # Check for old input parameters
+        if (
+            folder_paths is not None
+            or include_shared is not None
+            or follow_shortcuts is not None
+            or only_org_public is not None
+            or continue_on_failure is not None
+        ):
+            logger.exception(
+                "Google Drive connector received old input parameters. "
+                "Please visit the docs for help with the new setup: "
+                f"{SCOPE_DOC_URL}"
+            )
+            raise ValueError(
+                "Google Drive connector received old input parameters. "
+                "Please visit the docs for help with the new setup: "
+                f"{SCOPE_DOC_URL}"
+            )
+
+        if (
+            not include_shared_drives
+            and not include_my_drives
+            and not shared_folder_urls
+        ):
+            raise ValueError(
+                "At least one of include_shared_drives, include_my_drives,"
+                " or shared_folder_urls must be true"
+            )
+
         self.batch_size = batch_size
-        self.include_shared = include_shared
-        self.follow_shortcuts = follow_shortcuts
-        self.only_org_public = only_org_public
-        self.continue_on_failure = continue_on_failure
+
+        self.include_shared_drives = include_shared_drives
+        shared_drive_url_list = _extract_str_list_from_comma_str(shared_drive_urls)
+        self.shared_drive_ids = _extract_ids_from_urls(shared_drive_url_list)
+
+        self.include_my_drives = include_my_drives
+        self.my_drive_emails = _extract_str_list_from_comma_str(my_drive_emails)
+
+        shared_folder_url_list = _extract_str_list_from_comma_str(shared_folder_urls)
+        self.shared_folder_ids = _extract_ids_from_urls(shared_folder_url_list)
+
+        self.primary_admin_email: str | None = None
+        self.google_domain: str | None = None
+
         self.creds: OAuthCredentials | ServiceAccountCredentials | None = None
 
-    @staticmethod
-    def _process_folder_paths(
-        service: discovery.Resource,
-        folder_paths: list[str],
-        include_shared: bool,
-        follow_shortcuts: bool,
-    ) -> list[str]:
-        """['Folder/Sub Folder'] -> ['<FOLDER_ID>']"""
-        folder_ids: list[str] = []
-        for path in folder_paths:
-            folder_names = path.split("/")
-            parent_id = "root"
-            for folder_name in folder_names:
-                found_parent_id = _get_folder_id(
-                    service=service,
-                    parent_id=parent_id,
-                    folder_name=folder_name,
-                    include_shared=include_shared,
-                    follow_shortcuts=follow_shortcuts,
-                )
-                if found_parent_id is None:
-                    raise ValueError(
-                        (
-                            f"Folder '{folder_name}' in path '{path}' "
-                            "not found in Google Drive"
-                        )
-                    )
-                parent_id = found_parent_id
-            folder_ids.append(parent_id)
-
-        return folder_ids
+        self._TRAVERSED_PARENT_IDS: set[str] = set()
+
+    def _update_traversed_parent_ids(self, folder_id: str) -> None:
+        self._TRAVERSED_PARENT_IDS.add(folder_id)
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:
-        """Checks for two different types of credentials.
-        (1) A credential which holds a token acquired via a user going thorough
-        the Google OAuth flow.
-        (2) A credential which holds a service account key JSON file, which
-        can then be used to impersonate any user in the workspace.
-        """
-        creds, new_creds_dict = get_google_drive_creds(credentials)
-        self.creds = creds
+        primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]
+        self.google_domain = primary_admin_email.split("@")[1]
+        self.primary_admin_email = primary_admin_email
+
+        self.creds, new_creds_dict = get_google_drive_creds(credentials)
         return new_creds_dict
 
-    def _fetch_docs_from_drive(
+    def get_google_resource(
         self,
+        service_name: str = "drive",
+        service_version: str = "v3",
+        user_email: str | None = None,
+    ) -> Resource:
+        if isinstance(self.creds, ServiceAccountCredentials):
+            creds = self.creds.with_subject(user_email or self.primary_admin_email)
+            service = build(service_name, service_version, credentials=creds)
+        elif isinstance(self.creds, OAuthCredentials):
+            service = build(service_name, service_version, credentials=self.creds)
+        else:
+            raise PermissionError("No credentials found")
+
+        return service
+
+    def _get_all_user_emails(self) -> list[str]:
+        admin_service = self.get_google_resource("admin", "directory_v1")
+        emails = []
+        for user in execute_paginated_retrieval(
+            retrieval_function=admin_service.users().list,
+            list_key="users",
+            fields=USER_FIELDS,
+            domain=self.google_domain,
+        ):
+            if email := user.get("primaryEmail"):
+                emails.append(email)
+        return emails
+
+    def _fetch_drive_items(
+        self,
+        is_slim: bool,
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
-    ) -> GenerateDocumentsOutput:
-        if self.creds is None:
-            raise PermissionError("Not logged into Google Drive")
-
-        service = discovery.build("drive", "v3", credentials=self.creds)
-        folder_ids: Sequence[str | None] = self._process_folder_paths(
-            service, self.folder_paths, self.include_shared, self.follow_shortcuts
-        )
-        if not folder_ids:
-            folder_ids = [None]
-
-        file_batches = chain(
-            *[
-                get_all_files_batched(
-                    service=service,
-                    continue_on_failure=self.continue_on_failure,
-                    include_shared=self.include_shared,
-                    follow_shortcuts=self.follow_shortcuts,
-                    batch_size=self.batch_size,
-                    time_range_start=start,
-                    time_range_end=end,
-                    folder_id=folder_id,
-                    traverse_subfolders=True,
+    ) -> Iterator[GoogleDriveFileType]:
+        primary_drive_service = self.get_google_resource()
+
+        if self.include_shared_drives:
+            shared_drive_urls = self.shared_drive_ids
+            if not shared_drive_urls:
+                # if no parent ids are specified, get all shared drives using the admin account
+                for drive in execute_paginated_retrieval(
+                    retrieval_function=primary_drive_service.drives().list,
+                    list_key="drives",
+                    useDomainAdminAccess=True,
+                    fields="drives(id)",
+                ):
+                    shared_drive_urls.append(drive["id"])
+
+            # For each shared drive, retrieve all files
+            for shared_drive_id in shared_drive_urls:
+                for file in get_files_in_shared_drive(
+                    service=primary_drive_service,
+                    drive_id=shared_drive_id,
+                    is_slim=is_slim,
+                    cache_folders=bool(self.shared_folder_ids),
+                    update_traversed_ids_func=self._update_traversed_parent_ids,
+                    start=start,
+                    end=end,
+                ):
+                    yield file
+
+        if self.shared_folder_ids:
+            # Crawl all the shared parent ids for files
+            for folder_id in self.shared_folder_ids:
+                yield from crawl_folders_for_files(
+                    service=primary_drive_service,
+                    parent_id=folder_id,
+                    personal_drive=False,
+                    traversed_parent_ids=self._TRAVERSED_PARENT_IDS,
+                    update_traversed_ids_func=self._update_traversed_parent_ids,
+                    start=start,
+                    end=end,
                 )
-                for folder_id in folder_ids
-            ]
-        )
-        for files_batch in file_batches:
-            doc_batch = []
-            for file in files_batch:
-                try:
-                    # Skip files that are shortcuts
-                    if file.get("mimeType") == DRIVE_SHORTCUT_TYPE:
-                        logger.info("Ignoring Drive Shortcut Filetype")
-                        continue
-
-                    if self.only_org_public:
-                        if "permissions" not in file:
-                            continue
-                        if not any(
-                            permission["type"] == "domain"
-                            for permission in file["permissions"]
-                        ):
-                            continue
-                    try:
-                        text_contents = extract_text(file, service) or ""
-                    except HttpError as e:
-                        reason = (
-                            e.error_details[0]["reason"]
-                            if e.error_details
-                            else e.reason
-                        )
-                        message = (
-                            e.error_details[0]["message"]
-                            if e.error_details
-                            else e.reason
-                        )
-
-                        # these errors don't represent a failure in the connector, but simply files
-                        # that can't / shouldn't be indexed
-                        ERRORS_TO_CONTINUE_ON = [
-                            "cannotExportFile",
-                            "exportSizeLimitExceeded",
-                            "cannotDownloadFile",
-                        ]
-                        if e.status_code == 403 and reason in ERRORS_TO_CONTINUE_ON:
-                            logger.warning(
-                                f"Could not export file '{file['name']}' due to '{message}', skipping..."
-                            )
-                            continue
-
-                        raise
-
-                    doc_batch.append(
-                        Document(
-                            id=file["webViewLink"],
-                            sections=[
-                                Section(link=file["webViewLink"], text=text_contents)
-                            ],
-                            source=DocumentSource.GOOGLE_DRIVE,
-                            semantic_identifier=file["name"],
-                            doc_updated_at=datetime.fromisoformat(
-                                file["modifiedTime"]
-                            ).astimezone(timezone.utc),
-                            metadata={} if text_contents else {IGNORE_FOR_QA: "True"},
-                            additional_info=file.get("id"),
-                        )
-                    )
-                except Exception as e:
-                    if not self.continue_on_failure:
-                        raise e
-
-                    logger.exception(
-                        "Ran into exception when pulling a file from Google Drive"
-                    )
-
-            yield doc_batch
+
+        all_user_emails = []
+        # get all personal docs from each users' personal drive
+        if self.include_my_drives:
+            if isinstance(self.creds, ServiceAccountCredentials):
+                all_user_emails = self.my_drive_emails or []
+
+                # If using service account and no emails specified, fetch all users
+                if not all_user_emails:
+                    all_user_emails = self._get_all_user_emails()
+
+            elif self.primary_admin_email:
+                # If using OAuth, only fetch the primary admin email
+                all_user_emails = [self.primary_admin_email]
+
+            for email in all_user_emails:
+                logger.info(f"Fetching personal files for user: {email}")
+                user_drive_service = self.get_google_resource(user_email=email)
+
+                yield from get_files_in_my_drive(
+                    service=user_drive_service,
+                    email=email,
+                    is_slim=is_slim,
+                    start=start,
+                    end=end,
+                )
+
+    def _extract_docs_from_google_drive(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateDocumentsOutput:
+        doc_batch = []
+        for file in self._fetch_drive_items(
+            is_slim=False,
+            start=start,
+            end=end,
+        ):
+            user_email = file.get("owners", [{}])[0].get("emailAddress")
+            service = self.get_google_resource(user_email=user_email)
+            if doc := convert_drive_item_to_document(
+                file=file,
+                service=service,
+            ):
+                doc_batch.append(doc)
+            if len(doc_batch) >= self.batch_size:
+                yield doc_batch
+                doc_batch = []
+
+        yield doc_batch
 
     def load_from_state(self) -> GenerateDocumentsOutput:
-        yield from self._fetch_docs_from_drive()
+        try:
+            yield from self._extract_docs_from_google_drive()
+        except Exception as e:
+            if MISSING_SCOPES_ERROR_STR in str(e):
+                raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+            raise e
 
     def poll_source(
         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
     ) -> GenerateDocumentsOutput:
-        # need to subtract 10 minutes from start time to account for modifiedTime
-        # propogation if a document is modified, it takes some time for the API to
-        # reflect these changes if we do not have an offset, then we may "miss" the
-        # update when polling
-        yield from self._fetch_docs_from_drive(start, end)
-
-
-if __name__ == "__main__":
-    import json
-    import os
-
-    service_account_json_path = os.environ.get("GOOGLE_SERVICE_ACCOUNT_KEY_JSON_PATH")
-    if not service_account_json_path:
-        raise ValueError(
-            "Please set GOOGLE_SERVICE_ACCOUNT_KEY_JSON_PATH environment variable"
-        )
-    with open(service_account_json_path) as f:
-        creds = json.load(f)
-
-    credentials_dict = {
-        DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY: json.dumps(creds),
-    }
-    delegated_user = os.environ.get("GOOGLE_DRIVE_DELEGATED_USER")
-    if delegated_user:
-        credentials_dict[DB_CREDENTIALS_DICT_DELEGATED_USER_KEY] = delegated_user
-
-    connector = GoogleDriveConnector(include_shared=True, follow_shortcuts=True)
-    connector.load_credentials(credentials_dict)
-    document_batch_generator = connector.load_from_state()
-    for document_batch in document_batch_generator:
-        print(document_batch)
-        break
+        try:
+            yield from self._extract_docs_from_google_drive(start, end)
+        except Exception as e:
+            if MISSING_SCOPES_ERROR_STR in str(e):
+                raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+            raise e
+
+    def _extract_slim_docs_from_google_drive(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
+        slim_batch = []
+        for file in self._fetch_drive_items(
+            is_slim=True,
+            start=start,
+            end=end,
+        ):
+            slim_batch.append(
+                SlimDocument(
+                    id=file["webViewLink"],
+                    perm_sync_data={
+                        "doc_id": file.get("id"),
+                        "permissions": file.get("permissions", []),
+                        "permission_ids": file.get("permissionIds", []),
+                        "name": file.get("name"),
+                        "owner_email": file.get("owners", [{}])[0].get("emailAddress"),
+                    },
+                )
+            )
+            if len(slim_batch) >= SLIM_BATCH_SIZE:
+                yield slim_batch
+                slim_batch = []
+        yield slim_batch
+
+    def retrieve_all_slim_documents(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
+        try:
+            yield from self._extract_slim_docs_from_google_drive(start, end)
+        except Exception as e:
+            if MISSING_SCOPES_ERROR_STR in str(e):
+                raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+            raise e
diff --git a/backend/danswer/connectors/google_drive/connector_auth.py b/backend/danswer/connectors/google_drive/connector_auth.py
index 777deae990a..80cbda6772a 100644
--- a/backend/danswer/connectors/google_drive/connector_auth.py
+++ b/backend/danswer/connectors/google_drive/connector_auth.py
@@ -8,24 +8,16 @@
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
 from google_auth_oauthlib.flow import InstalledAppFlow  # type: ignore
+from googleapiclient.discovery import build  # type: ignore
 from sqlalchemy.orm import Session
 
-from danswer.configs.app_configs import ENTERPRISE_EDITION_ENABLED
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import KV_CRED_KEY
 from danswer.configs.constants import KV_GOOGLE_DRIVE_CRED_KEY
 from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
-from danswer.connectors.google_drive.constants import BASE_SCOPES
-from danswer.connectors.google_drive.constants import (
-    DB_CREDENTIALS_DICT_DELEGATED_USER_KEY,
-)
-from danswer.connectors.google_drive.constants import (
-    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
-)
-from danswer.connectors.google_drive.constants import DB_CREDENTIALS_DICT_TOKEN_KEY
-from danswer.connectors.google_drive.constants import FETCH_GROUPS_SCOPES
-from danswer.connectors.google_drive.constants import FETCH_PERMISSIONS_SCOPES
+from danswer.connectors.google_drive.constants import MISSING_SCOPES_ERROR_STR
+from danswer.connectors.google_drive.constants import ONYX_SCOPE_INSTRUCTIONS
 from danswer.db.credentials import update_credential_json
 from danswer.db.models import User
 from danswer.key_value_store.factory import get_kv_store
@@ -36,15 +28,14 @@
 
 logger = setup_logger()
 
-
-def build_gdrive_scopes() -> list[str]:
-    base_scopes: list[str] = BASE_SCOPES
-    permissions_scopes: list[str] = FETCH_PERMISSIONS_SCOPES
-    groups_scopes: list[str] = FETCH_GROUPS_SCOPES
-
-    if ENTERPRISE_EDITION_ENABLED:
-        return base_scopes + permissions_scopes + groups_scopes
-    return base_scopes + permissions_scopes
+GOOGLE_DRIVE_SCOPES = [
+    "https://www.googleapis.com/auth/drive.readonly",
+    "https://www.googleapis.com/auth/drive.metadata.readonly",
+    "https://www.googleapis.com/auth/admin.directory.group.readonly",
+    "https://www.googleapis.com/auth/admin.directory.user.readonly",
+]
+DB_CREDENTIALS_DICT_TOKEN_KEY = "google_drive_tokens"
+DB_CREDENTIALS_PRIMARY_ADMIN_KEY = "google_drive_primary_admin"
 
 
 def _build_frontend_google_drive_redirect() -> str:
@@ -52,7 +43,7 @@ def _build_frontend_google_drive_redirect() -> str:
 
 
 def get_google_drive_creds_for_authorized_user(
-    token_json_str: str, scopes: list[str] = build_gdrive_scopes()
+    token_json_str: str, scopes: list[str]
 ) -> OAuthCredentials | None:
     creds_json = json.loads(token_json_str)
     creds = OAuthCredentials.from_authorized_user_info(creds_json, scopes)
@@ -72,21 +63,15 @@ def get_google_drive_creds_for_authorized_user(
     return None
 
 
-def _get_google_drive_creds_for_service_account(
-    service_account_key_json_str: str, scopes: list[str] = build_gdrive_scopes()
-) -> ServiceAccountCredentials | None:
-    service_account_key = json.loads(service_account_key_json_str)
-    creds = ServiceAccountCredentials.from_service_account_info(
-        service_account_key, scopes=scopes
-    )
-    if not creds.valid or not creds.expired:
-        creds.refresh(Request())
-    return creds if creds.valid else None
-
-
 def get_google_drive_creds(
-    credentials: dict[str, str], scopes: list[str] = build_gdrive_scopes()
+    credentials: dict[str, str], scopes: list[str] = GOOGLE_DRIVE_SCOPES
 ) -> tuple[ServiceAccountCredentials | OAuthCredentials, dict[str, str] | None]:
+    """Checks for two different types of credentials.
+    (1) A credential which holds a token acquired via a user going thorough
+    the Google OAuth flow.
+    (2) A credential which holds a service account key JSON file, which
+    can then be used to impersonate any user in the workspace.
+    """
     oauth_creds = None
     service_creds = None
     new_creds_dict = None
@@ -100,26 +85,27 @@ def get_google_drive_creds(
         # (e.g. the token has been refreshed)
         new_creds_json_str = oauth_creds.to_json() if oauth_creds else ""
         if new_creds_json_str != access_token_json_str:
-            new_creds_dict = {DB_CREDENTIALS_DICT_TOKEN_KEY: new_creds_json_str}
-
-    elif DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY in credentials:
-        service_account_key_json_str = credentials[
-            DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY
-        ]
-        service_creds = _get_google_drive_creds_for_service_account(
-            service_account_key_json_str=service_account_key_json_str,
-            scopes=scopes,
+            new_creds_dict = {
+                DB_CREDENTIALS_DICT_TOKEN_KEY: new_creds_json_str,
+                DB_CREDENTIALS_PRIMARY_ADMIN_KEY: credentials[
+                    DB_CREDENTIALS_PRIMARY_ADMIN_KEY
+                ],
+            }
+
+    elif KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY in credentials:
+        service_account_key_json_str = credentials[KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY]
+        service_account_key = json.loads(service_account_key_json_str)
+
+        service_creds = ServiceAccountCredentials.from_service_account_info(
+            service_account_key, scopes=scopes
         )
 
-        # "Impersonate" a user if one is specified
-        delegated_user_email = cast(
-            str | None, credentials.get(DB_CREDENTIALS_DICT_DELEGATED_USER_KEY)
-        )
-        if delegated_user_email:
-            service_creds = (
-                service_creds.with_subject(delegated_user_email)
-                if service_creds
-                else None
+        if not service_creds.valid or not service_creds.expired:
+            service_creds.refresh(Request())
+
+        if not service_creds.valid:
+            raise PermissionError(
+                "Unable to access Google Drive - service account credentials are invalid."
             )
 
     creds: ServiceAccountCredentials | OAuthCredentials | None = (
@@ -146,7 +132,7 @@ def get_auth_url(credential_id: int) -> str:
     credential_json = json.loads(creds_str)
     flow = InstalledAppFlow.from_client_config(
         credential_json,
-        scopes=build_gdrive_scopes(),
+        scopes=GOOGLE_DRIVE_SCOPES,
         redirect_uri=_build_frontend_google_drive_redirect(),
     )
     auth_url, _ = flow.authorization_url(prompt="consent")
@@ -169,13 +155,34 @@ def update_credential_access_tokens(
     app_credentials = get_google_app_cred()
     flow = InstalledAppFlow.from_client_config(
         app_credentials.model_dump(),
-        scopes=build_gdrive_scopes(),
+        scopes=GOOGLE_DRIVE_SCOPES,
         redirect_uri=_build_frontend_google_drive_redirect(),
     )
     flow.fetch_token(code=auth_code)
     creds = flow.credentials
     token_json_str = creds.to_json()
-    new_creds_dict = {DB_CREDENTIALS_DICT_TOKEN_KEY: token_json_str}
+
+    # Get user email from Google API so we know who
+    # the primary admin is for this connector
+    try:
+        admin_service = build("drive", "v3", credentials=creds)
+        user_info = (
+            admin_service.about()
+            .get(
+                fields="user(emailAddress)",
+            )
+            .execute()
+        )
+        email = user_info.get("user", {}).get("emailAddress")
+    except Exception as e:
+        if MISSING_SCOPES_ERROR_STR in str(e):
+            raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+        raise e
+
+    new_creds_dict = {
+        DB_CREDENTIALS_DICT_TOKEN_KEY: token_json_str,
+        DB_CREDENTIALS_PRIMARY_ADMIN_KEY: email,
+    }
 
     if not update_credential_json(credential_id, new_creds_dict, user, db_session):
         return None
@@ -184,15 +191,15 @@ def update_credential_access_tokens(
 
 def build_service_account_creds(
     source: DocumentSource,
-    delegated_user_email: str | None = None,
+    primary_admin_email: str | None = None,
 ) -> CredentialBase:
     service_account_key = get_service_account_key()
 
     credential_dict = {
-        DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY: service_account_key.json(),
+        KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY: service_account_key.json(),
     }
-    if delegated_user_email:
-        credential_dict[DB_CREDENTIALS_DICT_DELEGATED_USER_KEY] = delegated_user_email
+    if primary_admin_email:
+        credential_dict[DB_CREDENTIALS_PRIMARY_ADMIN_KEY] = primary_admin_email
 
     return CredentialBase(
         credential_json=credential_dict,
diff --git a/backend/danswer/connectors/google_drive/constants.py b/backend/danswer/connectors/google_drive/constants.py
index 0cca65c13df..848a21fffe6 100644
--- a/backend/danswer/connectors/google_drive/constants.py
+++ b/backend/danswer/connectors/google_drive/constants.py
@@ -1,7 +1,36 @@
-DB_CREDENTIALS_DICT_TOKEN_KEY = "google_drive_tokens"
-DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY = "google_drive_service_account_key"
-DB_CREDENTIALS_DICT_DELEGATED_USER_KEY = "google_drive_delegated_user"
+UNSUPPORTED_FILE_TYPE_CONTENT = ""  # keep empty for now
+DRIVE_FOLDER_TYPE = "application/vnd.google-apps.folder"
+DRIVE_SHORTCUT_TYPE = "application/vnd.google-apps.shortcut"
+DRIVE_FILE_TYPE = "application/vnd.google-apps.file"
 
-BASE_SCOPES = ["https://www.googleapis.com/auth/drive.readonly"]
-FETCH_PERMISSIONS_SCOPES = ["https://www.googleapis.com/auth/drive.metadata.readonly"]
-FETCH_GROUPS_SCOPES = ["https://www.googleapis.com/auth/cloud-identity.groups.readonly"]
+FILE_FIELDS = (
+    "nextPageToken, files(mimeType, id, name, permissions, modifiedTime, webViewLink, "
+    "shortcutDetails, owners(emailAddress))"
+)
+SLIM_FILE_FIELDS = (
+    "nextPageToken, files(mimeType, id, name, permissions(emailAddress, type), "
+    "permissionIds, webViewLink, owners(emailAddress))"
+)
+FOLDER_FIELDS = "nextPageToken, files(id, name, permissions, modifiedTime, webViewLink, shortcutDetails)"
+USER_FIELDS = "nextPageToken, users(primaryEmail)"
+
+# these errors don't represent a failure in the connector, but simply files
+# that can't / shouldn't be indexed
+ERRORS_TO_CONTINUE_ON = [
+    "cannotExportFile",
+    "exportSizeLimitExceeded",
+    "cannotDownloadFile",
+]
+
+# Error message substrings
+MISSING_SCOPES_ERROR_STR = "client not authorized for any of the scopes requested"
+
+# Documentation and error messages
+SCOPE_DOC_URL = "https://docs.danswer.dev/connectors/google_drive/overview"
+ONYX_SCOPE_INSTRUCTIONS = (
+    "You have upgraded Danswer without updating the Google Drive scopes. "
+    f"Please refer to the documentation to learn how to update the scopes: {SCOPE_DOC_URL}"
+)
+
+# Batch sizes
+SLIM_BATCH_SIZE = 500
diff --git a/backend/danswer/connectors/google_drive/doc_conversion.py b/backend/danswer/connectors/google_drive/doc_conversion.py
new file mode 100644
index 00000000000..688190c2267
--- /dev/null
+++ b/backend/danswer/connectors/google_drive/doc_conversion.py
@@ -0,0 +1,115 @@
+import io
+from datetime import datetime
+from datetime import timezone
+
+from googleapiclient.discovery import Resource  # type: ignore
+from googleapiclient.errors import HttpError  # type: ignore
+
+from danswer.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
+from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import IGNORE_FOR_QA
+from danswer.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
+from danswer.connectors.google_drive.constants import ERRORS_TO_CONTINUE_ON
+from danswer.connectors.google_drive.constants import UNSUPPORTED_FILE_TYPE_CONTENT
+from danswer.connectors.google_drive.models import GDriveMimeType
+from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.file_processing.extract_file_text import docx_to_text
+from danswer.file_processing.extract_file_text import pptx_to_text
+from danswer.file_processing.extract_file_text import read_pdf_file
+from danswer.file_processing.unstructured import get_unstructured_api_key
+from danswer.file_processing.unstructured import unstructured_to_text
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def _extract_text(file: dict[str, str], service: Resource) -> str:
+    mime_type = file["mimeType"]
+
+    if mime_type not in set(item.value for item in GDriveMimeType):
+        # Unsupported file types can still have a title, finding this way is still useful
+        return UNSUPPORTED_FILE_TYPE_CONTENT
+
+    if mime_type in [
+        GDriveMimeType.DOC.value,
+        GDriveMimeType.PPT.value,
+        GDriveMimeType.SPREADSHEET.value,
+    ]:
+        export_mime_type = (
+            "text/plain"
+            if mime_type != GDriveMimeType.SPREADSHEET.value
+            else "text/csv"
+        )
+        return (
+            service.files()
+            .export(fileId=file["id"], mimeType=export_mime_type)
+            .execute()
+            .decode("utf-8")
+        )
+    elif mime_type in [
+        GDriveMimeType.PLAIN_TEXT.value,
+        GDriveMimeType.MARKDOWN.value,
+    ]:
+        return service.files().get_media(fileId=file["id"]).execute().decode("utf-8")
+    if mime_type in [
+        GDriveMimeType.WORD_DOC.value,
+        GDriveMimeType.POWERPOINT.value,
+        GDriveMimeType.PDF.value,
+    ]:
+        response = service.files().get_media(fileId=file["id"]).execute()
+        if get_unstructured_api_key():
+            return unstructured_to_text(
+                file=io.BytesIO(response), file_name=file.get("name", file["id"])
+            )
+
+        if mime_type == GDriveMimeType.WORD_DOC.value:
+            return docx_to_text(file=io.BytesIO(response))
+        elif mime_type == GDriveMimeType.PDF.value:
+            text, _ = read_pdf_file(file=io.BytesIO(response))
+            return text
+        elif mime_type == GDriveMimeType.POWERPOINT.value:
+            return pptx_to_text(file=io.BytesIO(response))
+
+    return UNSUPPORTED_FILE_TYPE_CONTENT
+
+
+def convert_drive_item_to_document(
+    file: GoogleDriveFileType, service: Resource
+) -> Document | None:
+    try:
+        # Skip files that are shortcuts
+        if file.get("mimeType") == DRIVE_SHORTCUT_TYPE:
+            logger.info("Ignoring Drive Shortcut Filetype")
+            return None
+        try:
+            text_contents = _extract_text(file, service) or ""
+        except HttpError as e:
+            reason = e.error_details[0]["reason"] if e.error_details else e.reason
+            message = e.error_details[0]["message"] if e.error_details else e.reason
+            if e.status_code == 403 and reason in ERRORS_TO_CONTINUE_ON:
+                logger.warning(
+                    f"Could not export file '{file['name']}' due to '{message}', skipping..."
+                )
+                return None
+
+            raise
+
+        return Document(
+            id=file["webViewLink"],
+            sections=[Section(link=file["webViewLink"], text=text_contents)],
+            source=DocumentSource.GOOGLE_DRIVE,
+            semantic_identifier=file["name"],
+            doc_updated_at=datetime.fromisoformat(file["modifiedTime"]).astimezone(
+                timezone.utc
+            ),
+            metadata={} if text_contents else {IGNORE_FOR_QA: "True"},
+            additional_info=file.get("id"),
+        )
+    except Exception as e:
+        if not CONTINUE_ON_CONNECTOR_FAILURE:
+            raise e
+
+        logger.exception("Ran into exception when pulling a file from Google Drive")
+    return None
diff --git a/backend/danswer/connectors/google_drive/file_retrieval.py b/backend/danswer/connectors/google_drive/file_retrieval.py
new file mode 100644
index 00000000000..ea4e7d49466
--- /dev/null
+++ b/backend/danswer/connectors/google_drive/file_retrieval.py
@@ -0,0 +1,192 @@
+from collections.abc import Callable
+from collections.abc import Iterator
+from datetime import datetime
+
+from googleapiclient.discovery import Resource  # type: ignore
+
+from danswer.connectors.google_drive.constants import DRIVE_FOLDER_TYPE
+from danswer.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
+from danswer.connectors.google_drive.constants import FILE_FIELDS
+from danswer.connectors.google_drive.constants import FOLDER_FIELDS
+from danswer.connectors.google_drive.constants import SLIM_FILE_FIELDS
+from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def _generate_time_range_filter(
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
+) -> str:
+    time_range_filter = ""
+    if start is not None:
+        time_start = datetime.utcfromtimestamp(start).isoformat() + "Z"
+        time_range_filter += f" and modifiedTime >= '{time_start}'"
+    if end is not None:
+        time_stop = datetime.utcfromtimestamp(end).isoformat() + "Z"
+        time_range_filter += f" and modifiedTime <= '{time_stop}'"
+    return time_range_filter
+
+
+def _get_folders_in_parent(
+    service: Resource,
+    parent_id: str | None = None,
+    personal_drive: bool = False,
+) -> Iterator[GoogleDriveFileType]:
+    # Follow shortcuts to folders
+    query = f"(mimeType = '{DRIVE_FOLDER_TYPE}' or mimeType = '{DRIVE_SHORTCUT_TYPE}')"
+    query += " and trashed = false"
+
+    if parent_id:
+        query += f" and '{parent_id}' in parents"
+
+    for file in execute_paginated_retrieval(
+        retrieval_function=service.files().list,
+        list_key="files",
+        corpora="user" if personal_drive else "allDrives",
+        supportsAllDrives=not personal_drive,
+        includeItemsFromAllDrives=not personal_drive,
+        fields=FOLDER_FIELDS,
+        q=query,
+    ):
+        yield file
+
+
+def _get_files_in_parent(
+    service: Resource,
+    parent_id: str,
+    personal_drive: bool,
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
+    is_slim: bool = False,
+) -> Iterator[GoogleDriveFileType]:
+    query = f"mimeType != '{DRIVE_FOLDER_TYPE}' and '{parent_id}' in parents"
+    query += " and trashed = false"
+    query += _generate_time_range_filter(start, end)
+
+    for file in execute_paginated_retrieval(
+        retrieval_function=service.files().list,
+        list_key="files",
+        corpora="user" if personal_drive else "allDrives",
+        supportsAllDrives=not personal_drive,
+        includeItemsFromAllDrives=not personal_drive,
+        fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
+        q=query,
+    ):
+        yield file
+
+
+def crawl_folders_for_files(
+    service: Resource,
+    parent_id: str,
+    personal_drive: bool,
+    traversed_parent_ids: set[str],
+    update_traversed_ids_func: Callable[[str], None],
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
+) -> Iterator[GoogleDriveFileType]:
+    """
+    This function starts crawling from any folder. It is slower though.
+    """
+    if parent_id in traversed_parent_ids:
+        print(f"Skipping subfolder since already traversed: {parent_id}")
+        return
+
+    update_traversed_ids_func(parent_id)
+
+    yield from _get_files_in_parent(
+        service=service,
+        personal_drive=personal_drive,
+        start=start,
+        end=end,
+        parent_id=parent_id,
+    )
+
+    for subfolder in _get_folders_in_parent(
+        service=service,
+        parent_id=parent_id,
+        personal_drive=personal_drive,
+    ):
+        logger.info("Fetching all files in subfolder: " + subfolder["name"])
+        yield from crawl_folders_for_files(
+            service=service,
+            parent_id=subfolder["id"],
+            personal_drive=personal_drive,
+            traversed_parent_ids=traversed_parent_ids,
+            update_traversed_ids_func=update_traversed_ids_func,
+            start=start,
+            end=end,
+        )
+
+
+def get_files_in_shared_drive(
+    service: Resource,
+    drive_id: str,
+    is_slim: bool = False,
+    cache_folders: bool = True,
+    update_traversed_ids_func: Callable[[str], None] = lambda _: None,
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
+) -> Iterator[GoogleDriveFileType]:
+    # If we know we are going to folder crawl later, we can cache the folders here
+    if cache_folders:
+        # Get all folders being queried and add them to the traversed set
+        query = f"mimeType = '{DRIVE_FOLDER_TYPE}'"
+        query += " and trashed = false"
+        for file in execute_paginated_retrieval(
+            retrieval_function=service.files().list,
+            list_key="files",
+            corpora="drive",
+            driveId=drive_id,
+            supportsAllDrives=True,
+            includeItemsFromAllDrives=True,
+            fields="nextPageToken, files(id)",
+            q=query,
+        ):
+            update_traversed_ids_func(file["id"])
+
+    # Get all files in the shared drive
+    query = f"mimeType != '{DRIVE_FOLDER_TYPE}'"
+    query += " and trashed = false"
+    query += _generate_time_range_filter(start, end)
+    for file in execute_paginated_retrieval(
+        retrieval_function=service.files().list,
+        list_key="files",
+        corpora="drive",
+        driveId=drive_id,
+        supportsAllDrives=True,
+        includeItemsFromAllDrives=True,
+        fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
+        q=query,
+    ):
+        yield file
+
+
+def get_files_in_my_drive(
+    service: Resource,
+    email: str,
+    is_slim: bool = False,
+    start: SecondsSinceUnixEpoch | None = None,
+    end: SecondsSinceUnixEpoch | None = None,
+) -> Iterator[GoogleDriveFileType]:
+    query = f"mimeType != '{DRIVE_FOLDER_TYPE}' and '{email}' in owners"
+    query += " and trashed = false"
+    query += _generate_time_range_filter(start, end)
+    for file in execute_paginated_retrieval(
+        retrieval_function=service.files().list,
+        list_key="files",
+        corpora="user",
+        fields=SLIM_FILE_FIELDS if is_slim else FILE_FIELDS,
+        q=query,
+    ):
+        yield file
+
+
+# Just in case we need to get the root folder id
+def get_root_folder_id(service: Resource) -> str:
+    # we dont paginate here because there is only one root folder per user
+    # https://developers.google.com/drive/api/guides/v2-to-v3-reference
+    return service.files().get(fileId="root", fields="id").execute()["id"]
diff --git a/backend/danswer/connectors/google_drive/google_utils.py b/backend/danswer/connectors/google_drive/google_utils.py
new file mode 100644
index 00000000000..5f772e5ad63
--- /dev/null
+++ b/backend/danswer/connectors/google_drive/google_utils.py
@@ -0,0 +1,35 @@
+from collections.abc import Callable
+from collections.abc import Iterator
+from typing import Any
+
+from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.utils.retry_wrapper import retry_builder
+
+
+# Google Drive APIs are quite flakey and may 500 for an
+# extended period of time. Trying to combat here by adding a very
+# long retry period (~20 minutes of trying every minute)
+add_retries = retry_builder(tries=50, max_delay=30)
+
+
+def execute_paginated_retrieval(
+    retrieval_function: Callable,
+    list_key: str,
+    **kwargs: Any,
+) -> Iterator[GoogleDriveFileType]:
+    """Execute a paginated retrieval from Google Drive API
+    Args:
+        retrieval_function: The specific list function to call (e.g., service.files().list)
+        **kwargs: Arguments to pass to the list function
+    """
+    next_page_token = ""
+    while next_page_token is not None:
+        request_kwargs = kwargs.copy()
+        if next_page_token:
+            request_kwargs["pageToken"] = next_page_token
+
+        results = add_retries(lambda: retrieval_function(**request_kwargs).execute())()
+
+        next_page_token = results.get("nextPageToken")
+        for item in results.get(list_key, []):
+            yield item
diff --git a/backend/danswer/connectors/google_drive/models.py b/backend/danswer/connectors/google_drive/models.py
new file mode 100644
index 00000000000..5bb06f3c206
--- /dev/null
+++ b/backend/danswer/connectors/google_drive/models.py
@@ -0,0 +1,18 @@
+from enum import Enum
+from typing import Any
+
+
+class GDriveMimeType(str, Enum):
+    DOC = "application/vnd.google-apps.document"
+    SPREADSHEET = "application/vnd.google-apps.spreadsheet"
+    PDF = "application/pdf"
+    WORD_DOC = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+    PPT = "application/vnd.google-apps.presentation"
+    POWERPOINT = (
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation"
+    )
+    PLAIN_TEXT = "text/plain"
+    MARKDOWN = "text/markdown"
+
+
+GoogleDriveFileType = dict[str, Any]
diff --git a/backend/danswer/connectors/interfaces.py b/backend/danswer/connectors/interfaces.py
index 4734212147e..c53b3de5f2f 100644
--- a/backend/danswer/connectors/interfaces.py
+++ b/backend/danswer/connectors/interfaces.py
@@ -56,7 +56,11 @@ def poll_source(
 
 class SlimConnector(BaseConnector):
     @abc.abstractmethod
-    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
+    def retrieve_all_slim_documents(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
         raise NotImplementedError
 
 
diff --git a/backend/danswer/connectors/salesforce/connector.py b/backend/danswer/connectors/salesforce/connector.py
index 78d73d44766..1e0fe9e1d3a 100644
--- a/backend/danswer/connectors/salesforce/connector.py
+++ b/backend/danswer/connectors/salesforce/connector.py
@@ -251,7 +251,11 @@ def poll_source(
         end_datetime = datetime.utcfromtimestamp(end)
         return self._fetch_from_salesforce(start=start_datetime, end=end_datetime)
 
-    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
+    def retrieve_all_slim_documents(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
         if self.sf_client is None:
             raise ConnectorMissingCredentialError("Salesforce")
         doc_metadata_list: list[SlimDocument] = []
diff --git a/backend/danswer/connectors/slack/connector.py b/backend/danswer/connectors/slack/connector.py
index 92b5a0b7558..22ace603bd4 100644
--- a/backend/danswer/connectors/slack/connector.py
+++ b/backend/danswer/connectors/slack/connector.py
@@ -391,7 +391,11 @@ def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None
         self.client = WebClient(token=bot_token)
         return None
 
-    def retrieve_all_slim_documents(self) -> GenerateSlimDocumentOutput:
+    def retrieve_all_slim_documents(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
         if self.client is None:
             raise ConnectorMissingCredentialError("Slack")
 
diff --git a/backend/danswer/db/credentials.py b/backend/danswer/db/credentials.py
index 80ebe1b1538..58be604a724 100644
--- a/backend/danswer/db/credentials.py
+++ b/backend/danswer/db/credentials.py
@@ -10,12 +10,10 @@
 
 from danswer.auth.schemas import UserRole
 from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
 from danswer.connectors.gmail.constants import (
     GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
 )
-from danswer.connectors.google_drive.constants import (
-    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
-)
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.models import Credential
 from danswer.db.models import Credential__UserGroup
@@ -442,7 +440,7 @@ def delete_google_drive_service_account_credentials(
 ) -> None:
     credentials = fetch_credentials(db_session=db_session, user=user)
     for credential in credentials:
-        if credential.credential_json.get(DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY):
+        if credential.credential_json.get(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY):
             db_session.delete(credential)
 
     db_session.commit()
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index 1ba0ab13e2c..b0866a826c1 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -9,6 +9,7 @@
 from fastapi import Request
 from fastapi import Response
 from fastapi import UploadFile
+from google.oauth2.credentials import Credentials  # type: ignore
 from pydantic import BaseModel
 from sqlalchemy.orm import Session
 
@@ -35,6 +36,7 @@
 )
 from danswer.connectors.gmail.connector_auth import upsert_google_app_gmail_cred
 from danswer.connectors.google_drive.connector_auth import build_service_account_creds
+from danswer.connectors.google_drive.connector_auth import DB_CREDENTIALS_DICT_TOKEN_KEY
 from danswer.connectors.google_drive.connector_auth import delete_google_app_cred
 from danswer.connectors.google_drive.connector_auth import delete_service_account_key
 from danswer.connectors.google_drive.connector_auth import get_auth_url
@@ -43,13 +45,13 @@
     get_google_drive_creds_for_authorized_user,
 )
 from danswer.connectors.google_drive.connector_auth import get_service_account_key
+from danswer.connectors.google_drive.connector_auth import GOOGLE_DRIVE_SCOPES
 from danswer.connectors.google_drive.connector_auth import (
     update_credential_access_tokens,
 )
 from danswer.connectors.google_drive.connector_auth import upsert_google_app_cred
 from danswer.connectors.google_drive.connector_auth import upsert_service_account_key
 from danswer.connectors.google_drive.connector_auth import verify_csrf
-from danswer.connectors.google_drive.constants import DB_CREDENTIALS_DICT_TOKEN_KEY
 from danswer.db.connector import create_connector
 from danswer.db.connector import delete_connector
 from danswer.db.connector import fetch_connector_by_id
@@ -294,7 +296,7 @@ def upsert_service_account_credential(
     try:
         credential_base = build_service_account_creds(
             DocumentSource.GOOGLE_DRIVE,
-            delegated_user_email=service_account_credential_request.google_drive_delegated_user,
+            primary_admin_email=service_account_credential_request.google_drive_primary_admin,
         )
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -320,7 +322,7 @@ def upsert_gmail_service_account_credential(
     try:
         credential_base = build_service_account_creds(
             DocumentSource.GMAIL,
-            delegated_user_email=service_account_credential_request.gmail_delegated_user,
+            primary_admin_email=service_account_credential_request.gmail_delegated_user,
         )
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -348,27 +350,14 @@ def check_drive_tokens(
         return AuthStatus(authenticated=False)
     token_json_str = str(db_credentials.credential_json[DB_CREDENTIALS_DICT_TOKEN_KEY])
     google_drive_creds = get_google_drive_creds_for_authorized_user(
-        token_json_str=token_json_str
+        token_json_str=token_json_str,
+        scopes=GOOGLE_DRIVE_SCOPES,
     )
     if google_drive_creds is None:
         return AuthStatus(authenticated=False)
     return AuthStatus(authenticated=True)
 
 
-@router.get("/admin/connector/google-drive/authorize/{credential_id}")
-def admin_google_drive_auth(
-    response: Response, credential_id: str, _: User = Depends(current_admin_user)
-) -> AuthUrl:
-    # set a cookie that we can read in the callback (used for `verify_csrf`)
-    response.set_cookie(
-        key=_GOOGLE_DRIVE_CREDENTIAL_ID_COOKIE_NAME,
-        value=credential_id,
-        httponly=True,
-        max_age=600,
-    )
-    return AuthUrl(auth_url=get_auth_url(credential_id=int(credential_id)))
-
-
 @router.post("/admin/connector/file/upload")
 def upload_files(
     files: list[UploadFile],
@@ -951,10 +940,11 @@ def google_drive_callback(
         )
     credential_id = int(credential_id_cookie)
     verify_csrf(credential_id, callback.state)
-    if (
-        update_credential_access_tokens(callback.code, credential_id, user, db_session)
-        is None
-    ):
+
+    credentials: Credentials | None = update_credential_access_tokens(
+        callback.code, credential_id, user, db_session
+    )
+    if credentials is None:
         raise HTTPException(
             status_code=500, detail="Unable to fetch Google Drive access tokens"
         )
diff --git a/backend/danswer/server/documents/models.py b/backend/danswer/server/documents/models.py
index fcbc0a76a12..e45d6eabff0 100644
--- a/backend/danswer/server/documents/models.py
+++ b/backend/danswer/server/documents/models.py
@@ -377,16 +377,16 @@ class GoogleServiceAccountKey(BaseModel):
 
 
 class GoogleServiceAccountCredentialRequest(BaseModel):
-    google_drive_delegated_user: str | None = None  # email of user to impersonate
+    google_drive_primary_admin: str | None = None  # email of user to impersonate
     gmail_delegated_user: str | None = None  # email of user to impersonate
 
     @model_validator(mode="after")
     def check_user_delegation(self) -> "GoogleServiceAccountCredentialRequest":
-        if (self.google_drive_delegated_user is None) == (
+        if (self.google_drive_primary_admin is None) == (
             self.gmail_delegated_user is None
         ):
             raise ValueError(
-                "Exactly one of google_drive_delegated_user or gmail_delegated_user must be set"
+                "Exactly one of google_drive_primary_admin or gmail_delegated_user must be set"
             )
         return self
 
diff --git a/backend/ee/danswer/background/celery/apps/beat.py b/backend/ee/danswer/background/celery/apps/beat.py
index bee219e2471..980eb5e3214 100644
--- a/backend/ee/danswer/background/celery/apps/beat.py
+++ b/backend/ee/danswer/background/celery/apps/beat.py
@@ -13,12 +13,12 @@
     {
         "name": "sync-external-doc-permissions",
         "task": "check_sync_external_doc_permissions_task",
-        "schedule": timedelta(seconds=5),  # TODO: optimize this
+        "schedule": timedelta(seconds=30),  # TODO: optimize this
     },
     {
         "name": "sync-external-group-permissions",
         "task": "check_sync_external_group_permissions_task",
-        "schedule": timedelta(seconds=5),  # TODO: optimize this
+        "schedule": timedelta(seconds=60),  # TODO: optimize this
     },
     {
         "name": "autogenerate_usage_report",
diff --git a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
index f1e805d46d7..d1df0cb0846 100644
--- a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
@@ -1,144 +1,119 @@
-from collections.abc import Iterator
 from datetime import datetime
 from datetime import timezone
 from typing import Any
-from typing import cast
 
-from googleapiclient.discovery import build  # type: ignore
-from googleapiclient.errors import HttpError  # type: ignore
 from sqlalchemy.orm import Session
 
 from danswer.access.models import ExternalAccess
-from danswer.connectors.factory import instantiate_connector
-from danswer.connectors.google_drive.connector_auth import (
-    get_google_drive_creds,
-)
-from danswer.connectors.interfaces import PollConnector
-from danswer.connectors.models import InputType
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
+from danswer.connectors.models import SlimDocument
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
-from danswer.utils.retry_wrapper import retry_builder
 from ee.danswer.db.document import upsert_document_external_perms__no_commit
 
-# Google Drive APIs are quite flakey and may 500 for an
-# extended period of time. Trying to combat here by adding a very
-# long retry period (~20 minutes of trying every minute)
-add_retries = retry_builder(tries=5, delay=5, max_delay=30)
-
-
 logger = setup_logger()
 
+_PERMISSION_ID_PERMISSION_MAP: dict[str, dict[str, Any]] = {}
 
-def _get_docs_with_additional_info(
-    db_session: Session,
-    cc_pair: ConnectorCredentialPair,
-) -> dict[str, Any]:
-    # Get all document ids that need their permissions updated
-    runnable_connector = instantiate_connector(
-        db_session=db_session,
-        source=cc_pair.connector.source,
-        input_type=InputType.POLL,
-        connector_specific_config=cc_pair.connector.connector_specific_config,
-        credential=cc_pair.credential,
-    )
-
-    assert isinstance(runnable_connector, PollConnector)
 
+def _get_slim_doc_generator(
+    cc_pair: ConnectorCredentialPair,
+    google_drive_connector: GoogleDriveConnector,
+) -> GenerateSlimDocumentOutput:
     current_time = datetime.now(timezone.utc)
     start_time = (
         cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc).timestamp()
         if cc_pair.last_time_perm_sync
         else 0.0
     )
-    cc_pair.last_time_perm_sync = current_time
 
-    doc_batch_generator = runnable_connector.poll_source(
+    return google_drive_connector.retrieve_all_slim_documents(
         start=start_time, end=current_time.timestamp()
     )
 
-    docs_with_additional_info = {
-        doc.id: doc.additional_info
-        for doc_batch in doc_batch_generator
-        for doc in doc_batch
-    }
-
-    return docs_with_additional_info
-
-
-def _fetch_permissions_paginated(
-    drive_service: Any, drive_file_id: str
-) -> Iterator[dict[str, Any]]:
-    next_token = None
-
-    # Get paginated permissions for the file id
-    while True:
-        try:
-            permissions_resp: dict[str, Any] = add_retries(
-                lambda: (
-                    drive_service.permissions()
-                    .list(
-                        fileId=drive_file_id,
-                        fields="permissions(emailAddress, type, domain)",
-                        supportsAllDrives=True,
-                        pageToken=next_token,
-                    )
-                    .execute()
-                )
-            )()
-        except HttpError as e:
-            if e.resp.status == 404:
-                logger.warning(f"Document with id {drive_file_id} not found: {e}")
-                break
-            elif e.resp.status == 403:
-                logger.warning(
-                    f"Access denied for retrieving document permissions: {e}"
-                )
-                break
-            else:
-                logger.error(f"Failed to fetch permissions: {e}")
-                raise
-
-        for permission in permissions_resp.get("permissions", []):
-            yield permission
-
-        next_token = permissions_resp.get("nextPageToken")
-        if not next_token:
-            break
-
-
-def _fetch_google_permissions_for_document_id(
-    db_session: Session,
-    drive_file_id: str,
-    credentials_json: dict[str, str],
-    company_google_domains: list[str],
-) -> ExternalAccess:
-    # Authenticate and construct service
-    google_drive_creds, _ = get_google_drive_creds(
-        credentials_json,
+
+def _fetch_permissions_for_permission_ids(
+    google_drive_connector: GoogleDriveConnector,
+    permission_ids: list[str],
+    permission_info: dict[str, Any],
+) -> list[dict[str, Any]]:
+    doc_id = permission_info.get("doc_id")
+    if not permission_info or not doc_id:
+        return []
+
+    # Check cache first for all permission IDs
+    permissions = [
+        _PERMISSION_ID_PERMISSION_MAP[pid]
+        for pid in permission_ids
+        if pid in _PERMISSION_ID_PERMISSION_MAP
+    ]
+
+    # If we found all permissions in cache, return them
+    if len(permissions) == len(permission_ids):
+        return permissions
+
+    owner_email = permission_info.get("owner_email")
+    drive_service = google_drive_connector.get_google_resource(user_email=owner_email)
+
+    # Otherwise, fetch all permissions and update cache
+    fetched_permissions = execute_paginated_retrieval(
+        retrieval_function=drive_service.permissions().list,
+        list_key="permissions",
+        fileId=doc_id,
+        fields="permissions(id, emailAddress, type, domain)",
+        supportsAllDrives=True,
     )
-    if not google_drive_creds.valid:
-        raise ValueError("Invalid Google Drive credentials")
 
-    drive_service = build("drive", "v3", credentials=google_drive_creds)
+    permissions_for_doc_id = []
+    # Update cache and return all permissions
+    for permission in fetched_permissions:
+        permissions_for_doc_id.append(permission)
+        _PERMISSION_ID_PERMISSION_MAP[permission["id"]] = permission
+
+    return permissions_for_doc_id
+
 
+def _get_permissions_from_slim_doc(
+    google_drive_connector: GoogleDriveConnector,
+    slim_doc: SlimDocument,
+) -> ExternalAccess:
+    permission_info = slim_doc.perm_sync_data or {}
+
+    permissions_list = permission_info.get("permissions", [])
+    if not permissions_list:
+        if permission_ids := permission_info.get("permission_ids"):
+            permissions_list = _fetch_permissions_for_permission_ids(
+                google_drive_connector=google_drive_connector,
+                permission_ids=permission_ids,
+                permission_info=permission_info,
+            )
+        if not permissions_list:
+            logger.warning(f"No permissions found for document {slim_doc.id}")
+            return ExternalAccess(
+                external_user_emails=set(),
+                external_user_group_ids=set(),
+                is_public=False,
+            )
+
+    company_domain = google_drive_connector.google_domain
     user_emails: set[str] = set()
     group_emails: set[str] = set()
     public = False
-    for permission in _fetch_permissions_paginated(drive_service, drive_file_id):
+    for permission in permissions_list:
         permission_type = permission["type"]
         if permission_type == "user":
             user_emails.add(permission["emailAddress"])
         elif permission_type == "group":
             group_emails.add(permission["emailAddress"])
-        elif permission_type == "domain":
-            if permission["domain"] in company_google_domains:
+        elif permission_type == "domain" and company_domain:
+            if permission["domain"] == company_domain:
                 public = True
         elif permission_type == "anyone":
             public = True
 
-    batch_add_non_web_user_if_not_exists__no_commit(db_session, list(user_emails))
-
     return ExternalAccess(
         external_user_emails=user_emails,
         external_user_group_ids=group_emails,
@@ -156,32 +131,26 @@ def gdrive_doc_sync(
     it in postgres so that when it gets created later, the permissions are
     already populated
     """
-    sync_details = cc_pair.auto_sync_options
-    if sync_details is None:
-        logger.error("Sync details not found for Google Drive")
-        raise ValueError("Sync details not found for Google Drive")
-
-    # Here we run the connector to grab all the ids
-    # this may grab ids before they are indexed but that is fine because
-    # we create a document in postgres to hold the permissions info
-    # until the indexing job has a chance to run
-    docs_with_additional_info = _get_docs_with_additional_info(
-        db_session=db_session,
-        cc_pair=cc_pair,
+    google_drive_connector = GoogleDriveConnector(
+        **cc_pair.connector.connector_specific_config
     )
-
-    for doc_id, doc_additional_info in docs_with_additional_info.items():
-        ext_access = _fetch_google_permissions_for_document_id(
-            db_session=db_session,
-            drive_file_id=doc_additional_info,
-            credentials_json=cc_pair.credential.credential_json,
-            company_google_domains=[
-                cast(dict[str, str], sync_details)["company_domain"]
-            ],
-        )
-        upsert_document_external_perms__no_commit(
-            db_session=db_session,
-            doc_id=doc_id,
-            external_access=ext_access,
-            source_type=cc_pair.connector.source,
-        )
+    google_drive_connector.load_credentials(cc_pair.credential.credential_json)
+
+    slim_doc_generator = _get_slim_doc_generator(cc_pair, google_drive_connector)
+
+    for slim_doc_batch in slim_doc_generator:
+        for slim_doc in slim_doc_batch:
+            ext_access = _get_permissions_from_slim_doc(
+                google_drive_connector=google_drive_connector,
+                slim_doc=slim_doc,
+            )
+            batch_add_non_web_user_if_not_exists__no_commit(
+                db_session=db_session,
+                emails=list(ext_access.external_user_emails),
+            )
+            upsert_document_external_perms__no_commit(
+                db_session=db_session,
+                doc_id=slim_doc.id,
+                external_access=ext_access,
+                source_type=cc_pair.connector.source,
+            )
diff --git a/backend/ee/danswer/external_permissions/google_drive/group_sync.py b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
index ab0f62a886b..c3afa962392 100644
--- a/backend/ee/danswer/external_permissions/google_drive/group_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
@@ -1,136 +1,48 @@
-from collections.abc import Iterator
-from typing import Any
-
-from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
-from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
-from googleapiclient.discovery import build  # type: ignore
-from googleapiclient.errors import HttpError  # type: ignore
 from sqlalchemy.orm import Session
 
-from danswer.connectors.google_drive.connector_auth import (
-    get_google_drive_creds,
-)
-from danswer.connectors.google_drive.constants import FETCH_GROUPS_SCOPES
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
-from danswer.utils.retry_wrapper import retry_builder
 from ee.danswer.db.external_perm import ExternalUserGroup
 from ee.danswer.db.external_perm import replace_user__ext_group_for_cc_pair__no_commit
 
 logger = setup_logger()
 
 
-# Google Drive APIs are quite flakey and may 500 for an
-# extended period of time. Trying to combat here by adding a very
-# long retry period (~20 minutes of trying every minute)
-add_retries = retry_builder(tries=5, delay=5, max_delay=30)
-
-
-def _fetch_groups_paginated(
-    google_drive_creds: ServiceAccountCredentials | OAuthCredentials,
-    identity_source: str | None = None,
-    customer_id: str | None = None,
-) -> Iterator[dict[str, Any]]:
-    # Note that Google Drive does not use of update the user_cache as the user email
-    # comes directly with the call to fetch the groups, therefore this is not a valid
-    # place to save on requests
-    if identity_source is None and customer_id is None:
-        raise ValueError(
-            "Either identity_source or customer_id must be provided to fetch groups"
-        )
-
-    cloud_identity_service = build(
-        "cloudidentity", "v1", credentials=google_drive_creds
-    )
-    parent = (
-        f"identitysources/{identity_source}"
-        if identity_source
-        else f"customers/{customer_id}"
-    )
-
-    while True:
-        try:
-            groups_resp: dict[str, Any] = add_retries(
-                lambda: (cloud_identity_service.groups().list(parent=parent).execute())
-            )()
-            for group in groups_resp.get("groups", []):
-                yield group
-
-            next_token = groups_resp.get("nextPageToken")
-            if not next_token:
-                break
-        except HttpError as e:
-            if e.resp.status == 404 or e.resp.status == 403:
-                break
-            logger.error(f"Error fetching groups: {e}")
-            raise
-
-
-def _fetch_group_members_paginated(
-    google_drive_creds: ServiceAccountCredentials | OAuthCredentials,
-    group_name: str,
-) -> Iterator[dict[str, Any]]:
-    cloud_identity_service = build(
-        "cloudidentity", "v1", credentials=google_drive_creds
-    )
-    next_token = None
-    while True:
-        try:
-            membership_info = add_retries(
-                lambda: (
-                    cloud_identity_service.groups()
-                    .memberships()
-                    .searchTransitiveMemberships(
-                        parent=group_name, pageToken=next_token
-                    )
-                    .execute()
-                )
-            )()
-
-            for member in membership_info.get("memberships", []):
-                yield member
-
-            next_token = membership_info.get("nextPageToken")
-            if not next_token:
-                break
-        except HttpError as e:
-            if e.resp.status == 404 or e.resp.status == 403:
-                break
-            logger.error(f"Error fetching group members: {e}")
-            raise
-
-
 def gdrive_group_sync(
     db_session: Session,
     cc_pair: ConnectorCredentialPair,
 ) -> None:
-    sync_details = cc_pair.auto_sync_options
-    if sync_details is None:
-        logger.error("Sync details not found for Google Drive")
-        raise ValueError("Sync details not found for Google Drive")
-
-    google_drive_creds, _ = get_google_drive_creds(
-        cc_pair.credential.credential_json,
-        scopes=FETCH_GROUPS_SCOPES,
+    google_drive_connector = GoogleDriveConnector(
+        **cc_pair.connector.connector_specific_config
     )
+    google_drive_connector.load_credentials(cc_pair.credential.credential_json)
+
+    admin_service = google_drive_connector.get_google_resource("admin", "directory_v1")
 
     danswer_groups: list[ExternalUserGroup] = []
-    for group in _fetch_groups_paginated(
-        google_drive_creds,
-        identity_source=sync_details.get("identity_source"),
-        customer_id=sync_details.get("customer_id"),
+    for group in execute_paginated_retrieval(
+        admin_service.groups().list,
+        list_key="groups",
+        domain=google_drive_connector.google_domain,
+        fields="groups(email)",
     ):
         # The id is the group email
-        group_email = group["groupKey"]["id"]
+        group_email = group["email"]
 
+        # Gather group member emails
         group_member_emails: list[str] = []
-        for member in _fetch_group_members_paginated(google_drive_creds, group["name"]):
-            member_keys = member["preferredMemberKey"]
-            member_emails = [member_key["id"] for member_key in member_keys]
-            for member_email in member_emails:
-                group_member_emails.append(member_email)
-
+        for member in execute_paginated_retrieval(
+            admin_service.members().list,
+            list_key="members",
+            groupKey=group_email,
+            fields="members(email)",
+        ):
+            group_member_emails.append(member["email"])
+
+        # Add group members to DB and get their IDs
         group_members = batch_add_non_web_user_if_not_exists__no_commit(
             db_session=db_session, emails=group_member_emails
         )
diff --git a/backend/ee/danswer/external_permissions/permission_sync.py b/backend/ee/danswer/external_permissions/permission_sync.py
index ba5bbbd4921..94a0b4bfa8e 100644
--- a/backend/ee/danswer/external_permissions/permission_sync.py
+++ b/backend/ee/danswer/external_permissions/permission_sync.py
@@ -59,6 +59,7 @@ def run_external_doc_permission_sync(
     source_type = cc_pair.connector.source
 
     doc_sync_func = DOC_PERMISSIONS_FUNC_MAP.get(source_type)
+    last_time_perm_sync = cc_pair.last_time_perm_sync
 
     if doc_sync_func is None:
         raise ValueError(
@@ -110,4 +111,5 @@ def run_external_doc_permission_sync(
         logger.info(f"Successfully synced docs for {source_type}")
     except Exception:
         logger.exception("Error Syncing Document Permissions")
+        cc_pair.last_time_perm_sync = last_time_perm_sync
         db_session.rollback()
diff --git a/backend/tests/daily/connectors/google_drive/conftest.py b/backend/tests/daily/connectors/google_drive/conftest.py
new file mode 100644
index 00000000000..0b516d0359c
--- /dev/null
+++ b/backend/tests/daily/connectors/google_drive/conftest.py
@@ -0,0 +1,98 @@
+import json
+import os
+from collections.abc import Callable
+
+import pytest
+
+from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.google_drive.connector_auth import DB_CREDENTIALS_DICT_TOKEN_KEY
+from danswer.connectors.google_drive.connector_auth import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
+)
+
+
+def load_env_vars(env_file: str = ".env") -> None:
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    env_path = os.path.join(current_dir, env_file)
+    try:
+        with open(env_path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    key, value = line.split("=", 1)
+                    os.environ[key] = value.strip()
+        print("Successfully loaded environment variables")
+    except FileNotFoundError:
+        print(f"File {env_file} not found")
+
+
+# Load environment variables at the module level
+load_env_vars()
+
+
+@pytest.fixture
+def google_drive_oauth_connector_factory() -> Callable[..., GoogleDriveConnector]:
+    def _connector_factory(
+        primary_admin_email: str = "admin@onyx-test.com",
+        include_shared_drives: bool = True,
+        shared_drive_urls: str | None = None,
+        include_my_drives: bool = True,
+        my_drive_emails: str | None = None,
+        shared_folder_urls: str | None = None,
+    ) -> GoogleDriveConnector:
+        connector = GoogleDriveConnector(
+            include_shared_drives=include_shared_drives,
+            shared_drive_urls=shared_drive_urls,
+            include_my_drives=include_my_drives,
+            my_drive_emails=my_drive_emails,
+            shared_folder_urls=shared_folder_urls,
+        )
+
+        json_string = os.environ["GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR"]
+        refried_json_string = json.loads(json_string)
+
+        credentials_json = {
+            DB_CREDENTIALS_DICT_TOKEN_KEY: refried_json_string,
+            DB_CREDENTIALS_PRIMARY_ADMIN_KEY: primary_admin_email,
+        }
+        connector.load_credentials(credentials_json)
+        return connector
+
+    return _connector_factory
+
+
+@pytest.fixture
+def google_drive_service_acct_connector_factory() -> (
+    Callable[..., GoogleDriveConnector]
+):
+    def _connector_factory(
+        primary_admin_email: str = "admin@onyx-test.com",
+        include_shared_drives: bool = True,
+        shared_drive_urls: str | None = None,
+        include_my_drives: bool = True,
+        my_drive_emails: str | None = None,
+        shared_folder_urls: str | None = None,
+    ) -> GoogleDriveConnector:
+        print("Creating GoogleDriveConnector with service account credentials")
+        connector = GoogleDriveConnector(
+            include_shared_drives=include_shared_drives,
+            shared_drive_urls=shared_drive_urls,
+            include_my_drives=include_my_drives,
+            my_drive_emails=my_drive_emails,
+            shared_folder_urls=shared_folder_urls,
+        )
+
+        json_string = os.environ["GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR"]
+        refried_json_string = json.loads(json_string)
+
+        # Load Service Account Credentials
+        connector.load_credentials(
+            {
+                KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY: refried_json_string,
+                DB_CREDENTIALS_PRIMARY_ADMIN_KEY: primary_admin_email,
+            }
+        )
+        return connector
+
+    return _connector_factory
diff --git a/backend/tests/daily/connectors/google_drive/helpers.py b/backend/tests/daily/connectors/google_drive/helpers.py
new file mode 100644
index 00000000000..a1bc8feec38
--- /dev/null
+++ b/backend/tests/daily/connectors/google_drive/helpers.py
@@ -0,0 +1,164 @@
+from collections.abc import Sequence
+
+from danswer.connectors.models import Document
+
+ALL_FILES = list(range(0, 60))
+SHARED_DRIVE_FILES = list(range(20, 25))
+
+
+_ADMIN_FILE_IDS = list(range(0, 5))
+_TEST_USER_1_FILE_IDS = list(range(5, 10))
+_TEST_USER_2_FILE_IDS = list(range(10, 15))
+_TEST_USER_3_FILE_IDS = list(range(15, 20))
+_SHARED_DRIVE_1_FILE_IDS = list(range(20, 25))
+_FOLDER_1_FILE_IDS = list(range(25, 30))
+_FOLDER_1_1_FILE_IDS = list(range(30, 35))
+_FOLDER_1_2_FILE_IDS = list(range(35, 40))
+_SHARED_DRIVE_2_FILE_IDS = list(range(40, 45))
+_FOLDER_2_FILE_IDS = list(range(45, 50))
+_FOLDER_2_1_FILE_IDS = list(range(50, 55))
+_FOLDER_2_2_FILE_IDS = list(range(55, 60))
+
+_PUBLIC_FOLDER_RANGE = _FOLDER_1_2_FILE_IDS
+_PUBLIC_FILE_IDS = list(range(55, 57))
+PUBLIC_RANGE = _PUBLIC_FOLDER_RANGE + _PUBLIC_FILE_IDS
+
+_SHARED_DRIVE_1_URL = "https://drive.google.com/drive/folders/0AC_OJ4BkMd4kUk9PVA"
+# Group 1 is given access to this folder
+_FOLDER_1_URL = (
+    "https://drive.google.com/drive/folders/1d3I7U3vUZMDziF1OQqYRkB8Jp2s_GWUn"
+)
+_FOLDER_1_1_URL = (
+    "https://drive.google.com/drive/folders/1aR33-zwzl_mnRAwH55GgtWTE-4A4yWWI"
+)
+_FOLDER_1_2_URL = (
+    "https://drive.google.com/drive/folders/1IO0X55VhvLXf4mdxzHxuKf4wxrDBB6jq"
+)
+_SHARED_DRIVE_2_URL = "https://drive.google.com/drive/folders/0ABKspIh7P4f4Uk9PVA"
+_FOLDER_2_URL = (
+    "https://drive.google.com/drive/folders/1lNpCJ1teu8Se0louwL0oOHK9nEalskof"
+)
+_FOLDER_2_1_URL = (
+    "https://drive.google.com/drive/folders/1XeDOMWwxTDiVr9Ig2gKum3Zq_Wivv6zY"
+)
+_FOLDER_2_2_URL = (
+    "https://drive.google.com/drive/folders/1RKlsexA8h7NHvBAWRbU27MJotic7KXe3"
+)
+
+_ADMIN_EMAIL = "admin@onyx-test.com"
+_TEST_USER_1_EMAIL = "test_user_1@onyx-test.com"
+_TEST_USER_2_EMAIL = "test_user_2@onyx-test.com"
+_TEST_USER_3_EMAIL = "test_user_3@onyx-test.com"
+
+# Dictionary for ranges
+DRIVE_ID_MAPPING: dict[str, list[int]] = {
+    "ADMIN": _ADMIN_FILE_IDS,
+    "TEST_USER_1": _TEST_USER_1_FILE_IDS,
+    "TEST_USER_2": _TEST_USER_2_FILE_IDS,
+    "TEST_USER_3": _TEST_USER_3_FILE_IDS,
+    "SHARED_DRIVE_1": _SHARED_DRIVE_1_FILE_IDS,
+    "FOLDER_1": _FOLDER_1_FILE_IDS,
+    "FOLDER_1_1": _FOLDER_1_1_FILE_IDS,
+    "FOLDER_1_2": _FOLDER_1_2_FILE_IDS,
+    "SHARED_DRIVE_2": _SHARED_DRIVE_2_FILE_IDS,
+    "FOLDER_2": _FOLDER_2_FILE_IDS,
+    "FOLDER_2_1": _FOLDER_2_1_FILE_IDS,
+    "FOLDER_2_2": _FOLDER_2_2_FILE_IDS,
+}
+
+# Dictionary for emails
+EMAIL_MAPPING: dict[str, str] = {
+    "ADMIN": _ADMIN_EMAIL,
+    "TEST_USER_1": _TEST_USER_1_EMAIL,
+    "TEST_USER_2": _TEST_USER_2_EMAIL,
+    "TEST_USER_3": _TEST_USER_3_EMAIL,
+}
+
+# Dictionary for URLs
+URL_MAPPING: dict[str, str] = {
+    "SHARED_DRIVE_1": _SHARED_DRIVE_1_URL,
+    "FOLDER_1": _FOLDER_1_URL,
+    "FOLDER_1_1": _FOLDER_1_1_URL,
+    "FOLDER_1_2": _FOLDER_1_2_URL,
+    "SHARED_DRIVE_2": _SHARED_DRIVE_2_URL,
+    "FOLDER_2": _FOLDER_2_URL,
+    "FOLDER_2_1": _FOLDER_2_1_URL,
+    "FOLDER_2_2": _FOLDER_2_2_URL,
+}
+
+# Dictionary for access permissions
+# All users have access to their own My Drive as well as public files
+ACCESS_MAPPING: dict[str, list[int]] = {
+    # Admin has access to everything in shared
+    "ADMIN": (
+        _ADMIN_FILE_IDS
+        + _SHARED_DRIVE_1_FILE_IDS
+        + _FOLDER_1_FILE_IDS
+        + _FOLDER_1_1_FILE_IDS
+        + _FOLDER_1_2_FILE_IDS
+        + _SHARED_DRIVE_2_FILE_IDS
+        + _FOLDER_2_FILE_IDS
+        + _FOLDER_2_1_FILE_IDS
+        + _FOLDER_2_2_FILE_IDS
+    ),
+    # This user has access to drive 1
+    # This user has redundant access to folder 1 because of group access
+    # This user has been given individual access to files in Admin's My Drive
+    "TEST_USER_1": (
+        _TEST_USER_1_FILE_IDS
+        + _SHARED_DRIVE_1_FILE_IDS
+        + _FOLDER_1_FILE_IDS
+        + _FOLDER_1_1_FILE_IDS
+        + _FOLDER_1_2_FILE_IDS
+        + list(range(0, 2))
+    ),
+    # Group 1 includes this user, giving access to folder 1
+    # This user has also been given access to folder 2-1
+    # This user has also been given individual access to files in folder 2
+    "TEST_USER_2": (
+        _TEST_USER_2_FILE_IDS
+        + _FOLDER_1_FILE_IDS
+        + _FOLDER_1_1_FILE_IDS
+        + _FOLDER_1_2_FILE_IDS
+        + _FOLDER_2_1_FILE_IDS
+        + list(range(45, 47))
+    ),
+    # This user can only see his own files and public files
+    "TEST_USER_3": _TEST_USER_3_FILE_IDS,
+}
+
+
+file_name_template = "file_{}.txt"
+file_text_template = "This is file {}"
+
+
+def print_discrepencies(expected: set[str], retrieved: set[str]) -> None:
+    if expected != retrieved:
+        print(expected)
+        print(retrieved)
+        print("Extra:")
+        print(retrieved - expected)
+        print("Missing:")
+        print(expected - retrieved)
+
+
+def assert_retrieved_docs_match_expected(
+    retrieved_docs: list[Document], expected_file_ids: Sequence[int]
+) -> None:
+    expected_file_names = {
+        file_name_template.format(file_id) for file_id in expected_file_ids
+    }
+    expected_file_texts = {
+        file_text_template.format(file_id) for file_id in expected_file_ids
+    }
+
+    retrieved_file_names = set([doc.semantic_identifier for doc in retrieved_docs])
+    retrieved_texts = set([doc.sections[0].text for doc in retrieved_docs])
+
+    # Check file names
+    print_discrepencies(expected_file_names, retrieved_file_names)
+    assert expected_file_names == retrieved_file_names
+
+    # Check file texts
+    print_discrepencies(expected_file_texts, retrieved_texts)
+    assert expected_file_texts == retrieved_texts
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
new file mode 100644
index 00000000000..f39b15600b4
--- /dev/null
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
@@ -0,0 +1,246 @@
+import time
+from collections.abc import Callable
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.models import Document
+from tests.daily.connectors.google_drive.helpers import (
+    assert_retrieved_docs_match_expected,
+)
+from tests.daily.connectors.google_drive.helpers import DRIVE_ID_MAPPING
+from tests.daily.connectors.google_drive.helpers import EMAIL_MAPPING
+from tests.daily.connectors.google_drive.helpers import URL_MAPPING
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_include_all(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_include_all")
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=True,
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should get everything in shared and admin's My Drive with oauth
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["ADMIN"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_include_shared_drives_only(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_include_shared_drives_only")
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=False,
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should only get shared drives
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_include_my_drives_only(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_include_my_drives_only")
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=True,
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should only get everyone's My Drives
+    expected_file_ids = DRIVE_ID_MAPPING["ADMIN"]
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_drive_one_only(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_drive_one_only")
+    drive_urls = [
+        URL_MAPPING["SHARED_DRIVE_1"],
+    ]
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=False,
+        shared_drive_urls=",".join([str(url) for url in drive_urls]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # We ignore shared_drive_urls if include_shared_drives is False
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_folder_and_shared_drive(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_folder_and_shared_drive")
+    drive_urls = [URL_MAPPING["SHARED_DRIVE_1"]]
+    folder_urls = [URL_MAPPING["FOLDER_2"]]
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=True,
+        shared_drive_urls=",".join([str(url) for url in drive_urls]),
+        shared_folder_urls=",".join([str(url) for url in folder_urls]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["ADMIN"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_folders_only(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_folders_only")
+    folder_urls = [
+        URL_MAPPING["FOLDER_1_1"],
+        URL_MAPPING["FOLDER_1_2"],
+        URL_MAPPING["FOLDER_2_1"],
+        URL_MAPPING["FOLDER_2_2"],
+    ]
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=False,
+        shared_folder_urls=",".join([str(url) for url in folder_urls]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_specific_emails(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_specific_emails")
+    my_drive_emails = [
+        EMAIL_MAPPING["TEST_USER_1"],
+        EMAIL_MAPPING["TEST_USER_3"],
+    ]
+    connector = google_drive_oauth_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=True,
+        my_drive_emails=",".join([str(email) for email in my_drive_emails]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # No matter who is specified, when using oauth, if include_my_drives is True,
+    # we will get all the files from the admin's My Drive
+    expected_file_ids = DRIVE_ID_MAPPING["ADMIN"]
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py b/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
new file mode 100644
index 00000000000..b36a53b30f6
--- /dev/null
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
@@ -0,0 +1,257 @@
+import time
+from collections.abc import Callable
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.models import Document
+from tests.daily.connectors.google_drive.helpers import (
+    assert_retrieved_docs_match_expected,
+)
+from tests.daily.connectors.google_drive.helpers import DRIVE_ID_MAPPING
+from tests.daily.connectors.google_drive.helpers import EMAIL_MAPPING
+from tests.daily.connectors.google_drive.helpers import URL_MAPPING
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_include_all(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_include_all")
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=True,
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should get everything
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["ADMIN"]
+        + DRIVE_ID_MAPPING["TEST_USER_1"]
+        + DRIVE_ID_MAPPING["TEST_USER_2"]
+        + DRIVE_ID_MAPPING["TEST_USER_3"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_include_shared_drives_only(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_include_shared_drives_only")
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=False,
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should only get shared drives
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_include_my_drives_only(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_include_my_drives_only")
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=True,
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should only get everyone's My Drives
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["ADMIN"]
+        + DRIVE_ID_MAPPING["TEST_USER_1"]
+        + DRIVE_ID_MAPPING["TEST_USER_2"]
+        + DRIVE_ID_MAPPING["TEST_USER_3"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_drive_one_only(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_drive_one_only")
+    urls = [URL_MAPPING["SHARED_DRIVE_1"]]
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=False,
+        shared_drive_urls=",".join([str(url) for url in urls]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # We ignore shared_drive_urls if include_shared_drives is False
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_folder_and_shared_drive(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_folder_and_shared_drive")
+    drive_urls = [
+        URL_MAPPING["SHARED_DRIVE_1"],
+    ]
+    folder_urls = [URL_MAPPING["FOLDER_2"]]
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=True,
+        shared_drive_urls=",".join([str(url) for url in drive_urls]),
+        shared_folder_urls=",".join([str(url) for url in folder_urls]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    # Should
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["ADMIN"]
+        + DRIVE_ID_MAPPING["TEST_USER_1"]
+        + DRIVE_ID_MAPPING["TEST_USER_2"]
+        + DRIVE_ID_MAPPING["TEST_USER_3"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_folders_only(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_folders_only")
+    folder_urls = [
+        URL_MAPPING["FOLDER_1_1"],
+        URL_MAPPING["FOLDER_1_2"],
+        URL_MAPPING["FOLDER_2_1"],
+        URL_MAPPING["FOLDER_2_2"],
+    ]
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=False,
+        shared_folder_urls=",".join([str(url) for url in folder_urls]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_specific_emails(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    print("\n\nRunning test_specific_emails")
+    my_drive_emails = [
+        EMAIL_MAPPING["TEST_USER_1"],
+        EMAIL_MAPPING["TEST_USER_3"],
+    ]
+    connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=True,
+        my_drive_emails=",".join([str(email) for email in my_drive_emails]),
+    )
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(0, time.time()):
+        retrieved_docs.extend(doc_batch)
+
+    expected_file_ids = (
+        DRIVE_ID_MAPPING["TEST_USER_1"] + DRIVE_ID_MAPPING["TEST_USER_3"]
+    )
+    assert_retrieved_docs_match_expected(
+        retrieved_docs=retrieved_docs,
+        expected_file_ids=expected_file_ids,
+    )
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py b/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
new file mode 100644
index 00000000000..e731c8b27ce
--- /dev/null
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
@@ -0,0 +1,174 @@
+import time
+from collections.abc import Callable
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from danswer.access.models import ExternalAccess
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from ee.danswer.external_permissions.google_drive.doc_sync import (
+    _get_permissions_from_slim_doc,
+)
+from tests.daily.connectors.google_drive.helpers import ACCESS_MAPPING
+from tests.daily.connectors.google_drive.helpers import DRIVE_ID_MAPPING
+from tests.daily.connectors.google_drive.helpers import EMAIL_MAPPING
+from tests.daily.connectors.google_drive.helpers import file_name_template
+from tests.daily.connectors.google_drive.helpers import print_discrepencies
+from tests.daily.connectors.google_drive.helpers import PUBLIC_RANGE
+
+
+def get_keys_available_to_user_from_access_map(
+    user_email: str,
+    group_map: dict[str, list[str]],
+    access_map: dict[str, ExternalAccess],
+) -> list[str]:
+    """
+    Extracts the names of the files available to the user from the access map
+    through their own email or group memberships or public access
+    """
+    group_emails_for_user = []
+    for group_email, user_in_group_email_list in group_map.items():
+        if user_email in user_in_group_email_list:
+            group_emails_for_user.append(group_email)
+
+    accessible_file_names_for_user = []
+    for file_name, external_access in access_map.items():
+        if external_access.is_public:
+            accessible_file_names_for_user.append(file_name)
+        elif user_email in external_access.external_user_emails:
+            accessible_file_names_for_user.append(file_name)
+        elif any(
+            group_email in external_access.external_user_group_ids
+            for group_email in group_emails_for_user
+        ):
+            accessible_file_names_for_user.append(file_name)
+    return accessible_file_names_for_user
+
+
+def assert_correct_access_for_user(
+    user_email: str,
+    expected_access_ids: list[int],
+    group_map: dict[str, list[str]],
+    retrieved_access_map: dict[str, ExternalAccess],
+) -> None:
+    """
+    compares the expected access range of the user to the keys available to the user
+    retrieved from the source
+    """
+    retrieved_keys_available_to_user = get_keys_available_to_user_from_access_map(
+        user_email, group_map, retrieved_access_map
+    )
+    retrieved_file_names = set(retrieved_keys_available_to_user)
+
+    # Combine public and user-specific access IDs
+    all_accessible_ids = expected_access_ids + PUBLIC_RANGE
+    expected_file_names = {file_name_template.format(i) for i in all_accessible_ids}
+
+    print_discrepencies(expected_file_names, retrieved_file_names)
+
+    assert expected_file_names == retrieved_file_names
+
+
+# This function is supposed to map to the group_sync.py file for the google drive connector
+# TODO: Call it directly
+def get_group_map(google_drive_connector: GoogleDriveConnector) -> dict[str, list[str]]:
+    admin_service = google_drive_connector.get_google_resource("admin", "directory_v1")
+
+    group_map: dict[str, list[str]] = {}
+    for group in execute_paginated_retrieval(
+        admin_service.groups().list,
+        list_key="groups",
+        domain=google_drive_connector.google_domain,
+        fields="groups(email)",
+    ):
+        # The id is the group email
+        group_email = group["email"]
+
+        # Gather group member emails
+        group_member_emails: list[str] = []
+        for member in execute_paginated_retrieval(
+            admin_service.members().list,
+            list_key="members",
+            groupKey=group_email,
+            fields="members(email)",
+        ):
+            group_member_emails.append(member["email"])
+        group_map[group_email] = group_member_emails
+    return group_map
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_all_permissions(
+    mock_get_api_key: MagicMock,
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    google_drive_connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=True,
+        include_my_drives=True,
+    )
+
+    access_map: dict[str, ExternalAccess] = {}
+    for slim_doc_batch in google_drive_connector.retrieve_all_slim_documents(
+        0, time.time()
+    ):
+        for slim_doc in slim_doc_batch:
+            access_map[
+                (slim_doc.perm_sync_data or {})["name"]
+            ] = _get_permissions_from_slim_doc(
+                google_drive_connector=google_drive_connector,
+                slim_doc=slim_doc,
+            )
+
+    for file_name, external_access in access_map.items():
+        print(file_name, external_access)
+
+    expected_file_range = (
+        DRIVE_ID_MAPPING["ADMIN"]
+        + DRIVE_ID_MAPPING["TEST_USER_1"]
+        + DRIVE_ID_MAPPING["TEST_USER_2"]
+        + DRIVE_ID_MAPPING["TEST_USER_3"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_1"]
+        + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + DRIVE_ID_MAPPING["SHARED_DRIVE_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2"]
+        + DRIVE_ID_MAPPING["FOLDER_2_1"]
+        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+    )
+
+    # Should get everything
+    assert len(access_map) == len(expected_file_range)
+
+    group_map = get_group_map(google_drive_connector)
+
+    print("groups:\n", group_map)
+
+    assert_correct_access_for_user(
+        user_email=EMAIL_MAPPING["ADMIN"],
+        expected_access_ids=ACCESS_MAPPING["ADMIN"],
+        group_map=group_map,
+        retrieved_access_map=access_map,
+    )
+    assert_correct_access_for_user(
+        user_email=EMAIL_MAPPING["TEST_USER_1"],
+        expected_access_ids=ACCESS_MAPPING["TEST_USER_1"],
+        group_map=group_map,
+        retrieved_access_map=access_map,
+    )
+
+    assert_correct_access_for_user(
+        user_email=EMAIL_MAPPING["TEST_USER_2"],
+        expected_access_ids=ACCESS_MAPPING["TEST_USER_2"],
+        group_map=group_map,
+        retrieved_access_map=access_map,
+    )
+    assert_correct_access_for_user(
+        user_email=EMAIL_MAPPING["TEST_USER_3"],
+        expected_access_ids=ACCESS_MAPPING["TEST_USER_3"],
+        group_map=group_map,
+        retrieved_access_map=access_map,
+    )
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index a5b6b3bd1ee..7f0e4c664f3 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -431,6 +431,12 @@ export default function AddConnector({
                   setSelectedFiles={setSelectedFiles}
                   selectedFiles={selectedFiles}
                   connector={connector}
+                  currentCredential={
+                    currentCredential ||
+                    liveGDriveCredential ||
+                    liveGmailCredential ||
+                    null
+                  }
                 />
               </Card>
             )}
diff --git a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
index 956e0c24597..05deec472a6 100644
--- a/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/ConnectorInput/ListInput.tsx
@@ -4,18 +4,22 @@ import { TextArrayField } from "@/components/admin/connectors/Field";
 import { useFormikContext } from "formik";
 
 interface ListInputProps {
-  field: ListOption;
+  name: string;
+  label: string | ((credential: any) => string);
+  description: string | ((credential: any) => string);
 }
 
-const ListInput: React.FC<ListInputProps> = ({ field }) => {
+const ListInput: React.FC<ListInputProps> = ({ name, label, description }) => {
   const { values } = useFormikContext<any>();
   return (
     <TextArrayField
-      name={field.name}
-      label={field.label}
+      name={name}
+      label={typeof label === "function" ? label(null) : label}
       values={values}
-      subtext={field.description}
-      placeholder={`Enter ${field.label.toLowerCase()}`}
+      subtext={
+        typeof description === "function" ? description(null) : description
+      }
+      placeholder={`Enter ${typeof label === "function" ? label(null) : label.toLowerCase()}`}
     />
   );
 };
diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
index 85237df2c7d..a6ac93441e6 100644
--- a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
@@ -13,6 +13,8 @@ import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import { AccessTypeForm } from "@/components/admin/connectors/AccessTypeForm";
 import { AccessTypeGroupSelector } from "@/components/admin/connectors/AccessTypeGroupSelector";
 import { ConfigurableSources } from "@/lib/types";
+import { Credential } from "@/lib/connectors/credentials";
+import CollapsibleSection from "@/app/admin/assistants/CollapsibleSection";
 
 export interface DynamicConnectionFormProps {
   config: ConnectionConfiguration;
@@ -20,19 +22,44 @@ export interface DynamicConnectionFormProps {
   setSelectedFiles: Dispatch<SetStateAction<File[]>>;
   values: any;
   connector: ConfigurableSources;
+  currentCredential: Credential<any> | null;
 }
 
-const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
-  config,
+interface RenderFieldProps {
+  field: any;
+  values: any;
+  selectedFiles: File[];
+  setSelectedFiles: Dispatch<SetStateAction<File[]>>;
+  connector: ConfigurableSources;
+  currentCredential: Credential<any> | null;
+}
+
+const RenderField: FC<RenderFieldProps> = ({
+  field,
+  values,
   selectedFiles,
   setSelectedFiles,
-  values,
   connector,
+  currentCredential,
 }) => {
-  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+  if (
+    field.visibleCondition &&
+    !field.visibleCondition(values, currentCredential)
+  ) {
+    return null;
+  }
+
+  const label =
+    typeof field.label === "function"
+      ? field.label(currentCredential)
+      : field.label;
+  const description =
+    typeof field.description === "function"
+      ? field.description(currentCredential)
+      : field.description;
 
-  const renderField = (field: any) => (
-    <div key={field.name}>
+  const fieldContent = (
+    <>
       {field.type === "file" ? (
         <FileUpload
           name={field.name}
@@ -42,46 +69,71 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
       ) : field.type === "zip" ? (
         <FileInput
           name={field.name}
-          label={field.label}
+          label={label}
           optional={field.optional}
-          description={field.description}
+          description={description}
         />
       ) : field.type === "list" ? (
-        <ListInput field={field} />
+        <ListInput name={field.name} label={label} description={description} />
       ) : field.type === "select" ? (
         <SelectInput
           name={field.name}
           optional={field.optional}
-          description={field.description}
+          description={description}
           options={field.options || []}
-          label={field.label}
+          label={label}
         />
       ) : field.type === "number" ? (
         <NumberInput
-          label={field.label}
+          label={label}
           optional={field.optional}
-          description={field.description}
+          description={description}
           name={field.name}
         />
       ) : field.type === "checkbox" ? (
         <AdminBooleanFormField
           checked={values[field.name]}
-          subtext={field.description}
+          subtext={description}
           name={field.name}
-          label={field.label}
+          label={label}
         />
       ) : (
         <TextFormField
-          subtext={field.description}
+          subtext={description}
           optional={field.optional}
           type={field.type}
-          label={field.label}
+          label={label}
           name={field.name}
+          isTextArea={true}
         />
       )}
-    </div>
+    </>
   );
 
+  if (
+    field.visibleCondition &&
+    field.visibleCondition(values, currentCredential)
+  ) {
+    return (
+      <CollapsibleSection prompt={label} key={field.name}>
+        {fieldContent}
+      </CollapsibleSection>
+    );
+  } else {
+    return <div key={field.name}>{fieldContent}</div>;
+  }
+};
+
+const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
+  config,
+  selectedFiles,
+  setSelectedFiles,
+  values,
+  connector,
+  currentCredential,
+}) => {
+  const [showAdvancedOptions, setShowAdvancedOptions] = useState(false);
+
   return (
     <>
       <h2 className="text-2xl font-bold text-text-800">{config.description}</h2>
@@ -97,7 +149,20 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
         name={"name"}
       />
 
-      {config.values.map((field) => !field.hidden && renderField(field))}
+      {config.values.map(
+        (field) =>
+          !field.hidden && (
+            <RenderField
+              key={field.name}
+              field={field}
+              values={values}
+              selectedFiles={selectedFiles}
+              setSelectedFiles={setSelectedFiles}
+              connector={connector}
+              currentCredential={currentCredential}
+            />
+          )
+      )}
 
       <AccessTypeForm connector={connector} />
       <AccessTypeGroupSelector />
@@ -108,7 +173,18 @@ const DynamicConnectionForm: FC<DynamicConnectionFormProps> = ({
             showAdvancedOptions={showAdvancedOptions}
             setShowAdvancedOptions={setShowAdvancedOptions}
           />
-          {showAdvancedOptions && config.advanced_values.map(renderField)}
+          {showAdvancedOptions &&
+            config.advanced_values.map((field) => (
+              <RenderField
+                key={field.name}
+                field={field}
+                values={values}
+                selectedFiles={selectedFiles}
+                setSelectedFiles={setSelectedFiles}
+                connector={connector}
+                currentCredential={currentCredential}
+              />
+            ))}
         </>
       )}
     </>
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
index 371bbef6dd1..03a73fe23e6 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
@@ -10,6 +10,7 @@ import { GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
 import Cookies from "js-cookie";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
+import { User } from "@/lib/types";
 import { Button as TremorButton } from "@tremor/react";
 import {
   Credential,
@@ -157,6 +158,7 @@ export const DriveJsonUploadSection = ({
   isAdmin,
 }: DriveJsonUploadSectionProps) => {
   const { mutate } = useSWRConfig();
+  const router = useRouter();
 
   if (serviceAccountCredentialData?.service_account_email) {
     return (
@@ -190,6 +192,7 @@ export const DriveJsonUploadSection = ({
                     message: "Successfully deleted service account key",
                     type: "success",
                   });
+                  router.refresh();
                 } else {
                   const errorMsg = await response.text();
                   setPopup({
@@ -307,9 +310,10 @@ interface DriveCredentialSectionProps {
   setPopup: (popupSpec: PopupSpec | null) => void;
   refreshCredentials: () => void;
   connectorExists: boolean;
+  user: User | null;
 }
 
-export const DriveOAuthSection = ({
+export const DriveAuthSection = ({
   googleDrivePublicCredential,
   googleDriveServiceAccountCredential,
   serviceAccountKeyData,
@@ -317,6 +321,7 @@ export const DriveOAuthSection = ({
   setPopup,
   refreshCredentials,
   connectorExists,
+  user,
 }: DriveCredentialSectionProps) => {
   const router = useRouter();
 
@@ -356,23 +361,23 @@ export const DriveOAuthSection = ({
     return (
       <div>
         <p className="text-sm mb-6">
-          When using a Google Drive Service Account, you can either have Danswer
-          act as the service account itself OR you can specify an account for
-          the service account to impersonate.
+          When using a Google Drive Service Account, you must specify the email
+          of the primary admin that you would like the service account to
+          impersonate.
           <br />
           <br />
-          If you want to use the service account itself, leave the{" "}
-          <b>&apos;User email to impersonate&apos;</b> field blank when
-          submitting. If you do choose this option, make sure you have shared
-          the documents you want to index with the service account.
+          Ideally, this account should be an owner/admin of the Google
+          Organization that owns the Google Drive(s) you want to index.
         </p>
 
         <Formik
           initialValues={{
-            google_drive_delegated_user: "",
+            google_drive_primary_admin: user?.email || "",
           }}
           validationSchema={Yup.object().shape({
-            google_drive_delegated_user: Yup.string().optional(),
+            google_drive_primary_admin: Yup.string().required(
+              "User email is required"
+            ),
           })}
           onSubmit={async (values, formikHelpers) => {
             formikHelpers.setSubmitting(true);
@@ -384,8 +389,7 @@ export const DriveOAuthSection = ({
                   "Content-Type": "application/json",
                 },
                 body: JSON.stringify({
-                  google_drive_delegated_user:
-                    values.google_drive_delegated_user,
+                  google_drive_primary_admin: values.google_drive_primary_admin,
                 }),
               }
             );
@@ -408,9 +412,9 @@ export const DriveOAuthSection = ({
           {({ isSubmitting }) => (
             <Form>
               <TextFormField
-                name="google_drive_delegated_user"
-                label="[Optional] User email to impersonate:"
-                subtext="If left blank, Danswer will use the service account itself."
+                name="google_drive_primary_admin"
+                label="User email to impersonate:"
+                subtext="Enter the email of the user whose Google Drive access you want to delegate to the service account."
               />
               <div className="flex">
                 <TremorButton type="submit" disabled={isSubmitting}>
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
index d8a14db03a1..518f9709d94 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
@@ -12,7 +12,7 @@ import {
   useConnectorCredentialIndexingStatus,
 } from "@/lib/hooks";
 import { Title } from "@tremor/react";
-import { DriveJsonUploadSection, DriveOAuthSection } from "./Credential";
+import { DriveJsonUploadSection, DriveAuthSection } from "./Credential";
 import {
   Credential,
   GoogleDriveCredentialJson,
@@ -22,7 +22,7 @@ import { GoogleDriveConfig } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
 
 const GDriveMain = ({}: {}) => {
-  const { isLoadingUser, isAdmin } = useUser();
+  const { isLoadingUser, isAdmin, user } = useUser();
 
   const {
     data: appCredentialData,
@@ -135,7 +135,7 @@ const GDriveMain = ({}: {}) => {
           <Title className="mb-2 mt-6 ml-auto mr-auto">
             Step 2: Authenticate with Danswer
           </Title>
-          <DriveOAuthSection
+          <DriveAuthSection
             setPopup={setPopup}
             refreshCredentials={refreshCredentials}
             googleDrivePublicCredential={googleDrivePublicCredential}
@@ -145,6 +145,7 @@ const GDriveMain = ({}: {}) => {
             appCredentialData={appCredentialData}
             serviceAccountKeyData={serviceAccountKeyData}
             connectorExists={googleDriveConnectorIndexingStatuses.length > 0}
+            user={user}
           />
         </>
       )}
diff --git a/web/src/components/admin/connectors/AccessTypeForm.tsx b/web/src/components/admin/connectors/AccessTypeForm.tsx
index 108b657b568..8993e28cdb3 100644
--- a/web/src/components/admin/connectors/AccessTypeForm.tsx
+++ b/web/src/components/admin/connectors/AccessTypeForm.tsx
@@ -9,6 +9,7 @@ import { useUser } from "@/components/user/UserProvider";
 import { useField } from "formik";
 import { AutoSyncOptions } from "./AutoSyncOptions";
 import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
+import { useEffect } from "react";
 
 function isValidAutoSyncSource(
   value: ConfigurableSources
@@ -28,6 +29,21 @@ export function AccessTypeForm({
   const isAutoSyncSupported = isValidAutoSyncSource(connector);
   const { isLoadingUser, isAdmin } = useUser();
 
+  useEffect(() => {
+    if (!isPaidEnterpriseEnabled) {
+      access_type_helpers.setValue("public");
+    } else if (isAutoSyncSupported) {
+      access_type_helpers.setValue("sync");
+    } else {
+      access_type_helpers.setValue("private");
+    }
+  }, [
+    isAutoSyncSupported,
+    isAdmin,
+    isPaidEnterpriseEnabled,
+    access_type_helpers,
+  ]);
+
   const options = [
     {
       name: "Private",
@@ -46,9 +62,9 @@ export function AccessTypeForm({
     });
   }
 
-  if (isAutoSyncSupported && isAdmin) {
+  if (isAutoSyncSupported && isAdmin && isPaidEnterpriseEnabled) {
     options.push({
-      name: "Auto Sync",
+      name: "Auto Sync Permissions",
       value: "sync",
       description:
         "We will automatically sync permissions from the source. A document will be searchable in Danswer if and only if the user performing the search has permission to access the document in the source.",
@@ -59,12 +75,13 @@ export function AccessTypeForm({
     <>
       {isPaidEnterpriseEnabled && isAdmin && (
         <>
-          <div className="flex gap-x-2 items-center">
+          <div>
             <label className="text-text-950 font-medium">Document Access</label>
+            <p className="text-sm text-text-500">
+              Control who has access to the documents indexed by this connector.
+            </p>
           </div>
-          <p className="text-sm text-text-500 mb-2">
-            Control who has access to the documents indexed by this connector.
-          </p>
+
           <DefaultDropdown
             options={options}
             selected={access_type.value}
@@ -75,11 +92,9 @@ export function AccessTypeForm({
           />
 
           {access_type.value === "sync" && isAutoSyncSupported && (
-            <div>
-              <AutoSyncOptions
-                connectorType={connector as ValidAutoSyncSources}
-              />
-            </div>
+            <AutoSyncOptions
+              connectorType={connector as ValidAutoSyncSources}
+            />
           )}
         </>
       )}
diff --git a/web/src/components/admin/connectors/ConnectorTitle.tsx b/web/src/components/admin/connectors/ConnectorTitle.tsx
index 269c72e905f..6e2da252aec 100644
--- a/web/src/components/admin/connectors/ConnectorTitle.tsx
+++ b/web/src/components/admin/connectors/ConnectorTitle.tsx
@@ -64,21 +64,6 @@ export const ConnectorTitle = ({
       "Jira Project URL",
       typedConnector.connector_specific_config.jira_project_url
     );
-  } else if (connector.source === "google_drive") {
-    const typedConnector = connector as Connector<GoogleDriveConfig>;
-    if (
-      typedConnector.connector_specific_config?.folder_paths &&
-      typedConnector.connector_specific_config?.folder_paths.length > 0
-    ) {
-      additionalMetadata.set(
-        "Folders",
-        typedConnector.connector_specific_config.folder_paths.join(", ")
-      );
-    }
-
-    if (!isPublic && owner) {
-      additionalMetadata.set("Owner", owner);
-    }
   } else if (connector.source === "slack") {
     const typedConnector = connector as Connector<SlackConfig>;
     if (
diff --git a/web/src/lib/connectors/AutoSyncOptionFields.tsx b/web/src/lib/connectors/AutoSyncOptionFields.tsx
index f6866a16991..4a8b44868e6 100644
--- a/web/src/lib/connectors/AutoSyncOptionFields.tsx
+++ b/web/src/lib/connectors/AutoSyncOptionFields.tsx
@@ -12,37 +12,6 @@ export const autoSyncConfigBySource: Record<
   >
 > = {
   confluence: {},
-  google_drive: {
-    customer_id: {
-      label: "Google Workspace Customer ID",
-      subtext: (
-        <>
-          The unique identifier for your Google Workspace account. To find this,
-          checkout the{" "}
-          <a
-            href="https://support.google.com/cloudidentity/answer/10070793"
-            target="_blank"
-            className="text-link"
-          >
-            guide from Google
-          </a>
-          .
-        </>
-      ),
-    },
-    company_domain: {
-      label: "Google Workspace Company Domain",
-      subtext: (
-        <>
-          The email domain for your Google Workspace account.
-          <br />
-          <br />
-          For example, if your email provided through Google Workspace looks
-          something like chris@danswer.ai, then your company domain is{" "}
-          <b>danswer.ai</b>
-        </>
-      ),
-    },
-  },
+  google_drive: {},
   slack: {},
 };
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index 4e7df383599..6e7ef1ad97f 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -2,6 +2,7 @@ import * as Yup from "yup";
 import { IsPublicGroupSelectorFormType } from "@/components/IsPublicGroupSelector";
 import { ConfigurableSources, ValidInputTypes, ValidSources } from "../types";
 import { AccessTypeGroupSelectorFormType } from "@/components/admin/connectors/AccessTypeGroupSelector";
+import { Credential } from "@/lib/connectors/credentials"; // Import Credential type
 
 export function isLoadState(connector_name: string): boolean {
   // TODO: centralize connector metadata like this somewhere instead of hardcoding it here
@@ -29,12 +30,18 @@ export type StringWithDescription = {
 };
 
 export interface Option {
-  label: string;
+  label: string | ((currentCredential: Credential<any> | null) => string);
   name: string;
-  description?: string;
+  description?:
+    | string
+    | ((currentCredential: Credential<any> | null) => string);
   query?: string;
   optional?: boolean;
   hidden?: boolean;
+  visibleCondition?: (
+    values: any,
+    currentCredential: Credential<any> | null
+  ) => boolean;
 }
 
 export interface SelectOption extends Option {
@@ -204,38 +211,59 @@ export const connectorConfigs: Record<
     description: "Configure Google Drive connector",
     values: [
       {
-        type: "list",
-        query: "Enter folder paths:",
-        label: "Folder Paths",
-        name: "folder_paths",
+        type: "checkbox",
+        label: "Include shared drives?",
+        description:
+          "This will allow Danswer to index everything in your shared drives.",
+        name: "include_shared_drives",
         optional: true,
+        default: true,
       },
       {
-        type: "checkbox",
-        query: "Include shared files?",
-        label: "Include Shared",
-        name: "include_shared",
-        optional: false,
-        default: false,
+        type: "text",
+        description:
+          "Enter a comma separated list of the URLs of the shared drives to index. Leave blank to index all shared drives.",
+        label: "Shared Drive URLs",
+        name: "shared_drive_urls",
+        visibleCondition: (values) => values.include_shared_drives,
+        optional: true,
       },
       {
         type: "checkbox",
-        query: "Follow shortcuts?",
-        label: "Follow Shortcuts",
-        name: "follow_shortcuts",
-        optional: false,
-        default: false,
+        label: (currentCredential) =>
+          currentCredential?.credential_json?.google_drive_tokens
+            ? "Include My Drive?"
+            : "Include Everyone's My Drive?",
+        description: (currentCredential) =>
+          currentCredential?.credential_json?.google_drive_tokens
+            ? "This will allow Danswer to index everything in your My Drive."
+            : "This will allow Danswer to index everything in everyone's My Drives.",
+        name: "include_my_drives",
+        optional: true,
+        default: true,
       },
       {
-        type: "checkbox",
-        query: "Only include organization public files?",
-        label: "Only Org Public",
-        name: "only_org_public",
-        optional: false,
-        default: false,
+        type: "text",
+        description:
+          "Enter a comma separated list of the emails of the users whose MyDrive you want to index. Leave blank to index all MyDrives.",
+        label: "My Drive Emails",
+        name: "my_drive_emails",
+        visibleCondition: (values, currentCredential) =>
+          values.include_my_drives &&
+          !currentCredential?.credential_json?.google_drive_tokens,
+        optional: true,
+      },
+    ],
+    advanced_values: [
+      {
+        type: "text",
+        description:
+          "Enter a comma separated list of the URLs of the folders located in Shared Drives to index. The files located in these folders (and all subfolders) will be indexed. Note: This will be in addition to the 'Include Shared Drives' and 'Shared Drive URLs' settings, so leave those blank if you only want to index the folders specified here.",
+        label: "Folder URLs",
+        name: "shared_folder_urls",
+        optional: true,
       },
     ],
-    advanced_values: [],
   },
   gmail: {
     description: "Configure Gmail connector",
@@ -1025,7 +1053,7 @@ export interface GitlabConfig {
 }
 
 export interface GoogleDriveConfig {
-  folder_paths?: string[];
+  parent_urls?: string[];
   include_shared?: boolean;
   follow_shortcuts?: boolean;
   only_org_public?: boolean;
diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts
index 532f8f6de76..73c788d3a96 100644
--- a/web/src/lib/connectors/credentials.ts
+++ b/web/src/lib/connectors/credentials.ts
@@ -58,6 +58,7 @@ export interface GmailCredentialJson {
 
 export interface GoogleDriveCredentialJson {
   google_drive_tokens: string;
+  google_drive_primary_admin: string;
 }
 
 export interface GmailServiceAccountCredentialJson {
@@ -67,7 +68,7 @@ export interface GmailServiceAccountCredentialJson {
 
 export interface GoogleDriveServiceAccountCredentialJson {
   google_drive_service_account_key: string;
-  google_drive_delegated_user: string;
+  google_drive_primary_admin: string;
 }
 
 export interface SlabCredentialJson {
@@ -331,7 +332,7 @@ export const credentialDisplayNames: Record<string, string> = {
 
   // Google Drive Service Account
   google_drive_service_account_key: "Google Drive Service Account Key",
-  google_drive_delegated_user: "Google Drive Delegated User",
+  google_drive_primary_admin: "Google Drive Delegated User",
 
   // Slab
   slab_bot_token: "Slab Bot Token",

From dc2dfeb5b822ebafbdf1b1d5f8f94590d6d1155a Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Thu, 31 Oct 2024 22:59:12 -0700
Subject: [PATCH 318/376] Fix pywikibot droppings (#2924)

* make pywikibot store its working files in a system provided temp directory

* move the config setting around

---------

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 backend/danswer/connectors/mediawiki/family.py              | 3 +++
 backend/danswer/connectors/mediawiki/wiki.py                | 4 +++-
 .../tests/unit/danswer/connectors/mediawiki/test_wiki.py    | 6 ++++--
 backend/throttle.ctrl                                       | 0
 4 files changed, 10 insertions(+), 3 deletions(-)
 delete mode 100644 backend/throttle.ctrl

diff --git a/backend/danswer/connectors/mediawiki/family.py b/backend/danswer/connectors/mediawiki/family.py
index 163bca2ef6b..7554dd0f3cd 100644
--- a/backend/danswer/connectors/mediawiki/family.py
+++ b/backend/danswer/connectors/mediawiki/family.py
@@ -3,6 +3,7 @@
 import builtins
 import functools
 import itertools
+import tempfile
 from typing import Any
 from unittest import mock
 from urllib.parse import urlparse
@@ -18,6 +19,8 @@
 
 logger = setup_logger()
 
+pywikibot.config.base_dir = tempfile.TemporaryDirectory().name
+
 
 @mock.patch.object(
     builtins, "print", lambda *args: logger.info("\t".join(map(str, args)))
diff --git a/backend/danswer/connectors/mediawiki/wiki.py b/backend/danswer/connectors/mediawiki/wiki.py
index 8a09fd6d500..27edea871dc 100644
--- a/backend/danswer/connectors/mediawiki/wiki.py
+++ b/backend/danswer/connectors/mediawiki/wiki.py
@@ -2,6 +2,7 @@
 
 import datetime
 import itertools
+import tempfile
 from collections.abc import Generator
 from collections.abc import Iterator
 from typing import Any
@@ -25,6 +26,8 @@
 
 logger = setup_logger()
 
+pywikibot.config.base_dir = tempfile.TemporaryDirectory().name
+
 
 def pywikibot_timestamp_to_utc_datetime(
     timestamp: pywikibot.time.Timestamp,
@@ -121,7 +124,6 @@ def __init__(
         self.batch_size = batch_size
 
         # short names can only have ascii letters and digits
-
         self.family = family_class_dispatch(hostname, "WikipediaConnector")()
         self.site = pywikibot.Site(fam=self.family, code=language_code)
         self.categories = [
diff --git a/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py b/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
index 78601446a66..9659c0afc0f 100644
--- a/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
+++ b/backend/tests/unit/danswer/connectors/mediawiki/test_wiki.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import datetime
+import tempfile
 from collections.abc import Iterable
 
 import pytest
@@ -9,7 +10,9 @@
 
 from danswer.connectors.mediawiki import wiki
 
-# These tests are disabled for now
+# Some of these tests are disabled for now due to flakiness with wikipedia as the backend
+
+pywikibot.config.base_dir = tempfile.TemporaryDirectory().name
 
 
 @pytest.fixture
@@ -17,7 +20,6 @@ def site() -> pywikibot.Site:
     return pywikibot.Site("en", "wikipedia")
 
 
-@pytest.mark.skip(reason="Test disabled")
 def test_pywikibot_timestamp_to_utc_datetime() -> None:
     timestamp_without_tzinfo = pywikibot.Timestamp(2023, 12, 27, 15, 38, 49)
     timestamp_min_timezone = timestamp_without_tzinfo.astimezone(datetime.timezone.min)
diff --git a/backend/throttle.ctrl b/backend/throttle.ctrl
deleted file mode 100644
index e69de29bb2d..00000000000

From 4a23c8702d51e46fac0ffc67a5d16f1337d00c35 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Fri, 1 Nov 2024 06:27:55 -0700
Subject: [PATCH 319/376] Quicky

---
 web/src/lib/connectors/connectors.tsx | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index 6e7ef1ad97f..8b6e285ec2b 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -1053,10 +1053,11 @@ export interface GitlabConfig {
 }
 
 export interface GoogleDriveConfig {
-  parent_urls?: string[];
-  include_shared?: boolean;
-  follow_shortcuts?: boolean;
-  only_org_public?: boolean;
+  include_shared_drives?: boolean;
+  shared_drive_urls?: string;
+  include_my_drives?: boolean;
+  my_drive_emails?: string;
+  shared_folder_urls?: string;
 }
 
 export interface GmailConfig {}

From 6d543f3d4fa53a2e5854bccda3e46613cddf4827 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 1 Nov 2024 12:34:30 -0700
Subject: [PATCH 320/376] Do not count API keys as users (#3022)

* don't count api keys as users

* typing
---
 backend/danswer/db/auth.py             | 9 +++++++--
 backend/danswer/server/manage/users.py | 6 +++---
 backend/ee/danswer/db/api_key.py       | 6 +++++-
 3 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/backend/danswer/db/auth.py b/backend/danswer/db/auth.py
index 53fba1e52dc..0a794f3c963 100644
--- a/backend/danswer/db/auth.py
+++ b/backend/danswer/db/auth.py
@@ -22,6 +22,7 @@
 from danswer.utils.variable_functionality import (
     fetch_versioned_implementation_with_fallback,
 )
+from ee.danswer.db.api_key import get_api_key_email_pattern
 
 
 def get_default_admin_user_emails() -> list[str]:
@@ -35,12 +36,16 @@ def get_default_admin_user_emails() -> list[str]:
     return get_default_admin_user_emails_fn()
 
 
-def get_total_users(db_session: Session) -> int:
+def get_total_users_count(db_session: Session) -> int:
     """
     Returns the total number of users in the system.
     This is the sum of users and invited users.
     """
-    user_count = db_session.query(User).count()
+    user_count = (
+        db_session.query(User)
+        .filter(~User.email.endswith(get_api_key_email_pattern()))  # type: ignore
+        .count()
+    )
     invited_users = len(get_invited_users())
     return user_count + invited_users
 
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index 9cb1cec1c1a..f0675c39282 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -37,7 +37,7 @@
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
-from danswer.db.auth import get_total_users
+from danswer.db.auth import get_total_users_count
 from danswer.db.engine import CURRENT_TENANT_ID_CONTEXTVAR
 from danswer.db.engine import get_session
 from danswer.db.models import AccessToken
@@ -226,7 +226,7 @@ def bulk_invite_users(
     try:
         logger.info("Registering tenant users")
         register_tenant_users(
-            CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users(db_session)
+            CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users_count(db_session)
         )
         if ENABLE_EMAIL_INVITES:
             try:
@@ -262,7 +262,7 @@ def remove_invited_user(
     try:
         if MULTI_TENANT:
             register_tenant_users(
-                CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users(db_session)
+                CURRENT_TENANT_ID_CONTEXTVAR.get(), get_total_users_count(db_session)
             )
     except Exception:
         logger.error(
diff --git a/backend/ee/danswer/db/api_key.py b/backend/ee/danswer/db/api_key.py
index c38f32a0f84..8bbdf7eaa9a 100644
--- a/backend/ee/danswer/db/api_key.py
+++ b/backend/ee/danswer/db/api_key.py
@@ -17,8 +17,12 @@
 from ee.danswer.server.api_key.models import APIKeyArgs
 
 
+def get_api_key_email_pattern() -> str:
+    return DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN
+
+
 def is_api_key_email_address(email: str) -> bool:
-    return email.endswith(f"{DANSWER_API_KEY_DUMMY_EMAIL_DOMAIN}")
+    return email.endswith(get_api_key_email_pattern())
 
 
 def fetch_api_keys(db_session: Session) -> list[ApiKeyDescriptor]:

From 753293cefbef221c671536bc18726aeb3f7efb39 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 1 Nov 2024 12:34:51 -0700
Subject: [PATCH 321/376] Basic multi tenant api key (#3004)

* basic multi tenant api key

* organization

* nit

* clean
---
 backend/ee/danswer/auth/api_key.py            | 33 +++++++-
 backend/ee/danswer/db/api_key.py              |  8 +-
 .../server/middleware/tenant_tracking.py      | 77 +++++++++++--------
 3 files changed, 84 insertions(+), 34 deletions(-)

diff --git a/backend/ee/danswer/auth/api_key.py b/backend/ee/danswer/auth/api_key.py
index 74e391ad1da..9ea827d27dc 100644
--- a/backend/ee/danswer/auth/api_key.py
+++ b/backend/ee/danswer/auth/api_key.py
@@ -1,5 +1,7 @@
 import secrets
 import uuid
+from urllib.parse import quote
+from urllib.parse import unquote
 
 from fastapi import Request
 from passlib.hash import sha256_crypt
@@ -30,8 +32,35 @@ class ApiKeyDescriptor(BaseModel):
     user_id: uuid.UUID
 
 
-def generate_api_key() -> str:
-    return _API_KEY_PREFIX + secrets.token_urlsafe(_API_KEY_LEN)
+def generate_api_key(tenant_id: str | None = None) -> str:
+    # For backwards compatibility, if no tenant_id, generate old style key
+    if not tenant_id:
+        return _API_KEY_PREFIX + secrets.token_urlsafe(_API_KEY_LEN)
+
+    encoded_tenant = quote(tenant_id)  # URL encode the tenant ID
+    return f"{_API_KEY_PREFIX}{encoded_tenant}.{secrets.token_urlsafe(_API_KEY_LEN)}"
+
+
+def extract_tenant_from_api_key_header(request: Request) -> str | None:
+    """Extract tenant ID from request. Returns None if auth is disabled or invalid format."""
+    raw_api_key_header = request.headers.get(
+        _API_KEY_HEADER_ALTERNATIVE_NAME
+    ) or request.headers.get(_API_KEY_HEADER_NAME)
+
+    if not raw_api_key_header or not raw_api_key_header.startswith(_BEARER_PREFIX):
+        return None
+
+    api_key = raw_api_key_header[len(_BEARER_PREFIX) :].strip()
+
+    if not api_key.startswith(_API_KEY_PREFIX):
+        return None
+
+    parts = api_key[len(_API_KEY_PREFIX) :].split(".", 1)
+    if len(parts) != 2:
+        return None
+
+    tenant_id = parts[0]
+    return unquote(tenant_id) if tenant_id else None
 
 
 def hash_api_key(api_key: str) -> str:
diff --git a/backend/ee/danswer/db/api_key.py b/backend/ee/danswer/db/api_key.py
index 8bbdf7eaa9a..4cc0774f129 100644
--- a/backend/ee/danswer/db/api_key.py
+++ b/backend/ee/danswer/db/api_key.py
@@ -15,6 +15,8 @@
 from ee.danswer.auth.api_key import generate_api_key
 from ee.danswer.auth.api_key import hash_api_key
 from ee.danswer.server.api_key.models import APIKeyArgs
+from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.configs import MULTI_TENANT
 
 
 def get_api_key_email_pattern() -> str:
@@ -64,7 +66,11 @@ def insert_api_key(
     db_session: Session, api_key_args: APIKeyArgs, user_id: uuid.UUID | None
 ) -> ApiKeyDescriptor:
     std_password_helper = PasswordHelper()
-    api_key = generate_api_key()
+
+    # Get tenant_id from context var (will be default schema for single tenant)
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+
+    api_key = generate_api_key(tenant_id if MULTI_TENANT else None)
     api_key_user_id = uuid.uuid4()
 
     display_name = api_key_args.name or UNNAMED_KEY_PLACEHOLDER
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index 8c076e57d5c..f9fe75425e0 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -10,6 +10,7 @@
 
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.db.engine import is_valid_schema_name
+from ee.danswer.auth.api_key import extract_tenant_from_api_key_header
 from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
@@ -21,40 +22,54 @@ async def set_tenant_id(
         request: Request, call_next: Callable[[Request], Awaitable[Response]]
     ) -> Response:
         try:
-            if not MULTI_TENANT:
-                tenant_id = POSTGRES_DEFAULT_SCHEMA
-            else:
-                token = request.cookies.get("fastapiusersauth")
-
-                if token:
-                    try:
-                        payload = jwt.decode(
-                            token,
-                            USER_AUTH_SECRET,
-                            audience=["fastapi-users:auth"],
-                            algorithms=["HS256"],
-                        )
-                        tenant_id = payload.get("tenant_id", POSTGRES_DEFAULT_SCHEMA)
-                        if not is_valid_schema_name(tenant_id):
-                            raise HTTPException(
-                                status_code=400, detail="Invalid tenant ID format"
-                            )
-                    except jwt.InvalidTokenError:
-                        tenant_id = POSTGRES_DEFAULT_SCHEMA
-                    except Exception as e:
-                        logger.error(
-                            f"Unexpected error in set_tenant_id_middleware: {str(e)}"
-                        )
-                        raise HTTPException(
-                            status_code=500, detail="Internal server error"
-                        )
-                else:
-                    tenant_id = POSTGRES_DEFAULT_SCHEMA
+            tenant_id = POSTGRES_DEFAULT_SCHEMA
+
+            if MULTI_TENANT:
+                tenant_id = _get_tenant_id_from_request(request, logger)
 
             CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
-            response = await call_next(request)
-            return response
+            return await call_next(request)
 
         except Exception as e:
             logger.error(f"Error in tenant ID middleware: {str(e)}")
             raise
+
+
+def _get_tenant_id_from_request(request: Request, logger: logging.LoggerAdapter) -> str:
+    # First check for API key
+    tenant_id = extract_tenant_from_api_key_header(request)
+    if tenant_id is not None:
+        return tenant_id
+
+    # Check for cookie-based auth
+    token = request.cookies.get("fastapiusersauth")
+    if not token:
+        return POSTGRES_DEFAULT_SCHEMA
+
+    try:
+        payload = jwt.decode(
+            token,
+            USER_AUTH_SECRET,
+            audience=["fastapi-users:auth"],
+            algorithms=["HS256"],
+        )
+        tenant_id_from_payload = payload.get("tenant_id", POSTGRES_DEFAULT_SCHEMA)
+
+        # Since payload.get() can return None, ensure we have a string
+        tenant_id = (
+            str(tenant_id_from_payload)
+            if tenant_id_from_payload is not None
+            else POSTGRES_DEFAULT_SCHEMA
+        )
+
+        if not is_valid_schema_name(tenant_id):
+            raise HTTPException(status_code=400, detail="Invalid tenant ID format")
+
+        return tenant_id
+
+    except jwt.InvalidTokenError:
+        return POSTGRES_DEFAULT_SCHEMA
+
+    except Exception as e:
+        logger.error(f"Unexpected error in set_tenant_id_middleware: {str(e)}")
+        raise HTTPException(status_code=500, detail="Internal server error")

From d66b81a902f31e24b5cd88d15700988196c70bb4 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 1 Nov 2024 12:34:52 -0700
Subject: [PATCH 322/376] Feat/certificate (#2998)

* first pass

* simplify

* remove now unneeded COPY command

* minor clean up

* k

* nit
---
 backend/Dockerfile                            | 11 ----
 .../docker_compose/docker-compose.dev.yml     | 36 ++++++----
 .../docker_compose/docker-compose.prod.yml    | 40 +++++------
 .../api_server-service-deployment.yaml        | 66 +++++++++----------
 .../kubernetes/background-deployment.yaml     | 47 ++++++++-----
 5 files changed, 108 insertions(+), 92 deletions(-)

diff --git a/backend/Dockerfile b/backend/Dockerfile
index f8feefdda76..2f8de6e7996 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -12,7 +12,6 @@ ARG DANSWER_VERSION=0.8-dev
 ENV DANSWER_VERSION=${DANSWER_VERSION} \
     DANSWER_RUNNING_IN_DOCKER="true"
 
-ARG CA_CERT_CONTENT=""
 
 RUN echo "DANSWER_VERSION: ${DANSWER_VERSION}"
 # Install system dependencies
@@ -39,15 +38,6 @@ RUN apt-get update && \
     apt-get clean
 
 
-# Conditionally write the CA certificate and update certificates
-RUN if [ -n "$CA_CERT_CONTENT" ]; then \
-    echo "Adding custom CA certificate"; \
-    echo "$CA_CERT_CONTENT" > /usr/local/share/ca-certificates/my-ca.crt && \
-    chmod 644 /usr/local/share/ca-certificates/my-ca.crt && \
-    update-ca-certificates; \
-else \
-    echo "No custom CA certificate provided"; \
-fi
 
 # Install Python dependencies
 # Remove py which is pulled in by retry, py is not needed and is a CVE
@@ -87,7 +77,6 @@ RUN apt-get update && \
 RUN python -c "from tokenizers import Tokenizer; \
 Tokenizer.from_pretrained('nomic-ai/nomic-embed-text-v1')"
 
-
 # Pre-downloading NLTK for setups with limited egress
 RUN python -c "import nltk; \
 nltk.download('stopwords', quiet=True); \
diff --git a/deployment/docker_compose/docker-compose.dev.yml b/deployment/docker_compose/docker-compose.dev.yml
index 7b31689c8f3..34be985c01d 100644
--- a/deployment/docker_compose/docker-compose.dev.yml
+++ b/deployment/docker_compose/docker-compose.dev.yml
@@ -79,8 +79,8 @@ services:
       # Leave this on pretty please? Nothing sensitive is collected!
       # https://docs.danswer.dev/more/telemetry
       - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-}
-      - LOG_LEVEL=${LOG_LEVEL:-info}  # Set to debug to get more fine-grained logs
-      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-}  # LiteLLM Verbose Logging
+      - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs
+      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging
       # Log all of Danswer prompts and interactions with the LLM
       - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
       # If set to `true` will enable additional logs about Vespa query performance
@@ -96,7 +96,7 @@ services:
 
       # Chat Configs
       - HARD_DELETE_CHATS=${HARD_DELETE_CHATS:-}
-      
+
       # Enterprise Edition only
       - ENABLE_PAID_ENTERPRISE_EDITION_FEATURES=${ENABLE_PAID_ENTERPRISE_EDITION_FEATURES:-false}
       - API_KEY_HASH_ROUNDS=${API_KEY_HASH_ROUNDS:-}
@@ -114,7 +114,12 @@ services:
     build:
       context: ../../backend
       dockerfile: Dockerfile
-    command: /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+    command: >
+      /bin/sh -c "
+      if [ -f /etc/ssl/certs/custom-ca.crt ]; then
+        update-ca-certificates;
+      fi &&
+      /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf"
     depends_on:
       - relational_db
       - index
@@ -197,8 +202,8 @@ services:
       # Leave this on pretty please? Nothing sensitive is collected!
       # https://docs.danswer.dev/more/telemetry
       - DISABLE_TELEMETRY=${DISABLE_TELEMETRY:-}
-      - LOG_LEVEL=${LOG_LEVEL:-info}  # Set to debug to get more fine-grained logs
-      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-}  # LiteLLM Verbose Logging
+      - LOG_LEVEL=${LOG_LEVEL:-info} # Set to debug to get more fine-grained logs
+      - LOG_ALL_MODEL_INTERACTIONS=${LOG_ALL_MODEL_INTERACTIONS:-} # LiteLLM Verbose Logging
       # Log all of Danswer prompts and interactions with the LLM
       - LOG_DANSWER_MODEL_INTERACTIONS=${LOG_DANSWER_MODEL_INTERACTIONS:-}
       - LOG_VESPA_TIMING_INFORMATION=${LOG_VESPA_TIMING_INFORMATION:-}
@@ -215,6 +220,11 @@ services:
       options:
         max-size: "50m"
         max-file: "6"
+    # Uncomment the following lines if you need to include a custom CA certificate
+    # Optional volume mount for CA certificate
+    # volumes:
+    #   # Maps to the CA_CERT_PATH environment variable in the Dockerfile
+    #   - ${CA_CERT_PATH:-./custom-ca.crt}:/etc/ssl/certs/custom-ca.crt:ro
 
   web_server:
     image: danswer/danswer-web-server:${IMAGE_TAG:-latest}
@@ -295,7 +305,7 @@ services:
       # Set to debug to get more fine-grained logs
       - LOG_LEVEL=${LOG_LEVEL:-info}
       - CLIENT_EMBEDDING_TIMEOUT=${CLIENT_EMBEDDING_TIMEOUT:-}
-      
+
       # Analytics Configs
       - SENTRY_DSN=${SENTRY_DSN:-}
     volumes:
@@ -338,7 +348,7 @@ services:
     image: nginx:1.23.4-alpine
     restart: always
     # nginx will immediately crash with `nginx: [emerg] host not found in upstream`
-    # if api_server / web_server are not up 
+    # if api_server / web_server are not up
     depends_on:
       - api_server
       - web_server
@@ -354,20 +364,20 @@ services:
       options:
         max-size: "50m"
         max-file: "6"
-    # The specified script waits for the api_server to start up. 
-    # Without this we've seen issues where nginx shows no error logs but 
+    # The specified script waits for the api_server to start up.
+    # Without this we've seen issues where nginx shows no error logs but
     # does not recieve any traffic
     # NOTE: we have to use dos2unix to remove Carriage Return chars from the file
     # in order to make this work on both Unix-like systems and windows
-    command: > 
+    command: >
       /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
-      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"  
+      && /etc/nginx/conf.d/run-nginx.sh app.conf.template.dev"
 
   cache:
     image: redis:7.4-alpine
     restart: always
     ports:
-      - '6379:6379'
+      - "6379:6379"
     # docker silently mounts /data even without an explicit volume mount, which enables
     # persistence. explicitly setting save and appendonly forces ephemeral behavior.
     command: redis-server --save "" --appendonly no
diff --git a/deployment/docker_compose/docker-compose.prod.yml b/deployment/docker_compose/docker-compose.prod.yml
index eaeaa7646b7..d4ba321736e 100644
--- a/deployment/docker_compose/docker-compose.prod.yml
+++ b/deployment/docker_compose/docker-compose.prod.yml
@@ -5,7 +5,8 @@ services:
       context: ../../backend
       dockerfile: Dockerfile
     command: >
-      /bin/sh -c "alembic upgrade head &&
+      /bin/sh -c "
+      alembic upgrade head &&
       echo \"Starting Danswer Api Server\" &&
       uvicorn danswer.main:app --host 0.0.0.0 --port 8080"
     depends_on:
@@ -29,14 +30,17 @@ services:
       options:
         max-size: "50m"
         max-file: "6"
-
-
   background:
     image: danswer/danswer-backend:${IMAGE_TAG:-latest}
     build:
       context: ../../backend
       dockerfile: Dockerfile
-    command: /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+    command: >
+      /bin/sh -c "
+      if [ -f /etc/ssl/certs/custom-ca.crt ]; then
+        update-ca-certificates;
+      fi &&
+      /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf"
     depends_on:
       - relational_db
       - index
@@ -60,6 +64,10 @@ services:
       options:
         max-size: "50m"
         max-file: "6"
+    # Uncomment the following lines if you need to include a custom CA certificate
+    # volumes:
+    #   # Maps to the CA_CERT_PATH environment variable in the Dockerfile
+    #   - ${CA_CERT_PATH:-./custom-ca.crt}:/etc/ssl/certs/custom-ca.crt:ro
 
   web_server:
     image: danswer/danswer-web-server:${IMAGE_TAG:-latest}
@@ -86,7 +94,6 @@ services:
         max-size: "50m"
         max-file: "6"
 
-
   relational_db:
     image: postgres:15.2-alpine
     command: -c 'max_connections=250'
@@ -102,7 +109,6 @@ services:
         max-size: "50m"
         max-file: "6"
 
-
   inference_model_server:
     image: danswer/danswer-model-server:${IMAGE_TAG:-latest}
     build:
@@ -129,7 +135,6 @@ services:
         max-size: "50m"
         max-file: "6"
 
-
   indexing_model_server:
     image: danswer/danswer-model-server:${IMAGE_TAG:-latest}
     build:
@@ -158,7 +163,6 @@ services:
         max-size: "50m"
         max-file: "6"
 
-
   # This container name cannot have an underscore in it due to Vespa expectations of the URL
   index:
     image: vespaengine/vespa:8.277.17
@@ -174,12 +178,11 @@ services:
         max-size: "50m"
         max-file: "6"
 
-
   nginx:
     image: nginx:1.23.4-alpine
     restart: always
     # nginx will immediately crash with `nginx: [emerg] host not found in upstream`
-    # if api_server / web_server are not up 
+    # if api_server / web_server are not up
     depends_on:
       - api_server
       - web_server
@@ -190,26 +193,25 @@ services:
       - ../data/nginx:/etc/nginx/conf.d
       - ../data/certbot/conf:/etc/letsencrypt
       - ../data/certbot/www:/var/www/certbot
-    # sleep a little bit to allow the web_server / api_server to start up. 
-    # Without this we've seen issues where nginx shows no error logs but 
+    # sleep a little bit to allow the web_server / api_server to start up.
+    # Without this we've seen issues where nginx shows no error logs but
     # does not recieve any traffic
     logging:
       driver: json-file
       options:
         max-size: "50m"
         max-file: "6"
-    # The specified script waits for the api_server to start up. 
-    # Without this we've seen issues where nginx shows no error logs but 
-    # does not recieve any traffic  
+    # The specified script waits for the api_server to start up.
+    # Without this we've seen issues where nginx shows no error logs but
+    # does not recieve any traffic
     # NOTE: we have to use dos2unix to remove Carriage Return chars from the file
     # in order to make this work on both Unix-like systems and windows
-    command: > 
+    command: >
       /bin/sh -c "dos2unix /etc/nginx/conf.d/run-nginx.sh 
       && /etc/nginx/conf.d/run-nginx.sh app.conf.template"
     env_file:
       - .env.nginx
 
-
   # follows https://pentacent.medium.com/nginx-and-lets-encrypt-with-docker-in-less-than-5-minutes-b4b8a60d3a71
   certbot:
     image: certbot/certbot
@@ -224,17 +226,15 @@ services:
         max-file: "6"
     entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
 
-
   cache:
     image: redis:7.4-alpine
     restart: always
     ports:
-      - '6379:6379'
+      - "6379:6379"
     # docker silently mounts /data even without an explicit volume mount, which enables
     # persistence. explicitly setting save and appendonly forces ephemeral behavior.
     command: redis-server --save "" --appendonly no
 
-
 volumes:
   db_volume:
   vespa_volume:
diff --git a/deployment/kubernetes/api_server-service-deployment.yaml b/deployment/kubernetes/api_server-service-deployment.yaml
index ccbbc906d61..0e1cd79af58 100644
--- a/deployment/kubernetes/api_server-service-deployment.yaml
+++ b/deployment/kubernetes/api_server-service-deployment.yaml
@@ -27,36 +27,36 @@ spec:
         app: api-server
     spec:
       containers:
-      - name: api-server
-        image: danswer/danswer-backend:latest
-        imagePullPolicy: IfNotPresent
-        command:
-          - "/bin/sh"
-          - "-c"
-          - |
-            alembic upgrade head &&
-            echo "Starting Danswer Api Server" &&
-            uvicorn danswer.main:app --host 0.0.0.0 --port 8080
-        ports:
-        - containerPort: 8080
-        # There are some extra values since this is shared between services
-        # There are no conflicts though, extra env variables are simply ignored
-        env:
-        - name: OAUTH_CLIENT_ID
-          valueFrom:
-            secretKeyRef:
-              name: danswer-secrets
-              key: google_oauth_client_id
-        - name: OAUTH_CLIENT_SECRET
-          valueFrom:
-            secretKeyRef:
-              name: danswer-secrets
-              key: google_oauth_client_secret
-        - name: REDIS_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: danswer-secrets
-              key: redis_password
-        envFrom:
-        - configMapRef:
-            name: env-configmap
+        - name: api-server
+          image: danswer/danswer-backend:latest
+          imagePullPolicy: IfNotPresent
+          command:
+            - "/bin/sh"
+            - "-c"
+            - |
+              alembic upgrade head &&
+              echo "Starting Danswer Api Server" &&
+              uvicorn danswer.main:app --host 0.0.0.0 --port 8080
+          ports:
+            - containerPort: 8080
+          # There are some extra values since this is shared between services
+          # There are no conflicts though, extra env variables are simply ignored
+          env:
+            - name: OAUTH_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: google_oauth_client_id
+            - name: OAUTH_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: google_oauth_client_secret
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+          envFrom:
+            - configMapRef:
+                name: env-configmap
diff --git a/deployment/kubernetes/background-deployment.yaml b/deployment/kubernetes/background-deployment.yaml
index dcb3c89a0c6..4989d889fdf 100644
--- a/deployment/kubernetes/background-deployment.yaml
+++ b/deployment/kubernetes/background-deployment.yaml
@@ -13,18 +13,35 @@ spec:
         app: background
     spec:
       containers:
-      - name: background
-        image: danswer/danswer-backend:latest
-        imagePullPolicy: IfNotPresent
-        command: ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
-        # There are some extra values since this is shared between services
-        # There are no conflicts though, extra env variables are simply ignored
-        env:
-        - name: REDIS_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: danswer-secrets
-              key: redis_password
-        envFrom:
-        - configMapRef:
-            name: env-configmap
\ No newline at end of file
+        - name: background
+          image: danswer/danswer-backend:latest
+          imagePullPolicy: IfNotPresent
+          command:
+            [
+              "/usr/bin/supervisord",
+              "-c",
+              "/etc/supervisor/conf.d/supervisord.conf",
+            ]
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+          envFrom:
+            - configMapRef:
+                name: env-configmap
+      # Uncomment the following lines if you need to include a custom CA certificate
+      # Optional volume mount for CA certificate
+      #   volumeMounts:
+      #     - name: my-ca-cert-volume
+      #       mountPath: /etc/ssl/certs/custom-ca.crt
+      #       subPath: my-ca.crt
+      # Optional volume for CA certificate
+      # volumes:
+      #   - name: my-cas-cert-volume
+      #     secret:
+      #       secretName: my-ca-cert
+      #       items:
+      #         - key: my-ca.crt
+      #           path: my-ca.crt

From ecf4923a3aeecd216515f6fc3f898f46712e9216 Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Fri, 1 Nov 2024 12:50:20 -0700
Subject: [PATCH 323/376] Fix answer with specified doc ids (#2703)

* Fix

Fix

Refactor

more

more

fix

refactor

Fix circular imports

Refactor

Move tests around

* Add quote support

* Testing

* More testing

* Fix image generation slowness

* Remove unused exception

* Fix UT

* fix stop generating

* minor typo

* minor logging updates for clarity

---------

Co-authored-by: pablodanswer <pablo@danswer.ai>
---
 backend/danswer/chat/models.py                |   2 +-
 backend/danswer/chat/process_message.py       |  92 ++-
 backend/danswer/llm/answering/answer.py       | 595 +++++-------------
 .../llm/answering/llm_response_handler.py     |  84 +++
 .../danswer/llm/answering/prompts/build.py    |  72 +--
 .../llm/answering/prompts/citations_prompt.py |  14 +-
 .../llm/answering/prompts/quotes_prompt.py    |  25 +-
 .../danswer/llm/answering/prune_and_merge.py  |   2 +-
 .../answer_response_handler.py                |  91 +++
 .../stream_processing/citation_processing.py  | 232 +++----
 .../stream_processing/quotes_processing.py    | 164 ++---
 .../answering/tool/tool_response_handler.py   | 207 ++++++
 backend/danswer/llm/utils.py                  |  22 +
 .../one_shot_answer/answer_question.py        |  37 +-
 .../danswer/server/features/persona/models.py |   2 +-
 backend/danswer/server/features/tool/api.py   |  14 +-
 .../server/query_and_chat/chat_backend.py     |  21 +-
 backend/danswer/tools/base_tool.py            |  59 ++
 backend/danswer/tools/built_in_tools.py       |  10 +-
 .../custom/custom_tool_prompt_builder.py      |  21 -
 backend/danswer/tools/tool.py                 |  27 +-
 .../custom/base_tool_types.py                 |   0
 .../custom/custom_tool.py                     |  42 +-
 .../custom/custom_tool_prompts.py             |   0
 .../custom/openapi_parsing.py                 |   0
 .../images/image_generation_tool.py           |  43 +-
 .../images/prompt.py                          |   0
 .../internet_search/internet_search_tool.py   |  48 +-
 .../internet_search/models.py                 |   0
 .../search/search_tool.py                     |  51 +-
 .../search/search_utils.py                    |   0
 .../search_like_tool_utils.py                 |  71 +++
 backend/danswer/tools/tool_runner.py          |   2 +-
 .../ee/danswer/server/query_and_chat/utils.py |   2 +-
 .../tests/dev_apis/test_simple_chat_api.py    |   3 +
 .../unit/danswer/llm/answering/conftest.py    | 113 ++++
 .../test_citation_processing.py               |  18 +-
 .../test_quote_processing.py                  | 447 +++++++------
 .../unit/danswer/llm/answering/test_answer.py | 405 ++++++++++++
 .../danswer/llm/answering/test_skip_gen_ai.py |  35 +-
 .../danswer/tools/custom/test_custom_tools.py |  24 +-
 41 files changed, 1987 insertions(+), 1110 deletions(-)
 create mode 100644 backend/danswer/llm/answering/llm_response_handler.py
 create mode 100644 backend/danswer/llm/answering/stream_processing/answer_response_handler.py
 create mode 100644 backend/danswer/llm/answering/tool/tool_response_handler.py
 create mode 100644 backend/danswer/tools/base_tool.py
 delete mode 100644 backend/danswer/tools/custom/custom_tool_prompt_builder.py
 rename backend/danswer/tools/{ => tool_implementations}/custom/base_tool_types.py (100%)
 rename backend/danswer/tools/{ => tool_implementations}/custom/custom_tool.py (88%)
 rename backend/danswer/tools/{ => tool_implementations}/custom/custom_tool_prompts.py (100%)
 rename backend/danswer/tools/{ => tool_implementations}/custom/openapi_parsing.py (100%)
 rename backend/danswer/tools/{ => tool_implementations}/images/image_generation_tool.py (84%)
 rename backend/danswer/tools/{ => tool_implementations}/images/prompt.py (100%)
 rename backend/danswer/tools/{ => tool_implementations}/internet_search/internet_search_tool.py (81%)
 rename backend/danswer/tools/{ => tool_implementations}/internet_search/models.py (100%)
 rename backend/danswer/tools/{ => tool_implementations}/search/search_tool.py (87%)
 rename backend/danswer/tools/{ => tool_implementations}/search/search_utils.py (100%)
 create mode 100644 backend/danswer/tools/tool_implementations/search_like_tool_utils.py
 create mode 100644 backend/tests/unit/danswer/llm/answering/conftest.py
 create mode 100644 backend/tests/unit/danswer/llm/answering/test_answer.py

diff --git a/backend/danswer/chat/models.py b/backend/danswer/chat/models.py
index 97d5b9e7275..d5925fc2ed9 100644
--- a/backend/danswer/chat/models.py
+++ b/backend/danswer/chat/models.py
@@ -10,7 +10,7 @@
 from danswer.search.enums import SearchType
 from danswer.search.models import RetrievalDocs
 from danswer.search.models import SearchResponse
-from danswer.tools.custom.base_tool_types import ToolResultType
+from danswer.tools.tool_implementations.custom.base_tool_types import ToolResultType
 
 
 class LlmDoc(BaseModel):
diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index f58a34c3243..0394f34b828 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -18,6 +18,7 @@
 from danswer.chat.models import MessageSpecificCitations
 from danswer.chat.models import QADocsResponse
 from danswer.chat.models import StreamingError
+from danswer.chat.models import StreamStopInfo
 from danswer.configs.app_configs import AZURE_DALLE_API_BASE
 from danswer.configs.app_configs import AZURE_DALLE_API_KEY
 from danswer.configs.app_configs import AZURE_DALLE_API_VERSION
@@ -77,31 +78,49 @@
 from danswer.server.query_and_chat.models import CreateChatMessageRequest
 from danswer.server.utils import get_json_line
 from danswer.tools.built_in_tools import get_built_in_tool_by_id
-from danswer.tools.custom.custom_tool import (
+from danswer.tools.force import ForceUseTool
+from danswer.tools.models import DynamicSchemaInfo
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool import Tool
+from danswer.tools.tool_implementations.custom.custom_tool import (
     build_custom_tools_from_openapi_schema_and_headers,
 )
-from danswer.tools.custom.custom_tool import CUSTOM_TOOL_RESPONSE_ID
-from danswer.tools.custom.custom_tool import CustomToolCallSummary
-from danswer.tools.force import ForceUseTool
-from danswer.tools.images.image_generation_tool import IMAGE_GENERATION_RESPONSE_ID
-from danswer.tools.images.image_generation_tool import ImageGenerationResponse
-from danswer.tools.images.image_generation_tool import ImageGenerationTool
-from danswer.tools.internet_search.internet_search_tool import (
+from danswer.tools.tool_implementations.custom.custom_tool import (
+    CUSTOM_TOOL_RESPONSE_ID,
+)
+from danswer.tools.tool_implementations.custom.custom_tool import CustomToolCallSummary
+from danswer.tools.tool_implementations.images.image_generation_tool import (
+    IMAGE_GENERATION_RESPONSE_ID,
+)
+from danswer.tools.tool_implementations.images.image_generation_tool import (
+    ImageGenerationResponse,
+)
+from danswer.tools.tool_implementations.images.image_generation_tool import (
+    ImageGenerationTool,
+)
+from danswer.tools.tool_implementations.internet_search.internet_search_tool import (
     INTERNET_SEARCH_RESPONSE_ID,
 )
-from danswer.tools.internet_search.internet_search_tool import (
+from danswer.tools.tool_implementations.internet_search.internet_search_tool import (
     internet_search_response_to_search_docs,
 )
-from danswer.tools.internet_search.internet_search_tool import InternetSearchResponse
-from danswer.tools.internet_search.internet_search_tool import InternetSearchTool
-from danswer.tools.models import DynamicSchemaInfo
-from danswer.tools.search.search_tool import FINAL_CONTEXT_DOCUMENTS_ID
-from danswer.tools.search.search_tool import SEARCH_RESPONSE_SUMMARY_ID
-from danswer.tools.search.search_tool import SearchResponseSummary
-from danswer.tools.search.search_tool import SearchTool
-from danswer.tools.search.search_tool import SECTION_RELEVANCE_LIST_ID
-from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
+from danswer.tools.tool_implementations.internet_search.internet_search_tool import (
+    InternetSearchResponse,
+)
+from danswer.tools.tool_implementations.internet_search.internet_search_tool import (
+    InternetSearchTool,
+)
+from danswer.tools.tool_implementations.search.search_tool import (
+    FINAL_CONTEXT_DOCUMENTS_ID,
+)
+from danswer.tools.tool_implementations.search.search_tool import (
+    SEARCH_RESPONSE_SUMMARY_ID,
+)
+from danswer.tools.tool_implementations.search.search_tool import SearchResponseSummary
+from danswer.tools.tool_implementations.search.search_tool import SearchTool
+from danswer.tools.tool_implementations.search.search_tool import (
+    SECTION_RELEVANCE_LIST_ID,
+)
 from danswer.tools.tool_runner import ToolCallFinalResult
 from danswer.tools.utils import compute_all_tool_tokens
 from danswer.tools.utils import explicit_tool_calling_supported
@@ -260,6 +279,7 @@ def _get_force_search_settings(
     | CustomToolResponse
     | MessageSpecificCitations
     | MessageResponseIDInfo
+    | StreamStopInfo
 )
 ChatPacketStream = Iterator[ChatPacket]
 
@@ -532,6 +552,13 @@ def stream_chat_message_objects(
             if not persona
             else PromptConfig.from_model(persona.prompts[0])
         )
+        answer_style_config = AnswerStyleConfig(
+            citation_config=CitationConfig(
+                all_docs_useful=selected_db_search_docs is not None
+            ),
+            document_pruning_config=document_pruning_config,
+            structured_response_format=new_msg_req.structured_response_format,
+        )
 
         # find out what tools to use
         search_tool: SearchTool | None = None
@@ -550,13 +577,16 @@ def stream_chat_message_objects(
                         llm=llm,
                         fast_llm=fast_llm,
                         pruning_config=document_pruning_config,
+                        answer_style_config=answer_style_config,
                         selected_sections=selected_sections,
                         chunks_above=new_msg_req.chunks_above,
                         chunks_below=new_msg_req.chunks_below,
                         full_doc=new_msg_req.full_doc,
-                        evaluation_type=LLMEvaluationType.BASIC
-                        if persona.llm_relevance_filter
-                        else LLMEvaluationType.SKIP,
+                        evaluation_type=(
+                            LLMEvaluationType.BASIC
+                            if persona.llm_relevance_filter
+                            else LLMEvaluationType.SKIP
+                        ),
                     )
                     tool_dict[db_tool_model.id] = [search_tool]
                 elif tool_cls.__name__ == ImageGenerationTool.__name__:
@@ -626,7 +656,11 @@ def stream_chat_message_objects(
                             "Internet search tool requires a Bing API key, please contact your Danswer admin to get it added!"
                         )
                     tool_dict[db_tool_model.id] = [
-                        InternetSearchTool(api_key=bing_api_key)
+                        InternetSearchTool(
+                            api_key=bing_api_key,
+                            answer_style_config=answer_style_config,
+                            prompt_config=prompt_config,
+                        )
                     ]
 
                 continue
@@ -667,13 +701,7 @@ def stream_chat_message_objects(
             is_connected=is_connected,
             question=final_msg.message,
             latest_query_files=latest_query_files,
-            answer_style_config=AnswerStyleConfig(
-                citation_config=CitationConfig(
-                    all_docs_useful=selected_db_search_docs is not None
-                ),
-                document_pruning_config=document_pruning_config,
-                structured_response_format=new_msg_req.structured_response_format,
-            ),
+            answer_style_config=answer_style_config,
             prompt_config=prompt_config,
             llm=(
                 llm
@@ -777,7 +805,8 @@ def stream_chat_message_objects(
                         response=custom_tool_response.tool_result,
                         tool_name=custom_tool_response.tool_name,
                     )
-
+            elif isinstance(packet, StreamStopInfo):
+                pass
             else:
                 if isinstance(packet, ToolCallFinalResult):
                     tool_result = packet
@@ -807,6 +836,7 @@ def stream_chat_message_objects(
 
     # Post-LLM answer processing
     try:
+        logger.debug("Post-LLM answer processing")
         message_specific_citations: MessageSpecificCitations | None = None
         if reference_db_search_docs:
             message_specific_citations = _translate_citations(
diff --git a/backend/danswer/llm/answering/answer.py b/backend/danswer/llm/answering/answer.py
index d2aeb1b14c4..c447db452ff 100644
--- a/backend/danswer/llm/answering/answer.py
+++ b/backend/danswer/llm/answering/answer.py
@@ -1,72 +1,44 @@
-import itertools
 from collections.abc import Callable
 from collections.abc import Iterator
-from typing import Any
-from typing import cast
 from uuid import uuid4
 
 from langchain.schema.messages import BaseMessage
 from langchain_core.messages import AIMessageChunk
-from langchain_core.messages import HumanMessage
+from langchain_core.messages import ToolCall
 
-from danswer.chat.chat_utils import llm_doc_from_inference_section
 from danswer.chat.models import AnswerQuestionPossibleReturn
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import DanswerAnswerPiece
-from danswer.chat.models import LlmDoc
-from danswer.chat.models import StreamStopInfo
-from danswer.chat.models import StreamStopReason
-from danswer.configs.chat_configs import QA_PROMPT_OVERRIDE
 from danswer.file_store.utils import InMemoryChatFile
+from danswer.llm.answering.llm_response_handler import LLMCall
+from danswer.llm.answering.llm_response_handler import LLMResponseHandlerManager
 from danswer.llm.answering.models import AnswerStyleConfig
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.answering.models import PromptConfig
-from danswer.llm.answering.models import StreamProcessor
 from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.answering.prompts.build import default_build_system_message
 from danswer.llm.answering.prompts.build import default_build_user_message
-from danswer.llm.answering.prompts.citations_prompt import (
-    build_citations_system_message,
+from danswer.llm.answering.stream_processing.answer_response_handler import (
+    AnswerResponseHandler,
 )
-from danswer.llm.answering.prompts.citations_prompt import build_citations_user_message
-from danswer.llm.answering.prompts.quotes_prompt import build_quotes_user_message
-from danswer.llm.answering.stream_processing.citation_processing import (
-    build_citation_processor,
+from danswer.llm.answering.stream_processing.answer_response_handler import (
+    CitationResponseHandler,
 )
-from danswer.llm.answering.stream_processing.quotes_processing import (
-    build_quotes_processor,
+from danswer.llm.answering.stream_processing.answer_response_handler import (
+    DummyAnswerResponseHandler,
+)
+from danswer.llm.answering.stream_processing.answer_response_handler import (
+    QuotesResponseHandler,
 )
-from danswer.llm.answering.stream_processing.utils import DocumentIdOrderMapping
 from danswer.llm.answering.stream_processing.utils import map_document_id_order
+from danswer.llm.answering.tool.tool_response_handler import ToolResponseHandler
 from danswer.llm.interfaces import LLM
-from danswer.llm.interfaces import ToolChoiceOptions
 from danswer.natural_language_processing.utils import get_tokenizer
-from danswer.tools.custom.custom_tool_prompt_builder import (
-    build_user_message_for_custom_tool_for_non_tool_calling_llm,
-)
-from danswer.tools.force import filter_tools_for_force_tool_use
 from danswer.tools.force import ForceUseTool
-from danswer.tools.images.image_generation_tool import IMAGE_GENERATION_RESPONSE_ID
-from danswer.tools.images.image_generation_tool import ImageGenerationResponse
-from danswer.tools.images.image_generation_tool import ImageGenerationTool
-from danswer.tools.images.prompt import build_image_generation_user_prompt
-from danswer.tools.internet_search.internet_search_tool import InternetSearchTool
-from danswer.tools.message import build_tool_message
-from danswer.tools.message import ToolCallSummary
-from danswer.tools.search.search_tool import FINAL_CONTEXT_DOCUMENTS_ID
-from danswer.tools.search.search_tool import SEARCH_DOC_CONTENT_ID
-from danswer.tools.search.search_tool import SEARCH_RESPONSE_SUMMARY_ID
-from danswer.tools.search.search_tool import SearchResponseSummary
-from danswer.tools.search.search_tool import SearchTool
+from danswer.tools.models import ToolResponse
 from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
-from danswer.tools.tool_runner import (
-    check_which_tools_should_run_for_non_tool_calling_llm,
-)
-from danswer.tools.tool_runner import ToolCallFinalResult
+from danswer.tools.tool_implementations.search.search_tool import SearchTool
 from danswer.tools.tool_runner import ToolCallKickoff
-from danswer.tools.tool_runner import ToolRunner
-from danswer.tools.tool_selection import select_single_tool_for_non_tool_calling_llm
 from danswer.tools.utils import explicit_tool_calling_supported
 from danswer.utils.logger import setup_logger
 
@@ -74,29 +46,9 @@
 logger = setup_logger()
 
 
-def _get_answer_stream_processor(
-    context_docs: list[LlmDoc],
-    doc_id_to_rank_map: DocumentIdOrderMapping,
-    answer_style_configs: AnswerStyleConfig,
-) -> StreamProcessor:
-    if answer_style_configs.citation_config:
-        return build_citation_processor(
-            context_docs=context_docs, doc_id_to_rank_map=doc_id_to_rank_map
-        )
-    if answer_style_configs.quotes_config:
-        return build_quotes_processor(
-            context_docs=context_docs, is_json_prompt=not (QA_PROMPT_OVERRIDE == "weak")
-        )
-
-    raise RuntimeError("Not implemented yet")
-
-
 AnswerStream = Iterator[AnswerQuestionPossibleReturn | ToolCallKickoff | ToolResponse]
 
 
-logger = setup_logger()
-
-
 class Answer:
     def __init__(
         self,
@@ -136,8 +88,6 @@ def __init__(
         self.tools = tools or []
         self.force_use_tool = force_use_tool
 
-        self.skip_explicit_tool_calling = skip_explicit_tool_calling
-
         self.message_history = message_history or []
         # used for QA flow where we only want to send a single message
         self.single_message_history = single_message_history
@@ -162,335 +112,141 @@ def __init__(
         self.skip_gen_ai_answer_generation = skip_gen_ai_answer_generation
         self._is_cancelled = False
 
-    def _update_prompt_builder_for_search_tool(
-        self, prompt_builder: AnswerPromptBuilder, final_context_documents: list[LlmDoc]
-    ) -> None:
-        if self.answer_style_config.citation_config:
-            prompt_builder.update_system_prompt(
-                build_citations_system_message(self.prompt_config)
-            )
-            prompt_builder.update_user_prompt(
-                build_citations_user_message(
-                    question=self.question,
-                    prompt_config=self.prompt_config,
-                    context_docs=final_context_documents,
-                    files=self.latest_query_files,
-                    all_doc_useful=(
-                        self.answer_style_config.citation_config.all_docs_useful
-                        if self.answer_style_config.citation_config
-                        else False
-                    ),
-                    history_message=self.single_message_history or "",
-                )
-            )
-        elif self.answer_style_config.quotes_config:
-            prompt_builder.update_user_prompt(
-                build_quotes_user_message(
-                    question=self.question,
-                    context_docs=final_context_documents,
-                    history_str=self.single_message_history or "",
-                    prompt=self.prompt_config,
-                )
+        self.using_tool_calling_llm = (
+            explicit_tool_calling_supported(
+                self.llm.config.model_provider, self.llm.config.model_name
             )
+            and not skip_explicit_tool_calling
+        )
 
-    def _raw_output_for_explicit_tool_calling_llms(
-        self,
-    ) -> Iterator[
-        str | StreamStopInfo | ToolCallKickoff | ToolResponse | ToolCallFinalResult
-    ]:
-        prompt_builder = AnswerPromptBuilder(self.message_history, self.llm.config)
-
-        tool_call_chunk: AIMessageChunk | None = None
-        if self.force_use_tool.force_use and self.force_use_tool.args is not None:
-            # if we are forcing a tool WITH args specified, we don't need to check which tools to run
-            # / need to generate the args
-            tool_call_chunk = AIMessageChunk(
-                content="",
-            )
-            tool_call_chunk.tool_calls = [
-                {
-                    "name": self.force_use_tool.tool_name,
-                    "args": self.force_use_tool.args,
-                    "id": str(uuid4()),
-                }
-            ]
-        else:
-            # if tool calling is supported, first try the raw message
-            # to see if we don't need to use any tools
-            prompt_builder.update_system_prompt(
-                default_build_system_message(self.prompt_config)
-            )
-            prompt_builder.update_user_prompt(
-                default_build_user_message(
-                    self.question, self.prompt_config, self.latest_query_files
-                )
-            )
-            prompt = prompt_builder.build()
-            final_tool_definitions = [
-                tool.tool_definition()
-                for tool in filter_tools_for_force_tool_use(
-                    self.tools, self.force_use_tool
-                )
-            ]
-
-            for message in self.llm.stream(
-                prompt=prompt,
-                tools=final_tool_definitions if final_tool_definitions else None,
-                tool_choice="required" if self.force_use_tool.force_use else None,
-                structured_response_format=self.answer_style_config.structured_response_format,
-            ):
-                if isinstance(message, AIMessageChunk) and (
-                    message.tool_call_chunks or message.tool_calls
-                ):
-                    if tool_call_chunk is None:
-                        tool_call_chunk = message
-                    else:
-                        tool_call_chunk += message  # type: ignore
-                else:
-                    if message.content:
-                        if self.is_cancelled:
-                            return
-                        yield cast(str, message.content)
-                    if (
-                        message.additional_kwargs.get("usage_metadata", {}).get("stop")
-                        == "length"
-                    ):
-                        yield StreamStopInfo(
-                            stop_reason=StreamStopReason.CONTEXT_LENGTH
-                        )
-
-            if not tool_call_chunk:
-                return  # no tool call needed
-
-        # if we have a tool call, we need to call the tool
-        tool_call_requests = tool_call_chunk.tool_calls
-        for tool_call_request in tool_call_requests:
-            known_tools_by_name = [
-                tool for tool in self.tools if tool.name == tool_call_request["name"]
-            ]
-
-            if not known_tools_by_name:
-                logger.error(
-                    "Tool call requested with unknown name field. \n"
-                    f"self.tools: {self.tools}"
-                    f"tool_call_request: {tool_call_request}"
-                )
-                if self.tools:
-                    tool = self.tools[0]
-                else:
-                    continue
-            else:
-                tool = known_tools_by_name[0]
-            tool_args = (
-                self.force_use_tool.args
-                if self.force_use_tool.tool_name == tool.name
-                and self.force_use_tool.args
-                else tool_call_request["args"]
-            )
+    def _get_tools_list(self) -> list[Tool]:
+        if not self.force_use_tool.force_use:
+            return self.tools
 
-            tool_runner = ToolRunner(tool, tool_args)
-            yield tool_runner.kickoff()
-            yield from tool_runner.tool_responses()
+        tool = next(
+            (t for t in self.tools if t.name == self.force_use_tool.tool_name), None
+        )
+        if tool is None:
+            raise RuntimeError(f"Tool '{self.force_use_tool.tool_name}' not found")
 
-            tool_call_summary = ToolCallSummary(
-                tool_call_request=tool_call_chunk,
-                tool_call_result=build_tool_message(
-                    tool_call_request, tool_runner.tool_message_content()
-                ),
+        logger.info(
+            f"Forcefully using tool='{tool.name}'"
+            + (
+                f" with args='{self.force_use_tool.args}'"
+                if self.force_use_tool.args is not None
+                else ""
             )
+        )
+        return [tool]
 
-            if tool.name in {SearchTool._NAME, InternetSearchTool._NAME}:
-                self._update_prompt_builder_for_search_tool(prompt_builder, [])
-            elif tool.name == ImageGenerationTool._NAME:
-                img_urls = [
-                    img_generation_result["url"]
-                    for img_generation_result in tool_runner.tool_final_result().tool_result
-                ]
-                prompt_builder.update_user_prompt(
-                    build_image_generation_user_prompt(
-                        query=self.question, img_urls=img_urls
-                    )
-                )
-            yield tool_runner.tool_final_result()
-            if not self.skip_gen_ai_answer_generation:
-                prompt = prompt_builder.build(tool_call_summary=tool_call_summary)
-
-                yield from self._process_llm_stream(
-                    prompt=prompt,
-                    # as of now, we don't support multiple tool calls in sequence, which is why
-                    # we don't need to pass this in here
-                    # tools=[tool.tool_definition() for tool in self.tools],
-                )
-
-            return
+    def _handle_specified_tool_call(
+        self, llm_calls: list[LLMCall], tool: Tool, tool_args: dict
+    ) -> AnswerStream:
+        current_llm_call = llm_calls[-1]
 
-    # This method processes the LLM stream and yields the content or stop information
-    def _process_llm_stream(
-        self,
-        prompt: Any,
-        tools: list[dict] | None = None,
-        tool_choice: ToolChoiceOptions | None = None,
-    ) -> Iterator[str | StreamStopInfo]:
-        for message in self.llm.stream(
-            prompt=prompt,
-            tools=tools,
-            tool_choice=tool_choice,
-            structured_response_format=self.answer_style_config.structured_response_format,
-        ):
-            if isinstance(message, AIMessageChunk):
-                if message.content:
-                    if self.is_cancelled:
-                        return StreamStopInfo(stop_reason=StreamStopReason.CANCELLED)
-                    yield cast(str, message.content)
-
-            if (
-                message.additional_kwargs.get("usage_metadata", {}).get("stop")
-                == "length"
-            ):
-                yield StreamStopInfo(stop_reason=StreamStopReason.CONTEXT_LENGTH)
-
-    def _raw_output_for_non_explicit_tool_calling_llms(
-        self,
-    ) -> Iterator[
-        str | StreamStopInfo | ToolCallKickoff | ToolResponse | ToolCallFinalResult
-    ]:
-        prompt_builder = AnswerPromptBuilder(self.message_history, self.llm.config)
-        chosen_tool_and_args: tuple[Tool, dict] | None = None
-
-        if self.force_use_tool.force_use:
-            # if we are forcing a tool, we don't need to check which tools to run
-            tool = next(
-                iter(
-                    [
-                        tool
-                        for tool in self.tools
-                        if tool.name == self.force_use_tool.tool_name
-                    ]
-                ),
-                None,
-            )
-            if not tool:
-                raise RuntimeError(f"Tool '{self.force_use_tool.tool_name}' not found")
+        # make a dummy tool handler
+        tool_handler = ToolResponseHandler([tool])
 
-            tool_args = (
-                self.force_use_tool.args
-                if self.force_use_tool.args is not None
-                else tool.get_args_for_non_tool_calling_llm(
-                    query=self.question,
-                    history=self.message_history,
-                    llm=self.llm,
-                    force_run=True,
-                )
-            )
+        dummy_tool_call_chunk = AIMessageChunk(content="")
+        dummy_tool_call_chunk.tool_calls = [
+            ToolCall(name=tool.name, args=tool_args, id=str(uuid4()))
+        ]
 
-            if tool_args is None:
-                raise RuntimeError(f"Tool '{tool.name}' did not return args")
+        response_handler_manager = LLMResponseHandlerManager(
+            tool_handler, DummyAnswerResponseHandler(), self.is_cancelled
+        )
+        yield from response_handler_manager.handle_llm_response(
+            iter([dummy_tool_call_chunk])
+        )
 
-            chosen_tool_and_args = (tool, tool_args)
+        new_llm_call = response_handler_manager.next_llm_call(current_llm_call)
+        if new_llm_call:
+            yield from self._get_response(llm_calls + [new_llm_call])
         else:
-            tool_options = check_which_tools_should_run_for_non_tool_calling_llm(
-                tools=self.tools,
-                query=self.question,
-                history=self.message_history,
-                llm=self.llm,
-            )
+            raise RuntimeError("Tool call handler did not return a new LLM call")
 
-            available_tools_and_args = [
-                (self.tools[ind], args)
-                for ind, args in enumerate(tool_options)
-                if args is not None
-            ]
+    def _get_response(self, llm_calls: list[LLMCall]) -> AnswerStream:
+        current_llm_call = llm_calls[-1]
 
-            logger.info(
-                f"Selecting single tool from tools: {[(tool.name, args) for tool, args in available_tools_and_args]}"
+        # handle the case where no decision has to be made; we simply run the tool
+        if (
+            current_llm_call.force_use_tool.force_use
+            and current_llm_call.force_use_tool.args is not None
+        ):
+            tool_name, tool_args = (
+                current_llm_call.force_use_tool.tool_name,
+                current_llm_call.force_use_tool.args,
             )
-
-            chosen_tool_and_args = (
-                select_single_tool_for_non_tool_calling_llm(
-                    tools_and_args=available_tools_and_args,
-                    history=self.message_history,
-                    query=self.question,
-                    llm=self.llm,
-                )
-                if available_tools_and_args
-                else None
+            tool = next(
+                (t for t in current_llm_call.tools if t.name == tool_name), None
             )
+            if not tool:
+                raise RuntimeError(f"Tool '{tool_name}' not found")
 
-            logger.notice(f"Chosen tool: {chosen_tool_and_args}")
+            yield from self._handle_specified_tool_call(llm_calls, tool, tool_args)
+            return
 
-        if not chosen_tool_and_args:
-            if self.skip_gen_ai_answer_generation:
-                raise ValueError(
-                    "skip_gen_ai_answer_generation is True, but no tool was chosen; no answer will be generated"
-                )
-            prompt_builder.update_system_prompt(
-                default_build_system_message(self.prompt_config)
-            )
-            prompt_builder.update_user_prompt(
-                default_build_user_message(
-                    self.question, self.prompt_config, self.latest_query_files
+        # special pre-logic for non-tool calling LLM case
+        if not self.using_tool_calling_llm and current_llm_call.tools:
+            chosen_tool_and_args = (
+                ToolResponseHandler.get_tool_call_for_non_tool_calling_llm(
+                    current_llm_call, self.llm
                 )
             )
-            prompt = prompt_builder.build()
-            yield from self._process_llm_stream(
-                prompt=prompt,
-                tools=None,
-            )
+            if chosen_tool_and_args:
+                tool, tool_args = chosen_tool_and_args
+                yield from self._handle_specified_tool_call(llm_calls, tool, tool_args)
+                return
+
+        # if we're skipping gen ai answer generation, we should break
+        # out unless we're forcing a tool call. If we don't, we might generate an
+        # answer, which is a no-no!
+        if (
+            self.skip_gen_ai_answer_generation
+            and not current_llm_call.force_use_tool.force_use
+        ):
             return
 
-        tool, tool_args = chosen_tool_and_args
-        tool_runner = ToolRunner(tool, tool_args)
-        yield tool_runner.kickoff()
+        # set up "handlers" to listen to the LLM response stream and
+        # feed back the processed results + handle tool call requests
+        # + figure out what the next LLM call should be
+        tool_call_handler = ToolResponseHandler(current_llm_call.tools)
 
-        if tool.name in {SearchTool._NAME, InternetSearchTool._NAME}:
-            final_context_documents = None
-            for response in tool_runner.tool_responses():
-                if response.id == FINAL_CONTEXT_DOCUMENTS_ID:
-                    final_context_documents = cast(list[LlmDoc], response.response)
-                yield response
+        search_result = SearchTool.get_search_result(current_llm_call) or []
 
-            if final_context_documents is None:
-                raise RuntimeError(
-                    f"{tool.name} did not return final context documents"
-                )
-
-            self._update_prompt_builder_for_search_tool(
-                prompt_builder, final_context_documents
+        answer_handler: AnswerResponseHandler
+        if self.answer_style_config.citation_config:
+            answer_handler = CitationResponseHandler(
+                context_docs=search_result,
+                doc_id_to_rank_map=map_document_id_order(search_result),
             )
-        elif tool.name == ImageGenerationTool._NAME:
-            img_urls = []
-            for response in tool_runner.tool_responses():
-                if response.id == IMAGE_GENERATION_RESPONSE_ID:
-                    img_generation_response = cast(
-                        list[ImageGenerationResponse], response.response
-                    )
-                    img_urls = [img.url for img in img_generation_response]
-
-                yield response
-
-            prompt_builder.update_user_prompt(
-                build_image_generation_user_prompt(
-                    query=self.question,
-                    img_urls=img_urls,
-                )
+        elif self.answer_style_config.quotes_config:
+            answer_handler = QuotesResponseHandler(
+                context_docs=search_result,
             )
         else:
-            prompt_builder.update_user_prompt(
-                HumanMessage(
-                    content=build_user_message_for_custom_tool_for_non_tool_calling_llm(
-                        self.question,
-                        tool.name,
-                        *tool_runner.tool_responses(),
-                    )
-                )
-            )
-        final = tool_runner.tool_final_result()
+            raise ValueError("No answer style config provided")
 
-        yield final
-        if not self.skip_gen_ai_answer_generation:
-            prompt = prompt_builder.build()
+        response_handler_manager = LLMResponseHandlerManager(
+            tool_call_handler, answer_handler, self.is_cancelled
+        )
 
-            yield from self._process_llm_stream(prompt=prompt, tools=None)
+        # DEBUG: good breakpoint
+        stream = self.llm.stream(
+            prompt=current_llm_call.prompt_builder.build(),
+            tools=[tool.tool_definition() for tool in current_llm_call.tools] or None,
+            tool_choice=(
+                "required"
+                if current_llm_call.tools and current_llm_call.force_use_tool.force_use
+                else None
+            ),
+            structured_response_format=self.answer_style_config.structured_response_format,
+        )
+        yield from response_handler_manager.handle_llm_response(stream)
+
+        new_llm_call = response_handler_manager.next_llm_call(current_llm_call)
+        if new_llm_call:
+            yield from self._get_response(llm_calls + [new_llm_call])
 
     @property
     def processed_streamed_output(self) -> AnswerStream:
@@ -498,94 +254,30 @@ def processed_streamed_output(self) -> AnswerStream:
             yield from self._processed_stream
             return
 
-        output_generator = (
-            self._raw_output_for_explicit_tool_calling_llms()
-            if explicit_tool_calling_supported(
-                self.llm.config.model_provider, self.llm.config.model_name
-            )
-            and not self.skip_explicit_tool_calling
-            else self._raw_output_for_non_explicit_tool_calling_llms()
+        prompt_builder = AnswerPromptBuilder(
+            user_message=default_build_user_message(
+                user_query=self.question,
+                prompt_config=self.prompt_config,
+                files=self.latest_query_files,
+            ),
+            message_history=self.message_history,
+            llm_config=self.llm.config,
+            single_message_history=self.single_message_history,
+        )
+        prompt_builder.update_system_prompt(
+            default_build_system_message(self.prompt_config)
+        )
+        llm_call = LLMCall(
+            prompt_builder=prompt_builder,
+            tools=self._get_tools_list(),
+            force_use_tool=self.force_use_tool,
+            files=self.latest_query_files,
+            tool_call_info=[],
+            using_tool_calling_llm=self.using_tool_calling_llm,
         )
-
-        def _process_stream(
-            stream: Iterator[ToolCallKickoff | ToolResponse | str | StreamStopInfo],
-        ) -> AnswerStream:
-            message = None
-
-            # special things we need to keep track of for the SearchTool
-            # raw results that will be displayed to the user
-            search_results: list[LlmDoc] | None = None
-            # processed docs to feed into the LLM
-            final_context_docs: list[LlmDoc] | None = None
-
-            for message in stream:
-                if isinstance(message, ToolCallKickoff) or isinstance(
-                    message, ToolCallFinalResult
-                ):
-                    yield message
-                elif isinstance(message, ToolResponse):
-                    if message.id == SEARCH_RESPONSE_SUMMARY_ID:
-                        # We don't need to run section merging in this flow, this variable is only used
-                        # below to specify the ordering of the documents for the purpose of matching
-                        # citations to the right search documents. The deduplication logic is more lightweight
-                        # there and we don't need to do it twice
-                        search_results = [
-                            llm_doc_from_inference_section(section)
-                            for section in cast(
-                                SearchResponseSummary, message.response
-                            ).top_sections
-                        ]
-                    elif message.id == FINAL_CONTEXT_DOCUMENTS_ID:
-                        final_context_docs = cast(list[LlmDoc], message.response)
-                        yield message
-
-                    elif (
-                        message.id == SEARCH_DOC_CONTENT_ID
-                        and not self._return_contexts
-                    ):
-                        continue
-
-                    yield message
-                else:
-                    # assumes all tool responses will come first, then the final answer
-                    break
-
-            if not self.skip_gen_ai_answer_generation:
-                process_answer_stream_fn = _get_answer_stream_processor(
-                    context_docs=final_context_docs or [],
-                    # if doc selection is enabled, then search_results will be None,
-                    # so we need to use the final_context_docs
-                    doc_id_to_rank_map=map_document_id_order(
-                        search_results or final_context_docs or []
-                    ),
-                    answer_style_configs=self.answer_style_config,
-                )
-
-                stream_stop_info = None
-
-                def _stream() -> Iterator[str]:
-                    nonlocal stream_stop_info
-                    for item in itertools.chain([message], stream):
-                        if isinstance(item, StreamStopInfo):
-                            stream_stop_info = item
-                            return
-
-                        # this should never happen, but we're seeing weird behavior here so handling for now
-                        if not isinstance(item, str):
-                            logger.error(
-                                f"Received non-string item in answer stream: {item}. Skipping."
-                            )
-                            continue
-
-                        yield item
-
-                yield from process_answer_stream_fn(_stream())
-
-                if stream_stop_info:
-                    yield stream_stop_info
 
         processed_stream = []
-        for processed_packet in _process_stream(output_generator):
+        for processed_packet in self._get_response([llm_call]):
             processed_stream.append(processed_packet)
             yield processed_packet
 
@@ -609,7 +301,6 @@ def citations(self) -> list[CitationInfo]:
 
         return citations
 
-    @property
     def is_cancelled(self) -> bool:
         if self._is_cancelled:
             return True
diff --git a/backend/danswer/llm/answering/llm_response_handler.py b/backend/danswer/llm/answering/llm_response_handler.py
new file mode 100644
index 00000000000..f8426844244
--- /dev/null
+++ b/backend/danswer/llm/answering/llm_response_handler.py
@@ -0,0 +1,84 @@
+from collections.abc import Callable
+from collections.abc import Generator
+from collections.abc import Iterator
+from typing import TYPE_CHECKING
+
+from langchain_core.messages import BaseMessage
+from pydantic.v1 import BaseModel as BaseModel__v1
+
+from danswer.chat.models import CitationInfo
+from danswer.chat.models import DanswerAnswerPiece
+from danswer.chat.models import DanswerQuotes
+from danswer.chat.models import StreamStopInfo
+from danswer.chat.models import StreamStopReason
+from danswer.file_store.models import InMemoryChatFile
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+from danswer.tools.force import ForceUseTool
+from danswer.tools.models import ToolCallFinalResult
+from danswer.tools.models import ToolCallKickoff
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool import Tool
+
+
+if TYPE_CHECKING:
+    from danswer.llm.answering.stream_processing.answer_response_handler import (
+        AnswerResponseHandler,
+    )
+    from danswer.llm.answering.tool.tool_response_handler import ToolResponseHandler
+
+
+ResponsePart = (
+    DanswerAnswerPiece
+    | CitationInfo
+    | DanswerQuotes
+    | ToolCallKickoff
+    | ToolResponse
+    | ToolCallFinalResult
+    | StreamStopInfo
+)
+
+
+class LLMCall(BaseModel__v1):
+    prompt_builder: AnswerPromptBuilder
+    tools: list[Tool]
+    force_use_tool: ForceUseTool
+    files: list[InMemoryChatFile]
+    tool_call_info: list[ToolCallKickoff | ToolResponse | ToolCallFinalResult]
+    using_tool_calling_llm: bool
+
+    class Config:
+        arbitrary_types_allowed = True
+
+
+class LLMResponseHandlerManager:
+    def __init__(
+        self,
+        tool_handler: "ToolResponseHandler",
+        answer_handler: "AnswerResponseHandler",
+        is_cancelled: Callable[[], bool],
+    ):
+        self.tool_handler = tool_handler
+        self.answer_handler = answer_handler
+        self.is_cancelled = is_cancelled
+
+    def handle_llm_response(
+        self,
+        stream: Iterator[BaseMessage],
+    ) -> Generator[ResponsePart, None, None]:
+        all_messages: list[BaseMessage] = []
+        for message in stream:
+            if self.is_cancelled():
+                yield StreamStopInfo(stop_reason=StreamStopReason.CANCELLED)
+                return
+            # tool handler doesn't do anything until the full message is received
+            # NOTE: still need to run list() to get this to run
+            list(self.tool_handler.handle_response_part(message, all_messages))
+            yield from self.answer_handler.handle_response_part(message, all_messages)
+            all_messages.append(message)
+
+        # potentially give back all info on the selected tool call + its result
+        yield from self.tool_handler.handle_response_part(None, all_messages)
+        yield from self.answer_handler.handle_response_part(None, all_messages)
+
+    def next_llm_call(self, llm_call: LLMCall) -> LLMCall | None:
+        return self.tool_handler.next_llm_call(llm_call)
diff --git a/backend/danswer/llm/answering/prompts/build.py b/backend/danswer/llm/answering/prompts/build.py
index f53d4481f6e..b5b774f522d 100644
--- a/backend/danswer/llm/answering/prompts/build.py
+++ b/backend/danswer/llm/answering/prompts/build.py
@@ -12,12 +12,12 @@
 from danswer.llm.interfaces import LLMConfig
 from danswer.llm.utils import build_content_with_imgs
 from danswer.llm.utils import check_message_tokens
+from danswer.llm.utils import message_to_prompt_and_imgs
 from danswer.llm.utils import translate_history_to_basemessages
 from danswer.natural_language_processing.utils import get_tokenizer
 from danswer.prompts.chat_prompts import CHAT_USER_CONTEXT_FREE_PROMPT
 from danswer.prompts.prompt_utils import add_date_time_to_prompt
 from danswer.prompts.prompt_utils import drop_messages_history_overflow
-from danswer.tools.message import ToolCallSummary
 
 
 def default_build_system_message(
@@ -54,18 +54,14 @@ def default_build_user_message(
 
 class AnswerPromptBuilder:
     def __init__(
-        self, message_history: list[PreviousMessage], llm_config: LLMConfig
+        self,
+        user_message: HumanMessage,
+        message_history: list[PreviousMessage],
+        llm_config: LLMConfig,
+        single_message_history: str | None = None,
     ) -> None:
         self.max_tokens = compute_max_llm_input_tokens(llm_config)
 
-        (
-            self.message_history,
-            self.history_token_cnts,
-        ) = translate_history_to_basemessages(message_history)
-
-        self.system_message_and_token_cnt: tuple[SystemMessage, int] | None = None
-        self.user_message_and_token_cnt: tuple[HumanMessage, int] | None = None
-
         llm_tokenizer = get_tokenizer(
             provider_type=llm_config.model_provider,
             model_name=llm_config.model_name,
@@ -74,6 +70,24 @@ def __init__(
             Callable[[str], list[int]], llm_tokenizer.encode
         )
 
+        self.raw_message_history = message_history
+        (
+            self.message_history,
+            self.history_token_cnts,
+        ) = translate_history_to_basemessages(message_history)
+
+        # for cases where like the QA flow where we want to condense the chat history
+        # into a single message rather than a sequence of User / Assistant messages
+        self.single_message_history = single_message_history
+
+        self.system_message_and_token_cnt: tuple[SystemMessage, int] | None = None
+        self.user_message_and_token_cnt = (
+            user_message,
+            check_message_tokens(user_message, self.llm_tokenizer_encode_func),
+        )
+
+        self.new_messages_and_token_cnts: list[tuple[BaseMessage, int]] = []
+
     def update_system_prompt(self, system_message: SystemMessage | None) -> None:
         if not system_message:
             self.system_message_and_token_cnt = None
@@ -85,18 +99,21 @@ def update_system_prompt(self, system_message: SystemMessage | None) -> None:
         )
 
     def update_user_prompt(self, user_message: HumanMessage) -> None:
-        if not user_message:
-            self.user_message_and_token_cnt = None
-            return
-
         self.user_message_and_token_cnt = (
             user_message,
             check_message_tokens(user_message, self.llm_tokenizer_encode_func),
         )
 
-    def build(
-        self, tool_call_summary: ToolCallSummary | None = None
-    ) -> list[BaseMessage]:
+    def append_message(self, message: BaseMessage) -> None:
+        """Append a new message to the message history."""
+        token_count = check_message_tokens(message, self.llm_tokenizer_encode_func)
+        self.new_messages_and_token_cnts.append((message, token_count))
+
+    def get_user_message_content(self) -> str:
+        query, _ = message_to_prompt_and_imgs(self.user_message_and_token_cnt[0])
+        return query
+
+    def build(self) -> list[BaseMessage]:
         if not self.user_message_and_token_cnt:
             raise ValueError("User message must be set before building prompt")
 
@@ -113,25 +130,8 @@ def build(
 
         final_messages_with_tokens.append(self.user_message_and_token_cnt)
 
-        if tool_call_summary:
-            final_messages_with_tokens.append(
-                (
-                    tool_call_summary.tool_call_request,
-                    check_message_tokens(
-                        tool_call_summary.tool_call_request,
-                        self.llm_tokenizer_encode_func,
-                    ),
-                )
-            )
-            final_messages_with_tokens.append(
-                (
-                    tool_call_summary.tool_call_result,
-                    check_message_tokens(
-                        tool_call_summary.tool_call_result,
-                        self.llm_tokenizer_encode_func,
-                    ),
-                )
-            )
+        if self.new_messages_and_token_cnts:
+            final_messages_with_tokens.extend(self.new_messages_and_token_cnts)
 
         return drop_messages_history_overflow(
             final_messages_with_tokens, self.max_tokens
diff --git a/backend/danswer/llm/answering/prompts/citations_prompt.py b/backend/danswer/llm/answering/prompts/citations_prompt.py
index a2248da0585..b7ca7797e88 100644
--- a/backend/danswer/llm/answering/prompts/citations_prompt.py
+++ b/backend/danswer/llm/answering/prompts/citations_prompt.py
@@ -6,7 +6,6 @@
 from danswer.db.models import Persona
 from danswer.db.persona import get_default_prompt__read_only
 from danswer.db.search_settings import get_multilingual_expansion
-from danswer.file_store.utils import InMemoryChatFile
 from danswer.llm.answering.models import PromptConfig
 from danswer.llm.factory import get_llms_for_persona
 from danswer.llm.factory import get_main_llm_from_tuple
@@ -14,6 +13,7 @@
 from danswer.llm.utils import build_content_with_imgs
 from danswer.llm.utils import check_number_of_tokens
 from danswer.llm.utils import get_max_input_tokens
+from danswer.llm.utils import message_to_prompt_and_imgs
 from danswer.prompts.chat_prompts import REQUIRE_CITATION_STATEMENT
 from danswer.prompts.constants import DEFAULT_IGNORE_STATEMENT
 from danswer.prompts.direct_qa_prompts import CITATIONS_PROMPT
@@ -132,10 +132,9 @@ def build_citations_system_message(
 
 
 def build_citations_user_message(
-    question: str,
+    message: HumanMessage,
     prompt_config: PromptConfig,
     context_docs: list[LlmDoc] | list[InferenceChunk],
-    files: list[InMemoryChatFile],
     all_doc_useful: bool,
     history_message: str = "",
 ) -> HumanMessage:
@@ -149,6 +148,7 @@ def build_citations_user_message(
         if history_message
         else ""
     )
+    query, img_urls = message_to_prompt_and_imgs(message)
 
     if context_docs:
         context_docs_str = build_complete_context_str(context_docs)
@@ -158,20 +158,22 @@ def build_citations_user_message(
             optional_ignore_statement=optional_ignore,
             context_docs_str=context_docs_str,
             task_prompt=task_prompt_with_reminder,
-            user_query=question,
+            user_query=query,
             history_block=history_block,
         )
     else:
         # if no context docs provided, assume we're in the tool calling flow
         user_prompt = CITATIONS_PROMPT_FOR_TOOL_CALLING.format(
             task_prompt=task_prompt_with_reminder,
-            user_query=question,
+            user_query=query,
             history_block=history_block,
         )
 
     user_prompt = user_prompt.strip()
     user_msg = HumanMessage(
-        content=build_content_with_imgs(user_prompt, files) if files else user_prompt
+        content=build_content_with_imgs(user_prompt, img_urls=img_urls)
+        if img_urls
+        else user_prompt
     )
 
     return user_msg
diff --git a/backend/danswer/llm/answering/prompts/quotes_prompt.py b/backend/danswer/llm/answering/prompts/quotes_prompt.py
index 07abc4356b6..42f736b627d 100644
--- a/backend/danswer/llm/answering/prompts/quotes_prompt.py
+++ b/backend/danswer/llm/answering/prompts/quotes_prompt.py
@@ -5,6 +5,7 @@
 from danswer.configs.chat_configs import QA_PROMPT_OVERRIDE
 from danswer.db.search_settings import get_multilingual_expansion
 from danswer.llm.answering.models import PromptConfig
+from danswer.llm.utils import message_to_prompt_and_imgs
 from danswer.prompts.direct_qa_prompts import CONTEXT_BLOCK
 from danswer.prompts.direct_qa_prompts import HISTORY_BLOCK
 from danswer.prompts.direct_qa_prompts import JSON_PROMPT
@@ -75,7 +76,7 @@ def _build_strong_llm_quotes_prompt(
 
 
 def build_quotes_user_message(
-    question: str,
+    message: HumanMessage,
     context_docs: list[LlmDoc] | list[InferenceChunk],
     history_str: str,
     prompt: PromptConfig,
@@ -86,28 +87,10 @@ def build_quotes_user_message(
         else _build_strong_llm_quotes_prompt
     )
 
-    return prompt_builder(
-        question=question,
-        context_docs=context_docs,
-        history_str=history_str,
-        prompt=prompt,
-    )
-
-
-def build_quotes_prompt(
-    question: str,
-    context_docs: list[LlmDoc] | list[InferenceChunk],
-    history_str: str,
-    prompt: PromptConfig,
-) -> HumanMessage:
-    prompt_builder = (
-        _build_weak_llm_quotes_prompt
-        if QA_PROMPT_OVERRIDE == "weak"
-        else _build_strong_llm_quotes_prompt
-    )
+    query, _ = message_to_prompt_and_imgs(message)
 
     return prompt_builder(
-        question=question,
+        question=query,
         context_docs=context_docs,
         history_str=history_str,
         prompt=prompt,
diff --git a/backend/danswer/llm/answering/prune_and_merge.py b/backend/danswer/llm/answering/prune_and_merge.py
index 0193de1f2aa..690a5d2280d 100644
--- a/backend/danswer/llm/answering/prune_and_merge.py
+++ b/backend/danswer/llm/answering/prune_and_merge.py
@@ -19,7 +19,7 @@
 from danswer.prompts.prompt_utils import build_doc_context_str
 from danswer.search.models import InferenceChunk
 from danswer.search.models import InferenceSection
-from danswer.tools.search.search_utils import section_to_dict
+from danswer.tools.tool_implementations.search.search_utils import section_to_dict
 from danswer.utils.logger import setup_logger
 
 
diff --git a/backend/danswer/llm/answering/stream_processing/answer_response_handler.py b/backend/danswer/llm/answering/stream_processing/answer_response_handler.py
new file mode 100644
index 00000000000..80a1446163b
--- /dev/null
+++ b/backend/danswer/llm/answering/stream_processing/answer_response_handler.py
@@ -0,0 +1,91 @@
+import abc
+from collections.abc import Generator
+
+from langchain_core.messages import BaseMessage
+
+from danswer.chat.models import CitationInfo
+from danswer.chat.models import LlmDoc
+from danswer.llm.answering.llm_response_handler import ResponsePart
+from danswer.llm.answering.stream_processing.citation_processing import (
+    CitationProcessor,
+)
+from danswer.llm.answering.stream_processing.quotes_processing import (
+    QuotesProcessor,
+)
+from danswer.llm.answering.stream_processing.utils import DocumentIdOrderMapping
+
+
+class AnswerResponseHandler(abc.ABC):
+    @abc.abstractmethod
+    def handle_response_part(
+        self,
+        response_item: BaseMessage | None,
+        previous_response_items: list[BaseMessage],
+    ) -> Generator[ResponsePart, None, None]:
+        raise NotImplementedError
+
+
+class DummyAnswerResponseHandler(AnswerResponseHandler):
+    def handle_response_part(
+        self,
+        response_item: BaseMessage | None,
+        previous_response_items: list[BaseMessage],
+    ) -> Generator[ResponsePart, None, None]:
+        # This is a dummy handler that returns nothing
+        yield from []
+
+
+class CitationResponseHandler(AnswerResponseHandler):
+    def __init__(
+        self, context_docs: list[LlmDoc], doc_id_to_rank_map: DocumentIdOrderMapping
+    ):
+        self.context_docs = context_docs
+        self.doc_id_to_rank_map = doc_id_to_rank_map
+        self.citation_processor = CitationProcessor(
+            context_docs=self.context_docs,
+            doc_id_to_rank_map=self.doc_id_to_rank_map,
+        )
+        self.processed_text = ""
+        self.citations: list[CitationInfo] = []
+
+    def handle_response_part(
+        self,
+        response_item: BaseMessage | None,
+        previous_response_items: list[BaseMessage],
+    ) -> Generator[ResponsePart, None, None]:
+        if response_item is None:
+            return
+
+        content = (
+            response_item.content if isinstance(response_item.content, str) else ""
+        )
+
+        # Process the new content through the citation processor
+        yield from self.citation_processor.process_token(content)
+
+
+class QuotesResponseHandler(AnswerResponseHandler):
+    def __init__(
+        self,
+        context_docs: list[LlmDoc],
+        is_json_prompt: bool = True,
+    ):
+        self.quotes_processor = QuotesProcessor(
+            context_docs=context_docs,
+            is_json_prompt=is_json_prompt,
+        )
+
+    def handle_response_part(
+        self,
+        response_item: BaseMessage | None,
+        previous_response_items: list[BaseMessage],
+    ) -> Generator[ResponsePart, None, None]:
+        if response_item is None:
+            yield from self.quotes_processor.process_token(None)
+            return
+
+        content = (
+            response_item.content if isinstance(response_item.content, str) else ""
+        )
+
+        yield from self.quotes_processor.process_token(content)
diff --git a/backend/danswer/llm/answering/stream_processing/citation_processing.py b/backend/danswer/llm/answering/stream_processing/citation_processing.py
index f1e5489550d..950ad207878 100644
--- a/backend/danswer/llm/answering/stream_processing/citation_processing.py
+++ b/backend/danswer/llm/answering/stream_processing/citation_processing.py
@@ -1,12 +1,10 @@
 import re
-from collections.abc import Iterator
+from collections.abc import Generator
 
-from danswer.chat.models import AnswerQuestionStreamReturn
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import DanswerAnswerPiece
 from danswer.chat.models import LlmDoc
 from danswer.configs.chat_configs import STOP_STREAM_PAT
-from danswer.llm.answering.models import StreamProcessor
 from danswer.llm.answering.stream_processing.utils import DocumentIdOrderMapping
 from danswer.prompts.constants import TRIPLE_BACKTICK
 from danswer.utils.logger import setup_logger
@@ -19,128 +17,104 @@ def in_code_block(llm_text: str) -> bool:
     return count % 2 != 0
 
 
-def extract_citations_from_stream(
-    tokens: Iterator[str],
-    context_docs: list[LlmDoc],
-    doc_id_to_rank_map: DocumentIdOrderMapping,
-    stop_stream: str | None = STOP_STREAM_PAT,
-) -> Iterator[DanswerAnswerPiece | CitationInfo]:
-    """
-    Key aspects:
-
-    1. Stream Processing:
-    - Processes tokens one by one, allowing for real-time handling of large texts.
-
-    2. Citation Detection:
-    - Uses regex to find citations in the format [number].
-    - Example: [1], [2], etc.
-
-    3. Citation Mapping:
-    - Maps detected citation numbers to actual document ranks using doc_id_to_rank_map.
-    - Example: [1] might become [3] if doc_id_to_rank_map maps it to 3.
-
-    4. Citation Formatting:
-    - Replaces citations with properly formatted versions.
-    - Adds links if available: [[1]](https://example.com)
-    - Handles cases where links are not available: [[1]]()
-
-    5. Duplicate Handling:
-    - Skips consecutive citations of the same document to avoid redundancy.
-
-    6. Output Generation:
-    - Yields DanswerAnswerPiece objects for regular text.
-    - Yields CitationInfo objects for each unique citation encountered.
-
-    7. Context Awareness:
-    - Uses context_docs to access document information for citations.
-
-    This function effectively processes a stream of text, identifies and reformats citations,
-    and provides both the processed text and citation information as output.
-    """
-    order_mapping = doc_id_to_rank_map.order_mapping
-    llm_out = ""
-    max_citation_num = len(context_docs)
-    citation_order = []
-    curr_segment = ""
-    cited_inds = set()
-    hold = ""
-
-    raw_out = ""
-    current_citations: list[int] = []
-    past_cite_count = 0
-    for raw_token in tokens:
-        raw_out += raw_token
-        if stop_stream:
-            next_hold = hold + raw_token
-            if stop_stream in next_hold:
-                break
-            if next_hold == stop_stream[: len(next_hold)]:
-                hold = next_hold
-                continue
+class CitationProcessor:
+    def __init__(
+        self,
+        context_docs: list[LlmDoc],
+        doc_id_to_rank_map: DocumentIdOrderMapping,
+        stop_stream: str | None = STOP_STREAM_PAT,
+    ):
+        self.context_docs = context_docs
+        self.doc_id_to_rank_map = doc_id_to_rank_map
+        self.stop_stream = stop_stream
+        self.order_mapping = doc_id_to_rank_map.order_mapping
+        self.llm_out = ""
+        self.max_citation_num = len(context_docs)
+        self.citation_order: list[int] = []
+        self.curr_segment = ""
+        self.cited_inds: set[int] = set()
+        self.hold = ""
+        self.current_citations: list[int] = []
+        self.past_cite_count = 0
+
+    def process_token(
+        self, token: str | None
+    ) -> Generator[DanswerAnswerPiece | CitationInfo, None, None]:
+        # None -> end of stream
+        if token is None:
+            yield DanswerAnswerPiece(answer_piece=self.curr_segment)
+            return
+
+        if self.stop_stream:
+            next_hold = self.hold + token
+            if self.stop_stream in next_hold:
+                return
+            if next_hold == self.stop_stream[: len(next_hold)]:
+                self.hold = next_hold
+                return
             token = next_hold
-            hold = ""
-        else:
-            token = raw_token
+            self.hold = ""
 
-        curr_segment += token
-        llm_out += token
+        self.curr_segment += token
+        self.llm_out += token
 
         # Handle code blocks without language tags
-        if "`" in curr_segment:
-            if curr_segment.endswith("`"):
-                continue
-            elif "```" in curr_segment:
-                piece_that_comes_after = curr_segment.split("```")[1][0]
-                if piece_that_comes_after == "\n" and in_code_block(llm_out):
-                    curr_segment = curr_segment.replace("```", "```plaintext")
+        if "`" in self.curr_segment:
+            if self.curr_segment.endswith("`"):
+                return
+            elif "```" in self.curr_segment:
+                piece_that_comes_after = self.curr_segment.split("```")[1][0]
+                if piece_that_comes_after == "\n" and in_code_block(self.llm_out):
+                    self.curr_segment = self.curr_segment.replace("```", "```plaintext")
 
         citation_pattern = r"\[(\d+)\]"
-
-        citations_found = list(re.finditer(citation_pattern, curr_segment))
+        citations_found = list(re.finditer(citation_pattern, self.curr_segment))
         possible_citation_pattern = r"(\[\d*$)"  # [1, [, etc
-        possible_citation_found = re.search(possible_citation_pattern, curr_segment)
+        possible_citation_found = re.search(
+            possible_citation_pattern, self.curr_segment
+        )
 
-        # `past_cite_count`: number of characters since past citation
-        # 5 to ensure a citation hasn't occured
-        if len(citations_found) == 0 and len(llm_out) - past_cite_count > 5:
-            current_citations = []
+        if len(citations_found) == 0 and len(self.llm_out) - self.past_cite_count > 5:
+            self.current_citations = []
 
-        if citations_found and not in_code_block(llm_out):
+        result = ""  # Initialize result here
+        if citations_found and not in_code_block(self.llm_out):
             last_citation_end = 0
             length_to_add = 0
             while len(citations_found) > 0:
                 citation = citations_found.pop(0)
                 numerical_value = int(citation.group(1))
 
-                if 1 <= numerical_value <= max_citation_num:
-                    context_llm_doc = context_docs[numerical_value - 1]
-                    real_citation_num = order_mapping[context_llm_doc.document_id]
+                if 1 <= numerical_value <= self.max_citation_num:
+                    context_llm_doc = self.context_docs[numerical_value - 1]
+                    real_citation_num = self.order_mapping[context_llm_doc.document_id]
 
-                    if real_citation_num not in citation_order:
-                        citation_order.append(real_citation_num)
+                    if real_citation_num not in self.citation_order:
+                        self.citation_order.append(real_citation_num)
 
-                    target_citation_num = citation_order.index(real_citation_num) + 1
+                    target_citation_num = (
+                        self.citation_order.index(real_citation_num) + 1
+                    )
 
                     # Skip consecutive citations of the same work
-                    if target_citation_num in current_citations:
+                    if target_citation_num in self.current_citations:
                         start, end = citation.span()
                         real_start = length_to_add + start
                         diff = end - start
-                        curr_segment = (
-                            curr_segment[: length_to_add + start]
-                            + curr_segment[real_start + diff :]
+                        self.curr_segment = (
+                            self.curr_segment[: length_to_add + start]
+                            + self.curr_segment[real_start + diff :]
                         )
                         length_to_add -= diff
                         continue
 
                     # Handle edge case where LLM outputs citation itself
-                    # by allowing it to generate citations on its own.
-                    if curr_segment.startswith("[["):
-                        match = re.match(r"\[\[(\d+)\]\]", curr_segment)
+                    if self.curr_segment.startswith("[["):
+                        match = re.match(r"\[\[(\d+)\]\]", self.curr_segment)
                         if match:
                             try:
                                 doc_id = int(match.group(1))
-                                context_llm_doc = context_docs[doc_id - 1]
+                                context_llm_doc = self.context_docs[doc_id - 1]
                                 yield CitationInfo(
                                     citation_num=target_citation_num,
                                     document_id=context_llm_doc.document_id,
@@ -150,75 +124,57 @@ def extract_citations_from_stream(
                                     f"Manual LLM citation didn't properly cite documents {e}"
                                 )
                         else:
-                            # Will continue attempt on next loops
                             logger.warning(
                                 "Manual LLM citation wasn't able to close brackets"
                             )
-
                         continue
 
                     link = context_llm_doc.link
 
                     # Replace the citation in the current segment
                     start, end = citation.span()
-                    curr_segment = (
-                        curr_segment[: start + length_to_add]
+                    self.curr_segment = (
+                        self.curr_segment[: start + length_to_add]
                         + f"[{target_citation_num}]"
-                        + curr_segment[end + length_to_add :]
+                        + self.curr_segment[end + length_to_add :]
                     )
 
-                    past_cite_count = len(llm_out)
-                    current_citations.append(target_citation_num)
+                    self.past_cite_count = len(self.llm_out)
+                    self.current_citations.append(target_citation_num)
 
-                    if target_citation_num not in cited_inds:
-                        cited_inds.add(target_citation_num)
+                    if target_citation_num not in self.cited_inds:
+                        self.cited_inds.add(target_citation_num)
                         yield CitationInfo(
                             citation_num=target_citation_num,
                             document_id=context_llm_doc.document_id,
                         )
 
                     if link:
-                        prev_length = len(curr_segment)
-                        curr_segment = (
-                            curr_segment[: start + length_to_add]
+                        prev_length = len(self.curr_segment)
+                        self.curr_segment = (
+                            self.curr_segment[: start + length_to_add]
                             + f"[[{target_citation_num}]]({link})"
-                            + curr_segment[end + length_to_add :]
+                            + self.curr_segment[end + length_to_add :]
                         )
-                        length_to_add += len(curr_segment) - prev_length
-
+                        length_to_add += len(self.curr_segment) - prev_length
                     else:
-                        prev_length = len(curr_segment)
-                        curr_segment = (
-                            curr_segment[: start + length_to_add]
+                        prev_length = len(self.curr_segment)
+                        self.curr_segment = (
+                            self.curr_segment[: start + length_to_add]
                             + f"[[{target_citation_num}]]()"
-                            + curr_segment[end + length_to_add :]
+                            + self.curr_segment[end + length_to_add :]
                         )
-                        length_to_add += len(curr_segment) - prev_length
+                        length_to_add += len(self.curr_segment) - prev_length
 
                     last_citation_end = end + length_to_add
 
             if last_citation_end > 0:
-                yield DanswerAnswerPiece(answer_piece=curr_segment[:last_citation_end])
-                curr_segment = curr_segment[last_citation_end:]
-        if possible_citation_found:
-            continue
-        yield DanswerAnswerPiece(answer_piece=curr_segment)
-        curr_segment = ""
-
-    if curr_segment:
-        yield DanswerAnswerPiece(answer_piece=curr_segment)
-
-
-def build_citation_processor(
-    context_docs: list[LlmDoc], doc_id_to_rank_map: DocumentIdOrderMapping
-) -> StreamProcessor:
-    def stream_processor(
-        tokens: Iterator[str],
-    ) -> AnswerQuestionStreamReturn:
-        yield from extract_citations_from_stream(
-            tokens=tokens,
-            context_docs=context_docs,
-            doc_id_to_rank_map=doc_id_to_rank_map,
-        )
+                result += self.curr_segment[:last_citation_end]
+                self.curr_segment = self.curr_segment[last_citation_end:]
+
+        if not possible_citation_found:
+            result += self.curr_segment
+            self.curr_segment = ""
 
-    return stream_processor
+        if result:
+            yield DanswerAnswerPiece(answer_piece=result)
diff --git a/backend/danswer/llm/answering/stream_processing/quotes_processing.py b/backend/danswer/llm/answering/stream_processing/quotes_processing.py
index 501a56b5aa7..33913133164 100644
--- a/backend/danswer/llm/answering/stream_processing/quotes_processing.py
+++ b/backend/danswer/llm/answering/stream_processing/quotes_processing.py
@@ -1,14 +1,11 @@
 import math
 import re
-from collections.abc import Callable
 from collections.abc import Generator
-from collections.abc import Iterator
 from json import JSONDecodeError
 from typing import Optional
 
 import regex
 
-from danswer.chat.models import AnswerQuestionStreamReturn
 from danswer.chat.models import DanswerAnswer
 from danswer.chat.models import DanswerAnswerPiece
 from danswer.chat.models import DanswerQuote
@@ -157,7 +154,7 @@ def separate_answer_quotes(
     return _extract_answer_quotes_freeform(clean_up_code_blocks(answer_raw))
 
 
-def process_answer(
+def _process_answer(
     answer_raw: str,
     docs: list[LlmDoc],
     is_json_prompt: bool = True,
@@ -195,7 +192,7 @@ def _stream_json_answer_end(answer_so_far: str, next_token: str) -> bool:
 def _extract_quotes_from_completed_token_stream(
     model_output: str, context_docs: list[LlmDoc], is_json_prompt: bool = True
 ) -> DanswerQuotes:
-    answer, quotes = process_answer(model_output, context_docs, is_json_prompt)
+    answer, quotes = _process_answer(model_output, context_docs, is_json_prompt)
     if answer:
         logger.notice(answer)
     elif model_output:
@@ -204,94 +201,101 @@ def _extract_quotes_from_completed_token_stream(
     return quotes
 
 
-def process_model_tokens(
-    tokens: Iterator[str],
-    context_docs: list[LlmDoc],
-    is_json_prompt: bool = True,
-) -> Generator[DanswerAnswerPiece | DanswerQuotes, None, None]:
-    """Used in the streaming case to process the model output
-    into an Answer and Quotes
-
-    Yields Answer tokens back out in a dict for streaming to frontend
-    When Answer section ends, yields dict with answer_finished key
-    Collects all the tokens at the end to form the complete model output"""
-    quote_pat = f"\n{QUOTE_PAT}"
-    # Sometimes worse model outputs new line instead of :
-    quote_loose = f"\n{quote_pat[:-1]}\n"
-    # Sometime model outputs two newlines before quote section
-    quote_pat_full = f"\n{quote_pat}"
-    model_output: str = ""
-    found_answer_start = False if is_json_prompt else True
-    found_answer_end = False
-    hold_quote = ""
-
-    for token in tokens:
-        model_previous = model_output
-        model_output += token
-
-        if not found_answer_start:
-            m = answer_pattern.search(model_output)
+class QuotesProcessor:
+    def __init__(
+        self,
+        context_docs: list[LlmDoc],
+        is_json_prompt: bool = True,
+    ):
+        self.context_docs = context_docs
+        self.is_json_prompt = is_json_prompt
+
+        self.found_answer_start = False if is_json_prompt else True
+        self.found_answer_end = False
+        self.hold_quote = ""
+        self.model_output = ""
+        self.hold = ""
+
+    def process_token(
+        self, token: str | None
+    ) -> Generator[DanswerAnswerPiece | DanswerQuotes, None, None]:
+        # None -> end of stream
+        if token is None:
+            if self.model_output:
+                yield _extract_quotes_from_completed_token_stream(
+                    model_output=self.model_output,
+                    context_docs=self.context_docs,
+                    is_json_prompt=self.is_json_prompt,
+                )
+            return
+
+        model_previous = self.model_output
+        self.model_output += token
+
+        if not self.found_answer_start:
+            m = answer_pattern.search(self.model_output)
             if m:
-                found_answer_start = True
+                self.found_answer_start = True
 
-                # Prevent heavy cases of hallucinations where model is never providing a JSON
-                # We want to quickly update the user - not stream forever
-                if is_json_prompt and len(model_output) > 70:
+                # Prevent heavy cases of hallucinations
+                if self.is_json_prompt and len(self.model_output) > 70:
                     logger.warning("LLM did not produce json as prompted")
-                    found_answer_end = True
-                    continue
-
-                remaining = model_output[m.end() :]
+                    self.found_answer_end = True
+                    return
+
+                remaining = self.model_output[m.end() :]
+
+                # Look for an unescaped quote, which means the answer is entirely contained
+                # in this token e.g. if the token is `{"answer": "blah", "qu`
+                quote_indices = [i for i, char in enumerate(remaining) if char == '"']
+                for quote_idx in quote_indices:
+                    # Check if quote is escaped by counting backslashes before it
+                    num_backslashes = 0
+                    pos = quote_idx - 1
+                    while pos >= 0 and remaining[pos] == "\\":
+                        num_backslashes += 1
+                        pos -= 1
+                    # If even number of backslashes, quote is not escaped
+                    if num_backslashes % 2 == 0:
+                        yield DanswerAnswerPiece(answer_piece=remaining[:quote_idx])
+                        return
+
+                # If no unescaped quote found, yield the remaining string
                 if len(remaining) > 0:
                     yield DanswerAnswerPiece(answer_piece=remaining)
-                continue
+                return
 
-        if found_answer_start and not found_answer_end:
-            if is_json_prompt and _stream_json_answer_end(model_previous, token):
-                found_answer_end = True
+        if self.found_answer_start and not self.found_answer_end:
+            if self.is_json_prompt and _stream_json_answer_end(model_previous, token):
+                self.found_answer_end = True
 
-                # return the remaining part of the answer e.g. token might be 'd.", ' and we should yield 'd.'
                 if token:
                     try:
                         answer_token_section = token.index('"')
                         yield DanswerAnswerPiece(
-                            answer_piece=hold_quote + token[:answer_token_section]
+                            answer_piece=self.hold_quote + token[:answer_token_section]
                         )
                     except ValueError:
                         logger.error("Quotation mark not found in token")
-                        yield DanswerAnswerPiece(answer_piece=hold_quote + token)
+                        yield DanswerAnswerPiece(answer_piece=self.hold_quote + token)
                 yield DanswerAnswerPiece(answer_piece=None)
-                continue
-            elif not is_json_prompt:
-                if quote_pat in hold_quote + token or quote_loose in hold_quote + token:
-                    found_answer_end = True
+                return
+
+            elif not self.is_json_prompt:
+                quote_pat = f"\n{QUOTE_PAT}"
+                quote_loose = f"\n{quote_pat[:-1]}\n"
+                quote_pat_full = f"\n{quote_pat}"
+
+                if (
+                    quote_pat in self.hold_quote + token
+                    or quote_loose in self.hold_quote + token
+                ):
+                    self.found_answer_end = True
                     yield DanswerAnswerPiece(answer_piece=None)
-                    continue
-                if hold_quote + token in quote_pat_full:
-                    hold_quote += token
-                    continue
-            yield DanswerAnswerPiece(answer_piece=hold_quote + token)
-            hold_quote = ""
-
-    logger.debug(f"Raw Model QnA Output: {model_output}")
-
-    yield _extract_quotes_from_completed_token_stream(
-        model_output=model_output,
-        context_docs=context_docs,
-        is_json_prompt=is_json_prompt,
-    )
-
-
-def build_quotes_processor(
-    context_docs: list[LlmDoc], is_json_prompt: bool
-) -> Callable[[Iterator[str]], AnswerQuestionStreamReturn]:
-    def stream_processor(
-        tokens: Iterator[str],
-    ) -> AnswerQuestionStreamReturn:
-        yield from process_model_tokens(
-            tokens=tokens,
-            context_docs=context_docs,
-            is_json_prompt=is_json_prompt,
-        )
-
-    return stream_processor
+                    return
+                if self.hold_quote + token in quote_pat_full:
+                    self.hold_quote += token
+                    return
+
+            yield DanswerAnswerPiece(answer_piece=self.hold_quote + token)
+            self.hold_quote = ""
diff --git a/backend/danswer/llm/answering/tool/tool_response_handler.py b/backend/danswer/llm/answering/tool/tool_response_handler.py
new file mode 100644
index 00000000000..08e7284f790
--- /dev/null
+++ b/backend/danswer/llm/answering/tool/tool_response_handler.py
@@ -0,0 +1,207 @@
+from collections.abc import Generator
+
+from langchain_core.messages import AIMessageChunk
+from langchain_core.messages import BaseMessage
+from langchain_core.messages import ToolCall
+
+from danswer.llm.answering.llm_response_handler import LLMCall
+from danswer.llm.answering.llm_response_handler import ResponsePart
+from danswer.llm.interfaces import LLM
+from danswer.tools.force import ForceUseTool
+from danswer.tools.message import build_tool_message
+from danswer.tools.message import ToolCallSummary
+from danswer.tools.models import ToolCallFinalResult
+from danswer.tools.models import ToolCallKickoff
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool import Tool
+from danswer.tools.tool_runner import (
+    check_which_tools_should_run_for_non_tool_calling_llm,
+)
+from danswer.tools.tool_runner import ToolRunner
+from danswer.tools.tool_selection import select_single_tool_for_non_tool_calling_llm
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
+
+
+class ToolResponseHandler:
+    def __init__(self, tools: list[Tool]):
+        self.tools = tools
+
+        self.tool_call_chunk: AIMessageChunk | None = None
+        self.tool_call_requests: list[ToolCall] = []
+
+        self.tool_runner: ToolRunner | None = None
+        self.tool_call_summary: ToolCallSummary | None = None
+
+        self.tool_kickoff: ToolCallKickoff | None = None
+        self.tool_responses: list[ToolResponse] = []
+        self.tool_final_result: ToolCallFinalResult | None = None
+
+    @classmethod
+    def get_tool_call_for_non_tool_calling_llm(
+        cls, llm_call: LLMCall, llm: LLM
+    ) -> tuple[Tool, dict] | None:
+        if llm_call.force_use_tool.force_use:
+            # if we are forcing a tool, we don't need to check which tools to run
+            tool = next(
+                (
+                    t
+                    for t in llm_call.tools
+                    if t.name == llm_call.force_use_tool.tool_name
+                ),
+                None,
+            )
+            if not tool:
+                raise RuntimeError(
+                    f"Tool '{llm_call.force_use_tool.tool_name}' not found"
+                )
+
+            tool_args = (
+                llm_call.force_use_tool.args
+                if llm_call.force_use_tool.args is not None
+                else tool.get_args_for_non_tool_calling_llm(
+                    query=llm_call.prompt_builder.get_user_message_content(),
+                    history=llm_call.prompt_builder.raw_message_history,
+                    llm=llm,
+                    force_run=True,
+                )
+            )
+
+            if tool_args is None:
+                raise RuntimeError(f"Tool '{tool.name}' did not return args")
+
+            return (tool, tool_args)
+        else:
+            tool_options = check_which_tools_should_run_for_non_tool_calling_llm(
+                tools=llm_call.tools,
+                query=llm_call.prompt_builder.get_user_message_content(),
+                history=llm_call.prompt_builder.raw_message_history,
+                llm=llm,
+            )
+
+            available_tools_and_args = [
+                (llm_call.tools[ind], args)
+                for ind, args in enumerate(tool_options)
+                if args is not None
+            ]
+
+            logger.info(
+                f"Selecting single tool from tools: {[(tool.name, args) for tool, args in available_tools_and_args]}"
+            )
+
+            chosen_tool_and_args = (
+                select_single_tool_for_non_tool_calling_llm(
+                    tools_and_args=available_tools_and_args,
+                    history=llm_call.prompt_builder.raw_message_history,
+                    query=llm_call.prompt_builder.get_user_message_content(),
+                    llm=llm,
+                )
+                if available_tools_and_args
+                else None
+            )
+
+            logger.notice(f"Chosen tool: {chosen_tool_and_args}")
+            return chosen_tool_and_args
+
+    def _handle_tool_call(self) -> Generator[ResponsePart, None, None]:
+        if not self.tool_call_chunk or not self.tool_call_chunk.tool_calls:
+            return
+
+        self.tool_call_requests = self.tool_call_chunk.tool_calls
+
+        selected_tool: Tool | None = None
+        selected_tool_call_request: ToolCall | None = None
+        for tool_call_request in self.tool_call_requests:
+            known_tools_by_name = [
+                tool for tool in self.tools if tool.name == tool_call_request["name"]
+            ]
+
+            if not known_tools_by_name:
+                logger.error(
+                    "Tool call requested with unknown name field. \n"
+                    f"self.tools: {self.tools}"
+                    f"tool_call_request: {tool_call_request}"
+                )
+                continue
+            else:
+                selected_tool = known_tools_by_name[0]
+                selected_tool_call_request = tool_call_request
+
+            if selected_tool and selected_tool_call_request:
+                break
+
+        if not selected_tool or not selected_tool_call_request:
+            return
+
+        logger.info(f"Selected tool: {selected_tool.name}")
+        logger.debug(f"Selected tool call request: {selected_tool_call_request}")
+        self.tool_runner = ToolRunner(selected_tool, selected_tool_call_request["args"])
+        self.tool_kickoff = self.tool_runner.kickoff()
+        yield self.tool_kickoff
+
+        for response in self.tool_runner.tool_responses():
+            self.tool_responses.append(response)
+            yield response
+
+        self.tool_final_result = self.tool_runner.tool_final_result()
+        yield self.tool_final_result
+
+        self.tool_call_summary = ToolCallSummary(
+            tool_call_request=self.tool_call_chunk,
+            tool_call_result=build_tool_message(
+                selected_tool_call_request, self.tool_runner.tool_message_content()
+            ),
+        )
+
+    def handle_response_part(
+        self,
+        response_item: BaseMessage | None,
+        previous_response_items: list[BaseMessage],
+    ) -> Generator[ResponsePart, None, None]:
+        if response_item is None:
+            yield from self._handle_tool_call()
+
+        if isinstance(response_item, AIMessageChunk) and (
+            response_item.tool_call_chunks or response_item.tool_calls
+        ):
+            if self.tool_call_chunk is None:
+                self.tool_call_chunk = response_item
+            else:
+                self.tool_call_chunk += response_item  # type: ignore
+
+        return
+
+    def next_llm_call(self, current_llm_call: LLMCall) -> LLMCall | None:
+        if (
+            self.tool_runner is None
+            or self.tool_call_summary is None
+            or self.tool_kickoff is None
+            or self.tool_final_result is None
+        ):
+            return None
+
+        tool_runner = self.tool_runner
+        new_prompt_builder = tool_runner.tool.build_next_prompt(
+            prompt_builder=current_llm_call.prompt_builder,
+            tool_call_summary=self.tool_call_summary,
+            tool_responses=self.tool_responses,
+            using_tool_calling_llm=current_llm_call.using_tool_calling_llm,
+        )
+        return LLMCall(
+            prompt_builder=new_prompt_builder,
+            tools=[],  # for now, only allow one tool call per response
+            force_use_tool=ForceUseTool(
+                force_use=False,
+                tool_name="",
+                args=None,
+            ),
+            files=current_llm_call.files,
+            using_tool_calling_llm=current_llm_call.using_tool_calling_llm,
+            tool_call_info=[
+                self.tool_kickoff,
+                *self.tool_responses,
+                self.tool_final_result,
+            ],
+        )
diff --git a/backend/danswer/llm/utils.py b/backend/danswer/llm/utils.py
index bad18214b95..af480f83955 100644
--- a/backend/danswer/llm/utils.py
+++ b/backend/danswer/llm/utils.py
@@ -203,6 +203,28 @@ def build_content_with_imgs(
     )
 
 
+def message_to_prompt_and_imgs(message: BaseMessage) -> tuple[str, list[str]]:
+    if isinstance(message.content, str):
+        return message.content, []
+
+    imgs = []
+    texts = []
+    for part in message.content:
+        if isinstance(part, dict):
+            if part.get("type") == "image_url":
+                img_url = part.get("image_url", {}).get("url")
+                if img_url:
+                    imgs.append(img_url)
+            elif part.get("type") == "text":
+                text = part.get("text")
+                if text:
+                    texts.append(text)
+        else:
+            texts.append(part)
+
+    return "".join(texts), imgs
+
+
 def dict_based_prompt_to_langchain_prompt(
     messages: list[dict[str, str]]
 ) -> list[BaseMessage]:
diff --git a/backend/danswer/one_shot_answer/answer_question.py b/backend/danswer/one_shot_answer/answer_question.py
index 1bfac570aee..9ece5f4bba2 100644
--- a/backend/danswer/one_shot_answer/answer_question.py
+++ b/backend/danswer/one_shot_answer/answer_question.py
@@ -52,12 +52,16 @@
 from danswer.server.query_and_chat.models import ChatMessageDetail
 from danswer.server.utils import get_json_line
 from danswer.tools.force import ForceUseTool
-from danswer.tools.search.search_tool import SEARCH_DOC_CONTENT_ID
-from danswer.tools.search.search_tool import SEARCH_RESPONSE_SUMMARY_ID
-from danswer.tools.search.search_tool import SearchResponseSummary
-from danswer.tools.search.search_tool import SearchTool
-from danswer.tools.search.search_tool import SECTION_RELEVANCE_LIST_ID
-from danswer.tools.tool import ToolResponse
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool_implementations.search.search_tool import SEARCH_DOC_CONTENT_ID
+from danswer.tools.tool_implementations.search.search_tool import (
+    SEARCH_RESPONSE_SUMMARY_ID,
+)
+from danswer.tools.tool_implementations.search.search_tool import SearchResponseSummary
+from danswer.tools.tool_implementations.search.search_tool import SearchTool
+from danswer.tools.tool_implementations.search.search_tool import (
+    SECTION_RELEVANCE_LIST_ID,
+)
 from danswer.tools.tool_runner import ToolCallKickoff
 from danswer.utils.logger import setup_logger
 from danswer.utils.timing import log_generator_function_time
@@ -202,30 +206,33 @@ def stream_answer_objects(
         max_tokens=max_document_tokens,
     )
 
+    answer_config = AnswerStyleConfig(
+        citation_config=CitationConfig() if use_citations else None,
+        quotes_config=QuotesConfig() if not use_citations else None,
+        document_pruning_config=document_pruning_config,
+    )
+
     search_tool = SearchTool(
         db_session=db_session,
         user=user,
-        evaluation_type=LLMEvaluationType.SKIP
-        if DISABLE_LLM_DOC_RELEVANCE
-        else query_req.evaluation_type,
+        evaluation_type=(
+            LLMEvaluationType.SKIP
+            if DISABLE_LLM_DOC_RELEVANCE
+            else query_req.evaluation_type
+        ),
         persona=persona,
         retrieval_options=query_req.retrieval_options,
         prompt_config=prompt_config,
         llm=llm,
         fast_llm=fast_llm,
         pruning_config=document_pruning_config,
+        answer_style_config=answer_config,
         bypass_acl=bypass_acl,
         chunks_above=query_req.chunks_above,
         chunks_below=query_req.chunks_below,
         full_doc=query_req.full_doc,
     )
 
-    answer_config = AnswerStyleConfig(
-        citation_config=CitationConfig() if use_citations else None,
-        quotes_config=QuotesConfig() if not use_citations else None,
-        document_pruning_config=document_pruning_config,
-    )
-
     answer = Answer(
         question=query_msg.message,
         answer_style_config=answer_config,
diff --git a/backend/danswer/server/features/persona/models.py b/backend/danswer/server/features/persona/models.py
index 0e69777a02c..5fa99952b1f 100644
--- a/backend/danswer/server/features/persona/models.py
+++ b/backend/danswer/server/features/persona/models.py
@@ -9,7 +9,7 @@
 from danswer.search.enums import RecencyBiasSetting
 from danswer.server.features.document_set.models import DocumentSet
 from danswer.server.features.prompt.models import PromptSnapshot
-from danswer.server.features.tool.api import ToolSnapshot
+from danswer.server.features.tool.models import ToolSnapshot
 from danswer.server.models import MinimalUserSnapshot
 from danswer.utils.logger import setup_logger
 
diff --git a/backend/danswer/server/features/tool/api.py b/backend/danswer/server/features/tool/api.py
index 7e15c048826..48f857780ba 100644
--- a/backend/danswer/server/features/tool/api.py
+++ b/backend/danswer/server/features/tool/api.py
@@ -18,10 +18,16 @@
 from danswer.server.features.tool.models import CustomToolCreate
 from danswer.server.features.tool.models import CustomToolUpdate
 from danswer.server.features.tool.models import ToolSnapshot
-from danswer.tools.custom.openapi_parsing import MethodSpec
-from danswer.tools.custom.openapi_parsing import openapi_to_method_specs
-from danswer.tools.custom.openapi_parsing import validate_openapi_schema
-from danswer.tools.images.image_generation_tool import ImageGenerationTool
+from danswer.tools.tool_implementations.custom.openapi_parsing import MethodSpec
+from danswer.tools.tool_implementations.custom.openapi_parsing import (
+    openapi_to_method_specs,
+)
+from danswer.tools.tool_implementations.custom.openapi_parsing import (
+    validate_openapi_schema,
+)
+from danswer.tools.tool_implementations.images.image_generation_tool import (
+    ImageGenerationTool,
+)
 from danswer.tools.utils import is_image_generation_available
 
 router = APIRouter(prefix="/tool")
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index ac64231dcf2..e14f6f25c5d 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -1,5 +1,6 @@
 import asyncio
 import io
+import json
 import uuid
 from collections.abc import Callable
 from collections.abc import Generator
@@ -283,13 +284,14 @@ def delete_chat_session_by_id(
         raise HTTPException(status_code=400, detail=str(e))
 
 
-async def is_disconnected(request: Request) -> Callable[[], bool]:
+async def is_connected(request: Request) -> Callable[[], bool]:
     main_loop = asyncio.get_event_loop()
 
-    def is_disconnected_sync() -> bool:
+    def is_connected_sync() -> bool:
         future = asyncio.run_coroutine_threadsafe(request.is_disconnected(), main_loop)
         try:
-            return not future.result(timeout=0.01)
+            is_connected = not future.result(timeout=0.01)
+            return is_connected
         except asyncio.TimeoutError:
             logger.error("Asyncio timed out")
             return True
@@ -300,7 +302,7 @@ def is_disconnected_sync() -> bool:
             )
             return True
 
-    return is_disconnected_sync
+    return is_connected_sync
 
 
 @router.post("/send-message")
@@ -309,7 +311,7 @@ def handle_new_chat_message(
     request: Request,
     user: User | None = Depends(current_user),
     _: None = Depends(check_token_rate_limits),
-    is_disconnected_func: Callable[[], bool] = Depends(is_disconnected),
+    is_connected_func: Callable[[], bool] = Depends(is_connected),
 ) -> StreamingResponse:
     """
     This endpoint is both used for all the following purposes:
@@ -325,7 +327,7 @@ def handle_new_chat_message(
         request (Request): The current HTTP request context.
         user (User | None): The current user, obtained via dependency injection.
         _ (None): Rate limit check is run if user/group/global rate limits are enabled.
-        is_disconnected_func (Callable[[], bool]): Function to check client disconnection,
+        is_connected_func (Callable[[], bool]): Function to check client disconnection,
             used to stop the streaming response if the client disconnects.
 
     Returns:
@@ -340,8 +342,6 @@ def handle_new_chat_message(
     ):
         raise HTTPException(status_code=400, detail="Empty chat message is invalid")
 
-    import json
-
     def stream_generator() -> Generator[str, None, None]:
         try:
             for packet in stream_chat_message(
@@ -354,7 +354,7 @@ def stream_generator() -> Generator[str, None, None]:
                 custom_tool_additional_headers=get_custom_tool_additional_request_headers(
                     request.headers
                 ),
-                is_connected=is_disconnected_func,
+                is_connected=is_connected_func,
             ):
                 yield json.dumps(packet) if isinstance(packet, dict) else packet
 
@@ -362,6 +362,9 @@ def stream_generator() -> Generator[str, None, None]:
             logger.exception(f"Error in chat message streaming: {e}")
             yield json.dumps({"error": str(e)})
 
+        finally:
+            logger.debug("Stream generator finished")
+
     return StreamingResponse(stream_generator(), media_type="text/event-stream")
 
 
diff --git a/backend/danswer/tools/base_tool.py b/backend/danswer/tools/base_tool.py
new file mode 100644
index 00000000000..73902504462
--- /dev/null
+++ b/backend/danswer/tools/base_tool.py
@@ -0,0 +1,59 @@
+from typing import cast
+from typing import TYPE_CHECKING
+
+from langchain_core.messages import HumanMessage
+
+from danswer.llm.utils import message_to_prompt_and_imgs
+from danswer.tools.tool import Tool
+
+if TYPE_CHECKING:
+    from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+    from danswer.tools.tool_implementations.custom.custom_tool import (
+        CustomToolCallSummary,
+    )
+    from danswer.tools.message import ToolCallSummary
+    from danswer.tools.models import ToolResponse
+
+
+def build_user_message_for_non_tool_calling_llm(
+    message: HumanMessage,
+    tool_name: str,
+    *args: "ToolResponse",
+) -> str:
+    query, _ = message_to_prompt_and_imgs(message)
+
+    tool_run_summary = cast("CustomToolCallSummary", args[0].response).tool_result
+    return f"""
+Here's the result from the {tool_name} tool:
+
+{tool_run_summary}
+
+Now respond to the following:
+
+{query}
+""".strip()
+
+
+class BaseTool(Tool):
+    def build_next_prompt(
+        self,
+        prompt_builder: "AnswerPromptBuilder",
+        tool_call_summary: "ToolCallSummary",
+        tool_responses: list["ToolResponse"],
+        using_tool_calling_llm: bool,
+    ) -> "AnswerPromptBuilder":
+        if using_tool_calling_llm:
+            prompt_builder.append_message(tool_call_summary.tool_call_request)
+            prompt_builder.append_message(tool_call_summary.tool_call_result)
+        else:
+            prompt_builder.update_user_prompt(
+                HumanMessage(
+                    content=build_user_message_for_non_tool_calling_llm(
+                        prompt_builder.user_message_and_token_cnt[0],
+                        self.name,
+                        *tool_responses,
+                    )
+                )
+            )
+
+        return prompt_builder
diff --git a/backend/danswer/tools/built_in_tools.py b/backend/danswer/tools/built_in_tools.py
index 99b2ae3bbb6..fb64381f1d0 100644
--- a/backend/danswer/tools/built_in_tools.py
+++ b/backend/danswer/tools/built_in_tools.py
@@ -9,9 +9,13 @@
 
 from danswer.db.models import Persona
 from danswer.db.models import Tool as ToolDBModel
-from danswer.tools.images.image_generation_tool import ImageGenerationTool
-from danswer.tools.internet_search.internet_search_tool import InternetSearchTool
-from danswer.tools.search.search_tool import SearchTool
+from danswer.tools.tool_implementations.images.image_generation_tool import (
+    ImageGenerationTool,
+)
+from danswer.tools.tool_implementations.internet_search.internet_search_tool import (
+    InternetSearchTool,
+)
+from danswer.tools.tool_implementations.search.search_tool import SearchTool
 from danswer.tools.tool import Tool
 from danswer.utils.logger import setup_logger
 
diff --git a/backend/danswer/tools/custom/custom_tool_prompt_builder.py b/backend/danswer/tools/custom/custom_tool_prompt_builder.py
deleted file mode 100644
index 8016363acc9..00000000000
--- a/backend/danswer/tools/custom/custom_tool_prompt_builder.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from typing import cast
-
-from danswer.tools.custom.custom_tool import CustomToolCallSummary
-from danswer.tools.models import ToolResponse
-
-
-def build_user_message_for_custom_tool_for_non_tool_calling_llm(
-    query: str,
-    tool_name: str,
-    *args: ToolResponse,
-) -> str:
-    tool_run_summary = cast(CustomToolCallSummary, args[0].response).tool_result
-    return f"""
-Here's the result from the {tool_name} tool:
-
-{tool_run_summary}
-
-Now respond to the following:
-
-{query}
-""".strip()
diff --git a/backend/danswer/tools/tool.py b/backend/danswer/tools/tool.py
index 29e5311fc15..1b1c43ab8da 100644
--- a/backend/danswer/tools/tool.py
+++ b/backend/danswer/tools/tool.py
@@ -1,11 +1,17 @@
 import abc
 from collections.abc import Generator
 from typing import Any
+from typing import TYPE_CHECKING
 
 from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
-from danswer.tools.models import ToolResponse
+
+
+if TYPE_CHECKING:
+    from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+    from danswer.tools.message import ToolCallSummary
+    from danswer.tools.models import ToolResponse
 
 
 class Tool(abc.ABC):
@@ -32,7 +38,7 @@ def tool_definition(self) -> dict:
 
     @abc.abstractmethod
     def build_tool_message_content(
-        self, *args: ToolResponse
+        self, *args: "ToolResponse"
     ) -> str | list[str | dict[str, Any]]:
         raise NotImplementedError
 
@@ -51,13 +57,26 @@ def get_args_for_non_tool_calling_llm(
     """Actual execution of the tool"""
 
     @abc.abstractmethod
-    def run(self, **kwargs: Any) -> Generator[ToolResponse, None, None]:
+    def run(self, **kwargs: Any) -> Generator["ToolResponse", None, None]:
         raise NotImplementedError
 
     @abc.abstractmethod
-    def final_result(self, *args: ToolResponse) -> JSON_ro:
+    def final_result(self, *args: "ToolResponse") -> JSON_ro:
         """
         This is the "final summary" result of the tool.
         It is the result that will be stored in the database.
         """
         raise NotImplementedError
+
+    """Some tools may want to modify the prompt based on the tool call summary and tool responses.
+    Default behavior is to continue with just the raw tool call request/result passed to the LLM."""
+
+    @abc.abstractmethod
+    def build_next_prompt(
+        self,
+        prompt_builder: "AnswerPromptBuilder",
+        tool_call_summary: "ToolCallSummary",
+        tool_responses: list["ToolResponse"],
+        using_tool_calling_llm: bool,
+    ) -> "AnswerPromptBuilder":
+        raise NotImplementedError
diff --git a/backend/danswer/tools/custom/base_tool_types.py b/backend/danswer/tools/tool_implementations/custom/base_tool_types.py
similarity index 100%
rename from backend/danswer/tools/custom/base_tool_types.py
rename to backend/danswer/tools/tool_implementations/custom/base_tool_types.py
diff --git a/backend/danswer/tools/custom/custom_tool.py b/backend/danswer/tools/tool_implementations/custom/custom_tool.py
similarity index 88%
rename from backend/danswer/tools/custom/custom_tool.py
rename to backend/danswer/tools/tool_implementations/custom/custom_tool.py
index ee431af70e1..a1fb4bb699e 100644
--- a/backend/danswer/tools/custom/custom_tool.py
+++ b/backend/danswer/tools/tool_implementations/custom/custom_tool.py
@@ -11,24 +11,34 @@
 from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.interfaces import LLM
-from danswer.tools.custom.base_tool_types import ToolResultType
-from danswer.tools.custom.custom_tool_prompts import (
-    SHOULD_USE_CUSTOM_TOOL_SYSTEM_PROMPT,
-)
-from danswer.tools.custom.custom_tool_prompts import SHOULD_USE_CUSTOM_TOOL_USER_PROMPT
-from danswer.tools.custom.custom_tool_prompts import TOOL_ARG_SYSTEM_PROMPT
-from danswer.tools.custom.custom_tool_prompts import TOOL_ARG_USER_PROMPT
-from danswer.tools.custom.custom_tool_prompts import USE_TOOL
-from danswer.tools.custom.openapi_parsing import MethodSpec
-from danswer.tools.custom.openapi_parsing import openapi_to_method_specs
-from danswer.tools.custom.openapi_parsing import openapi_to_url
-from danswer.tools.custom.openapi_parsing import REQUEST_BODY
-from danswer.tools.custom.openapi_parsing import validate_openapi_schema
+from danswer.tools.base_tool import BaseTool
 from danswer.tools.models import CHAT_SESSION_ID_PLACEHOLDER
 from danswer.tools.models import DynamicSchemaInfo
 from danswer.tools.models import MESSAGE_ID_PLACEHOLDER
-from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool_implementations.custom.base_tool_types import ToolResultType
+from danswer.tools.tool_implementations.custom.custom_tool_prompts import (
+    SHOULD_USE_CUSTOM_TOOL_SYSTEM_PROMPT,
+)
+from danswer.tools.tool_implementations.custom.custom_tool_prompts import (
+    SHOULD_USE_CUSTOM_TOOL_USER_PROMPT,
+)
+from danswer.tools.tool_implementations.custom.custom_tool_prompts import (
+    TOOL_ARG_SYSTEM_PROMPT,
+)
+from danswer.tools.tool_implementations.custom.custom_tool_prompts import (
+    TOOL_ARG_USER_PROMPT,
+)
+from danswer.tools.tool_implementations.custom.custom_tool_prompts import USE_TOOL
+from danswer.tools.tool_implementations.custom.openapi_parsing import MethodSpec
+from danswer.tools.tool_implementations.custom.openapi_parsing import (
+    openapi_to_method_specs,
+)
+from danswer.tools.tool_implementations.custom.openapi_parsing import openapi_to_url
+from danswer.tools.tool_implementations.custom.openapi_parsing import REQUEST_BODY
+from danswer.tools.tool_implementations.custom.openapi_parsing import (
+    validate_openapi_schema,
+)
 from danswer.utils.headers import header_list_to_header_dict
 from danswer.utils.headers import HeaderItemDict
 from danswer.utils.logger import setup_logger
@@ -43,7 +53,7 @@ class CustomToolCallSummary(BaseModel):
     tool_result: ToolResultType
 
 
-class CustomTool(Tool):
+class CustomTool(BaseTool):
     def __init__(
         self,
         method_spec: MethodSpec,
diff --git a/backend/danswer/tools/custom/custom_tool_prompts.py b/backend/danswer/tools/tool_implementations/custom/custom_tool_prompts.py
similarity index 100%
rename from backend/danswer/tools/custom/custom_tool_prompts.py
rename to backend/danswer/tools/tool_implementations/custom/custom_tool_prompts.py
diff --git a/backend/danswer/tools/custom/openapi_parsing.py b/backend/danswer/tools/tool_implementations/custom/openapi_parsing.py
similarity index 100%
rename from backend/danswer/tools/custom/openapi_parsing.py
rename to backend/danswer/tools/tool_implementations/custom/openapi_parsing.py
diff --git a/backend/danswer/tools/images/image_generation_tool.py b/backend/danswer/tools/tool_implementations/images/image_generation_tool.py
similarity index 84%
rename from backend/danswer/tools/images/image_generation_tool.py
rename to backend/danswer/tools/tool_implementations/images/image_generation_tool.py
index 3584d50f77e..3da53751812 100644
--- a/backend/danswer/tools/images/image_generation_tool.py
+++ b/backend/danswer/tools/tool_implementations/images/image_generation_tool.py
@@ -11,12 +11,17 @@
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
 from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.interfaces import LLM
 from danswer.llm.utils import build_content_with_imgs
 from danswer.llm.utils import message_to_string
 from danswer.prompts.constants import GENERAL_SEP_PAT
+from danswer.tools.message import ToolCallSummary
+from danswer.tools.models import ToolResponse
 from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
+from danswer.tools.tool_implementations.images.prompt import (
+    build_image_generation_user_prompt,
+)
 from danswer.utils.headers import build_llm_extra_headers
 from danswer.utils.logger import setup_logger
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
@@ -112,7 +117,10 @@ def tool_definition(self) -> dict:
                         },
                         "shape": {
                             "type": "string",
-                            "description": "Optional. Image shape: 'square', 'portrait', or 'landscape'",
+                            "description": (
+                                "Optional - only specify if you want a specific shape."
+                                " Image shape: 'square', 'portrait', or 'landscape'."
+                            ),
                             "enum": [shape.value for shape in ImageShape],
                         },
                     },
@@ -258,3 +266,34 @@ def final_result(self, *args: ToolResponse) -> JSON_ro:
             image_generation_response.model_dump()
             for image_generation_response in image_generation_responses
         ]
+
+    def build_next_prompt(
+        self,
+        prompt_builder: AnswerPromptBuilder,
+        tool_call_summary: ToolCallSummary,
+        tool_responses: list[ToolResponse],
+        using_tool_calling_llm: bool,
+    ) -> AnswerPromptBuilder:
+        img_generation_response = cast(
+            list[ImageGenerationResponse] | None,
+            next(
+                (
+                    response.response
+                    for response in tool_responses
+                    if response.id == IMAGE_GENERATION_RESPONSE_ID
+                ),
+                None,
+            ),
+        )
+        if img_generation_response is None:
+            raise ValueError("No image generation response found")
+
+        img_urls = [img.url for img in img_generation_response]
+        prompt_builder.update_user_prompt(
+            build_image_generation_user_prompt(
+                query=prompt_builder.get_user_message_content(),
+                img_urls=img_urls,
+            )
+        )
+
+        return prompt_builder
diff --git a/backend/danswer/tools/images/prompt.py b/backend/danswer/tools/tool_implementations/images/prompt.py
similarity index 100%
rename from backend/danswer/tools/images/prompt.py
rename to backend/danswer/tools/tool_implementations/images/prompt.py
diff --git a/backend/danswer/tools/internet_search/internet_search_tool.py b/backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py
similarity index 81%
rename from backend/danswer/tools/internet_search/internet_search_tool.py
rename to backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py
index 70b4483b996..12142bc4852 100644
--- a/backend/danswer/tools/internet_search/internet_search_tool.py
+++ b/backend/danswer/tools/tool_implementations/internet_search/internet_search_tool.py
@@ -11,18 +11,31 @@
 from danswer.configs.constants import DocumentSource
 from danswer.configs.model_configs import GEN_AI_HISTORY_CUTOFF
 from danswer.key_value_store.interface import JSON_ro
+from danswer.llm.answering.models import AnswerStyleConfig
 from danswer.llm.answering.models import PreviousMessage
+from danswer.llm.answering.models import PromptConfig
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.interfaces import LLM
 from danswer.llm.utils import message_to_string
 from danswer.prompts.chat_prompts import INTERNET_SEARCH_QUERY_REPHRASE
 from danswer.prompts.constants import GENERAL_SEP_PAT
 from danswer.search.models import SearchDoc
 from danswer.secondary_llm_flows.query_expansion import history_based_query_rephrase
-from danswer.tools.internet_search.models import InternetSearchResponse
-from danswer.tools.internet_search.models import InternetSearchResult
-from danswer.tools.search.search_tool import FINAL_CONTEXT_DOCUMENTS_ID
+from danswer.tools.message import ToolCallSummary
+from danswer.tools.models import ToolResponse
 from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
+from danswer.tools.tool_implementations.internet_search.models import (
+    InternetSearchResponse,
+)
+from danswer.tools.tool_implementations.internet_search.models import (
+    InternetSearchResult,
+)
+from danswer.tools.tool_implementations.search_like_tool_utils import (
+    build_next_prompt_for_search_like_tool,
+)
+from danswer.tools.tool_implementations.search_like_tool_utils import (
+    FINAL_CONTEXT_DOCUMENTS_ID,
+)
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -97,8 +110,17 @@ class InternetSearchTool(Tool):
     _DISPLAY_NAME = "[Beta] Internet Search Tool"
     _DESCRIPTION = "Perform an internet search for up-to-date information."
 
-    def __init__(self, api_key: str, num_results: int = 10) -> None:
+    def __init__(
+        self,
+        api_key: str,
+        answer_style_config: AnswerStyleConfig,
+        prompt_config: PromptConfig,
+        num_results: int = 10,
+    ) -> None:
         self.api_key = api_key
+        self.answer_style_config = answer_style_config
+        self.prompt_config = prompt_config
+
         self.host = "https://api.bing.microsoft.com/v7.0"
         self.headers = {
             "Ocp-Apim-Subscription-Key": api_key,
@@ -231,3 +253,19 @@ def run(self, **kwargs: str) -> Generator[ToolResponse, None, None]:
     def final_result(self, *args: ToolResponse) -> JSON_ro:
         search_response = cast(InternetSearchResponse, args[0].response)
         return search_response.model_dump()
+
+    def build_next_prompt(
+        self,
+        prompt_builder: AnswerPromptBuilder,
+        tool_call_summary: ToolCallSummary,
+        tool_responses: list[ToolResponse],
+        using_tool_calling_llm: bool,
+    ) -> AnswerPromptBuilder:
+        return build_next_prompt_for_search_like_tool(
+            prompt_builder=prompt_builder,
+            tool_call_summary=tool_call_summary,
+            tool_responses=tool_responses,
+            using_tool_calling_llm=using_tool_calling_llm,
+            answer_style_config=self.answer_style_config,
+            prompt_config=self.prompt_config,
+        )
diff --git a/backend/danswer/tools/internet_search/models.py b/backend/danswer/tools/tool_implementations/internet_search/models.py
similarity index 100%
rename from backend/danswer/tools/internet_search/models.py
rename to backend/danswer/tools/tool_implementations/internet_search/models.py
diff --git a/backend/danswer/tools/search/search_tool.py b/backend/danswer/tools/tool_implementations/search/search_tool.py
similarity index 87%
rename from backend/danswer/tools/search/search_tool.py
rename to backend/danswer/tools/tool_implementations/search/search_tool.py
index 96ab7b843f6..6eda3013ab3 100644
--- a/backend/danswer/tools/search/search_tool.py
+++ b/backend/danswer/tools/tool_implementations/search/search_tool.py
@@ -17,10 +17,13 @@
 from danswer.db.models import Persona
 from danswer.db.models import User
 from danswer.key_value_store.interface import JSON_ro
+from danswer.llm.answering.llm_response_handler import LLMCall
+from danswer.llm.answering.models import AnswerStyleConfig
 from danswer.llm.answering.models import ContextualPruningConfig
 from danswer.llm.answering.models import DocumentPruningConfig
 from danswer.llm.answering.models import PreviousMessage
 from danswer.llm.answering.models import PromptConfig
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.answering.prompts.citations_prompt import compute_max_llm_input_tokens
 from danswer.llm.answering.prune_and_merge import prune_and_merge_sections
 from danswer.llm.answering.prune_and_merge import prune_sections
@@ -35,9 +38,16 @@
 from danswer.search.pipeline import SearchPipeline
 from danswer.secondary_llm_flows.choose_search import check_if_need_search
 from danswer.secondary_llm_flows.query_expansion import history_based_query_rephrase
-from danswer.tools.search.search_utils import llm_doc_to_dict
+from danswer.tools.message import ToolCallSummary
+from danswer.tools.models import ToolResponse
 from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
+from danswer.tools.tool_implementations.search.search_utils import llm_doc_to_dict
+from danswer.tools.tool_implementations.search_like_tool_utils import (
+    build_next_prompt_for_search_like_tool,
+)
+from danswer.tools.tool_implementations.search_like_tool_utils import (
+    FINAL_CONTEXT_DOCUMENTS_ID,
+)
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
@@ -45,7 +55,6 @@
 SEARCH_RESPONSE_SUMMARY_ID = "search_response_summary"
 SEARCH_DOC_CONTENT_ID = "search_doc_content"
 SECTION_RELEVANCE_LIST_ID = "section_relevance_list"
-FINAL_CONTEXT_DOCUMENTS_ID = "final_context_documents"
 SEARCH_EVALUATION_ID = "llm_doc_eval"
 
 
@@ -85,6 +94,7 @@ def __init__(
         llm: LLM,
         fast_llm: LLM,
         pruning_config: DocumentPruningConfig,
+        answer_style_config: AnswerStyleConfig,
         evaluation_type: LLMEvaluationType,
         # if specified, will not actually run a search and will instead return these
         # sections. Used when the user selects specific docs to talk to
@@ -136,6 +146,7 @@ def __init__(
 
         num_chunk_multiple = self.chunks_above + self.chunks_below + 1
 
+        self.answer_style_config = answer_style_config
         self.contextual_pruning_config = (
             ContextualPruningConfig.from_doc_pruning_config(
                 num_chunk_multiple=num_chunk_multiple, doc_pruning_config=pruning_config
@@ -353,4 +364,36 @@ def final_result(self, *args: ToolResponse) -> JSON_ro:
         # NOTE: need to do this json.loads(doc.json()) stuff because there are some
         # subfields that are not serializable by default (datetime)
         # this forces pydantic to make them JSON serializable for us
-        return [json.loads(doc.json()) for doc in final_docs]
+        return [json.loads(doc.model_dump_json()) for doc in final_docs]
+
+    def build_next_prompt(
+        self,
+        prompt_builder: AnswerPromptBuilder,
+        tool_call_summary: ToolCallSummary,
+        tool_responses: list[ToolResponse],
+        using_tool_calling_llm: bool,
+    ) -> AnswerPromptBuilder:
+        return build_next_prompt_for_search_like_tool(
+            prompt_builder=prompt_builder,
+            tool_call_summary=tool_call_summary,
+            tool_responses=tool_responses,
+            using_tool_calling_llm=using_tool_calling_llm,
+            answer_style_config=self.answer_style_config,
+            prompt_config=self.prompt_config,
+        )
+
+    """Other utility functions"""
+
+    @classmethod
+    def get_search_result(cls, llm_call: LLMCall) -> list[LlmDoc] | None:
+        if not llm_call.tool_call_info:
+            return None
+
+        for yield_item in llm_call.tool_call_info:
+            if (
+                isinstance(yield_item, ToolResponse)
+                and yield_item.id == FINAL_CONTEXT_DOCUMENTS_ID
+            ):
+                return cast(list[LlmDoc], yield_item.response)
+
+        return None
diff --git a/backend/danswer/tools/search/search_utils.py b/backend/danswer/tools/tool_implementations/search/search_utils.py
similarity index 100%
rename from backend/danswer/tools/search/search_utils.py
rename to backend/danswer/tools/tool_implementations/search/search_utils.py
diff --git a/backend/danswer/tools/tool_implementations/search_like_tool_utils.py b/backend/danswer/tools/tool_implementations/search_like_tool_utils.py
new file mode 100644
index 00000000000..6701f1602ea
--- /dev/null
+++ b/backend/danswer/tools/tool_implementations/search_like_tool_utils.py
@@ -0,0 +1,71 @@
+from typing import cast
+
+from danswer.chat.models import LlmDoc
+from danswer.llm.answering.models import AnswerStyleConfig
+from danswer.llm.answering.models import PromptConfig
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+from danswer.llm.answering.prompts.citations_prompt import (
+    build_citations_system_message,
+)
+from danswer.llm.answering.prompts.citations_prompt import build_citations_user_message
+from danswer.llm.answering.prompts.quotes_prompt import build_quotes_user_message
+from danswer.tools.message import ToolCallSummary
+from danswer.tools.models import ToolResponse
+
+
+FINAL_CONTEXT_DOCUMENTS_ID = "final_context_documents"
+
+
+def build_next_prompt_for_search_like_tool(
+    prompt_builder: AnswerPromptBuilder,
+    tool_call_summary: ToolCallSummary,
+    tool_responses: list[ToolResponse],
+    using_tool_calling_llm: bool,
+    answer_style_config: AnswerStyleConfig,
+    prompt_config: PromptConfig,
+) -> AnswerPromptBuilder:
+    if not using_tool_calling_llm:
+        final_context_docs_response = next(
+            response
+            for response in tool_responses
+            if response.id == FINAL_CONTEXT_DOCUMENTS_ID
+        )
+        final_context_documents = cast(
+            list[LlmDoc], final_context_docs_response.response
+        )
+    else:
+        # if using tool calling llm, then the final context documents are the tool responses
+        final_context_documents = []
+
+    if answer_style_config.citation_config:
+        prompt_builder.update_system_prompt(
+            build_citations_system_message(prompt_config)
+        )
+        prompt_builder.update_user_prompt(
+            build_citations_user_message(
+                message=prompt_builder.user_message_and_token_cnt[0],
+                prompt_config=prompt_config,
+                context_docs=final_context_documents,
+                all_doc_useful=(
+                    answer_style_config.citation_config.all_docs_useful
+                    if answer_style_config.citation_config
+                    else False
+                ),
+                history_message=prompt_builder.single_message_history or "",
+            )
+        )
+    elif answer_style_config.quotes_config:
+        prompt_builder.update_user_prompt(
+            build_quotes_user_message(
+                message=prompt_builder.user_message_and_token_cnt[0],
+                context_docs=final_context_documents,
+                history_str=prompt_builder.single_message_history or "",
+                prompt=prompt_config,
+            )
+        )
+
+    if using_tool_calling_llm:
+        prompt_builder.append_message(tool_call_summary.tool_call_request)
+        prompt_builder.append_message(tool_call_summary.tool_call_result)
+
+    return prompt_builder
diff --git a/backend/danswer/tools/tool_runner.py b/backend/danswer/tools/tool_runner.py
index 58b94bdb0c8..fb3eb8b9932 100644
--- a/backend/danswer/tools/tool_runner.py
+++ b/backend/danswer/tools/tool_runner.py
@@ -6,8 +6,8 @@
 from danswer.llm.interfaces import LLM
 from danswer.tools.models import ToolCallFinalResult
 from danswer.tools.models import ToolCallKickoff
+from danswer.tools.models import ToolResponse
 from danswer.tools.tool import Tool
-from danswer.tools.tool import ToolResponse
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
 
 
diff --git a/backend/ee/danswer/server/query_and_chat/utils.py b/backend/ee/danswer/server/query_and_chat/utils.py
index a2f7253517a..be5507b01c2 100644
--- a/backend/ee/danswer/server/query_and_chat/utils.py
+++ b/backend/ee/danswer/server/query_and_chat/utils.py
@@ -12,7 +12,7 @@
 from danswer.db.models import User
 from danswer.db.persona import get_prompts_by_ids
 from danswer.one_shot_answer.models import PersonaConfig
-from danswer.tools.custom.custom_tool import (
+from danswer.tools.tool_implementations.custom.custom_tool import (
     build_custom_tools_from_openapi_schema_and_headers,
 )
 
diff --git a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
index 10d1950ae03..0ed40c758d0 100644
--- a/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
+++ b/backend/tests/integration/tests/dev_apis/test_simple_chat_api.py
@@ -142,6 +142,9 @@ def test_using_reference_docs_with_simple_with_history_api_flow(reset: None) ->
     assert response.status_code == 200
     response_json = response.json()
 
+    # make sure there is an answer
+    assert response_json["answer"]
+
     # since we only gave it one search doc, all responses should only contain that doc
     assert response_json["final_context_doc_indices"] == [0]
     assert response_json["llm_selected_doc_indices"] == [0]
diff --git a/backend/tests/unit/danswer/llm/answering/conftest.py b/backend/tests/unit/danswer/llm/answering/conftest.py
new file mode 100644
index 00000000000..a0077b53917
--- /dev/null
+++ b/backend/tests/unit/danswer/llm/answering/conftest.py
@@ -0,0 +1,113 @@
+import json
+from datetime import datetime
+from unittest.mock import MagicMock
+
+import pytest
+from langchain_core.messages import SystemMessage
+
+from danswer.chat.models import LlmDoc
+from danswer.configs.constants import DocumentSource
+from danswer.llm.answering.models import AnswerStyleConfig
+from danswer.llm.answering.models import CitationConfig
+from danswer.llm.answering.models import PromptConfig
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
+from danswer.llm.interfaces import LLMConfig
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool_implementations.search.search_tool import SearchTool
+from danswer.tools.tool_implementations.search_like_tool_utils import (
+    FINAL_CONTEXT_DOCUMENTS_ID,
+)
+
+QUERY = "Test question"
+DEFAULT_SEARCH_ARGS = {"query": "search"}
+
+
+@pytest.fixture
+def answer_style_config() -> AnswerStyleConfig:
+    return AnswerStyleConfig(citation_config=CitationConfig())
+
+
+@pytest.fixture
+def prompt_config() -> PromptConfig:
+    return PromptConfig(
+        system_prompt="System prompt",
+        task_prompt="Task prompt",
+        datetime_aware=False,
+        include_citations=True,
+    )
+
+
+@pytest.fixture
+def mock_llm() -> MagicMock:
+    mock_llm_obj = MagicMock()
+    mock_llm_obj.config = LLMConfig(
+        model_provider="openai",
+        model_name="gpt-4o",
+        temperature=0.0,
+        api_key=None,
+        api_base=None,
+        api_version=None,
+    )
+    return mock_llm_obj
+
+
+@pytest.fixture
+def mock_search_results() -> list[LlmDoc]:
+    return [
+        LlmDoc(
+            content="Search result 1",
+            source_type=DocumentSource.WEB,
+            metadata={"id": "doc1"},
+            document_id="doc1",
+            blurb="Blurb 1",
+            semantic_identifier="Semantic ID 1",
+            updated_at=datetime(2023, 1, 1),
+            link="https://example.com/doc1",
+            source_links={0: "https://example.com/doc1"},
+        ),
+        LlmDoc(
+            content="Search result 2",
+            source_type=DocumentSource.WEB,
+            metadata={"id": "doc2"},
+            document_id="doc2",
+            blurb="Blurb 2",
+            semantic_identifier="Semantic ID 2",
+            updated_at=datetime(2023, 1, 2),
+            link="https://example.com/doc2",
+            source_links={0: "https://example.com/doc2"},
+        ),
+    ]
+
+
+@pytest.fixture
+def mock_search_tool(mock_search_results: list[LlmDoc]) -> MagicMock:
+    mock_tool = MagicMock(spec=SearchTool)
+    mock_tool.name = "search"
+    mock_tool.build_tool_message_content.return_value = "search_response"
+    mock_tool.get_args_for_non_tool_calling_llm.return_value = DEFAULT_SEARCH_ARGS
+    mock_tool.final_result.return_value = [
+        json.loads(doc.model_dump_json()) for doc in mock_search_results
+    ]
+    mock_tool.run.return_value = [
+        ToolResponse(id=FINAL_CONTEXT_DOCUMENTS_ID, response=mock_search_results)
+    ]
+    mock_tool.tool_definition.return_value = {
+        "type": "function",
+        "function": {
+            "name": "search",
+            "description": "Search for information",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {"type": "string", "description": "The search query"},
+                },
+                "required": ["query"],
+            },
+        },
+    }
+    mock_post_search_tool_prompt_builder = MagicMock(spec=AnswerPromptBuilder)
+    mock_post_search_tool_prompt_builder.build.return_value = [
+        SystemMessage(content="Updated system prompt"),
+    ]
+    mock_tool.build_next_prompt.return_value = mock_post_search_tool_prompt_builder
+    return mock_tool
diff --git a/backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py b/backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py
index 12e3254d6d6..e6a5fe1f027 100644
--- a/backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py
+++ b/backend/tests/unit/danswer/llm/answering/stream_processing/test_citation_processing.py
@@ -7,7 +7,7 @@
 from danswer.chat.models import LlmDoc
 from danswer.configs.constants import DocumentSource
 from danswer.llm.answering.stream_processing.citation_processing import (
-    extract_citations_from_stream,
+    CitationProcessor,
 )
 from danswer.llm.answering.stream_processing.utils import DocumentIdOrderMapping
 
@@ -70,14 +70,16 @@ def process_text(
 ) -> tuple[str, list[CitationInfo]]:
     mock_docs, mock_doc_id_to_rank_map = mock_data
     mapping = DocumentIdOrderMapping(order_mapping=mock_doc_id_to_rank_map)
-    result = list(
-        extract_citations_from_stream(
-            tokens=iter(tokens),
-            context_docs=mock_docs,
-            doc_id_to_rank_map=mapping,
-            stop_stream=None,
-        )
+    processor = CitationProcessor(
+        context_docs=mock_docs,
+        doc_id_to_rank_map=mapping,
+        stop_stream=None,
     )
+    result: list[DanswerAnswerPiece | CitationInfo] = []
+    for token in tokens:
+        result.extend(processor.process_token(token))
+    result.extend(processor.process_token(None))
+
     final_answer_text = ""
     citations = []
     for piece in result:
diff --git a/backend/tests/unit/danswer/llm/answering/stream_processing/test_quote_processing.py b/backend/tests/unit/danswer/llm/answering/stream_processing/test_quote_processing.py
index e80c5c4f657..c154c5a5b0c 100644
--- a/backend/tests/unit/danswer/llm/answering/stream_processing/test_quote_processing.py
+++ b/backend/tests/unit/danswer/llm/answering/stream_processing/test_quote_processing.py
@@ -6,7 +6,7 @@
 from danswer.chat.models import LlmDoc
 from danswer.configs.constants import DocumentSource
 from danswer.llm.answering.stream_processing.quotes_processing import (
-    process_model_tokens,
+    QuotesProcessor,
 )
 
 mock_docs = [
@@ -25,179 +25,202 @@
 ]
 
 
-tokens_with_quotes = [
-    "{",
-    "\n  ",
-    '"answer": "Yes',
-    ", Danswer allows",
-    " customized prompts. This",
-    " feature",
-    " is currently being",
-    " developed and implemente",
-    "d to",
-    " improve",
-    " the accuracy",
-    " of",
-    " Language",
-    " Models (",
-    "LL",
-    "Ms) for",
-    " different",
-    " companies",
-    ".",
-    " The custom",
-    "ized prompts feature",
-    " woul",
-    "d allow users to ad",
-    "d person",
-    "alized prom",
-    "pts through",
-    " an",
-    " interface or",
-    " metho",
-    "d,",
-    " which would then be used to",
-    " train",
-    " the LLM.",
-    " This enhancement",
-    " aims to make",
-    " Danswer more",
-    " adaptable to",
-    " different",
-    " business",
-    " contexts",
-    " by",
-    " tail",
-    "oring it",
-    " to the specific language",
-    " an",
-    "d terminology",
-    " used within",
-    " a",
-    " company.",
-    " Additionally",
-    ",",
-    " Danswer already",
-    " supports creating",
-    " custom AI",
-    " Assistants with",
-    " different",
-    " prom",
-    "pts and backing",
-    " knowledge",
-    " sets",
-    ",",
-    " which",
-    " is",
-    " a form",
-    " of prompt",
-    " customization. However, it",
-    "'s important to nLogging Details LiteLLM-Success Call: Noneote that some",
-    " aspects",
-    " of prompt",
-    " customization,",
-    " such as for",
-    " Sl",
-    "ack",
-    "b",
-    "ots, may",
-    " still",
-    " be in",
-    " development or have",
-    ' limitations.",',
-    '\n  "quotes": [',
-    '\n    "We',
-    " woul",
-    "d like to ad",
-    "d customized prompts for",
-    " different",
-    " companies to improve the accuracy of",
-    " Language",
-    " Model",
-    " (LLM)",
-    '.",\n    "A',
-    " new",
-    " feature that",
-    " allows users to add personalize",
-    "d prompts.",
-    " This would involve",
-    " creating",
-    " an interface or method for",
-    " users to input",
-    " their",
-    " own",
-    " prom",
-    "pts,",
-    " which would then be used to",
-    ' train the LLM.",',
-    '\n    "Create',
-    " custom AI Assistants with",
-    " different prompts and backing knowledge",
-    ' sets.",',
-    '\n    "This',
-    " PR",
-    " fixes",
-    " https",
-    "://github.com/dan",
-    "swer-ai/dan",
-    "swer/issues/1",
-    "584",
-    " by",
-    " setting",
-    " the system",
-    " default",
-    " prompt for",
-    " sl",
-    "ackbots const",
-    "rained by",
-    " ",
-    "document sets",
-    ".",
-    " It",
-    " probably",
-    " isn",
-    "'t ideal",
-    " -",
-    " it",
-    " might",
-    " be pref",
-    "erable to be",
-    " able to select",
-    " a prompt for",
-    " the",
-    " slackbot from",
-    " the",
-    " admin",
-    " panel",
-    " -",
-    " but it sol",
-    "ves the immediate problem",
-    " of",
-    " the slack",
-    " listener",
-    " cr",
-    "ashing when",
-    " configure",
-    "d this",
-    ' way."\n  ]',
-    "\n}",
-    "",
-]
+def _process_tokens(
+    processor: QuotesProcessor, tokens: list[str]
+) -> tuple[str, list[str]]:
+    """Process a list of tokens and return the answer and quotes.
+
+    Args:
+        processor: QuotesProcessor instance
+        tokens: List of tokens to process
+
+    Returns:
+        Tuple of (answer_text, list_of_quotes)
+    """
+    answer = ""
+    quotes: list[str] = []
+
+    # need to add a None to the end to simulate the end of the stream
+    for token in tokens + [None]:
+        for output in processor.process_token(token):
+            if isinstance(output, DanswerAnswerPiece):
+                if output.answer_piece:
+                    answer += output.answer_piece
+            elif isinstance(output, DanswerQuotes):
+                quotes.extend(q.quote for q in output.quotes)
+
+    return answer, quotes
 
 
 def test_process_model_tokens_answer() -> None:
-    gen = process_model_tokens(tokens=iter(tokens_with_quotes), context_docs=mock_docs)
+    tokens_with_quotes = [
+        "{",
+        "\n  ",
+        '"answer": "Yes',
+        ", Danswer allows",
+        " customized prompts. This",
+        " feature",
+        " is currently being",
+        " developed and implemente",
+        "d to",
+        " improve",
+        " the accuracy",
+        " of",
+        " Language",
+        " Models (",
+        "LL",
+        "Ms) for",
+        " different",
+        " companies",
+        ".",
+        " The custom",
+        "ized prompts feature",
+        " woul",
+        "d allow users to ad",
+        "d person",
+        "alized prom",
+        "pts through",
+        " an",
+        " interface or",
+        " metho",
+        "d,",
+        " which would then be used to",
+        " train",
+        " the LLM.",
+        " This enhancement",
+        " aims to make",
+        " Danswer more",
+        " adaptable to",
+        " different",
+        " business",
+        " contexts",
+        " by",
+        " tail",
+        "oring it",
+        " to the specific language",
+        " an",
+        "d terminology",
+        " used within",
+        " a",
+        " company.",
+        " Additionally",
+        ",",
+        " Danswer already",
+        " supports creating",
+        " custom AI",
+        " Assistants with",
+        " different",
+        " prom",
+        "pts and backing",
+        " knowledge",
+        " sets",
+        ",",
+        " which",
+        " is",
+        " a form",
+        " of prompt",
+        " customization. However, it",
+        "'s important to nLogging Details LiteLLM-Success Call: Noneote that some",
+        " aspects",
+        " of prompt",
+        " customization,",
+        " such as for",
+        " Sl",
+        "ack",
+        "b",
+        "ots, may",
+        " still",
+        " be in",
+        " development or have",
+        ' limitations.",',
+        '\n  "quotes": [',
+        '\n    "We',
+        " woul",
+        "d like to ad",
+        "d customized prompts for",
+        " different",
+        " companies to improve the accuracy of",
+        " Language",
+        " Model",
+        " (LLM)",
+        '.",\n    "A',
+        " new",
+        " feature that",
+        " allows users to add personalize",
+        "d prompts.",
+        " This would involve",
+        " creating",
+        " an interface or method for",
+        " users to input",
+        " their",
+        " own",
+        " prom",
+        "pts,",
+        " which would then be used to",
+        ' train the LLM.",',
+        '\n    "Create',
+        " custom AI Assistants with",
+        " different prompts and backing knowledge",
+        ' sets.",',
+        '\n    "This',
+        " PR",
+        " fixes",
+        " https",
+        "://github.com/dan",
+        "swer-ai/dan",
+        "swer/issues/1",
+        "584",
+        " by",
+        " setting",
+        " the system",
+        " default",
+        " prompt for",
+        " sl",
+        "ackbots const",
+        "rained by",
+        " ",
+        "document sets",
+        ".",
+        " It",
+        " probably",
+        " isn",
+        "'t ideal",
+        " -",
+        " it",
+        " might",
+        " be pref",
+        "erable to be",
+        " able to select",
+        " a prompt for",
+        " the",
+        " slackbot from",
+        " the",
+        " admin",
+        " panel",
+        " -",
+        " but it sol",
+        "ves the immediate problem",
+        " of",
+        " the slack",
+        " listener",
+        " cr",
+        "ashing when",
+        " configure",
+        "d this",
+        ' way."\n  ]',
+        "\n}",
+        "",
+    ]
+
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens_with_quotes)
 
     s_json = "".join(tokens_with_quotes)
     j = json.loads(s_json)
     expected_answer = j["answer"]
-    actual = ""
-    for o in gen:
-        if isinstance(o, DanswerAnswerPiece):
-            if o.answer_piece:
-                actual += o.answer_piece
-
-    assert expected_answer == actual
+    assert expected_answer == answer
+    # NOTE: no quotes, since the docs don't match the quotes
+    assert len(quotes) == 0
 
 
 def test_simple_json_answer() -> None:
@@ -214,16 +237,11 @@ def test_simple_json_answer() -> None:
         "\n",
         "```",
     ]
-    gen = process_model_tokens(tokens=iter(tokens), context_docs=mock_docs)
-
-    expected_answer = "This is a simple answer."
-    actual = "".join(
-        o.answer_piece
-        for o in gen
-        if isinstance(o, DanswerAnswerPiece) and o.answer_piece
-    )
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens)
 
-    assert expected_answer == actual
+    assert "This is a simple answer." == answer
+    assert len(quotes) == 0
 
 
 def test_json_answer_with_quotes() -> None:
@@ -242,16 +260,21 @@ def test_json_answer_with_quotes() -> None:
         "\n",
         "```",
     ]
-    gen = process_model_tokens(tokens=iter(tokens), context_docs=mock_docs)
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens)
+
+    assert "This is a split answer." == answer
+    assert len(quotes) == 0
 
-    expected_answer = "This is a split answer."
-    actual = "".join(
-        o.answer_piece
-        for o in gen
-        if isinstance(o, DanswerAnswerPiece) and o.answer_piece
-    )
 
-    assert expected_answer == actual
+def test_json_answer_with_quotes_one_chunk() -> None:
+    tokens = ['```json\n{"answer": "z",\n"quotes": ["Document"]\n}\n```']
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens)
+
+    assert "z" == answer
+    assert len(quotes) == 1
+    assert quotes[0] == "Document"
 
 
 def test_json_answer_split_tokens() -> None:
@@ -271,16 +294,11 @@ def test_json_answer_split_tokens() -> None:
         "\n",
         "```",
     ]
-    gen = process_model_tokens(tokens=iter(tokens), context_docs=mock_docs)
-
-    expected_answer = "This is a split answer."
-    actual = "".join(
-        o.answer_piece
-        for o in gen
-        if isinstance(o, DanswerAnswerPiece) and o.answer_piece
-    )
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens)
 
-    assert expected_answer == actual
+    assert "This is a split answer." == answer
+    assert len(quotes) == 0
 
 
 def test_lengthy_prefixed_json_with_quotes() -> None:
@@ -298,23 +316,12 @@ def test_lengthy_prefixed_json_with_quotes() -> None:
         "\n",
         "```",
     ]
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens)
 
-    gen = process_model_tokens(tokens=iter(tokens), context_docs=mock_docs)
-
-    actual_answer = ""
-    actual_count = 0
-    for o in gen:
-        if isinstance(o, DanswerAnswerPiece):
-            if o.answer_piece:
-                actual_answer += o.answer_piece
-            continue
-
-        if isinstance(o, DanswerQuotes):
-            for q in o.quotes:
-                assert q.quote == "Document"
-                actual_count += 1
-    assert "This is a simple answer." == actual_answer
-    assert 1 == actual_count
+    assert "This is a simple answer." == answer
+    assert len(quotes) == 1
+    assert quotes[0] == "Document"
 
 
 def test_prefixed_json_with_quotes() -> None:
@@ -331,21 +338,9 @@ def test_prefixed_json_with_quotes() -> None:
         "\n",
         "```",
     ]
+    processor = QuotesProcessor(context_docs=mock_docs)
+    answer, quotes = _process_tokens(processor, tokens)
 
-    gen = process_model_tokens(tokens=iter(tokens), context_docs=mock_docs)
-
-    actual_answer = ""
-    actual_count = 0
-    for o in gen:
-        if isinstance(o, DanswerAnswerPiece):
-            if o.answer_piece:
-                actual_answer += o.answer_piece
-            continue
-
-        if isinstance(o, DanswerQuotes):
-            for q in o.quotes:
-                assert q.quote == "Document"
-                actual_count += 1
-
-    assert "This is a simple answer." == actual_answer
-    assert 1 == actual_count
+    assert "This is a simple answer." == answer
+    assert len(quotes) == 1
+    assert quotes[0] == "Document"
diff --git a/backend/tests/unit/danswer/llm/answering/test_answer.py b/backend/tests/unit/danswer/llm/answering/test_answer.py
new file mode 100644
index 00000000000..96c791cd47b
--- /dev/null
+++ b/backend/tests/unit/danswer/llm/answering/test_answer.py
@@ -0,0 +1,405 @@
+import json
+from typing import cast
+from unittest.mock import MagicMock
+from unittest.mock import Mock
+
+import pytest
+from langchain_core.messages import AIMessageChunk
+from langchain_core.messages import BaseMessage
+from langchain_core.messages import HumanMessage
+from langchain_core.messages import SystemMessage
+from langchain_core.messages import ToolCall
+from langchain_core.messages import ToolCallChunk
+
+from danswer.chat.models import CitationInfo
+from danswer.chat.models import DanswerAnswerPiece
+from danswer.chat.models import DanswerQuote
+from danswer.chat.models import DanswerQuotes
+from danswer.chat.models import LlmDoc
+from danswer.chat.models import StreamStopInfo
+from danswer.chat.models import StreamStopReason
+from danswer.llm.answering.answer import Answer
+from danswer.llm.answering.models import AnswerStyleConfig
+from danswer.llm.answering.models import PromptConfig
+from danswer.llm.answering.models import QuotesConfig
+from danswer.llm.interfaces import LLM
+from danswer.tools.force import ForceUseTool
+from danswer.tools.models import ToolCallFinalResult
+from danswer.tools.models import ToolCallKickoff
+from danswer.tools.models import ToolResponse
+from tests.unit.danswer.llm.answering.conftest import DEFAULT_SEARCH_ARGS
+from tests.unit.danswer.llm.answering.conftest import QUERY
+
+
+@pytest.fixture
+def answer_instance(
+    mock_llm: LLM, answer_style_config: AnswerStyleConfig, prompt_config: PromptConfig
+) -> Answer:
+    return Answer(
+        question=QUERY,
+        answer_style_config=answer_style_config,
+        llm=mock_llm,
+        prompt_config=prompt_config,
+        force_use_tool=ForceUseTool(force_use=False, tool_name="", args=None),
+    )
+
+
+def test_basic_answer(answer_instance: Answer) -> None:
+    mock_llm = cast(Mock, answer_instance.llm)
+    mock_llm.stream.return_value = [
+        AIMessageChunk(content="This is a "),
+        AIMessageChunk(content="mock answer."),
+    ]
+
+    output = list(answer_instance.processed_streamed_output)
+    assert len(output) == 2
+    assert isinstance(output[0], DanswerAnswerPiece)
+    assert isinstance(output[1], DanswerAnswerPiece)
+
+    full_answer = "".join(
+        piece.answer_piece
+        for piece in output
+        if isinstance(piece, DanswerAnswerPiece) and piece.answer_piece is not None
+    )
+    assert full_answer == "This is a mock answer."
+
+    assert answer_instance.llm_answer == "This is a mock answer."
+    assert answer_instance.citations == []
+
+    assert mock_llm.stream.call_count == 1
+    mock_llm.stream.assert_called_once_with(
+        prompt=[
+            SystemMessage(content="System prompt"),
+            HumanMessage(content="Task prompt\n\nQUERY:\nTest question"),
+        ],
+        tools=None,
+        tool_choice=None,
+        structured_response_format=None,
+    )
+
+
+@pytest.mark.parametrize(
+    "force_use_tool, expected_tool_args",
+    [
+        (
+            ForceUseTool(force_use=False, tool_name="", args=None),
+            DEFAULT_SEARCH_ARGS,
+        ),
+        (
+            ForceUseTool(
+                force_use=True, tool_name="search", args={"query": "forced search"}
+            ),
+            {"query": "forced search"},
+        ),
+    ],
+)
+def test_answer_with_search_call(
+    answer_instance: Answer,
+    mock_search_results: list[LlmDoc],
+    mock_search_tool: MagicMock,
+    force_use_tool: ForceUseTool,
+    expected_tool_args: dict,
+) -> None:
+    answer_instance.tools = [mock_search_tool]
+    answer_instance.force_use_tool = force_use_tool
+
+    # Set up the LLM mock to return search results and then an answer
+    mock_llm = cast(Mock, answer_instance.llm)
+
+    stream_side_effect: list[list[BaseMessage]] = []
+
+    if not force_use_tool.force_use:
+        tool_call_chunk = AIMessageChunk(content="")
+        tool_call_chunk.tool_calls = [
+            ToolCall(
+                id="search",
+                name="search",
+                args=expected_tool_args,
+            )
+        ]
+        tool_call_chunk.tool_call_chunks = [
+            ToolCallChunk(
+                id="search",
+                name="search",
+                args=json.dumps(expected_tool_args),
+                index=0,
+            )
+        ]
+        stream_side_effect.append([tool_call_chunk])
+
+    stream_side_effect.append(
+        [
+            AIMessageChunk(content="Based on the search results, "),
+            AIMessageChunk(content="the answer is abc[1]. "),
+            AIMessageChunk(content="This is some other stuff."),
+        ],
+    )
+    mock_llm.stream.side_effect = stream_side_effect
+
+    # Process the output
+    output = list(answer_instance.processed_streamed_output)
+    print(output)
+
+    # Updated assertions
+    assert len(output) == 7
+    assert output[0] == ToolCallKickoff(
+        tool_name="search", tool_args=expected_tool_args
+    )
+    assert output[1] == ToolResponse(
+        id="final_context_documents",
+        response=mock_search_results,
+    )
+    assert output[2] == ToolCallFinalResult(
+        tool_name="search",
+        tool_args=expected_tool_args,
+        tool_result=[json.loads(doc.model_dump_json()) for doc in mock_search_results],
+    )
+    assert output[3] == DanswerAnswerPiece(answer_piece="Based on the search results, ")
+    expected_citation = CitationInfo(citation_num=1, document_id="doc1")
+    assert output[4] == expected_citation
+    assert output[5] == DanswerAnswerPiece(
+        answer_piece="the answer is abc[[1]](https://example.com/doc1). "
+    )
+    assert output[6] == DanswerAnswerPiece(answer_piece="This is some other stuff.")
+
+    expected_answer = (
+        "Based on the search results, "
+        "the answer is abc[[1]](https://example.com/doc1). "
+        "This is some other stuff."
+    )
+    full_answer = "".join(
+        piece.answer_piece
+        for piece in output
+        if isinstance(piece, DanswerAnswerPiece) and piece.answer_piece is not None
+    )
+    assert full_answer == expected_answer
+
+    assert answer_instance.llm_answer == expected_answer
+    assert len(answer_instance.citations) == 1
+    assert answer_instance.citations[0] == expected_citation
+
+    # Verify LLM calls
+    if not force_use_tool.force_use:
+        assert mock_llm.stream.call_count == 2
+        first_call, second_call = mock_llm.stream.call_args_list
+
+        # First call should include the search tool definition
+        assert len(first_call.kwargs["tools"]) == 1
+        assert (
+            first_call.kwargs["tools"][0]
+            == mock_search_tool.tool_definition.return_value
+        )
+
+        # Second call should not include tools (as we're just generating the final answer)
+        assert "tools" not in second_call.kwargs or not second_call.kwargs["tools"]
+        # Second call should use the returned prompt from build_next_prompt
+        assert (
+            second_call.kwargs["prompt"]
+            == mock_search_tool.build_next_prompt.return_value.build.return_value
+        )
+
+        # Verify that tool_definition was called on the mock_search_tool
+        mock_search_tool.tool_definition.assert_called_once()
+    else:
+        assert mock_llm.stream.call_count == 1
+
+        call = mock_llm.stream.call_args_list[0]
+        assert (
+            call.kwargs["prompt"]
+            == mock_search_tool.build_next_prompt.return_value.build.return_value
+        )
+
+
+def test_answer_with_search_no_tool_calling(
+    answer_instance: Answer,
+    mock_search_results: list[LlmDoc],
+    mock_search_tool: MagicMock,
+) -> None:
+    answer_instance.tools = [mock_search_tool]
+
+    # Set up the LLM mock to return an answer
+    mock_llm = cast(Mock, answer_instance.llm)
+    mock_llm.stream.return_value = [
+        AIMessageChunk(content="Based on the search results, "),
+        AIMessageChunk(content="the answer is abc[1]. "),
+        AIMessageChunk(content="This is some other stuff."),
+    ]
+
+    # Force non-tool calling behavior
+    answer_instance.using_tool_calling_llm = False
+
+    # Process the output
+    output = list(answer_instance.processed_streamed_output)
+
+    # Assertions
+    assert len(output) == 7
+    assert output[0] == ToolCallKickoff(
+        tool_name="search", tool_args=DEFAULT_SEARCH_ARGS
+    )
+    assert output[1] == ToolResponse(
+        id="final_context_documents",
+        response=mock_search_results,
+    )
+    assert output[2] == ToolCallFinalResult(
+        tool_name="search",
+        tool_args=DEFAULT_SEARCH_ARGS,
+        tool_result=[json.loads(doc.model_dump_json()) for doc in mock_search_results],
+    )
+    assert output[3] == DanswerAnswerPiece(answer_piece="Based on the search results, ")
+    expected_citation = CitationInfo(citation_num=1, document_id="doc1")
+    assert output[4] == expected_citation
+    assert output[5] == DanswerAnswerPiece(
+        answer_piece="the answer is abc[[1]](https://example.com/doc1). "
+    )
+    assert output[6] == DanswerAnswerPiece(answer_piece="This is some other stuff.")
+
+    expected_answer = (
+        "Based on the search results, "
+        "the answer is abc[[1]](https://example.com/doc1). "
+        "This is some other stuff."
+    )
+    assert answer_instance.llm_answer == expected_answer
+    assert len(answer_instance.citations) == 1
+    assert answer_instance.citations[0] == expected_citation
+
+    # Verify LLM calls
+    assert mock_llm.stream.call_count == 1
+    call_args = mock_llm.stream.call_args
+
+    # Verify that no tools were passed to the LLM
+    assert "tools" not in call_args.kwargs or not call_args.kwargs["tools"]
+
+    # Verify that the prompt was built correctly
+    assert (
+        call_args.kwargs["prompt"]
+        == mock_search_tool.build_next_prompt.return_value.build.return_value
+    )
+
+    # Verify that get_args_for_non_tool_calling_llm was called on the mock_search_tool
+    mock_search_tool.get_args_for_non_tool_calling_llm.assert_called_once_with(
+        f"Task prompt\n\nQUERY:\n{QUERY}", [], answer_instance.llm
+    )
+
+    # Verify that the search tool's run method was called
+    mock_search_tool.run.assert_called_once()
+
+
+def test_answer_with_search_call_quotes_enabled(
+    answer_instance: Answer,
+    mock_search_results: list[LlmDoc],
+    mock_search_tool: MagicMock,
+) -> None:
+    answer_instance.tools = [mock_search_tool]
+    answer_instance.force_use_tool = ForceUseTool(
+        force_use=False, tool_name="", args=None
+    )
+    answer_instance.answer_style_config.citation_config = None
+    answer_instance.answer_style_config.quotes_config = QuotesConfig()
+
+    # Set up the LLM mock to return search results and then an answer
+    mock_llm = cast(Mock, answer_instance.llm)
+
+    tool_call_chunk = AIMessageChunk(content="")
+    tool_call_chunk.tool_calls = [
+        ToolCall(
+            id="search",
+            name="search",
+            args=DEFAULT_SEARCH_ARGS,
+        )
+    ]
+    tool_call_chunk.tool_call_chunks = [
+        ToolCallChunk(
+            id="search",
+            name="search",
+            args=json.dumps(DEFAULT_SEARCH_ARGS),
+            index=0,
+        )
+    ]
+
+    # needs to be short due to the "anti-hallucination" check in QuotesProcessor
+    answer_content = "z"
+    quote_content = mock_search_results[0].content
+    mock_llm.stream.side_effect = [
+        [tool_call_chunk],
+        [
+            AIMessageChunk(
+                content=(
+                    '{"answer": "'
+                    + answer_content
+                    + '", "quotes": ["'
+                    + quote_content
+                    + '"]}'
+                )
+            ),
+        ],
+    ]
+
+    # Process the output
+    output = list(answer_instance.processed_streamed_output)
+
+    # Assertions
+    assert len(output) == 5
+    assert output[0] == ToolCallKickoff(
+        tool_name="search", tool_args=DEFAULT_SEARCH_ARGS
+    )
+    assert output[1] == ToolResponse(
+        id="final_context_documents",
+        response=mock_search_results,
+    )
+    assert output[2] == ToolCallFinalResult(
+        tool_name="search",
+        tool_args=DEFAULT_SEARCH_ARGS,
+        tool_result=[json.loads(doc.model_dump_json()) for doc in mock_search_results],
+    )
+    assert output[3] == DanswerAnswerPiece(answer_piece=answer_content)
+    assert output[4] == DanswerQuotes(
+        quotes=[
+            DanswerQuote(
+                quote=quote_content,
+                document_id=mock_search_results[0].document_id,
+                link=mock_search_results[0].link,
+                source_type=mock_search_results[0].source_type,
+                semantic_identifier=mock_search_results[0].semantic_identifier,
+                blurb=mock_search_results[0].blurb,
+            )
+        ]
+    )
+
+    assert answer_instance.llm_answer == answer_content
+
+
+def test_is_cancelled(answer_instance: Answer) -> None:
+    # Set up the LLM mock to return multiple chunks
+    mock_llm = Mock()
+    answer_instance.llm = mock_llm
+    mock_llm.stream.return_value = [
+        AIMessageChunk(content="This is the "),
+        AIMessageChunk(content="first part."),
+        AIMessageChunk(content="This should not be seen."),
+    ]
+
+    # Create a mutable object to control is_connected behavior
+    connection_status = {"connected": True}
+    answer_instance.is_connected = lambda: connection_status["connected"]
+
+    # Process the output
+    output = []
+    for i, chunk in enumerate(answer_instance.processed_streamed_output):
+        output.append(chunk)
+        # Simulate disconnection after the second chunk
+        if i == 1:
+            connection_status["connected"] = False
+
+    assert len(output) == 3
+    assert output[0] == DanswerAnswerPiece(answer_piece="This is the ")
+    assert output[1] == DanswerAnswerPiece(answer_piece="first part.")
+    assert output[2] == StreamStopInfo(stop_reason=StreamStopReason.CANCELLED)
+
+    # Verify that the stream was cancelled
+    assert answer_instance.is_cancelled() is True
+
+    # Verify that the final answer only contains the streamed parts
+    assert answer_instance.llm_answer == "This is the first part."
+
+    # Verify LLM calls
+    mock_llm.stream.assert_called_once()
diff --git a/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py b/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py
index 998b2932cbb..7bd4a498bd7 100644
--- a/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py
+++ b/backend/tests/unit/danswer/llm/answering/test_skip_gen_ai.py
@@ -6,8 +6,11 @@
 from pytest_mock import MockerFixture
 
 from danswer.llm.answering.answer import Answer
+from danswer.llm.answering.models import AnswerStyleConfig
+from danswer.llm.answering.models import PromptConfig
 from danswer.one_shot_answer.answer_question import AnswerObjectIterator
 from danswer.tools.force import ForceUseTool
+from danswer.tools.tool_implementations.search.search_tool import SearchTool
 from tests.regression.answer_quality.run_qa import _process_and_write_query_results
 
 
@@ -24,39 +27,43 @@
         },
     ],
 )
-def test_skip_gen_ai_answer_generation_flag(config: dict[str, Any]) -> None:
-    search_tool = Mock()
-    search_tool.name = "search"
-    search_tool.run = Mock()
-    search_tool.run.return_value = [Mock()]
+def test_skip_gen_ai_answer_generation_flag(
+    config: dict[str, Any],
+    mock_search_tool: SearchTool,
+    answer_style_config: AnswerStyleConfig,
+    prompt_config: PromptConfig,
+) -> None:
+    question = config["question"]
+    skip_gen_ai_answer_generation = config["skip_gen_ai_answer_generation"]
+
     mock_llm = Mock()
     mock_llm.config = Mock()
     mock_llm.config.model_name = "gpt-4o-mini"
     mock_llm.stream = Mock()
     mock_llm.stream.return_value = [Mock()]
     answer = Answer(
-        question=config["question"],
-        answer_style_config=Mock(),
-        prompt_config=Mock(),
+        question=question,
+        answer_style_config=answer_style_config,
+        prompt_config=prompt_config,
         llm=mock_llm,
         single_message_history="history",
-        tools=[search_tool],
+        tools=[mock_search_tool],
         force_use_tool=(
             ForceUseTool(
-                tool_name=search_tool.name,
-                args={"query": config["question"]},
+                tool_name=mock_search_tool.name,
+                args={"query": question},
                 force_use=True,
             )
         ),
         skip_explicit_tool_calling=True,
         return_contexts=True,
-        skip_gen_ai_answer_generation=config["skip_gen_ai_answer_generation"],
+        skip_gen_ai_answer_generation=skip_gen_ai_answer_generation,
     )
     count = 0
     for _ in cast(AnswerObjectIterator, answer.processed_streamed_output):
         count += 1
-    assert count == 2
-    if not config["skip_gen_ai_answer_generation"]:
+    assert count == 3 if skip_gen_ai_answer_generation else 4
+    if not skip_gen_ai_answer_generation:
         mock_llm.stream.assert_called_once()
     else:
         mock_llm.stream.assert_not_called()
diff --git a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
index 6139f41e62a..f56336809e4 100644
--- a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
+++ b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
@@ -5,14 +5,18 @@
 
 import pytest
 
-from danswer.tools.custom.custom_tool import (
+from danswer.tools.models import DynamicSchemaInfo
+from danswer.tools.models import ToolResponse
+from danswer.tools.tool_implementations.custom.custom_tool import (
     build_custom_tools_from_openapi_schema_and_headers,
 )
-from danswer.tools.custom.custom_tool import CUSTOM_TOOL_RESPONSE_ID
-from danswer.tools.custom.custom_tool import CustomToolCallSummary
-from danswer.tools.custom.custom_tool import validate_openapi_schema
-from danswer.tools.models import DynamicSchemaInfo
-from danswer.tools.tool import ToolResponse
+from danswer.tools.tool_implementations.custom.custom_tool import (
+    CUSTOM_TOOL_RESPONSE_ID,
+)
+from danswer.tools.tool_implementations.custom.custom_tool import CustomToolCallSummary
+from danswer.tools.tool_implementations.custom.custom_tool import (
+    validate_openapi_schema,
+)
 from danswer.utils.headers import HeaderItemDict
 
 
@@ -78,7 +82,7 @@ def setUp(self) -> None:
             chat_session_id=uuid.uuid4(), message_id=20
         )
 
-    @patch("danswer.tools.custom.custom_tool.requests.request")
+    @patch("danswer.tools.tool_implementations.custom.custom_tool.requests.request")
     def test_custom_tool_run_get(self, mock_request: unittest.mock.MagicMock) -> None:
         """
         Test the GET method of a custom tool.
@@ -106,7 +110,7 @@ def test_custom_tool_run_get(self, mock_request: unittest.mock.MagicMock) -> Non
             "Tool name in response does not match expected value",
         )
 
-    @patch("danswer.tools.custom.custom_tool.requests.request")
+    @patch("danswer.tools.tool_implementations.custom.custom_tool.requests.request")
     def test_custom_tool_run_post(self, mock_request: unittest.mock.MagicMock) -> None:
         """
         Test the POST method of a custom tool.
@@ -136,7 +140,7 @@ def test_custom_tool_run_post(self, mock_request: unittest.mock.MagicMock) -> No
             "Tool name in response does not match expected value",
         )
 
-    @patch("danswer.tools.custom.custom_tool.requests.request")
+    @patch("danswer.tools.tool_implementations.custom.custom_tool.requests.request")
     def test_custom_tool_with_headers(
         self, mock_request: unittest.mock.MagicMock
     ) -> None:
@@ -164,7 +168,7 @@ def test_custom_tool_with_headers(
             "GET", expected_url, json=None, headers=expected_headers
         )
 
-    @patch("danswer.tools.custom.custom_tool.requests.request")
+    @patch("danswer.tools.tool_implementations.custom.custom_tool.requests.request")
     def test_custom_tool_with_empty_headers(
         self, mock_request: unittest.mock.MagicMock
     ) -> None:

From 772313236fe0f664286c6e8a2e26e16603efc0db Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 1 Nov 2024 14:16:50 -0700
Subject: [PATCH 324/376] minor foreign key update (#3007)

---
 ..._nullable_search_settings_for_historic_.py | 70 +++++++++++++++++++
 .../background/indexing/run_indexing.py       |  6 ++
 backend/danswer/db/models.py                  |  7 +-
 3 files changed, 80 insertions(+), 3 deletions(-)
 create mode 100644 backend/alembic/versions/5b29123cd710_nullable_search_settings_for_historic_.py

diff --git a/backend/alembic/versions/5b29123cd710_nullable_search_settings_for_historic_.py b/backend/alembic/versions/5b29123cd710_nullable_search_settings_for_historic_.py
new file mode 100644
index 00000000000..58164cd4c14
--- /dev/null
+++ b/backend/alembic/versions/5b29123cd710_nullable_search_settings_for_historic_.py
@@ -0,0 +1,70 @@
+"""nullable search settings for historic index attempts
+
+Revision ID: 5b29123cd710
+Revises: 949b4a92a401
+Create Date: 2024-10-30 19:37:59.630704
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "5b29123cd710"
+down_revision = "949b4a92a401"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Drop the existing foreign key constraint
+    op.drop_constraint(
+        "fk_index_attempt_search_settings", "index_attempt", type_="foreignkey"
+    )
+
+    # Modify the column to be nullable
+    op.alter_column(
+        "index_attempt", "search_settings_id", existing_type=sa.INTEGER(), nullable=True
+    )
+
+    # Add back the foreign key with ON DELETE SET NULL
+    op.create_foreign_key(
+        "fk_index_attempt_search_settings",
+        "index_attempt",
+        "search_settings",
+        ["search_settings_id"],
+        ["id"],
+        ondelete="SET NULL",
+    )
+
+
+def downgrade() -> None:
+    # Warning: This will delete all index attempts that don't have search settings
+    op.execute(
+        """
+        DELETE FROM index_attempt
+        WHERE search_settings_id IS NULL
+    """
+    )
+
+    # Drop foreign key constraint
+    op.drop_constraint(
+        "fk_index_attempt_search_settings", "index_attempt", type_="foreignkey"
+    )
+
+    # Modify the column to be not nullable
+    op.alter_column(
+        "index_attempt",
+        "search_settings_id",
+        existing_type=sa.INTEGER(),
+        nullable=False,
+    )
+
+    # Add back the foreign key without ON DELETE SET NULL
+    op.create_foreign_key(
+        "fk_index_attempt_search_settings",
+        "index_attempt",
+        "search_settings",
+        ["search_settings_id"],
+        ["id"],
+    )
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index d95a6a70d57..252309e3faa 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -118,7 +118,13 @@ def _run_indexing(
     """
     start_time = time.time()
 
+    if index_attempt.search_settings is None:
+        raise ValueError(
+            "Search settings must be set for indexing. This should not be possible."
+        )
+
     search_settings = index_attempt.search_settings
+
     index_name = search_settings.index_name
 
     # Only update cc-pair status for primary index jobs
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index bee69353437..3f63d1d0dba 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -734,9 +734,10 @@ class IndexAttempt(Base):
     full_exception_trace: Mapped[str | None] = mapped_column(Text, default=None)
     # Nullable because in the past, we didn't allow swapping out embedding models live
     search_settings_id: Mapped[int] = mapped_column(
-        ForeignKey("search_settings.id"),
-        nullable=False,
+        ForeignKey("search_settings.id", ondelete="SET NULL"),
+        nullable=True,
     )
+
     time_created: Mapped[datetime.datetime] = mapped_column(
         DateTime(timezone=True),
         server_default=func.now(),
@@ -756,7 +757,7 @@ class IndexAttempt(Base):
         "ConnectorCredentialPair", back_populates="index_attempts"
     )
 
-    search_settings: Mapped[SearchSettings] = relationship(
+    search_settings: Mapped[SearchSettings | None] = relationship(
         "SearchSettings", back_populates="index_attempts"
     )
 

From 645402c71a89218de73ed13ebddfa936164fa272 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Fri, 1 Nov 2024 16:20:06 -0700
Subject: [PATCH 325/376] Tremor -> Shadcn (#2983)

* initialization

* button + input updates

* migrate dividers + buttons

* migrate badges

* minor updates

* migrate cards

* fix compiling

* begin date picker + badge transfer

* remove tremor

* fully swapped

* nits

* list item + configuration updates

* clean build

* update colors

* nits
---
 web/components.json                           |   20 +
 web/package-lock.json                         | 1852 ++++++++++++++---
 web/package.json                              |   18 +-
 web/src/app/admin/add-connector/page.tsx      |   13 +-
 .../app/admin/assistants/AssistantEditor.tsx  |   30 +-
 web/src/app/admin/assistants/PersonaTable.tsx |    6 +-
 .../assistants/[id]/DeletePersonaButton.tsx   |    7 +-
 web/src/app/admin/assistants/[id]/page.tsx    |    9 +-
 web/src/app/admin/assistants/new/page.tsx     |    6 +-
 web/src/app/admin/assistants/page.tsx         |    8 +-
 .../admin/bot/SlackBotConfigCreationForm.tsx  |   10 +-
 web/src/app/admin/bot/SlackBotTokensForm.tsx  |    9 +-
 web/src/app/admin/bot/[id]/page.tsx           |    2 +-
 web/src/app/admin/bot/page.tsx                |   27 +-
 .../document-processing/page.tsx              |   14 +-
 .../llm/ConfiguredLLMProviderDisplay.tsx      |   14 +-
 .../llm/CustomLLMProviderUpdateForm.tsx       |   21 +-
 .../configuration/llm/LLMConfiguration.tsx    |   11 +-
 .../llm/LLMProviderUpdateForm.tsx             |   17 +-
 .../configuration/search/UpgradingPage.tsx    |    9 +-
 .../app/admin/configuration/search/page.tsx   |   13 +-
 .../connector/[ccPairId]/ConfigDisplay.tsx    |   47 +-
 .../connector/[ccPairId]/DeletionButton.tsx   |    5 +-
 .../[ccPairId]/IndexingAttemptsTable.tsx      |   29 +-
 .../[ccPairId]/ModifyStatusButtonCluster.tsx  |    8 +-
 .../connector/[ccPairId]/ReIndexButton.tsx    |   15 +-
 .../app/admin/connector/[ccPairId]/page.tsx   |   18 +-
 .../[connector]/AddConnectorPage.tsx          |   15 +-
 .../[connector]/ConnectorWrapper.tsx          |    2 +-
 .../[connector]/pages/gdrive/Credential.tsx   |    2 +-
 .../pages/gdrive/GoogleDrivePage.tsx          |    2 +-
 .../[connector]/pages/gmail/Credential.tsx    |    6 +-
 .../[connector]/pages/gmail/GmailPage.tsx     |    2 +-
 .../feedback/DocumentFeedbackTable.tsx        |   14 +-
 web/src/app/admin/documents/feedback/page.tsx |    2 +-
 .../sets/DocumentSetCreationForm.tsx          |   12 +-
 .../documents/sets/[documentSetId]/page.tsx   |    6 +-
 web/src/app/admin/documents/sets/new/page.tsx |    6 +-
 web/src/app/admin/documents/sets/page.tsx     |   52 +-
 .../admin/embeddings/RerankingFormPage.tsx    |   15 +-
 .../embeddings/modals/AlreadyPickedModal.tsx  |    5 +-
 .../modals/ChangeCredentialsModal.tsx         |   33 +-
 .../modals/DeleteCredentialsModal.tsx         |   16 +-
 .../embeddings/modals/ModelSelectionModal.tsx |    8 +-
 .../modals/ProviderCreationModal.tsx          |   12 +-
 .../embeddings/modals/SelectModelModal.tsx    |    5 +-
 .../embeddings/pages/CloudEmbeddingPage.tsx   |   29 +-
 .../embeddings/pages/EmbeddingFormPage.tsx    |   22 +-
 .../embeddings/pages/OpenEmbeddingPage.tsx    |   16 +-
 .../indexing/[id]/IndexAttemptErrorsTable.tsx |   18 +-
 web/src/app/admin/indexing/[id]/page.tsx      |    2 +-
 .../status/CCPairIndexingStatusTable.tsx      |  328 ++-
 web/src/app/admin/indexing/status/page.tsx    |    8 +-
 .../prompt-library/modals/AddPromptModal.tsx  |   11 +-
 .../prompt-library/modals/EditPromptModal.tsx |   12 +-
 .../admin/prompt-library/promptLibrary.tsx    |   14 +-
 .../admin/prompt-library/promptSection.tsx    |   14 +-
 web/src/app/admin/settings/SettingsForm.tsx   |   13 +-
 web/src/app/admin/settings/page.tsx           |    2 +-
 .../CreateRateLimitModal.tsx                  |   10 +-
 .../TokenRateLimitTables.tsx                  |   28 +-
 web/src/app/admin/token-rate-limits/page.tsx  |  104 +-
 web/src/app/admin/tools/ToolEditor.tsx        |   15 +-
 web/src/app/admin/tools/ToolsTable.tsx        |   16 +-
 .../tools/edit/[toolId]/DeleteToolButton.tsx  |    6 +-
 .../app/admin/tools/edit/[toolId]/page.tsx    |    8 +-
 web/src/app/admin/tools/new/page.tsx          |    6 +-
 web/src/app/admin/tools/page.tsx              |    8 +-
 web/src/app/admin/users/page.tsx              |    3 +-
 web/src/app/assistants/edit/[id]/page.tsx     |    8 +-
 .../assistants/gallery/AssistantsGallery.tsx  |    9 +-
 .../assistants/mine/AssistantSharingModal.tsx |    5 +-
 .../app/assistants/mine/AssistantsList.tsx    |    9 +-
 web/src/app/assistants/new/page.tsx           |    6 +-
 web/src/app/auth/create-account/page.tsx      |    4 +-
 web/src/app/auth/error/page.tsx               |    4 +-
 web/src/app/auth/login/EmailPasswordForm.tsx  |    2 +-
 web/src/app/auth/login/page.tsx               |    8 +-
 web/src/app/auth/signup/page.tsx              |    2 +-
 web/src/app/auth/verify-email/Verify.tsx      |    2 +-
 .../app/auth/waiting-on-verification/page.tsx |    2 +-
 web/src/app/chat/ChatPage.tsx                 |    4 +-
 web/src/app/chat/ChatPopup.tsx                |    8 +-
 .../chat/documentSidebar/DocumentSidebar.tsx  |    9 +-
 .../app/chat/input/SelectedFilterDisplay.tsx  |    2 +-
 .../chat/modal/MakePublicAssistantModal.tsx   |   10 +-
 .../app/chat/modal/SetDefaultModelModal.tsx   |    6 +-
 .../app/chat/modal/ShareChatSessionModal.tsx  |   34 +-
 .../app/chat/modal/configuration/LlmTab.tsx   |    2 +-
 .../shared/[chatId]/SharedChatDisplay.tsx     |    8 +-
 .../ee/admin/api-key/DanswerApiKeyForm.tsx    |   10 +-
 web/src/app/ee/admin/api-key/page.tsx         |   33 +-
 .../ee/admin/groups/UserGroupCreationForm.tsx |   13 +-
 .../app/ee/admin/groups/UserGroupsTable.tsx   |   18 +-
 .../admin/groups/[groupId]/GroupDisplay.tsx   |   77 +-
 web/src/app/ee/admin/groups/page.tsx          |   11 +-
 .../admin/performance/DateRangeSelector.tsx   |  132 +-
 .../CustomAnalyticsUpdateForm.tsx             |    6 +-
 .../performance/custom-analytics/page.tsx     |    5 +-
 web/src/app/ee/admin/performance/lib.ts       |    3 +-
 .../query-history/FeedbackBadge.tsx           |   10 +-
 .../query-history/QueryHistoryTable.tsx       |   81 +-
 .../performance/query-history/[id]/page.tsx   |   21 +-
 .../performance/usage/DanswerBotChart.tsx     |   20 +-
 .../admin/performance/usage/FeedbackChart.tsx |   21 +-
 .../usage/QueryPerformanceChart.tsx           |   20 +-
 .../admin/performance/usage/UsageReports.tsx  |  173 +-
 .../app/ee/admin/performance/usage/page.tsx   |   12 +-
 .../StandardAnswerCreationForm.tsx            |   10 +-
 web/src/app/ee/admin/standard-answer/page.tsx |   25 +-
 .../admin/whitelabeling/WhitelabelingForm.tsx |   20 +-
 web/src/app/globals.css                       |   53 +
 web/src/app/layout.tsx                        |   10 +-
 web/src/components/AdvancedOptionsToggle.tsx  |    6 +-
 .../components/DanswerInitializingLoader.tsx  |    5 +-
 web/src/components/ErrorCallout.tsx           |    7 +-
 web/src/components/IsPublicGroupSelector.tsx  |    9 +-
 web/src/components/Modal.tsx                  |    4 +-
 web/src/components/Status.tsx                 |   26 +-
 web/src/components/SwitchModelModal.tsx       |    5 +-
 web/src/components/admin/CardSection.tsx      |   21 +
 web/src/components/admin/Title.tsx            |    4 +-
 .../connectors/AccessTypeGroupSelector.tsx    |   13 +-
 .../admin/connectors/AutoSyncOptions.tsx      |    4 +-
 .../admin/connectors/CredentialForm.tsx       |    3 +-
 web/src/components/admin/connectors/Field.tsx |   50 +-
 web/src/components/admin/users/BulkAdd.tsx    |    6 +-
 .../admin/users/InvitedUserTable.tsx          |   21 +-
 .../admin/users/SignedUpUserTable.tsx         |   92 +-
 .../credentials/CredentialSection.tsx         |    2 +-
 .../credentials/actions/CreateCredential.tsx  |    8 +-
 .../credentials/actions/EditCredential.tsx    |    5 +-
 .../credentials/actions/ModifyCredential.tsx  |    4 +-
 .../embedding/CustomEmbeddingModelForm.tsx    |    7 +-
 .../components/embedding/CustomModelForm.tsx  |    2 +-
 .../embedding/FailedReIndexAttempts.tsx       |   43 +-
 .../components/embedding/ModelSelector.tsx    |    9 +-
 .../embedding/ReindexingProgressTable.tsx     |   19 +-
 .../initialSetup/welcome/WelcomeModal.tsx     |    3 +-
 web/src/components/llm/ApiKeyForm.tsx         |   76 +-
 .../components/search/DateRangeSelector.tsx   |    2 +-
 web/src/components/search/SearchBar.tsx       |    4 +-
 web/src/components/search/SearchSection.tsx   |    3 +-
 .../components/search/filtering/Filters.tsx   |    2 +-
 .../StandardAnswerCategoryDropdown.tsx        |    4 +-
 web/src/components/table/DraggableRow.tsx     |    2 +-
 web/src/components/table/DraggableTable.tsx   |   12 +-
 .../components/table/DraggableTableBody.tsx   |    2 +-
 web/src/components/table/StaticRow.tsx        |    2 +-
 web/src/components/ui/alert.tsx               |   60 +
 web/src/components/ui/areaChart.tsx           |  129 ++
 web/src/components/ui/badge.tsx               |   75 +
 web/src/components/ui/button.tsx              |  132 ++
 web/src/components/ui/calendar.tsx            |   67 +
 web/src/components/ui/callout.tsx             |   38 +
 web/src/components/ui/card.tsx                |   86 +
 web/src/components/ui/chart.tsx               |  365 ++++
 web/src/components/ui/input.tsx               |   25 +
 web/src/components/ui/popover.tsx             |   31 +
 web/src/components/ui/select.tsx              |  165 ++
 web/src/components/ui/separator.tsx           |   31 +
 web/src/components/ui/table.tsx               |  120 ++
 web/src/components/ui/tabs.tsx                |   55 +
 web/src/components/ui/text.tsx                |   11 +
 web/src/components/ui/textarea.tsx            |   24 +
 web/src/components/ui/title.tsx               |   15 +
 web/src/lib/hooks.ts                          |    2 +-
 web/src/lib/search/interfaces.ts              |    2 +-
 web/src/lib/search/utils.ts                   |    2 +-
 web/src/lib/utils.ts                          |    6 +
 170 files changed, 4601 insertions(+), 1401 deletions(-)
 create mode 100644 web/components.json
 create mode 100644 web/src/components/admin/CardSection.tsx
 create mode 100644 web/src/components/ui/alert.tsx
 create mode 100644 web/src/components/ui/areaChart.tsx
 create mode 100644 web/src/components/ui/badge.tsx
 create mode 100644 web/src/components/ui/button.tsx
 create mode 100644 web/src/components/ui/calendar.tsx
 create mode 100644 web/src/components/ui/callout.tsx
 create mode 100644 web/src/components/ui/card.tsx
 create mode 100644 web/src/components/ui/chart.tsx
 create mode 100644 web/src/components/ui/input.tsx
 create mode 100644 web/src/components/ui/popover.tsx
 create mode 100644 web/src/components/ui/select.tsx
 create mode 100644 web/src/components/ui/separator.tsx
 create mode 100644 web/src/components/ui/table.tsx
 create mode 100644 web/src/components/ui/tabs.tsx
 create mode 100644 web/src/components/ui/text.tsx
 create mode 100644 web/src/components/ui/textarea.tsx
 create mode 100644 web/src/components/ui/title.tsx
 create mode 100644 web/src/lib/utils.ts

diff --git a/web/components.json b/web/components.json
new file mode 100644
index 00000000000..bfcba8e4a85
--- /dev/null
+++ b/web/components.json
@@ -0,0 +1,20 @@
+{
+  "$schema": "https://ui.shadcn.com/schema.json",
+  "style": "default",
+  "rsc": true,
+  "tsx": true,
+  "tailwind": {
+    "config": "tailwind.config.js",
+    "css": "src/app/globals.css",
+    "baseColor": "neutral",
+    "cssVariables": false,
+    "prefix": ""
+  },
+  "aliases": {
+    "components": "@/components",
+    "utils": "@/lib/utils",
+    "ui": "@/components/ui",
+    "lib": "@/lib",
+    "hooks": "@/hooks"
+  }
+}
\ No newline at end of file
diff --git a/web/package-lock.json b/web/package-lock.json
index 8d914d5b661..abf460023fc 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -11,13 +11,17 @@
         "@dnd-kit/core": "^6.1.0",
         "@dnd-kit/modifiers": "^7.0.0",
         "@dnd-kit/sortable": "^8.0.0",
+        "@headlessui/tailwindcss": "^0.2.1",
         "@phosphor-icons/react": "^2.0.8",
         "@radix-ui/react-dialog": "^1.0.5",
-        "@radix-ui/react-popover": "^1.0.7",
+        "@radix-ui/react-popover": "^1.1.2",
+        "@radix-ui/react-select": "^2.1.2",
+        "@radix-ui/react-separator": "^1.1.0",
+        "@radix-ui/react-slot": "^1.1.0",
+        "@radix-ui/react-tabs": "^1.1.1",
         "@radix-ui/react-tooltip": "^1.0.7",
         "@sentry/nextjs": "^8.34.0",
         "@stripe/stripe-js": "^4.6.0",
-        "@tremor/react": "^3.9.2",
         "@types/js-cookie": "^3.0.3",
         "@types/lodash": "^4.17.0",
         "@types/node": "18.15.11",
@@ -26,9 +30,13 @@
         "@types/react-dom": "18.0.11",
         "@types/uuid": "^9.0.8",
         "autoprefixer": "^10.4.14",
+        "class-variance-authority": "^0.7.0",
+        "clsx": "^2.1.1",
+        "date-fns": "^3.6.0",
         "formik": "^2.2.9",
         "js-cookie": "^3.0.5",
         "lodash": "^4.17.21",
+        "lucide-react": "^0.454.0",
         "mdast-util-find-and-replace": "^3.0.1",
         "next": "^14.2.3",
         "npm": "^10.8.0",
@@ -36,22 +44,26 @@
         "posthog-js": "^1.176.0",
         "prismjs": "^1.29.0",
         "react": "^18.3.1",
+        "react-day-picker": "^8.10.1",
         "react-dom": "^18.3.1",
         "react-dropzone": "^14.2.3",
         "react-icons": "^4.8.0",
         "react-loader-spinner": "^5.4.5",
         "react-markdown": "^9.0.1",
         "react-select": "^5.8.0",
+        "recharts": "^2.13.1",
         "rehype-prism-plus": "^2.0.0",
         "remark-gfm": "^4.0.0",
         "semver": "^7.5.4",
         "sharp": "^0.33.5",
         "stripe": "^17.0.0",
         "swr": "^2.1.5",
+        "tailwind-merge": "^2.5.4",
         "tailwindcss": "^3.3.1",
+        "tailwindcss-animate": "^1.0.7",
         "typescript": "5.0.3",
         "uuid": "^9.0.1",
-        "yup": "^1.1.1"
+        "yup": "^1.4.0"
       },
       "devDependencies": {
         "@tailwindcss/typography": "^0.5.10",
@@ -761,20 +773,6 @@
         "@floating-ui/utils": "^0.2.0"
       }
     },
-    "node_modules/@floating-ui/react": {
-      "version": "0.19.2",
-      "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.19.2.tgz",
-      "integrity": "sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w==",
-      "dependencies": {
-        "@floating-ui/react-dom": "^1.3.0",
-        "aria-hidden": "^1.1.3",
-        "tabbable": "^6.0.1"
-      },
-      "peerDependencies": {
-        "react": ">=16.8.0",
-        "react-dom": ">=16.8.0"
-      }
-    },
     "node_modules/@floating-ui/react-dom": {
       "version": "2.0.9",
       "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.0.9.tgz",
@@ -787,43 +785,16 @@
         "react-dom": ">=16.8.0"
       }
     },
-    "node_modules/@floating-ui/react/node_modules/@floating-ui/react-dom": {
-      "version": "1.3.0",
-      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-1.3.0.tgz",
-      "integrity": "sha512-htwHm67Ji5E/pROEAr7f8IKFShuiCKHwUC/UY4vC3I5jiSvGFAYnSYiZO5MlGmads+QqvUkR9ANHEguGrDv72g==",
-      "dependencies": {
-        "@floating-ui/dom": "^1.2.1"
-      },
-      "peerDependencies": {
-        "react": ">=16.8.0",
-        "react-dom": ">=16.8.0"
-      }
-    },
     "node_modules/@floating-ui/utils": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.2.tgz",
       "integrity": "sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw=="
     },
-    "node_modules/@headlessui/react": {
-      "version": "1.7.19",
-      "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-1.7.19.tgz",
-      "integrity": "sha512-Ll+8q3OlMJfJbAKM/+/Y2q6PPYbryqNTXDbryx7SXLIDamkF6iQFbriYHga0dY44PvDhvvBWCx1Xj4U5+G4hOw==",
-      "dependencies": {
-        "@tanstack/react-virtual": "^3.0.0-beta.60",
-        "client-only": "^0.0.1"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "peerDependencies": {
-        "react": "^16 || ^17 || ^18",
-        "react-dom": "^16 || ^17 || ^18"
-      }
-    },
     "node_modules/@headlessui/tailwindcss": {
-      "version": "0.2.0",
-      "resolved": "https://registry.npmjs.org/@headlessui/tailwindcss/-/tailwindcss-0.2.0.tgz",
-      "integrity": "sha512-fpL830Fln1SykOCboExsWr3JIVeQKieLJ3XytLe/tt1A0XzqUthOftDmjcCYLW62w7mQI7wXcoPXr3tZ9QfGxw==",
+      "version": "0.2.1",
+      "resolved": "https://registry.npmjs.org/@headlessui/tailwindcss/-/tailwindcss-0.2.1.tgz",
+      "integrity": "sha512-2+5+NZ+RzMyrVeCZOxdbvkUSssSxGvcUxphkIfSVLpRiKsj+/63T2TOL9dBYMXVfj/CGr6hMxSRInzXv6YY7sA==",
+      "license": "MIT",
       "engines": {
         "node": ">=10"
       },
@@ -2025,6 +1996,12 @@
         "@opentelemetry/api": "^1.3.0"
       }
     },
+    "node_modules/@radix-ui/number": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.0.tgz",
+      "integrity": "sha512-V3gRzhVNU1ldS5XhAPTom1fOIo4ccrjjJgmE+LI2h/WaFpHmx0MQApT+KZHnx8abG6Avtfcz4WoEciMnpFT3HQ==",
+      "license": "MIT"
+    },
     "node_modules/@radix-ui/primitive": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.0.1.tgz",
@@ -2056,6 +2033,85 @@
         }
       }
     },
+    "node_modules/@radix-ui/react-collection": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.0.tgz",
+      "integrity": "sha512-GZsZslMJEyo1VKm5L1ZJY8tGDxZNPAoUeQUIbKeJfoi7Q4kmig5AsgLMYYuyYbfjd8fBmFORAIwYAkXMnXZgZw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-slot": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-collection/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
+      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-collection/node_modules/@radix-ui/react-context": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
+      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-collection/node_modules/@radix-ui/react-primitive": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
+      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-slot": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@radix-ui/react-compose-refs": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
@@ -2114,8 +2170,1321 @@
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-direction": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.0.tgz",
+      "integrity": "sha512-BUuBvgThEiAXh2DWu93XsT+a3aWrGqolGlqqw5VU1kG7p/ZH2cuDlM1sRLNnY3QcBS69UIz2mcKhMxDsdewhjg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
+      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-escape-keydown": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
+      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz",
+      "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-id": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
+      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.2.tgz",
+      "integrity": "sha512-u2HRUyWW+lOiA2g0Le0tMmT55FGOEWHwPFt1EPfbLly7uXQExFo5duNKqG2DzmFXIdqOeNd+TpE8baHWJCyP9w==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.1",
+        "@radix-ui/react-dismissable-layer": "1.1.1",
+        "@radix-ui/react-focus-guards": "1.1.1",
+        "@radix-ui/react-focus-scope": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-popper": "1.2.0",
+        "@radix-ui/react-portal": "1.1.2",
+        "@radix-ui/react-presence": "1.1.1",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-slot": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0",
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.6.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/primitive": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-arrow": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.0.tgz",
+      "integrity": "sha512-FmlW1rCg7hBpEBwFbjHwCW6AmWLQM6g/v0Sn8XbP9NvmSZ2San1FpQeyPtufzOMSIx7Y4dzjlHoifhp+7NkZhw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
+      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.1.tgz",
+      "integrity": "sha512-QSxg29lfr/xcev6kSz7MAlmDnzbP1eI/Dwn3Tp1ip0KT5CUELsxkekFEMVBEoykI3oV39hKT4TKZzBNMbcTZYQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-escape-keydown": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz",
+      "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.0.tgz",
+      "integrity": "sha512-200UD8zylvEyL8Bx+z76RJnASR2gRMuxlgFCPAe/Q/679a/r0eK3MBVYMb7vZODZcffZBdob1EGnky78xmVvcA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-id": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
+      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-popper": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.0.tgz",
+      "integrity": "sha512-ZnRMshKF43aBxVWPWvbj21+7TQCvhuULWJ4gNIKYpRlQt5xGRhLx66tMp8pya2UkGHTSlhpXwmjqltDYHhw7Vg==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0",
+        "@radix-ui/react-use-rect": "1.1.0",
+        "@radix-ui/react-use-size": "1.1.0",
+        "@radix-ui/rect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-context": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
+      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-portal": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.2.tgz",
+      "integrity": "sha512-WeDYLGPxJb/5EGBoedyJbT0MpoULmwnIPMJMSldkuiMsBAv7N1cRdsTWZWht9vpPOiN3qyiGAtbK2is47/uMFg==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-presence": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.1.tgz",
+      "integrity": "sha512-IeFXVi4YS1K0wVZzXNrbaaUvIJ3qdY+/Ih4eHFhWA9SwGR9UDX7Ck8abvL57C4cv3wwMvUE0OG69Qc3NCcTe/A==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-primitive": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
+      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-slot": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
+      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
+      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.0.tgz",
+      "integrity": "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
+      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-rect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.0.tgz",
+      "integrity": "sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/rect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-size": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.0.tgz",
+      "integrity": "sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/rect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.0.tgz",
+      "integrity": "sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/react-remove-scroll": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz",
+      "integrity": "sha512-I2U4JVEsQenxDAKaVa3VZ/JeJZe0/2DxPWL8Tj8yLKctQJQiZM52pn/GWFpSp8dftjM3pSAHVJZscAnC/y+ySQ==",
+      "license": "MIT",
+      "dependencies": {
+        "react-remove-scroll-bar": "^2.3.6",
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.1.0",
+        "use-callback-ref": "^1.3.0",
+        "use-sidecar": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-popper": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
+      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.0.3",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1",
+        "@radix-ui/react-use-rect": "1.0.1",
+        "@radix-ui/react-use-size": "1.0.1",
+        "@radix-ui/rect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-portal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
+      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-presence": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
+      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-primitive": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
+      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-slot": "1.0.2"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-primitive/node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.0.tgz",
+      "integrity": "sha512-EA6AMGeq9AEeQDeSH0aZgG198qkfHSbvWTf1HvoDmOB5bBG/qTxjYMWUKMnYiV6J/iP/J8MEFSuB2zRU2n7ODA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-collection": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.0",
+        "@radix-ui/react-direction": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/primitive": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
+      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-context": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
+      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-id": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
+      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-primitive": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
+      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-slot": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
+      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
+      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
+      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.1.2.tgz",
+      "integrity": "sha512-rZJtWmorC7dFRi0owDmoijm6nSJH1tVw64QGiNIZ9PNLyBDtG+iAq+XGsya052At4BfarzY/Dhv9wrrUr6IMZA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/number": "1.1.0",
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-collection": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.1",
+        "@radix-ui/react-direction": "1.1.0",
+        "@radix-ui/react-dismissable-layer": "1.1.1",
+        "@radix-ui/react-focus-guards": "1.1.1",
+        "@radix-ui/react-focus-scope": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-popper": "1.2.0",
+        "@radix-ui/react-portal": "1.1.2",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-slot": "1.1.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0",
+        "@radix-ui/react-use-previous": "1.1.0",
+        "@radix-ui/react-visually-hidden": "1.1.0",
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.6.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/primitive": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-arrow": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.0.tgz",
+      "integrity": "sha512-FmlW1rCg7hBpEBwFbjHwCW6AmWLQM6g/v0Sn8XbP9NvmSZ2San1FpQeyPtufzOMSIx7Y4dzjlHoifhp+7NkZhw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
+      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.1.tgz",
+      "integrity": "sha512-QSxg29lfr/xcev6kSz7MAlmDnzbP1eI/Dwn3Tp1ip0KT5CUELsxkekFEMVBEoykI3oV39hKT4TKZzBNMbcTZYQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-escape-keydown": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz",
+      "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.0.tgz",
+      "integrity": "sha512-200UD8zylvEyL8Bx+z76RJnASR2gRMuxlgFCPAe/Q/679a/r0eK3MBVYMb7vZODZcffZBdob1EGnky78xmVvcA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-id": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
+      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-popper": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.0.tgz",
+      "integrity": "sha512-ZnRMshKF43aBxVWPWvbj21+7TQCvhuULWJ4gNIKYpRlQt5xGRhLx66tMp8pya2UkGHTSlhpXwmjqltDYHhw7Vg==",
+      "license": "MIT",
+      "dependencies": {
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0",
+        "@radix-ui/react-use-rect": "1.1.0",
+        "@radix-ui/react-use-size": "1.1.0",
+        "@radix-ui/rect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-context": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
+      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-portal": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.2.tgz",
+      "integrity": "sha512-WeDYLGPxJb/5EGBoedyJbT0MpoULmwnIPMJMSldkuiMsBAv7N1cRdsTWZWht9vpPOiN3qyiGAtbK2is47/uMFg==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-primitive": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
+      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-slot": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
+      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
+      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.0.tgz",
+      "integrity": "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-callback-ref": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
+      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-rect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.0.tgz",
+      "integrity": "sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/rect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-size": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.0.tgz",
+      "integrity": "sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-visually-hidden": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.1.0.tgz",
+      "integrity": "sha512-N8MDZqtgCgG5S3aV60INAB475osJousYpZ4cTJ2cFbMpdHS5Y6loLTH8LPtkj2QN0x93J30HT/M3qJXM0+lyeQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/rect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.0.tgz",
+      "integrity": "sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz",
+      "integrity": "sha512-I2U4JVEsQenxDAKaVa3VZ/JeJZe0/2DxPWL8Tj8yLKctQJQiZM52pn/GWFpSp8dftjM3pSAHVJZscAnC/y+ySQ==",
+      "license": "MIT",
+      "dependencies": {
+        "react-remove-scroll-bar": "^2.3.6",
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.1.0",
+        "use-callback-ref": "^1.3.0",
+        "use-sidecar": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-separator": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.0.tgz",
+      "integrity": "sha512-3uBAs+egzvJBDZAzvb/n4NxxOYpnspmWxO2u5NbZ8Y6FM/NdrGSF9bop3Cf6F6C71z1rTSn8KV0Fo2ZVd79lGA==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-separator/node_modules/@radix-ui/react-primitive": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
+      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
+      "license": "MIT",
+      "dependencies": {
+        "@radix-ui/react-slot": "1.1.0"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2126,43 +3495,32 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-dismissable-layer": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
-      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
+    "node_modules/@radix-ui/react-slot": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.1.0.tgz",
+      "integrity": "sha512-FUCf5XMfmW4dtYl69pdS4DbxKy8nj4M7SafBgPllysxmdachynNflAdp/gCsnYWNDnge6tI9onzMp5ARYc1KNw==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-escape-keydown": "1.0.3"
+        "@radix-ui/react-compose-refs": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-focus-guards": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
-      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
+    "node_modules/@radix-ui/react-slot/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
+      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2170,21 +3528,26 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-focus-scope": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz",
-      "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==",
+    "node_modules/@radix-ui/react-tabs": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.1.tgz",
+      "integrity": "sha512-3GBUDmP2DvzmtYLMsHmpA1GtR46ZDZ+OreXM/N+kkQJOPIgytFWWTfDQmBQKBvaFS0Vno0FktdbVzN28KGrMdw==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-context": "1.1.1",
+        "@radix-ui/react-direction": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-presence": "1.1.1",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-roving-focus": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2195,17 +3558,35 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-id": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
-      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/primitive": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
+      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
+      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2213,65 +3594,38 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.0.7.tgz",
-      "integrity": "sha512-shtvVnlsxT6faMnK/a7n0wptwBD23xc1Z5mdrtKLwVEfsEMXodS0r5s0/g5P0hX//EKYZS2sxUjqfzlg52ZSnQ==",
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-id": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
+      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-dismissable-layer": "1.0.5",
-        "@radix-ui/react-focus-guards": "1.0.1",
-        "@radix-ui/react-focus-scope": "1.0.4",
-        "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-popper": "1.1.3",
-        "@radix-ui/react-portal": "1.0.4",
-        "@radix-ui/react-presence": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-slot": "1.0.2",
-        "@radix-ui/react-use-controllable-state": "1.0.1",
-        "aria-hidden": "^1.1.1",
-        "react-remove-scroll": "2.5.5"
+        "@radix-ui/react-use-layout-effect": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popper": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
-      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-presence": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.1.tgz",
+      "integrity": "sha512-IeFXVi4YS1K0wVZzXNrbaaUvIJ3qdY+/Ih4eHFhWA9SwGR9UDX7Ck8abvL57C4cv3wwMvUE0OG69Qc3NCcTe/A==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@floating-ui/react-dom": "^2.0.0",
-        "@radix-ui/react-arrow": "1.0.3",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1",
-        "@radix-ui/react-use-rect": "1.0.1",
-        "@radix-ui/react-use-size": "1.0.1",
-        "@radix-ui/rect": "1.0.1"
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2282,19 +3636,19 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-portal": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
-      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-primitive": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
+      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+        "@radix-ui/react-slot": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2305,64 +3659,47 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-presence": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
-      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
-      },
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
+      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
+      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-primitive": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
-      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
+      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-slot": "1.0.2"
+        "@radix-ui/react-use-callback-ref": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-slot": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
-      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1"
-      },
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
+      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
+      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2404,6 +3741,25 @@
         }
       }
     },
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
+      },
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@radix-ui/react-use-callback-ref": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
@@ -2474,6 +3830,21 @@
         }
       }
     },
+    "node_modules/@radix-ui/react-use-previous": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.0.tgz",
+      "integrity": "sha512-Z/e78qg2YFnnXcW88A4JmTtm4ADckLno6F7OXotmkQfeuCVaKuYzqAATPhVzl3delXE7CxIV8shofPn3jPc5Og==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@radix-ui/react-use-rect": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz",
@@ -3173,50 +4544,6 @@
         "tailwindcss": ">=3.0.0 || insiders"
       }
     },
-    "node_modules/@tanstack/react-virtual": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.5.0.tgz",
-      "integrity": "sha512-rtvo7KwuIvqK9zb0VZ5IL7fiJAEnG+0EiFZz8FUOs+2mhGqdGmjKIaT1XU7Zq0eFqL0jonLlhbayJI/J2SA/Bw==",
-      "dependencies": {
-        "@tanstack/virtual-core": "3.5.0"
-      },
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/tannerlinsley"
-      },
-      "peerDependencies": {
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      }
-    },
-    "node_modules/@tanstack/virtual-core": {
-      "version": "3.5.0",
-      "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.5.0.tgz",
-      "integrity": "sha512-KnPRCkQTyqhanNC0K63GBG3wA8I+D1fQuVnAvcBF8f13akOKeQp1gSbu6f77zCxhEk727iV5oQnbHLYzHrECLg==",
-      "funding": {
-        "type": "github",
-        "url": "https://github.com/sponsors/tannerlinsley"
-      }
-    },
-    "node_modules/@tremor/react": {
-      "version": "3.16.3",
-      "resolved": "https://registry.npmjs.org/@tremor/react/-/react-3.16.3.tgz",
-      "integrity": "sha512-XiufPz4RRdrHrhwL7Cfcd9XoUEPyN/Q4jwj3kw1OQmFD1sYMCS2pAzzSP62k7zq02Z0QIPBuVK5p7/KQ+h4esQ==",
-      "dependencies": {
-        "@floating-ui/react": "^0.19.2",
-        "@headlessui/react": "^1.7.19",
-        "@headlessui/tailwindcss": "^0.2.0",
-        "date-fns": "^3.6.0",
-        "react-day-picker": "^8.10.1",
-        "react-transition-state": "^2.1.1",
-        "recharts": "^2.12.7",
-        "tailwind-merge": "^1.14.0"
-      },
-      "peerDependencies": {
-        "react": "^18.0.0",
-        "react-dom": ">=16.6.0"
-      }
-    },
     "node_modules/@types/connect": {
       "version": "3.4.36",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz",
@@ -4411,6 +5738,27 @@
       "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.1.tgz",
       "integrity": "sha512-cuSVIHi9/9E/+821Qjdvngor+xpnlwnuwIyZOaLmHBVdXL+gP+I6QQB9VkO7RI77YIcTV+S1W9AreJ5eN63JBA=="
     },
+    "node_modules/class-variance-authority": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.0.tgz",
+      "integrity": "sha512-jFI8IQw4hczaL4ALINxqLEXQbWcNjoSkloa4IaufXCJr6QawJyw7tuRysRsrE8w2p/4gGaxKIt/hX3qz/IbD1A==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "clsx": "2.0.0"
+      },
+      "funding": {
+        "url": "https://joebell.co.uk"
+      }
+    },
+    "node_modules/class-variance-authority/node_modules/clsx": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.0.0.tgz",
+      "integrity": "sha512-rQ1+kcj+ttHG0MKVGBUXwayCCF1oh39BF5COIpRzuCEv8Mwjv0XucrI2ExNTOn9IlLifGClWQcU9BrZORvtw6Q==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/client-only": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
@@ -4420,6 +5768,7 @@
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
       "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
+      "license": "MIT",
       "engines": {
         "node": ">=6"
       }
@@ -4745,6 +6094,7 @@
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-3.6.0.tgz",
       "integrity": "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==",
+      "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/kossnocorp"
@@ -7125,6 +8475,15 @@
         "node": "14 || >=16.14"
       }
     },
+    "node_modules/lucide-react": {
+      "version": "0.454.0",
+      "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-0.454.0.tgz",
+      "integrity": "sha512-hw7zMDwykCLnEzgncEEjHeA6+45aeEzRYuKHuyRSOPkhko+J3ySGjGIzu+mmMfDFG1vazHepMaYFYHbTFAZAAQ==",
+      "license": "ISC",
+      "peerDependencies": {
+        "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0-rc"
+      }
+    },
     "node_modules/magic-string": {
       "version": "0.30.12",
       "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.12.tgz",
@@ -11306,6 +12665,7 @@
       "version": "8.10.1",
       "resolved": "https://registry.npmjs.org/react-day-picker/-/react-day-picker-8.10.1.tgz",
       "integrity": "sha512-TMx7fNbhLk15eqcMt+7Z7S2KF7mfTId/XJDjKE8f+IUcFn0l08/kI4FiYTL/0yuOLmEcbR4Fwe3GJf/NiiMnPA==",
+      "license": "MIT",
       "funding": {
         "type": "individual",
         "url": "https://github.com/sponsors/gpbl"
@@ -11521,15 +12881,6 @@
         "react-dom": ">=16.6.0"
       }
     },
-    "node_modules/react-transition-state": {
-      "version": "2.1.1",
-      "resolved": "https://registry.npmjs.org/react-transition-state/-/react-transition-state-2.1.1.tgz",
-      "integrity": "sha512-kQx5g1FVu9knoz1T1WkapjUgFz08qQ/g1OmuWGi3/AoEFfS0kStxrPlZx81urjCXdz2d+1DqLpU6TyLW/Ro04Q==",
-      "peerDependencies": {
-        "react": ">=16.8.0",
-        "react-dom": ">=16.8.0"
-      }
-    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
@@ -11550,14 +12901,15 @@
       }
     },
     "node_modules/recharts": {
-      "version": "2.12.7",
-      "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.12.7.tgz",
-      "integrity": "sha512-hlLJMhPQfv4/3NBSAyq3gzGg4h2v69RJh6KU7b3pXYNNAELs9kEoXOjbkxdXpALqKBoVmVptGfLpxdaVYqjmXQ==",
+      "version": "2.13.1",
+      "resolved": "https://registry.npmjs.org/recharts/-/recharts-2.13.1.tgz",
+      "integrity": "sha512-87LdsmgK/MHLmWQfTC6yDysno2cOigi/+2KRCwy0D8NDu1IOdtTGS8lMovA0VIvJ7kf3zdp1IiwznHZWSPJhYw==",
+      "license": "MIT",
       "dependencies": {
         "clsx": "^2.0.0",
         "eventemitter3": "^4.0.1",
         "lodash": "^4.17.21",
-        "react-is": "^16.10.2",
+        "react-is": "^18.3.1",
         "react-smooth": "^4.0.0",
         "recharts-scale": "^0.4.4",
         "tiny-invariant": "^1.3.1",
@@ -11579,6 +12931,12 @@
         "decimal.js-light": "^2.4.1"
       }
     },
+    "node_modules/recharts/node_modules/react-is": {
+      "version": "18.3.1",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
+      "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
+      "license": "MIT"
+    },
     "node_modules/reflect.getprototypeof": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.6.tgz",
@@ -12529,15 +13887,11 @@
         "react": "^16.11.0 || ^17.0.0 || ^18.0.0"
       }
     },
-    "node_modules/tabbable": {
-      "version": "6.2.0",
-      "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz",
-      "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="
-    },
     "node_modules/tailwind-merge": {
-      "version": "1.14.0",
-      "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-1.14.0.tgz",
-      "integrity": "sha512-3mFKyCo/MBcgyOTlrY8T7odzZFx+w+qKSMAmdFzRvqBfLlSigU6TZnlFHK0lkMwj9Bj8OYU+9yW9lmGuS0QEnQ==",
+      "version": "2.5.4",
+      "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.4.tgz",
+      "integrity": "sha512-0q8cfZHMu9nuYP/b5Shb7Y7Sh1B7Nnl5GqNr1U+n2p6+mybvRtayrQ+0042Z5byvTA8ihjlP8Odo8/VnHbZu4Q==",
+      "license": "MIT",
       "funding": {
         "type": "github",
         "url": "https://github.com/sponsors/dcastil"
@@ -12579,6 +13933,15 @@
         "node": ">=14.0.0"
       }
     },
+    "node_modules/tailwindcss-animate": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/tailwindcss-animate/-/tailwindcss-animate-1.0.7.tgz",
+      "integrity": "sha512-bl6mpH3T7I3UFxuvDEXLxy/VuFxBk5bbzplh7tXI68mwMokNYd1t9qPBHlnyTwfa4JGC4zP516I1hYYtQ/vspA==",
+      "license": "MIT",
+      "peerDependencies": {
+        "tailwindcss": ">=3.0.0 || insiders"
+      }
+    },
     "node_modules/tailwindcss/node_modules/postcss-selector-parser": {
       "version": "6.0.16",
       "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.16.tgz",
@@ -13562,6 +14925,7 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/yup/-/yup-1.4.0.tgz",
       "integrity": "sha512-wPbgkJRCqIf+OHyiTBQoJiP5PFuAXaWiJK6AmYkzQAh5/c2K9hzSApBZG5wV9KoKSePF7sAxmNSvh/13YHkFDg==",
+      "license": "MIT",
       "dependencies": {
         "property-expr": "^2.0.5",
         "tiny-case": "^1.0.3",
diff --git a/web/package.json b/web/package.json
index 418eaf4f9ac..ea6201642a8 100644
--- a/web/package.json
+++ b/web/package.json
@@ -12,13 +12,17 @@
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/modifiers": "^7.0.0",
     "@dnd-kit/sortable": "^8.0.0",
+    "@headlessui/tailwindcss": "^0.2.1",
     "@phosphor-icons/react": "^2.0.8",
     "@radix-ui/react-dialog": "^1.0.5",
-    "@radix-ui/react-popover": "^1.0.7",
+    "@radix-ui/react-popover": "^1.1.2",
+    "@radix-ui/react-select": "^2.1.2",
+    "@radix-ui/react-separator": "^1.1.0",
+    "@radix-ui/react-slot": "^1.1.0",
+    "@radix-ui/react-tabs": "^1.1.1",
     "@radix-ui/react-tooltip": "^1.0.7",
     "@sentry/nextjs": "^8.34.0",
     "@stripe/stripe-js": "^4.6.0",
-    "@tremor/react": "^3.9.2",
     "@types/js-cookie": "^3.0.3",
     "@types/lodash": "^4.17.0",
     "@types/node": "18.15.11",
@@ -27,9 +31,13 @@
     "@types/react-dom": "18.0.11",
     "@types/uuid": "^9.0.8",
     "autoprefixer": "^10.4.14",
+    "class-variance-authority": "^0.7.0",
+    "clsx": "^2.1.1",
+    "date-fns": "^3.6.0",
     "formik": "^2.2.9",
     "js-cookie": "^3.0.5",
     "lodash": "^4.17.21",
+    "lucide-react": "^0.454.0",
     "mdast-util-find-and-replace": "^3.0.1",
     "next": "^14.2.3",
     "npm": "^10.8.0",
@@ -37,22 +45,26 @@
     "posthog-js": "^1.176.0",
     "prismjs": "^1.29.0",
     "react": "^18.3.1",
+    "react-day-picker": "^8.10.1",
     "react-dom": "^18.3.1",
     "react-dropzone": "^14.2.3",
     "react-icons": "^4.8.0",
     "react-loader-spinner": "^5.4.5",
     "react-markdown": "^9.0.1",
     "react-select": "^5.8.0",
+    "recharts": "^2.13.1",
     "rehype-prism-plus": "^2.0.0",
     "remark-gfm": "^4.0.0",
     "semver": "^7.5.4",
     "sharp": "^0.33.5",
     "stripe": "^17.0.0",
     "swr": "^2.1.5",
+    "tailwind-merge": "^2.5.4",
     "tailwindcss": "^3.3.1",
+    "tailwindcss-animate": "^1.0.7",
     "typescript": "5.0.3",
     "uuid": "^9.0.1",
-    "yup": "^1.1.1"
+    "yup": "^1.4.0"
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.10",
diff --git a/web/src/app/admin/add-connector/page.tsx b/web/src/app/admin/add-connector/page.tsx
index ecf459d612f..dc1e0721f3f 100644
--- a/web/src/app/admin/add-connector/page.tsx
+++ b/web/src/app/admin/add-connector/page.tsx
@@ -4,7 +4,8 @@ import { AdminPageTitle } from "@/components/admin/Title";
 import { ConnectorIcon } from "@/components/icons/icons";
 import { SourceCategory, SourceMetadata } from "@/lib/search/interfaces";
 import { listSourceMetadata } from "@/lib/sources";
-import { Title, Text, Button } from "@tremor/react";
+import Title from "@/components/ui/title";
+import { Button } from "@/components/ui/button";
 import Link from "next/link";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 
@@ -32,9 +33,7 @@ function SourceTile({
       href={sourceMetadata.adminUrl}
     >
       <SourceIcon sourceType={sourceMetadata.internalName} iconSize={24} />
-      <Text className="font-medium text-sm mt-2">
-        {sourceMetadata.displayName}
-      </Text>
+      <p className="font-medium text-sm mt-2">{sourceMetadata.displayName}</p>
     </Link>
   );
 }
@@ -101,9 +100,7 @@ export default function Page() {
         title="Add Connector"
         farRightElement={
           <Link href="/admin/indexing/status">
-            <Button color="green" size="xs">
-              See Connectors
-            </Button>
+            <Button variant="success-reverse">See Connectors</Button>
           </Link>
         }
       />
@@ -125,7 +122,7 @@ export default function Page() {
             <div className="flex mt-8">
               <Title>{category}</Title>
             </div>
-            <Text>{getCategoryDescription(category as SourceCategory)}</Text>
+            <p>{getCategoryDescription(category as SourceCategory)}</p>
             <div className="flex flex-wrap gap-4 p-4">
               {sources.map((source, sourceInd) => (
                 <SourceTile
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 69b2d9c1276..51c2cb4c3ed 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -3,7 +3,8 @@
 import { generateRandomIconShape, createSVG } from "@/lib/assistantIconUtils";
 
 import { CCPairBasicInfo, DocumentSet, User } from "@/lib/types";
-import { Button, Divider, Italic } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
 import { IsPublicGroupSelector } from "@/components/IsPublicGroupSelector";
 import {
   ArrayHelpers,
@@ -39,13 +40,13 @@ import {
 import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { useEffect, useState } from "react";
-import { FiInfo, FiPlus, FiX } from "react-icons/fi";
+import { FiInfo, FiX } from "react-icons/fi";
 import * as Yup from "yup";
 import { FullLLMProvider } from "../configuration/llm/interfaces";
 import CollapsibleSection from "./CollapsibleSection";
 import { SuccessfulPersonaUpdateRedirectType } from "./enums";
 import { Persona, StarterMessage } from "./interfaces";
-import { buildFinalPrompt, createPersona, updatePersona } from "./lib";
+import { createPersona, updatePersona } from "./lib";
 import { Popover } from "@/components/popover/Popover";
 import {
   CameraIcon,
@@ -627,7 +628,10 @@ export function AssistantEditor({
                   otherwise specified below.
                   {admin &&
                     user?.preferences.default_model &&
-                    `  Your current (user-specific) default model is ${getDisplayNameForModel(destructureValue(user?.preferences?.default_model!).modelName)}`}
+                    `  Your current (user-specific) default model is ${getDisplayNameForModel(
+                      destructureValue(user?.preferences?.default_model!)
+                        .modelName
+                    )}`}
                 </p>
                 {admin ? (
                   <div className="mb-2 flex items-starts">
@@ -869,7 +873,7 @@ export function AssistantEditor({
                                     )}
                                   />
                                 ) : (
-                                  <Italic className="text-sm">
+                                  <p className="text-sm italic">
                                     No Document Sets available.{" "}
                                     {user?.role !== "admin" && (
                                       <>
@@ -878,7 +882,7 @@ export function AssistantEditor({
                                         Danswer for assistance.
                                       </>
                                     )}
-                                  </Italic>
+                                  </p>
                                 )}
 
                                 <div className="mt-4  flex flex-col gap-y-4">
@@ -972,7 +976,7 @@ export function AssistantEditor({
                   )}
                 </div>
               </div>
-              <Divider />
+              <Separator />
               <AdvancedOptionsToggle
                 showAdvancedOptions={showAdvancedOptions}
                 setShowAdvancedOptions={setShowAdvancedOptions}
@@ -1138,11 +1142,9 @@ export function AssistantEditor({
                                 message: "",
                               });
                             }}
-                            className="text-white mt-3"
-                            size="xs"
-                            type="button"
-                            icon={FiPlus}
-                            color="green"
+                            className="mt-3"
+                            size="sm"
+                            variant="next"
                           >
                             Add New
                           </Button>
@@ -1166,9 +1168,7 @@ export function AssistantEditor({
 
               <div className="flex">
                 <Button
-                  className="mx-auto"
-                  color="green"
-                  size="md"
+                  variant="submit"
                   type="submit"
                   disabled={isSubmitting || isRequestSuccessful}
                 >
diff --git a/web/src/app/admin/assistants/PersonaTable.tsx b/web/src/app/admin/assistants/PersonaTable.tsx
index 6adf22a525e..c5dcfc2690e 100644
--- a/web/src/app/admin/assistants/PersonaTable.tsx
+++ b/web/src/app/admin/assistants/PersonaTable.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { Persona } from "./interfaces";
 import { useRouter } from "next/navigation";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
@@ -160,7 +160,9 @@ export function PersonasTable() {
                     }
                   }
                 }}
-                className={`px-1 py-0.5 rounded flex ${isEditable ? "hover:bg-hover cursor-pointer" : ""} select-none w-fit`}
+                className={`px-1 py-0.5 rounded flex ${
+                  isEditable ? "hover:bg-hover cursor-pointer" : ""
+                } select-none w-fit`}
               >
                 <div className="my-auto w-12">
                   {!persona.is_visible ? (
diff --git a/web/src/app/admin/assistants/[id]/DeletePersonaButton.tsx b/web/src/app/admin/assistants/[id]/DeletePersonaButton.tsx
index 0bbc268fae4..dc12ca5d9e1 100644
--- a/web/src/app/admin/assistants/[id]/DeletePersonaButton.tsx
+++ b/web/src/app/admin/assistants/[id]/DeletePersonaButton.tsx
@@ -1,7 +1,6 @@
 "use client";
 
-import { Button } from "@tremor/react";
-import { FiTrash } from "react-icons/fi";
+import { Button } from "@/components/ui/button";
 import { deletePersona } from "../lib";
 import { useRouter } from "next/navigation";
 import { SuccessfulPersonaUpdateRedirectType } from "../enums";
@@ -17,8 +16,7 @@ export function DeletePersonaButton({
 
   return (
     <Button
-      size="xs"
-      color="red"
+      variant="destructive"
       onClick={async () => {
         const response = await deletePersona(personaId);
         if (response.ok) {
@@ -31,7 +29,6 @@ export function DeletePersonaButton({
           alert(`Failed to delete persona - ${await response.text()}`);
         }
       }}
-      icon={FiTrash}
     >
       Delete
     </Button>
diff --git a/web/src/app/admin/assistants/[id]/page.tsx b/web/src/app/admin/assistants/[id]/page.tsx
index fab6f9f038b..bacfa625149 100644
--- a/web/src/app/admin/assistants/[id]/page.tsx
+++ b/web/src/app/admin/assistants/[id]/page.tsx
@@ -1,12 +1,14 @@
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { AssistantEditor } from "../AssistantEditor";
 import { BackButton } from "@/components/BackButton";
-import { Card, Title } from "@tremor/react";
+
 import { DeletePersonaButton } from "./DeletePersonaButton";
 import { fetchAssistantEditorInfoSS } from "@/lib/assistants/fetchPersonaEditorInfoSS";
 import { SuccessfulPersonaUpdateRedirectType } from "../enums";
 import { RobotIcon } from "@/components/icons/icons";
 import { AdminPageTitle } from "@/components/admin/Title";
+import CardSection from "@/components/admin/CardSection";
+import Title from "@/components/ui/title";
 
 export default async function Page({ params }: { params: { id: string } }) {
   const [values, error] = await fetchAssistantEditorInfoSS(params.id);
@@ -19,16 +21,17 @@ export default async function Page({ params }: { params: { id: string } }) {
   } else {
     body = (
       <>
-        <Card>
+        <CardSection>
           <AssistantEditor
             {...values}
             defaultPublic={true}
             redirectType={SuccessfulPersonaUpdateRedirectType.ADMIN}
           />
-        </Card>
+        </CardSection>
 
         <div className="mt-12">
           <Title>Delete Assistant</Title>
+
           <div className="flex mt-6">
             <DeletePersonaButton
               personaId={values.existingPersona!.id}
diff --git a/web/src/app/admin/assistants/new/page.tsx b/web/src/app/admin/assistants/new/page.tsx
index c770056321f..a61d9ab54bf 100644
--- a/web/src/app/admin/assistants/new/page.tsx
+++ b/web/src/app/admin/assistants/new/page.tsx
@@ -2,7 +2,7 @@ import { AssistantEditor } from "../AssistantEditor";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { RobotIcon } from "@/components/icons/icons";
 import { BackButton } from "@/components/BackButton";
-import { Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { fetchAssistantEditorInfoSS } from "@/lib/assistants/fetchPersonaEditorInfoSS";
 import { SuccessfulPersonaUpdateRedirectType } from "../enums";
@@ -17,14 +17,14 @@ export default async function Page() {
     );
   } else {
     body = (
-      <Card>
+      <CardSection>
         <AssistantEditor
           {...values}
           admin
           defaultPublic={true}
           redirectType={SuccessfulPersonaUpdateRedirectType.ADMIN}
         />
-      </Card>
+      </CardSection>
     );
   }
 
diff --git a/web/src/app/admin/assistants/page.tsx b/web/src/app/admin/assistants/page.tsx
index 6c13bbb66dd..66139dbd030 100644
--- a/web/src/app/admin/assistants/page.tsx
+++ b/web/src/app/admin/assistants/page.tsx
@@ -1,7 +1,9 @@
 import { PersonasTable } from "./PersonaTable";
 import { FiPlusSquare } from "react-icons/fi";
 import Link from "next/link";
-import { Divider, Text, Title } from "@tremor/react";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Separator } from "@/components/ui/separator";
 import { AssistantsIcon } from "@/components/icons/icons";
 import { AdminPageTitle } from "@/components/admin/Title";
 
@@ -25,7 +27,7 @@ export default async function Page() {
       </div>
 
       <div>
-        <Divider />
+        <Separator />
 
         <Title>Create an Assistant</Title>
         <Link
@@ -38,7 +40,7 @@ export default async function Page() {
           </div>
         </Link>
 
-        <Divider />
+        <Separator />
 
         <Title>Existing Assistants</Title>
         <PersonasTable />
diff --git a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
index 9fb5aa063d2..cf4e018dc66 100644
--- a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
+++ b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
@@ -16,7 +16,9 @@ import {
   isPersonaASlackBotPersona,
   updateSlackBotConfig,
 } from "./lib";
-import { Button, Card, Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
+import CardSection from "@/components/admin/CardSection";
+import { Button } from "@/components/ui/button";
 import { useRouter } from "next/navigation";
 import { Persona } from "../assistants/interfaces";
 import { useState } from "react";
@@ -52,7 +54,7 @@ export const SlackBotCreationForm = ({
 
   return (
     <div>
-      <Card>
+      <CardSection>
         {popup}
         <Formik
           initialValues={{
@@ -259,7 +261,7 @@ export const SlackBotCreationForm = ({
                   </div>
                 </div>
 
-                <Divider />
+                <Separator />
 
                 <AdvancedOptionsToggle
                   showAdvancedOptions={showAdvancedOptions}
@@ -378,7 +380,7 @@ export const SlackBotCreationForm = ({
             </Form>
           )}
         </Formik>
-      </Card>
+      </CardSection>
     </div>
   );
 };
diff --git a/web/src/app/admin/bot/SlackBotTokensForm.tsx b/web/src/app/admin/bot/SlackBotTokensForm.tsx
index ed6b9d11f24..f290787ce3d 100644
--- a/web/src/app/admin/bot/SlackBotTokensForm.tsx
+++ b/web/src/app/admin/bot/SlackBotTokensForm.tsx
@@ -4,7 +4,8 @@ import { PopupSpec } from "@/components/admin/connectors/Popup";
 import { SlackBotTokens } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { setSlackBotTokens } from "./lib";
-import { Button, Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
+import { Button } from "@/components/ui/button";
 
 interface SlackBotTokensFormProps {
   onClose: () => void;
@@ -18,7 +19,7 @@ export const SlackBotTokensForm = ({
   existingTokens,
 }: SlackBotTokensFormProps) => {
   return (
-    <Card>
+    <CardSection>
       <Formik
         initialValues={existingTokens || { app_token: "", bot_token: "" }}
         validationSchema={Yup.object().shape({
@@ -57,13 +58,13 @@ export const SlackBotTokensForm = ({
               type="password"
             />
             <div className="flex">
-              <Button type="submit" disabled={isSubmitting}>
+              <Button type="submit" disabled={isSubmitting} variant="submit">
                 Set Tokens
               </Button>
             </div>
           </Form>
         )}
       </Formik>
-    </Card>
+    </CardSection>
   );
 };
diff --git a/web/src/app/admin/bot/[id]/page.tsx b/web/src/app/admin/bot/[id]/page.tsx
index 61fb6ee2e0d..f0ae638f980 100644
--- a/web/src/app/admin/bot/[id]/page.tsx
+++ b/web/src/app/admin/bot/[id]/page.tsx
@@ -4,7 +4,7 @@ import { SlackBotCreationForm } from "../SlackBotConfigCreationForm";
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { DocumentSet, SlackBotConfig } from "@/lib/types";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { BackButton } from "@/components/BackButton";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import {
diff --git a/web/src/app/admin/bot/page.tsx b/web/src/app/admin/bot/page.tsx
index 62f74277d1c..af1f12d56bc 100644
--- a/web/src/app/admin/bot/page.tsx
+++ b/web/src/app/admin/bot/page.tsx
@@ -11,20 +11,20 @@ import { deleteSlackBotConfig, isPersonaASlackBotPersona } from "./lib";
 import { SlackBotTokensForm } from "./SlackBotTokensForm";
 import { AdminPageTitle } from "@/components/admin/Title";
 import {
-  Button,
   Table,
   TableBody,
   TableCell,
   TableHead,
-  TableHeaderCell,
+  TableHeader,
   TableRow,
-  Text,
-  Title,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
 import { FiArrowUpRight, FiChevronDown, FiChevronUp } from "react-icons/fi";
 import Link from "next/link";
 import { InstantSSRAutoRefresh } from "@/components/SSRAutoRefresh";
 import { ErrorCallout } from "@/components/ErrorCallout";
+import { Button } from "@/components/ui/button";
 
 const numToDisplay = 50;
 
@@ -53,14 +53,14 @@ const SlackBotConfigsTable = ({
   return (
     <div>
       <Table>
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Channels</TableHeaderCell>
-            <TableHeaderCell>Assistant</TableHeaderCell>
-            <TableHeaderCell>Document Sets</TableHeaderCell>
-            <TableHeaderCell>Delete</TableHeaderCell>
+            <TableHead>Channels</TableHead>
+            <TableHead>Assistant</TableHead>
+            <TableHead>Document Sets</TableHead>
+            <TableHead>Delete</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {slackBotConfigs
             .slice(numToDisplay * (page - 1), numToDisplay * page)
@@ -235,8 +235,7 @@ const Main = () => {
             onClick={() => {
               setSlackBotTokensModalIsOpen(!slackBotTokensModalIsOpen);
             }}
-            color="blue"
-            size="xs"
+            variant="outline"
             className="mt-2"
             icon={slackBotTokensModalIsOpen ? FiChevronUp : FiChevronDown}
           >
@@ -268,7 +267,7 @@ const Main = () => {
           <div className="mb-2"></div>
 
           <Link className="flex mb-3 w-fit" href="/admin/bot/new">
-            <Button className="my-auto" color="green" size="xs">
+            <Button className="my-auto" variant="next">
               New Slack Bot Configuration
             </Button>
           </Link>
diff --git a/web/src/app/admin/configuration/document-processing/page.tsx b/web/src/app/admin/configuration/document-processing/page.tsx
index 37934181e17..1381e8c4061 100644
--- a/web/src/app/admin/configuration/document-processing/page.tsx
+++ b/web/src/app/admin/configuration/document-processing/page.tsx
@@ -1,7 +1,8 @@
 "use client";
 
 import { useState } from "react";
-import { Button, Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
+import { Button } from "@/components/ui/button";
 import { DocumentIcon2 } from "@/components/icons/icons";
 import useSWR from "swr";
 import { ThreeDotsLoader } from "@/components/Loading";
@@ -53,7 +54,7 @@ function Main() {
   }
   return (
     <div className="container mx-auto p-4">
-      <Card className="mb-8 max-w-2xl bg-white text-text shadow-lg rounded-lg">
+      <CardSection className="mb-8 max-w-2xl bg-white text-text shadow-lg rounded-lg">
         <h3 className="text-2xl text-text-800 font-bold mb-4 text-text border-b border-b-border pb-2">
           Process with Unstructured API
         </h3>
@@ -99,12 +100,7 @@ function Main() {
           <div className="flex space-x-4 mt-6">
             {isApiKeySet ? (
               <>
-                <Button
-                  color="red"
-                  onClick={handleDelete}
-                  variant="secondary"
-                  className="bg-red-100 text-red-600 hover:bg-red-400 transition duration-200"
-                >
+                <Button onClick={handleDelete} variant="destructive">
                   Delete API Key
                 </Button>
                 <p className="text-text-600 my-auto">
@@ -121,7 +117,7 @@ function Main() {
             )}
           </div>
         </div>
-      </Card>
+      </CardSection>
     </div>
   );
 }
diff --git a/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx b/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
index 850eb9690b7..ea6149e08ad 100644
--- a/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
+++ b/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
@@ -6,7 +6,8 @@ import { CustomLLMProviderUpdateForm } from "./CustomLLMProviderUpdateForm";
 import { useState } from "react";
 import { LLM_PROVIDERS_ADMIN_URL } from "./constants";
 import { mutate } from "swr";
-import { Badge, Button } from "@tremor/react";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
 import isEqual from "lodash/isEqual";
 
 function LLMProviderUpdateModal({
@@ -112,21 +113,16 @@ function LLMProviderDisplay({
         {existingLlmProvider && (
           <div className="my-auto ml-3">
             {existingLlmProvider.is_default_provider ? (
-              <Badge color="orange" size="xs">
-                Default
-              </Badge>
+              <Badge variant="orange">Default</Badge>
             ) : (
-              <Badge color="green" size="xs">
-                Enabled
-              </Badge>
+              <Badge variant="success">Enabled</Badge>
             )}
           </div>
         )}
 
         <div className="ml-auto">
           <Button
-            color={existingLlmProvider ? "green" : "blue"}
-            size="xs"
+            variant={existingLlmProvider ? "outline" : "next"}
             onClick={() => setFormIsVisible(true)}
           >
             {existingLlmProvider ? "Edit" : "Set up"}
diff --git a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
index cc059ebe3d0..4301c34d3e7 100644
--- a/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/CustomLLMProviderUpdateForm.tsx
@@ -1,5 +1,7 @@
 import { LoadingAnimation } from "@/components/Loading";
-import { Button, Divider, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import {
   ArrayHelpers,
@@ -232,7 +234,7 @@ export function CustomLLMProviderUpdateForm({
               placeholder="Name of the custom provider"
             />
 
-            <Divider />
+            <Separator />
 
             <SubLabel>
               Fill in the following as is needed. Refer to the LiteLLM
@@ -357,8 +359,7 @@ export function CustomLLMProviderUpdateForm({
                       arrayHelpers.push(["", ""]);
                     }}
                     className="mt-3"
-                    color="green"
-                    size="xs"
+                    variant="next"
                     type="button"
                     icon={FiPlus}
                   >
@@ -368,7 +369,7 @@ export function CustomLLMProviderUpdateForm({
               )}
             />
 
-            <Divider />
+            <Separator />
 
             {!existingLlmProvider?.deployment_name && (
               <TextArrayField
@@ -395,7 +396,7 @@ export function CustomLLMProviderUpdateForm({
               />
             )}
 
-            <Divider />
+            <Separator />
 
             <TextFormField
               name="default_model_name"
@@ -418,7 +419,7 @@ export function CustomLLMProviderUpdateForm({
               />
             )}
 
-            <Divider />
+            <Separator />
 
             <AdvancedOptionsToggle
               showAdvancedOptions={showAdvancedOptions}
@@ -441,7 +442,7 @@ export function CustomLLMProviderUpdateForm({
               )}
 
               <div className="flex w-full mt-4">
-                <Button type="submit" size="xs">
+                <Button type="submit" variant="submit">
                   {isTesting ? (
                     <LoadingAnimation text="Testing" />
                   ) : existingLlmProvider ? (
@@ -453,9 +454,7 @@ export function CustomLLMProviderUpdateForm({
                 {existingLlmProvider && (
                   <Button
                     type="button"
-                    color="red"
-                    className="ml-3"
-                    size="xs"
+                    variant="destructive"
                     icon={FiTrash}
                     onClick={async () => {
                       const response = await fetch(
diff --git a/web/src/app/admin/configuration/llm/LLMConfiguration.tsx b/web/src/app/admin/configuration/llm/LLMConfiguration.tsx
index f8fb7512740..a4161e63a21 100644
--- a/web/src/app/admin/configuration/llm/LLMConfiguration.tsx
+++ b/web/src/app/admin/configuration/llm/LLMConfiguration.tsx
@@ -4,7 +4,10 @@ import { Modal } from "@/components/Modal";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { useState } from "react";
 import useSWR from "swr";
-import { Button, Callout, Text, Title } from "@tremor/react";
+import { Callout } from "@/components/ui/callout";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Button } from "@/components/ui/button";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { FullLLMProvider, WellKnownLLMProviderDescriptor } from "./interfaces";
 import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
@@ -76,7 +79,7 @@ function DefaultLLMProviderDisplay({
         </div>
 
         <div className="ml-auto">
-          <Button color="blue" size="xs" onClick={() => setFormIsVisible(true)}>
+          <Button variant="navigate" onClick={() => setFormIsVisible(true)}>
             Set up
           </Button>
         </div>
@@ -117,7 +120,7 @@ function AddCustomLLMProvider({
   }
 
   return (
-    <Button size="xs" onClick={() => setFormIsVisible(true)}>
+    <Button variant="navigate" onClick={() => setFormIsVisible(true)}>
       Add Custom LLM Provider
     </Button>
   );
@@ -154,7 +157,7 @@ export function LLMConfiguration() {
           />
         </>
       ) : (
-        <Callout title="No LLM providers configured yet" color="yellow">
+        <Callout type="warning" title="No LLM providers configured yet">
           Please set one up below in order to start using Danswer!
         </Callout>
       )}
diff --git a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
index 7968fc5d7bf..59c0bb0fd1a 100644
--- a/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
+++ b/web/src/app/admin/configuration/llm/LLMProviderUpdateForm.tsx
@@ -1,6 +1,8 @@
 import { LoadingAnimation } from "@/components/Loading";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
-import { Button, Divider, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
 import { Form, Formik } from "formik";
 import { FiTrash } from "react-icons/fi";
 import { LLM_PROVIDERS_ADMIN_URL } from "./constants";
@@ -144,7 +146,9 @@ export function LLMProviderUpdateForm({
         }
 
         const response = await fetch(
-          `${LLM_PROVIDERS_ADMIN_URL}${existingLlmProvider ? "" : "?is_creation=true"}`,
+          `${LLM_PROVIDERS_ADMIN_URL}${
+            existingLlmProvider ? "" : "?is_creation=true"
+          }`,
           {
             method: "PUT",
             headers: {
@@ -273,7 +277,7 @@ export function LLMProviderUpdateForm({
 
           {!(hideAdvanced && llmProviderDescriptor.name != "azure") && (
             <>
-              <Divider />
+              <Separator />
 
               {llmProviderDescriptor.llm_names.length > 0 ? (
                 <SelectorFormField
@@ -332,7 +336,7 @@ export function LLMProviderUpdateForm({
 
               {llmProviderDescriptor.name != "azure" && (
                 <>
-                  <Divider />
+                  <Separator />
 
                   <AdvancedOptionsToggle
                     showAdvancedOptions={showAdvancedOptions}
@@ -383,7 +387,7 @@ export function LLMProviderUpdateForm({
             {testError && <Text className="text-error mt-2">{testError}</Text>}
 
             <div className="flex w-full mt-4">
-              <Button type="submit" size="xs">
+              <Button type="submit" variant="submit">
                 {isTesting ? (
                   <LoadingAnimation text="Testing" />
                 ) : existingLlmProvider ? (
@@ -395,9 +399,8 @@ export function LLMProviderUpdateForm({
               {existingLlmProvider && (
                 <Button
                   type="button"
-                  color="red"
+                  variant="destructive"
                   className="ml-3"
-                  size="xs"
                   icon={FiTrash}
                   onClick={async () => {
                     const response = await fetch(
diff --git a/web/src/app/admin/configuration/search/UpgradingPage.tsx b/web/src/app/admin/configuration/search/UpgradingPage.tsx
index 60177661c9d..e48976c406f 100644
--- a/web/src/app/admin/configuration/search/UpgradingPage.tsx
+++ b/web/src/app/admin/configuration/search/UpgradingPage.tsx
@@ -6,7 +6,9 @@ import {
   FailedConnectorIndexingStatus,
   ValidStatuses,
 } from "@/lib/types";
-import { Button, Text, Title } from "@tremor/react";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Button } from "@/components/ui/button";
 import { useMemo, useState } from "react";
 import useSWR, { mutate } from "swr";
 import { ReindexingProgressTable } from "../../../../components/embedding/ReindexingProgressTable";
@@ -111,7 +113,7 @@ export default function UpgradingPage({
               be lost.
             </div>
             <div className="flex">
-              <Button onClick={onCancel} className="mt-3 mx-auto" color="green">
+              <Button onClick={onCancel} variant="submit">
                 Confirm
               </Button>
             </div>
@@ -129,8 +131,7 @@ export default function UpgradingPage({
             </div>
 
             <Button
-              color="red"
-              size="xs"
+              variant="destructive"
               className="mt-4"
               onClick={() => setIsCancelling(true)}
             >
diff --git a/web/src/app/admin/configuration/search/page.tsx b/web/src/app/admin/configuration/search/page.tsx
index a66e45d9023..7c5a0cdd733 100644
--- a/web/src/app/admin/configuration/search/page.tsx
+++ b/web/src/app/admin/configuration/search/page.tsx
@@ -3,7 +3,9 @@
 import { ThreeDotsLoader } from "@/components/Loading";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { errorHandlingFetcher } from "@/lib/fetcher";
-import { Button, Card, Text, Title } from "@tremor/react";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Button } from "@/components/ui/button";
 import useSWR from "swr";
 import { ModelPreview } from "../../../../components/embedding/ModelSelector";
 import {
@@ -28,6 +30,7 @@ import { SavedSearchSettings } from "../../embeddings/interfaces";
 import UpgradingPage from "./UpgradingPage";
 import { useContext } from "react";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
+import CardSection from "@/components/admin/CardSection";
 
 function Main() {
   const settings = useContext(SettingsContext);
@@ -101,7 +104,7 @@ function Main() {
 
           <Title className="mb-2 mt-8 !text-2xl">Post-processing</Title>
 
-          <Card className="!mr-auto mt-8 !w-96">
+          <CardSection className="!mr-auto mt-8 !w-96">
             {searchSettings && (
               <>
                 <div className="px-1 w-full rounded-lg">
@@ -154,10 +157,12 @@ function Main() {
                 </div>
               </>
             )}
-          </Card>
+          </CardSection>
 
           <Link href="/admin/embeddings">
-            <Button className="mt-8">Update Search Settings</Button>
+            <Button variant="navigate" className="mt-8">
+              Update Search Settings
+            </Button>
           </Link>
         </>
       ) : (
diff --git a/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx b/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
index 7bd42947116..f03b74ba848 100644
--- a/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/ConfigDisplay.tsx
@@ -1,6 +1,7 @@
+import CardSection from "@/components/admin/CardSection";
 import { getNameFromPath } from "@/lib/fileUtils";
 import { ValidSources } from "@/lib/types";
-import { List, ListItem, Card, Title } from "@tremor/react";
+import Title from "@/components/ui/title";
 
 function convertObjectToString(obj: any): string | any {
   // Check if obj is an object and not an array or null
@@ -73,28 +74,37 @@ export function AdvancedConfigDisplay({
   return (
     <>
       <Title className="mt-8 mb-2">Advanced Configuration</Title>
-      <Card>
-        <List>
+      <CardSection>
+        <ul className="w-full text-sm divide-y divide-neutral-200 dark:divide-neutral-700">
           {pruneFreq && (
-            <ListItem key={0}>
+            <li
+              key={0}
+              className="w-full flex justify-between items-center py-2"
+            >
               <span>Pruning Frequency</span>
               <span>{formatPruneFrequency(pruneFreq)}</span>
-            </ListItem>
+            </li>
           )}
           {refreshFreq && (
-            <ListItem key={1}>
+            <li
+              key={1}
+              className="w-full flex justify-between items-center py-2"
+            >
               <span>Refresh Frequency</span>
               <span>{formatRefreshFrequency(refreshFreq)}</span>
-            </ListItem>
+            </li>
           )}
           {indexingStart && (
-            <ListItem key={2}>
+            <li
+              key={2}
+              className="w-full flex justify-between items-center py-2"
+            >
               <span>Indexing Start</span>
               <span>{formatDate(indexingStart)}</span>
-            </ListItem>
+            </li>
           )}
-        </List>
-      </Card>
+        </ul>
+      </CardSection>
     </>
   );
 }
@@ -116,16 +126,19 @@ export function ConfigDisplay({
   return (
     <>
       <Title className="mb-2">Configuration</Title>
-      <Card>
-        <List>
+      <CardSection>
+        <ul className="w-full text-sm divide-y divide-neutral-200 dark:divide-neutral-700">
           {configEntries.map(([key, value]) => (
-            <ListItem key={key}>
+            <li
+              key={key}
+              className="w-full flex justify-between items-center py-2"
+            >
               <span>{key}</span>
               <span>{convertObjectToString(value) || "-"}</span>
-            </ListItem>
+            </li>
           ))}
-        </List>
-      </Card>
+        </ul>
+      </CardSection>
     </>
   );
 }
diff --git a/web/src/app/admin/connector/[ccPairId]/DeletionButton.tsx b/web/src/app/admin/connector/[ccPairId]/DeletionButton.tsx
index 7ea03747bd3..ccef14b5a35 100644
--- a/web/src/app/admin/connector/[ccPairId]/DeletionButton.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/DeletionButton.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { FiTrash } from "react-icons/fi";
@@ -30,8 +30,7 @@ export function DeletionButton({ ccPair }: { ccPair: CCPairFullInfo }) {
     <div>
       {popup}
       <Button
-        size="xs"
-        color="red"
+        variant="destructive"
         onClick={() =>
           deleteCCPair(
             ccPair.connector.id,
diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index 51b7f8881ee..4494e782ded 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -5,12 +5,12 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Text,
-  Callout,
-} from "@tremor/react";
+  TableHeader,
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
+import { Callout } from "@/components/ui/callout";
 import { CCPairFullInfo, PaginatedIndexAttempts } from "./types";
 import { IndexAttemptStatus } from "@/components/Status";
 import { PageSelector } from "@/components/PageSelector";
@@ -206,8 +206,8 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
       <Callout
         className="mt-4"
         title="No indexing attempts scheduled yet"
-        icon={FiInfo}
-        color="blue"
+        icon={"info"}
+        type="notice"
       >
         Index attempts are scheduled in the background, and may take some time
         to appear. Try refreshing the page in ~30 seconds!
@@ -231,12 +231,12 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
         )}
 
       <Table>
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Time Started</TableHeaderCell>
-            <TableHeaderCell>Status</TableHeaderCell>
-            <TableHeaderCell>New Doc Cnt</TableHeaderCell>
-            <TableHeaderCell>
+            <TableHead>Time Started</TableHead>
+            <TableHead>Status</TableHead>
+            <TableHead>New Doc Cnt</TableHead>
+            <TableHead>
               <div className="w-fit">
                 <Tooltip
                   width="max-w-sm"
@@ -248,10 +248,10 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
                   </span>
                 </Tooltip>
               </div>
-            </TableHeaderCell>
-            <TableHeaderCell>Error Message</TableHeaderCell>
+            </TableHead>
+            <TableHead>Error Message</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {currentPageData.index_attempts.map((indexAttempt) => {
             const docsPerMinute =
@@ -266,7 +266,6 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
                 <TableCell>
                   <IndexAttemptStatus
                     status={indexAttempt.status || "not_started"}
-                    size="xs"
                   />
                   {docsPerMinute ? (
                     <div className="text-xs mt-1">
diff --git a/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx b/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx
index 10460459e32..71d26a8eb47 100644
--- a/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/ModifyStatusButtonCluster.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { CCPairFullInfo, ConnectorCredentialPairStatus } from "./types";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { mutate } from "swr";
@@ -19,8 +19,7 @@ export function ModifyStatusButtonCluster({
       {popup}
       {ccPair.status === ConnectorCredentialPairStatus.PAUSED ? (
         <Button
-          color="green"
-          size="xs"
+          variant="success-reverse"
           onClick={() =>
             setCCPairStatus(
               ccPair.id,
@@ -35,8 +34,7 @@ export function ModifyStatusButtonCluster({
         </Button>
       ) : (
         <Button
-          color="red"
-          size="xs"
+          variant="default"
           onClick={() =>
             setCCPairStatus(
               ccPair.id,
diff --git a/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx b/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx
index 75bb2eca95d..af0e2a8f4aa 100644
--- a/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/ReIndexButton.tsx
@@ -2,11 +2,13 @@
 
 import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { runConnector } from "@/lib/connector";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 import { mutate } from "swr";
 import { buildCCPairInfoUrl } from "./lib";
 import { useState } from "react";
 import { Modal } from "@/components/Modal";
+import { Separator } from "@/components/ui/separator";
 
 function ReIndexPopup({
   connectorId,
@@ -45,9 +47,8 @@ function ReIndexPopup({
     <Modal title="Run Indexing" onOutsideClick={hide}>
       <div>
         <Button
+          variant="submit"
           className="ml-auto"
-          color="green"
-          size="xs"
           onClick={() => {
             triggerIndexing(false);
             hide();
@@ -61,12 +62,11 @@ function ReIndexPopup({
           have been added since the last successful indexing run.
         </Text>
 
-        <Divider />
+        <Separator />
 
         <Button
+          variant="submit"
           className="ml-auto"
-          color="green"
-          size="xs"
           onClick={() => {
             triggerIndexing(true);
             hide();
@@ -120,9 +120,8 @@ export function ReIndexButton({
       )}
       {popup}
       <Button
+        variant="success-reverse"
         className="ml-auto"
-        color="green"
-        size="xs"
         onClick={() => {
           setReIndexPopupVisible(true);
         }}
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index e2576cc9b81..b703d75109d 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -12,7 +12,9 @@ import { updateConnectorCredentialPairName } from "@/lib/connector";
 import { credentialTemplates } from "@/lib/connectors/credentials";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { ValidSources } from "@/lib/types";
-import { Button, Divider, Title } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Title from "@/components/ui/title";
+import { Separator } from "@/components/ui/separator";
 import { useRouter } from "next/navigation";
 import { useCallback, useEffect, useRef, useState } from "react";
 import useSWR, { mutate } from "swr";
@@ -157,10 +159,10 @@ function Main({ ccPairId }: { ccPairId: number }) {
               onChange={handleNameChange}
               className="text-3xl w-full ring ring-1 ring-neutral-800 text-emphasis font-bold"
             />
-            <Button onClick={handleUpdateName} className="ml-2">
+            <Button onClick={handleUpdateName}>
               <CheckmarkIcon className="text-neutral-200" />
             </Button>
-            <Button onClick={() => resetEditing()} className="ml-2">
+            <Button onClick={() => resetEditing()}>
               <XIcon className="text-neutral-200" />
             </Button>
           </div>
@@ -169,7 +171,9 @@ function Main({ ccPairId }: { ccPairId: number }) {
             onClick={() =>
               ccPair.is_editable_for_current_user && startEditing()
             }
-            className={`group flex ${ccPair.is_editable_for_current_user ? "cursor-pointer" : ""} text-3xl text-emphasis gap-x-2 items-center font-bold`}
+            className={`group flex ${
+              ccPair.is_editable_for_current_user ? "cursor-pointer" : ""
+            } text-3xl text-emphasis gap-x-2 items-center font-bold`}
           >
             {ccPair.name}
             {ccPair.is_editable_for_current_user && (
@@ -229,7 +233,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
       {credentialTemplates[ccPair.connector.source] &&
         ccPair.is_editable_for_current_user && (
           <>
-            <Divider />
+            <Separator />
 
             <Title className="mb-2">Credentials</Title>
 
@@ -240,7 +244,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
             />
           </>
         )}
-      <Divider />
+      <Separator />
       <ConfigDisplay
         connectorSpecificConfig={ccPair.connector.connector_specific_config}
         sourceType={ccPair.connector.source}
@@ -262,7 +266,7 @@ function Main({ ccPairId }: { ccPairId: number }) {
         </div>
         <IndexingAttemptsTable ccPair={ccPair} />
       </div>
-      <Divider />
+      <Separator />
       <div className="flex mt-4">
         <div className="mx-auto">
           {ccPair.is_editable_for_current_user && (
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index 7f0e4c664f3..95474914678 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -4,7 +4,7 @@ import { errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR, { mutate } from "swr";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 
-import { Card, Title } from "@tremor/react";
+import Title from "@/components/ui/title";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { buildSimilarCredentialInfoURL } from "@/app/admin/connector/[ccPairId]/lib";
 import { usePopup } from "@/components/admin/connectors/Popup";
@@ -42,6 +42,7 @@ import {
 import { Formik } from "formik";
 import NavigationRow from "./NavigationRow";
 import { useRouter } from "next/navigation";
+import CardSection from "@/components/admin/CardSection";
 export interface AdvancedConfig {
   refreshFreq: number;
   pruneFreq: number;
@@ -363,7 +364,7 @@ export default function AddConnector({
             />
 
             {formStep == 0 && (
-              <Card>
+              <CardSection>
                 <Title className="mb-2 text-lg">Select a credential</Title>
 
                 {connector == "google_drive" ? (
@@ -420,11 +421,11 @@ export default function AddConnector({
                       )}
                   </>
                 )}
-              </Card>
+              </CardSection>
             )}
 
             {formStep == 1 && (
-              <Card className="w-full py-8 flex gap-y-6 flex-col max-w-3xl px-12 mx-auto">
+              <CardSection className="w-full py-8 flex gap-y-6 flex-col max-w-3xl px-12 mx-auto">
                 <DynamicConnectionForm
                   values={formikProps.values}
                   config={configuration}
@@ -438,13 +439,13 @@ export default function AddConnector({
                     null
                   }
                 />
-              </Card>
+              </CardSection>
             )}
 
             {formStep === 2 && (
-              <Card>
+              <CardSection>
                 <AdvancedFormPage />
-              </Card>
+              </CardSection>
             )}
 
             <NavigationRow
diff --git a/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx b/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx
index 282aafbfbb6..e48a5875261 100644
--- a/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx
+++ b/web/src/app/admin/connectors/[connector]/ConnectorWrapper.tsx
@@ -5,7 +5,7 @@ import AddConnector from "./AddConnectorPage";
 import { FormProvider } from "@/components/context/FormContext";
 import Sidebar from "./Sidebar";
 import { HeaderTitle } from "@/components/header/HeaderTitle";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { isValidSource } from "@/lib/sources";
 
 export default function ConnectorWrapper({
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
index 03a73fe23e6..8cf94d1b656 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
@@ -11,7 +11,7 @@ import Cookies from "js-cookie";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
 import { User } from "@/lib/types";
-import { Button as TremorButton } from "@tremor/react";
+import { Button as TremorButton } from "@/components/ui/button";
 import {
   Credential,
   GoogleDriveCredentialJson,
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
index 518f9709d94..a4efe283683 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/GoogleDrivePage.tsx
@@ -11,7 +11,7 @@ import {
   usePublicCredentials,
   useConnectorCredentialIndexingStatus,
 } from "@/lib/hooks";
-import { Title } from "@tremor/react";
+import Title from "@/components/ui/title";
 import { DriveJsonUploadSection, DriveAuthSection } from "./Credential";
 import {
   Credential,
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
index 51502bd443f..061ddddcde4 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
@@ -10,7 +10,7 @@ import { GMAIL_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
 import Cookies from "js-cookie";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
-import { Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
 import {
   Credential,
   GmailCredentialJson,
@@ -353,7 +353,7 @@ export const GmailOAuthSection = ({
           the documents you want to index with the service account.
         </p>
 
-        <Card>
+        <CardSection>
           <Formik
             initialValues={{
               gmail_delegated_user: "",
@@ -415,7 +415,7 @@ export const GmailOAuthSection = ({
               </Form>
             )}
           </Formik>
-        </Card>
+        </CardSection>
       </div>
     );
   }
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
index 1a7fbe5de69..cf646916099 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
@@ -15,7 +15,7 @@ import {
   usePublicCredentials,
   useConnectorCredentialIndexingStatus,
 } from "@/lib/hooks";
-import { Title } from "@tremor/react";
+import Title from "@/components/ui/title";
 import { GmailConfig } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
 
diff --git a/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx b/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx
index 09713b104ff..7e02beef689 100644
--- a/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx
+++ b/web/src/app/admin/documents/feedback/DocumentFeedbackTable.tsx
@@ -4,10 +4,10 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
+  TableHeader,
   TableBody,
   TableCell,
-} from "@tremor/react";
+} from "@/components/ui/table";
 import { PageSelector } from "@/components/PageSelector";
 import { DocumentBoostStatus } from "@/lib/types";
 import { updateHiddenStatus } from "../lib";
@@ -94,13 +94,13 @@ export const DocumentFeedbackTable = ({
   return (
     <div>
       <Table className="overflow-visible">
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Document Name</TableHeaderCell>
-            <TableHeaderCell>Is Searchable?</TableHeaderCell>
-            <TableHeaderCell>Score</TableHeaderCell>
+            <TableHead>Document Name</TableHead>
+            <TableHead>Is Searchable?</TableHead>
+            <TableHead>Score</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {documents
             .slice((page - 1) * numToDisplay, page * numToDisplay)
diff --git a/web/src/app/admin/documents/feedback/page.tsx b/web/src/app/admin/documents/feedback/page.tsx
index 30f425bc725..d66d3f60a99 100644
--- a/web/src/app/admin/documents/feedback/page.tsx
+++ b/web/src/app/admin/documents/feedback/page.tsx
@@ -6,7 +6,7 @@ import { useMostReactedToDocuments } from "@/lib/hooks";
 import { DocumentFeedbackTable } from "./DocumentFeedbackTable";
 import { numPages, numToDisplay } from "./constants";
 import { AdminPageTitle } from "@/components/admin/Title";
-import { Title } from "@tremor/react";
+import Title from "@/components/ui/title";
 
 const Main = () => {
   const {
diff --git a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
index 6c61953fb94..1691b9c46d8 100644
--- a/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
+++ b/web/src/app/admin/documents/sets/DocumentSetCreationForm.tsx
@@ -16,7 +16,8 @@ import {
 } from "@/lib/types";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { ConnectorTitle } from "@/components/admin/connectors/ConnectorTitle";
-import { Button, Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
 import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import { IsPublicGroupSelector } from "@/components/IsPublicGroupSelector";
 import React, { useEffect, useState } from "react";
@@ -133,7 +134,7 @@ export const DocumentSetCreationForm = ({
                 />
               )}
 
-              <Divider />
+              <Separator />
 
               {user?.role === UserRole.CURATOR ? (
                 <>
@@ -238,11 +239,13 @@ export const DocumentSetCreationForm = ({
 
                         return nonVisibleCcPairs.length > 0 ? (
                           <>
-                            <Divider />
+                            <Separator />
                             <h2 className="mb-1 font-medium text-base">
                               These connectors are not available to the{" "}
                               {userGroups && userGroups.length > 1
-                                ? `group${props.values.groups.length > 1 ? "s" : ""} you have selected`
+                                ? `group${
+                                    props.values.groups.length > 1 ? "s" : ""
+                                  } you have selected`
                                 : "group you curate"}
                               :
                             </h2>
@@ -339,6 +342,7 @@ export const DocumentSetCreationForm = ({
               <div className="flex mt-6">
                 <Button
                   type="submit"
+                  variant="submit"
                   disabled={props.isSubmitting}
                   className="w-64 mx-auto"
                 >
diff --git a/web/src/app/admin/documents/sets/[documentSetId]/page.tsx b/web/src/app/admin/documents/sets/[documentSetId]/page.tsx
index 960e11cc6c1..ebce7d2027a 100644
--- a/web/src/app/admin/documents/sets/[documentSetId]/page.tsx
+++ b/web/src/app/admin/documents/sets/[documentSetId]/page.tsx
@@ -10,7 +10,7 @@ import { ThreeDotsLoader } from "@/components/Loading";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { BookmarkIcon } from "@/components/icons/icons";
 import { BackButton } from "@/components/BackButton";
-import { Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
 import { DocumentSetCreationForm } from "../DocumentSetCreationForm";
 import { useRouter } from "next/navigation";
 import { usePopup } from "@/components/admin/connectors/Popup";
@@ -77,7 +77,7 @@ function Main({ documentSetId }: { documentSetId: number }) {
         title={documentSet.name}
       />
 
-      <Card>
+      <CardSection>
         <DocumentSetCreationForm
           ccPairs={ccPairs}
           userGroups={userGroups}
@@ -88,7 +88,7 @@ function Main({ documentSetId }: { documentSetId: number }) {
           setPopup={setPopup}
           existingDocumentSet={documentSet}
         />
-      </Card>
+      </CardSection>
     </div>
   );
 }
diff --git a/web/src/app/admin/documents/sets/new/page.tsx b/web/src/app/admin/documents/sets/new/page.tsx
index c44f4b9502f..828f5128c4b 100644
--- a/web/src/app/admin/documents/sets/new/page.tsx
+++ b/web/src/app/admin/documents/sets/new/page.tsx
@@ -9,11 +9,11 @@ import {
 } from "@/lib/hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { Card } from "@tremor/react";
 import { BackButton } from "@/components/BackButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { useRouter } from "next/navigation";
 import { refreshDocumentSets } from "../hooks";
+import CardSection from "@/components/admin/CardSection";
 
 function Main() {
   const { popup, setPopup } = usePopup();
@@ -45,7 +45,7 @@ function Main() {
     <>
       {popup}
 
-      <Card>
+      <CardSection>
         <DocumentSetCreationForm
           ccPairs={ccPairs}
           userGroups={userGroups}
@@ -55,7 +55,7 @@ function Main() {
           }}
           setPopup={setPopup}
         />
-      </Card>
+      </CardSection>
     </>
   );
 }
diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx
index a4a3a227048..9db9c15ee7b 100644
--- a/web/src/app/admin/documents/sets/page.tsx
+++ b/web/src/app/admin/documents/sets/page.tsx
@@ -7,15 +7,13 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Title,
-  Divider,
-  Badge,
-  Button,
-  Text,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
 import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
 import { ConnectorIndexingStatus, DocumentSet } from "@/lib/types";
 import { useState } from "react";
@@ -35,6 +33,8 @@ import {
 import { DeleteButton } from "@/components/DeleteButton";
 import Link from "next/link";
 import { useRouter } from "next/navigation";
+import { TableHeader } from "@/components/ui/table";
+import { Badge } from "@/components/ui/badge";
 
 const numToDisplay = 50;
 
@@ -134,15 +134,15 @@ const DocumentSetTable = ({
     <div>
       <Title>Existing Document Sets</Title>
       <Table className="overflow-visible mt-2">
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Name</TableHeaderCell>
-            <TableHeaderCell>Connectors</TableHeaderCell>
-            <TableHeaderCell>Status</TableHeaderCell>
-            <TableHeaderCell>Public</TableHeaderCell>
-            <TableHeaderCell>Delete</TableHeaderCell>
+            <TableHead>Name</TableHead>
+            <TableHead>Connectors</TableHead>
+            <TableHead>Status</TableHead>
+            <TableHead>Public</TableHead>
+            <TableHead>Delete</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {sortedDocumentSets
             .slice((page - 1) * numToDisplay, page * numToDisplay)
@@ -188,32 +188,32 @@ const DocumentSetTable = ({
                   </TableCell>
                   <TableCell>
                     {documentSet.is_up_to_date ? (
-                      <Badge size="md" color="green" icon={FiCheckCircle}>
+                      <Badge size="md" variant="success" icon={FiCheckCircle}>
                         Up to Date
                       </Badge>
                     ) : documentSet.cc_pair_descriptors.length > 0 ? (
-                      <Badge size="md" color="amber" icon={FiClock}>
+                      <Badge size="md" variant="in_progress" icon={FiClock}>
                         Syncing
                       </Badge>
                     ) : (
-                      <Badge size="md" color="red" icon={FiAlertTriangle}>
+                      <Badge
+                        size="md"
+                        variant="destructive"
+                        icon={FiAlertTriangle}
+                      >
                         Deleting
                       </Badge>
                     )}
                   </TableCell>
                   <TableCell>
                     {documentSet.is_public ? (
-                      <Badge
-                        size="md"
-                        color={isEditable ? "green" : "gray"}
-                        icon={FiUnlock}
-                      >
+                      <Badge size="md" variant="success" icon={FiUnlock}>
                         Public
                       </Badge>
                     ) : (
                       <Badge
                         size="md"
-                        color={isEditable ? "blue" : "gray"}
+                        variant={isEditable ? "purple" : "outline"}
                         icon={FiLock}
                       >
                         Private
@@ -322,15 +322,13 @@ const Main = () => {
 
       <div className="flex mb-6">
         <Link href="/admin/documents/sets/new">
-          <Button size="xs" color="green" className="ml-2 my-auto">
-            New Document Set
-          </Button>
+          <Button variant="navigate">New Document Set</Button>
         </Link>
       </div>
 
       {documentSets.length > 0 && (
         <>
-          <Divider />
+          <Separator />
           <DocumentSetTable
             documentSets={documentSets}
             editableDocumentSets={editableDocumentSets}
diff --git a/web/src/app/admin/embeddings/RerankingFormPage.tsx b/web/src/app/admin/embeddings/RerankingFormPage.tsx
index 67f5fc1614b..6bf887bc230 100644
--- a/web/src/app/admin/embeddings/RerankingFormPage.tsx
+++ b/web/src/app/admin/embeddings/RerankingFormPage.tsx
@@ -20,7 +20,7 @@ import {
   MixedBreadIcon,
 } from "@/components/icons/icons";
 import { Modal } from "@/components/Modal";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 
@@ -108,7 +108,9 @@ const RerankingDetailsForm = forwardRef<
                   </button>
                 )}
                 <div
-                  className={`${originalRerankingDetails.rerank_model_name && "px-2 ml-2"}`}
+                  className={`${
+                    originalRerankingDetails.rerank_model_name && "px-2 ml-2"
+                  }`}
                 >
                   <button
                     onClick={() => setModelTab("cloud")}
@@ -267,8 +269,7 @@ const RerankingDetailsForm = forwardRef<
                       <div className="flex justify-end">
                         <Button
                           onClick={() => setShowGpuWarningModalModel(null)}
-                          color="blue"
-                          size="xs"
+                          variant="submit"
                         >
                           Understood
                         </Button>
@@ -348,8 +349,7 @@ const RerankingDetailsForm = forwardRef<
                           onClick={() => {
                             setShowLiteLLMConfigurationModal(false);
                           }}
-                          color="blue"
-                          size="xs"
+                          variant="submit"
                         >
                           Update
                         </Button>
@@ -397,8 +397,7 @@ const RerankingDetailsForm = forwardRef<
                       <div className="flex w-full justify-end mt-4">
                         <Button
                           onClick={() => setIsApiKeyModalOpen(false)}
-                          color="blue"
-                          size="xs"
+                          variant="submit"
                         >
                           Update
                         </Button>
diff --git a/web/src/app/admin/embeddings/modals/AlreadyPickedModal.tsx b/web/src/app/admin/embeddings/modals/AlreadyPickedModal.tsx
index d6e29424114..8aebcdce0d8 100644
--- a/web/src/app/admin/embeddings/modals/AlreadyPickedModal.tsx
+++ b/web/src/app/admin/embeddings/modals/AlreadyPickedModal.tsx
@@ -1,6 +1,7 @@
 import React from "react";
 import { Modal } from "@/components/Modal";
-import { Button, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 
 import { CloudEmbeddingModel } from "../../../../components/embedding/interfaces";
 
@@ -22,7 +23,7 @@ export function AlreadyPickedModal({
           You can select a different one if you want!
         </Text>
         <div className="flex mt-8 justify-between">
-          <Button color="blue" onClick={onClose}>
+          <Button variant="submit" onClick={onClose}>
             Close
           </Button>
         </div>
diff --git a/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx b/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
index f769099e77d..044155fce3a 100644
--- a/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ChangeCredentialsModal.tsx
@@ -1,6 +1,9 @@
 import React, { useRef, useState } from "react";
 import { Modal } from "@/components/Modal";
-import { Button, Text, Callout, Subtitle, Divider } from "@tremor/react";
+import { Callout } from "@/components/ui/callout";
+import Text from "@/components/ui/text";
+import { Separator } from "@/components/ui/separator";
+import { Button } from "@/components/ui/button";
 import { Label } from "@/components/admin/connectors/Field";
 import { CloudEmbeddingProvider } from "../../../../components/embedding/interfaces";
 import {
@@ -142,7 +145,9 @@ export function ChangeCredentialsModal({
         const errorData = await updateResponse.json();
         throw new Error(
           errorData.detail ||
-            `Failed to update provider- check your ${isProxy ? "API URL" : "API key"}`
+            `Failed to update provider- check your ${
+              isProxy ? "API URL" : "API key"
+            }`
         );
       }
 
@@ -157,7 +162,9 @@ export function ChangeCredentialsModal({
     <Modal
       width="max-w-3xl"
       icon={provider.icon}
-      title={`Modify your ${provider.provider_type} ${isProxy ? "Configuration" : "key"}`}
+      title={`Modify your ${provider.provider_type} ${
+        isProxy ? "Configuration" : "key"
+      }`}
       onOutsideClick={onCancel}
     >
       <>
@@ -221,7 +228,7 @@ export function ChangeCredentialsModal({
                   />
 
                   {deletionError && (
-                    <Callout title="Error" color="red" className="mt-4">
+                    <Callout type="danger" title="Error" className="mt-4">
                       {deletionError}
                     </Callout>
                   )}
@@ -251,38 +258,42 @@ export function ChangeCredentialsModal({
               )}
 
               {testError && (
-                <Callout title="Error" color="red" className="my-4">
+                <Callout type="danger" title="Error" className="my-4">
                   {testError}
                 </Callout>
               )}
 
               <Button
                 className="mr-auto mt-4"
-                color="blue"
+                variant="submit"
                 onClick={() => handleSubmit()}
                 disabled={!apiKey}
               >
                 Update Configuration
               </Button>
 
-              <Divider />
+              <Separator />
             </div>
           </>
         )}
 
-        <Subtitle className="mt-4 font-bold text-lg mb-2">
+        <Text className="mt-4 font-bold text-lg mb-2">
           You can delete your configuration.
-        </Subtitle>
+        </Text>
         <Text className="mb-2">
           This is only possible if you have already switched to a different
           embedding type!
         </Text>
 
-        <Button className="mr-auto" onClick={handleDelete} color="red">
+        <Button
+          className="mr-auto"
+          onClick={handleDelete}
+          variant="destructive"
+        >
           Delete Configuration
         </Button>
         {deletionError && (
-          <Callout title="Error" color="red" className="mt-4">
+          <Callout type="danger" title="Error" className="mt-4">
             {deletionError}
           </Callout>
         )}
diff --git a/web/src/app/admin/embeddings/modals/DeleteCredentialsModal.tsx b/web/src/app/admin/embeddings/modals/DeleteCredentialsModal.tsx
index 2d77d0febd3..608ce8239f6 100644
--- a/web/src/app/admin/embeddings/modals/DeleteCredentialsModal.tsx
+++ b/web/src/app/admin/embeddings/modals/DeleteCredentialsModal.tsx
@@ -1,6 +1,8 @@
 import React from "react";
 import { Modal } from "@/components/Modal";
-import { Button, Text, Callout } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Button } from "@/components/ui/button";
+import { Callout } from "@/components/ui/callout";
 import { CloudEmbeddingProvider } from "../../../../components/embedding/interfaces";
 
 export function DeleteCredentialsModal({
@@ -23,16 +25,12 @@ export function DeleteCredentialsModal({
           You&apos;re about to delete your {modelProvider.provider_type}{" "}
           credentials. Are you sure?
         </Text>
-        <Callout
-          title="Point of No Return"
-          color="red"
-          className="mt-4"
-        ></Callout>
+        <Callout type="danger" title="Point of No Return" className="mt-4" />
         <div className="flex mt-8 justify-between">
-          <Button color="gray" onClick={onCancel}>
-            Keep Credentaisl
+          <Button variant="secondary" onClick={onCancel}>
+            Keep Credentials
           </Button>
-          <Button color="red" onClick={onConfirm}>
+          <Button variant="destructive" onClick={onConfirm}>
             Delete Credentials
           </Button>
         </div>
diff --git a/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx b/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
index 3e4c4936832..ca341a9b308 100644
--- a/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ModelSelectionModal.tsx
@@ -1,5 +1,7 @@
 import { Modal } from "@/components/Modal";
-import { Button, Text, Callout } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Callout } from "@/components/ui/callout";
+import { Button } from "@/components/ui/button";
 import { HostedEmbeddingModel } from "../../../../components/embedding/interfaces";
 
 export function ModelSelectionConfirmationModal({
@@ -38,7 +40,7 @@ export function ModelSelectionConfirmationModal({
           </Text>
 
           {isCustom && (
-            <Callout title="IMPORTANT" color="yellow" className="mt-4">
+            <Callout type="warning" title="IMPORTANT" className="mt-4">
               We&apos;ve detected that this is a custom-specified embedding
               model. Since we have to download the model files before verifying
               the configuration&apos;s correctness, we won&apos;t be able to let
@@ -50,7 +52,7 @@ export function ModelSelectionConfirmationModal({
           )}
 
           <div className="flex mt-8">
-            <Button className="mx-auto" color="green" onClick={onConfirm}>
+            <Button className="mx-auto" variant="submit" onClick={onConfirm}>
               Yes
             </Button>
           </div>
diff --git a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
index 8229d199675..4ca22e2501e 100644
--- a/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
+++ b/web/src/app/admin/embeddings/modals/ProviderCreationModal.tsx
@@ -1,5 +1,7 @@
 import React, { useRef, useState } from "react";
-import { Text, Button, Callout } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Callout } from "@/components/ui/callout";
+import { Button } from "@/components/ui/button";
 import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { Label, TextFormField } from "@/components/admin/connectors/Field";
@@ -259,7 +261,9 @@ export function ProviderCreationModal({
                 ) : (
                   <TextFormField
                     name="api_key"
-                    label={`API Key ${isProxy ? "(for non-local deployments)" : ""}`}
+                    label={`API Key ${
+                      isProxy ? "(for non-local deployments)" : ""
+                    }`}
                     placeholder="API Key"
                     type="password"
                   />
@@ -276,14 +280,14 @@ export function ProviderCreationModal({
               </div>
 
               {errorMsg && (
-                <Callout title="Error" color="red">
+                <Callout title="Error" type="danger">
                   {errorMsg}
                 </Callout>
               )}
 
               <Button
                 type="submit"
-                color="blue"
+                variant="submit"
                 className="w-full"
                 disabled={isSubmitting}
               >
diff --git a/web/src/app/admin/embeddings/modals/SelectModelModal.tsx b/web/src/app/admin/embeddings/modals/SelectModelModal.tsx
index ed84fdc93cb..7b9347a3771 100644
--- a/web/src/app/admin/embeddings/modals/SelectModelModal.tsx
+++ b/web/src/app/admin/embeddings/modals/SelectModelModal.tsx
@@ -1,6 +1,7 @@
 import React from "react";
 import { Modal } from "@/components/Modal";
-import { Button, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 import { CloudEmbeddingModel } from "../../../../components/embedding/interfaces";
 
 export function SelectModelModal({
@@ -27,7 +28,7 @@ export function SelectModelModal({
           Are you sure?
         </Text>
         <div className="flex mt-8 justify-end">
-          <Button color="green" onClick={onConfirm}>
+          <Button variant="submit" onClick={onConfirm}>
             Yes
           </Button>
         </div>
diff --git a/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx b/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
index 6818101de43..309c10a2a9e 100644
--- a/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
+++ b/web/src/app/admin/embeddings/pages/CloudEmbeddingPage.tsx
@@ -1,7 +1,7 @@
 "use client";
 
-import { Card, Text, Title } from "@tremor/react";
-
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
 import {
   CloudEmbeddingProvider,
   CloudEmbeddingModel,
@@ -21,6 +21,7 @@ import { deleteSearchSettings } from "./utils";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
 import { AdvancedSearchConfiguration } from "../interfaces";
+import CardSection from "@/components/admin/CardSection";
 
 export default function CloudEmbeddingPage({
   currentModel,
@@ -209,7 +210,7 @@ export default function CloudEmbeddingPage({
             )}
 
             {!liteLLMProvider && (
-              <Card className="mt-2 w-full max-w-4xl bg-gray-50 border border-gray-200">
+              <CardSection className="mt-2 w-full max-w-4xl bg-gray-50 border border-gray-200">
                 <div className="p-4">
                   <Text className="text-lg font-semibold mb-2">
                     API URL Required
@@ -227,7 +228,7 @@ export default function CloudEmbeddingPage({
                     </Text>
                   </div>
                 </div>
-              </Card>
+              </CardSection>
             )}
             {liteLLMProvider && (
               <>
@@ -252,7 +253,7 @@ export default function CloudEmbeddingPage({
                     ))}
                 </div>
 
-                <Card
+                <CardSection
                   className={`mt-2 w-full max-w-4xl ${
                     currentModel.provider_type === EmbeddingProvider.LITELLM
                       ? "border-2 border-blue-500"
@@ -269,7 +270,7 @@ export default function CloudEmbeddingPage({
                     }
                     setShowTentativeModel={setShowTentativeModel}
                   />
-                </Card>
+                </CardSection>
               </>
             )}
           </div>
@@ -312,7 +313,7 @@ export default function CloudEmbeddingPage({
                 Configure Azure OpenAI
               </button>
               <div className="mt-2 w-full max-w-4xl">
-                <Card className="p-4 border border-gray-200 rounded-lg shadow-sm">
+                <CardSection className="p-4 border border-gray-200 rounded-lg shadow-sm">
                   <Text className="text-base font-medium mb-2">
                     Configure Azure OpenAI for Embeddings
                   </Text>
@@ -327,7 +328,7 @@ export default function CloudEmbeddingPage({
                       name, and deployment name.
                     </Text>
                   </div>
-                </Card>
+                </CardSection>
               </div>
             </>
           ) : (
@@ -338,7 +339,7 @@ export default function CloudEmbeddingPage({
                 </Text>
 
                 {azureProviderDetails ? (
-                  <Card className="bg-white shadow-sm border border-gray-200 rounded-lg">
+                  <CardSection className="bg-white shadow-sm border border-gray-200 rounded-lg">
                     <div className="p-4 space-y-3">
                       <div className="flex justify-between">
                         <span className="font-medium">API Version:</span>
@@ -361,17 +362,17 @@ export default function CloudEmbeddingPage({
                     >
                       Delete Current Azure Provider
                     </button>
-                  </Card>
+                  </CardSection>
                 ) : (
-                  <Card className="bg-gray-50 border border-gray-200 rounded-lg">
+                  <CardSection className="bg-gray-50 border border-gray-200 rounded-lg">
                     <div className="p-4 text-gray-500 text-center">
                       No Azure provider has been configured yet.
                     </div>
-                  </Card>
+                  </CardSection>
                 )}
               </div>
 
-              <Card
+              <CardSection
                 className={`mt-2 w-full max-w-4xl ${
                   currentModel.provider_type === EmbeddingProvider.AZURE
                     ? "border-2 border-blue-500"
@@ -390,7 +391,7 @@ export default function CloudEmbeddingPage({
                     setShowTentativeModel={setShowTentativeModel}
                   />
                 )}
-              </Card>
+              </CardSection>
             </>
           )}
         </div>
diff --git a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
index 4947e8759d5..8b4f2955789 100644
--- a/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
+++ b/web/src/app/admin/embeddings/pages/EmbeddingFormPage.tsx
@@ -4,7 +4,8 @@ import { HealthCheckBanner } from "@/components/health/healthcheck";
 
 import { EmbeddingModelSelection } from "../EmbeddingModelSelectionForm";
 import { useEffect, useMemo, useState } from "react";
-import { Button, Card, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Button } from "@/components/ui/button";
 import { ArrowLeft, ArrowRight, WarningCircle } from "@phosphor-icons/react";
 import {
   CloudEmbeddingModel,
@@ -26,6 +27,8 @@ import { useEmbeddingFormContext } from "@/components/context/EmbeddingContext";
 import { Modal } from "@/components/Modal";
 
 import { useRouter } from "next/navigation";
+import CardSection from "@/components/admin/CardSection";
+import { CardDescription } from "@/components/ui/card";
 export default function EmbeddingForm() {
   const { formStep, nextFormStep, prevFormStep } = useEmbeddingFormContext();
   const { popup, setPopup } = usePopup();
@@ -313,7 +316,7 @@ export default function EmbeddingForm() {
               take hours or days. You can monitor the progress of the
               re-indexing on this page while the models are being switched.
             </Text>
-            <Card>
+            <CardSection>
               <EmbeddingModelSelection
                 updateCurrentModel={updateCurrentModel}
                 setModelTab={setModelTab}
@@ -323,7 +326,7 @@ export default function EmbeddingForm() {
                 updateSelectedProvider={updateSelectedProvider}
                 advancedEmbeddingDetails={advancedEmbeddingDetails}
               />
-            </Card>
+            </CardSection>
             <div className="mt-4 flex w-full justify-end">
               <button
                 className="enabled:cursor-pointer disabled:cursor-not-allowed disabled:bg-blue-200 bg-blue-400 flex gap-x-1 items-center text-white py-2.5 px-3.5 text-sm font-regular rounded-sm"
@@ -360,7 +363,10 @@ export default function EmbeddingForm() {
                 <li>Nomic nomic-embed-text-v1 for self-hosted</li>
               </div>
               <div className="flex mt-4 justify-between">
-                <Button color="green" onClick={() => setShowPoorModel(false)}>
+                <Button
+                  variant="secondary"
+                  onClick={() => setShowPoorModel(false)}
+                >
                   Cancel update
                 </Button>
                 <Button
@@ -388,7 +394,7 @@ export default function EmbeddingForm() {
               effect immediately for all new searches.
             </Text>
 
-            <Card>
+            <CardSection>
               <RerankingDetailsForm
                 setModelTab={setModelTab}
                 modelTab={
@@ -400,7 +406,7 @@ export default function EmbeddingForm() {
                 originalRerankingDetails={originalRerankingDetails}
                 setRerankingDetails={setRerankingDetails}
               />
-            </Card>
+            </CardSection>
 
             <div className={`mt-4 w-full grid grid-cols-3`}>
               <button
@@ -437,12 +443,12 @@ export default function EmbeddingForm() {
               require re-indexing documents.
             </Text>
 
-            <Card>
+            <CardSection>
               <AdvancedEmbeddingFormPage
                 advancedEmbeddingDetails={advancedEmbeddingDetails}
                 updateAdvancedEmbeddingDetails={updateAdvancedEmbeddingDetails}
               />
-            </Card>
+            </CardSection>
 
             <div className={`mt-4 grid  grid-cols-3 w-full `}>
               <button
diff --git a/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx b/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
index 1b573cdd3df..54317605541 100644
--- a/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
+++ b/web/src/app/admin/embeddings/pages/OpenEmbeddingPage.tsx
@@ -1,5 +1,8 @@
 "use client";
-import { Button, Card, Text, Title } from "@tremor/react";
+
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
 import { ModelSelector } from "../../../../components/embedding/ModelSelector";
 import {
   AVAILABLE_MODELS,
@@ -8,6 +11,7 @@ import {
 } from "../../../../components/embedding/interfaces";
 import { CustomModelForm } from "../../../../components/embedding/CustomModelForm";
 import { useState } from "react";
+import CardSection from "@/components/admin/CardSection";
 export default function OpenEmbeddingPage({
   onSelectOpenSource,
   selectedProvider,
@@ -58,15 +62,19 @@ export default function OpenEmbeddingPage({
         to the Danswer team.
       </Text>
       {!configureModel && (
-        <Button onClick={() => setConfigureModel(true)} className="mt-4">
+        <Button
+          onClick={() => setConfigureModel(true)}
+          className="mt-4"
+          variant="secondary"
+        >
           Configure custom model
         </Button>
       )}
       {configureModel && (
         <div className="w-full flex">
-          <Card className="mt-4 2xl:w-4/6 mx-auto">
+          <CardSection className="mt-4 2xl:w-4/6 mx-auto">
             <CustomModelForm onSubmit={onSelectOpenSource} />
-          </Card>
+          </CardSection>
         </div>
       )}
     </div>
diff --git a/web/src/app/admin/indexing/[id]/IndexAttemptErrorsTable.tsx b/web/src/app/admin/indexing/[id]/IndexAttemptErrorsTable.tsx
index 6ee8efef511..de68a46069b 100644
--- a/web/src/app/admin/indexing/[id]/IndexAttemptErrorsTable.tsx
+++ b/web/src/app/admin/indexing/[id]/IndexAttemptErrorsTable.tsx
@@ -9,12 +9,12 @@ import {
   TableBody,
   TableCell,
   TableHead,
-  TableHeaderCell,
   TableRow,
-  Text,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
 import { useState } from "react";
 import { IndexAttemptError } from "./types";
+import { TableHeader } from "@/components/ui/table";
 
 const NUM_IN_PAGE = 8;
 
@@ -100,14 +100,14 @@ export function IndexAttemptErrorsTable({
       )}
 
       <Table>
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Timestamp</TableHeaderCell>
-            <TableHeaderCell>Batch Number</TableHeaderCell>
-            <TableHeaderCell>Document Summaries</TableHeaderCell>
-            <TableHeaderCell>Error Message</TableHeaderCell>
+            <TableHead>Timestamp</TableHead>
+            <TableHead>Batch Number</TableHead>
+            <TableHead>Document Summaries</TableHead>
+            <TableHead>Error Message</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {indexAttemptErrors
             .slice(NUM_IN_PAGE * (page - 1), NUM_IN_PAGE * page)
diff --git a/web/src/app/admin/indexing/[id]/page.tsx b/web/src/app/admin/indexing/[id]/page.tsx
index 041f7bda9ba..a5fa79abe42 100644
--- a/web/src/app/admin/indexing/[id]/page.tsx
+++ b/web/src/app/admin/indexing/[id]/page.tsx
@@ -4,7 +4,7 @@ import { BackButton } from "@/components/BackButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { errorHandlingFetcher } from "@/lib/fetcher";
-import { Title } from "@tremor/react";
+import Title from "@/components/ui/title";
 import useSWR from "swr";
 import { IndexAttemptErrorsTable } from "./IndexAttemptErrorsTable";
 import { buildIndexingErrorsUrl } from "./lib";
diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
index 7d60d786986..118b9157860 100644
--- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
+++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
@@ -2,12 +2,12 @@ import React, { useState, useMemo, useEffect, useRef } from "react";
 import {
   Table,
   TableRow,
-  TableHeaderCell,
+  TableHead,
   TableBody,
   TableCell,
-  Badge,
-  Button,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import { Badge } from "@/components/ui/badge";
+import { Button } from "@/components/ui/button";
 import { IndexAttemptStatus } from "@/components/Status";
 import { timeAgo } from "@/lib/time";
 import {
@@ -24,6 +24,7 @@ import {
   FiLock,
   FiUnlock,
   FiRefreshCw,
+  FiPauseCircle,
 } from "react-icons/fi";
 import { Tooltip } from "@/components/tooltip/Tooltip";
 import { SourceIcon } from "@/components/SourceIcon";
@@ -144,30 +145,14 @@ function ConnectorRow({
       ccPairsIndexingStatus.cc_pair_status ===
       ConnectorCredentialPairStatus.DELETING
     ) {
-      return (
-        <Badge
-          color="red"
-          className="w-fit px-2 py-1 rounded-full border border-red-500"
-        >
-          <div className="flex text-xs items-center gap-x-1">
-            <div className="w-3 h-3 rounded-full bg-red-500"></div>
-            Deleting
-          </div>
-        </Badge>
-      );
+      return <Badge variant="destructive">Deleting</Badge>;
     } else if (
       ccPairsIndexingStatus.cc_pair_status ===
       ConnectorCredentialPairStatus.PAUSED
     ) {
       return (
-        <Badge
-          color="yellow"
-          className="w-fit px-2 py-1 rounded-full border border-yellow-500"
-        >
-          <div className="flex text-xs items-center gap-x-1">
-            <div className="w-3 h-3 rounded-full bg-yellow-500"></div>
-            Paused
-          </div>
+        <Badge icon={FiPauseCircle} variant="paused">
+          Paused
         </Badge>
       );
     }
@@ -176,38 +161,20 @@ function ConnectorRow({
     switch (ccPairsIndexingStatus.last_status) {
       case "in_progress":
         return (
-          <Badge
-            color="green"
-            className="w-fit px-2 py-1 rounded-full border border-green-500"
-          >
-            <div className="flex text-xs items-center gap-x-1">
-              <div className="w-3 h-3 rounded-full bg-green-500"></div>
-              Indexing
-            </div>
+          <Badge circle variant="success">
+            Indexing
           </Badge>
         );
       case "not_started":
         return (
-          <Badge
-            color="purple"
-            className="w-fit px-2 py-1 rounded-full border border-purple-500"
-          >
-            <div className="flex text-xs items-center gap-x-1">
-              <div className="w-3 h-3 rounded-full bg-purple-500"></div>
-              Scheduled
-            </div>
+          <Badge circle variant="secondary">
+            Scheduled
           </Badge>
         );
       default:
         return (
-          <Badge
-            color="green"
-            className="w-fit px-2 py-1 rounded-full border border-green-500"
-          >
-            <div className="flex text-xs items-center gap-x-1">
-              <div className="w-3 h-3 rounded-full bg-green-500"></div>
-              Active
-            </div>
+          <Badge circle variant="success">
+            Active
           </Badge>
         );
     }
@@ -215,9 +182,9 @@ function ConnectorRow({
 
   return (
     <TableRow
-      className={`hover:bg-hover-light ${
-        invisible ? "invisible h-0 !-mb-10" : "border border-border !border-b"
-      }  w-full cursor-pointer relative`}
+      className={`border hover:bg-hover-light ${
+        invisible ? "invisible !h-0 !-mb-10" : "!border !border-border"
+      }  w-full cursor-pointer relative `}
       onClick={() => {
         router.push(`/admin/connector/${ccPairsIndexingStatus.cc_pair_id}`);
       }}
@@ -234,23 +201,23 @@ function ConnectorRow({
       {isPaidEnterpriseFeaturesEnabled && (
         <TableCell>
           {ccPairsIndexingStatus.access_type === "public" ? (
-            <Badge
-              size="md"
-              color={isEditable ? "green" : "gray"}
-              icon={FiUnlock}
-            >
+            <Badge variant="success" icon={FiUnlock}>
               Public
             </Badge>
           ) : ccPairsIndexingStatus.access_type === "sync" ? (
             <Badge
-              size="md"
+              variant="outline"
               color={isEditable ? "orange" : "gray"}
               icon={FiRefreshCw}
             >
               Sync
             </Badge>
           ) : (
-            <Badge size="md" color={isEditable ? "blue" : "gray"} icon={FiLock}>
+            <Badge
+              size="md"
+              variant={isEditable ? "outline" : "default"}
+              icon={FiLock}
+            >
               Private
             </Badge>
           )}
@@ -261,7 +228,6 @@ function ConnectorRow({
         <IndexAttemptStatus
           status={ccPairsIndexingStatus.last_finished_status || null}
           errorMsg={ccPairsIndexingStatus?.latest_index_attempt?.error_msg}
-          size="xs"
         />
       </TableCell>
       <TableCell>
@@ -393,138 +359,132 @@ export function CCPairIndexingStatusTable({
   };
   const shouldExpand =
     Object.values(connectorsToggled).filter(Boolean).length <
-    sortedSources.length / 2;
+    sortedSources.length;
 
   return (
-    <div className="-mt-20">
-      <Table>
-        <ConnectorRow
-          invisible
-          ccPairsIndexingStatus={{
-            cc_pair_id: 1,
+    <Table>
+      <ConnectorRow
+        invisible
+        ccPairsIndexingStatus={{
+          cc_pair_id: 1,
+          name: "Sample File Connector",
+          cc_pair_status: ConnectorCredentialPairStatus.ACTIVE,
+          last_status: "success",
+          connector: {
             name: "Sample File Connector",
-            cc_pair_status: ConnectorCredentialPairStatus.ACTIVE,
-            last_status: "success",
-            connector: {
-              name: "Sample File Connector",
-              source: "file",
-              input_type: "poll",
-              connector_specific_config: {
-                file_locations: ["/path/to/sample/file.txt"],
-              },
-              refresh_freq: 86400,
-              prune_freq: null,
-              indexing_start: new Date("2023-07-01T12:00:00Z"),
-              id: 1,
-              credential_ids: [],
-              time_created: "2023-07-01T12:00:00Z",
-              time_updated: "2023-07-01T12:00:00Z",
-            },
-            credential: {
-              id: 1,
-              name: "Sample Credential",
-              source: "file",
-              user_id: "1",
-              time_created: "2023-07-01T12:00:00Z",
-              time_updated: "2023-07-01T12:00:00Z",
-              credential_json: {},
-              admin_public: false,
+            source: "file",
+            input_type: "poll",
+            connector_specific_config: {
+              file_locations: ["/path/to/sample/file.txt"],
             },
-            access_type: "public",
-            docs_indexed: 1000,
-            last_success: "2023-07-01T12:00:00Z",
-            last_finished_status: "success",
-            latest_index_attempt: null,
-            owner: "1",
-            error_msg: "",
-            deletion_attempt: null,
-            is_deletable: true,
-            in_progress: false,
-            groups: [], // Add this line
-          }}
-          isEditable={false}
+            refresh_freq: 86400,
+            prune_freq: null,
+            indexing_start: new Date("2023-07-01T12:00:00Z"),
+            id: 1,
+            credential_ids: [],
+            time_created: "2023-07-01T12:00:00Z",
+            time_updated: "2023-07-01T12:00:00Z",
+          },
+          credential: {
+            id: 1,
+            name: "Sample Credential",
+            source: "file",
+            user_id: "1",
+            time_created: "2023-07-01T12:00:00Z",
+            time_updated: "2023-07-01T12:00:00Z",
+            credential_json: {},
+            admin_public: false,
+          },
+          access_type: "public",
+          docs_indexed: 1000,
+          last_success: "2023-07-01T12:00:00Z",
+          last_finished_status: "success",
+          latest_index_attempt: null,
+          owner: "1",
+          error_msg: "",
+          deletion_attempt: null,
+          is_deletable: true,
+          in_progress: false,
+          groups: [], // Add this line
+        }}
+        isEditable={false}
+      />
+      <div className="flex -mt-12 items-center w-0 m4 gap-x-2">
+        <input
+          type="text"
+          ref={searchInputRef}
+          placeholder="Search connectors..."
+          value={searchTerm}
+          onChange={(e) => setSearchTerm(e.target.value)}
+          className="ml-1 w-96 h-9 flex-none rounded-md border border-border bg-background-50 px-3 py-1 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
         />
-        <div className="flex items-center w-0 mt-4 gap-x-2">
-          <input
-            type="text"
-            ref={searchInputRef}
-            placeholder="Search connectors..."
-            value={searchTerm}
-            onChange={(e) => setSearchTerm(e.target.value)}
-            className="ml-1 w-96 h-9 flex-none rounded-md border border-border bg-background-50 px-3 py-1 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
-          />
-
-          <Button className="h-9" onClick={() => toggleSources()}>
-            {!shouldExpand ? "Collapse All" : "Expand All"}
-          </Button>
-        </div>
 
-        <TableBody>
-          {sortedSources
-            .filter(
-              (source) =>
-                source != "not_applicable" && source != "ingestion_api"
-            )
-            .map((source, ind) => {
-              const sourceMatches = source
-                .toLowerCase()
-                .includes(searchTerm.toLowerCase());
-              const matchingConnectors = groupedStatuses[source].filter(
-                (status) =>
-                  (status.name || "")
-                    .toLowerCase()
-                    .includes(searchTerm.toLowerCase())
-              );
-              if (sourceMatches || matchingConnectors.length > 0) {
-                return (
-                  <React.Fragment key={ind}>
-                    <br className="mt-4" />
-
-                    <SummaryRow
-                      source={source}
-                      summary={groupSummaries[source]}
-                      isOpen={connectorsToggled[source] || false}
-                      onToggle={() => toggleSource(source)}
-                    />
-
-                    {connectorsToggled[source] && (
-                      <>
-                        <TableRow className="border border-border">
-                          <TableHeaderCell>Name</TableHeaderCell>
-                          <TableHeaderCell>Last Indexed</TableHeaderCell>
-                          <TableHeaderCell>Activity</TableHeaderCell>
-                          {isPaidEnterpriseFeaturesEnabled && (
-                            <TableHeaderCell>Permissions</TableHeaderCell>
+        <Button className="h-9" onClick={() => toggleSources()}>
+          {!shouldExpand ? "Collapse All" : "Expand All"}
+        </Button>
+      </div>
+
+      <TableBody>
+        {sortedSources
+          .filter(
+            (source) => source != "not_applicable" && source != "ingestion_api"
+          )
+          .map((source, ind) => {
+            const sourceMatches = source
+              .toLowerCase()
+              .includes(searchTerm.toLowerCase());
+            const matchingConnectors = groupedStatuses[source].filter(
+              (status) =>
+                (status.name || "")
+                  .toLowerCase()
+                  .includes(searchTerm.toLowerCase())
+            );
+            if (sourceMatches || matchingConnectors.length > 0) {
+              return (
+                <React.Fragment key={ind}>
+                  <br className="mt-4" />
+
+                  <SummaryRow
+                    source={source}
+                    summary={groupSummaries[source]}
+                    isOpen={connectorsToggled[source] || false}
+                    onToggle={() => toggleSource(source)}
+                  />
+
+                  {connectorsToggled[source] && (
+                    <>
+                      <TableRow className="border border-border">
+                        <TableHead>Name</TableHead>
+                        <TableHead>Last Indexed</TableHead>
+                        <TableHead>Activity</TableHead>
+                        {isPaidEnterpriseFeaturesEnabled && (
+                          <TableHead>Permissions</TableHead>
+                        )}
+                        <TableHead>Total Docs</TableHead>
+                        <TableHead>Last Status</TableHead>
+                        <TableHead></TableHead>
+                      </TableRow>
+                      {(sourceMatches
+                        ? groupedStatuses[source]
+                        : matchingConnectors
+                      ).map((ccPairsIndexingStatus) => (
+                        <ConnectorRow
+                          key={ccPairsIndexingStatus.cc_pair_id}
+                          ccPairsIndexingStatus={ccPairsIndexingStatus}
+                          isEditable={editableCcPairsIndexingStatuses.some(
+                            (e) =>
+                              e.cc_pair_id === ccPairsIndexingStatus.cc_pair_id
                           )}
-                          <TableHeaderCell>Total Docs</TableHeaderCell>
-                          <TableHeaderCell>Last Status</TableHeaderCell>
-                          <TableHeaderCell></TableHeaderCell>
-                        </TableRow>
-                        {(sourceMatches
-                          ? groupedStatuses[source]
-                          : matchingConnectors
-                        ).map((ccPairsIndexingStatus) => (
-                          <ConnectorRow
-                            key={ccPairsIndexingStatus.cc_pair_id}
-                            ccPairsIndexingStatus={ccPairsIndexingStatus}
-                            isEditable={editableCcPairsIndexingStatuses.some(
-                              (e) =>
-                                e.cc_pair_id ===
-                                ccPairsIndexingStatus.cc_pair_id
-                            )}
-                          />
-                        ))}
-                      </>
-                    )}
-                  </React.Fragment>
-                );
-              }
-              return null;
-            })}
-        </TableBody>
-
-        <div className="invisible w-full pb-40" />
-      </Table>
-    </div>
+                        />
+                      ))}
+                    </>
+                  )}
+                </React.Fragment>
+              );
+            }
+            return null;
+          })}
+      </TableBody>
+    </Table>
   );
 }
diff --git a/web/src/app/admin/indexing/status/page.tsx b/web/src/app/admin/indexing/status/page.tsx
index 5b77d5151a0..7e6d8ec94c4 100644
--- a/web/src/app/admin/indexing/status/page.tsx
+++ b/web/src/app/admin/indexing/status/page.tsx
@@ -5,9 +5,10 @@ import { NotebookIcon } from "@/components/icons/icons";
 import { CCPairIndexingStatusTable } from "./CCPairIndexingStatusTable";
 import { AdminPageTitle } from "@/components/admin/Title";
 import Link from "next/link";
-import { Button, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { useConnectorCredentialIndexingStatus } from "@/lib/hooks";
 import { usePopupFromQuery } from "@/components/popup/PopupFromQuery";
+import { Button } from "@/components/ui/button";
 
 function Main() {
   const {
@@ -91,12 +92,11 @@ export default function Status() {
         title="Existing Connectors"
         farRightElement={
           <Link href="/admin/add-connector">
-            <Button color="green" size="xs">
-              Add Connector
-            </Button>
+            <Button variant="success-reverse">Add Connector</Button>
           </Link>
         }
       />
+
       <Main />
     </div>
   );
diff --git a/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx b/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
index eaa3c7d27c5..1d6ca466367 100644
--- a/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
+++ b/web/src/app/admin/prompt-library/modals/AddPromptModal.tsx
@@ -1,7 +1,7 @@
 import React from "react";
 import { Formik, Form } from "formik";
 import * as Yup from "yup";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 
 import { BookstackIcon } from "@/components/icons/icons";
 import { AddPromptModalProps } from "../interfaces";
@@ -15,7 +15,7 @@ const AddPromptSchema = Yup.object().shape({
 
 const AddPromptModal = ({ onClose, onSubmit }: AddPromptModalProps) => {
   return (
-    <Modal onOutsideClick={onClose} width="max-w-xl">
+    <Modal onOutsideClick={onClose} width="w-full max-w-3xl">
       <Formik
         initialValues={{
           title: "",
@@ -51,7 +51,12 @@ const AddPromptModal = ({ onClose, onSubmit }: AddPromptModalProps) => {
               placeholder="Enter a prompt (e.g. 'help me rewrite the following politely and concisely for professional communication')"
             />
 
-            <Button type="submit" className="w-full" disabled={isSubmitting}>
+            <Button
+              type="submit"
+              className="w-full"
+              disabled={isSubmitting}
+              variant="submit"
+            >
               Add prompt
             </Button>
           </Form>
diff --git a/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx b/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
index 996b70b9fb4..c12006b07fc 100644
--- a/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
+++ b/web/src/app/admin/prompt-library/modals/EditPromptModal.tsx
@@ -2,10 +2,11 @@ import React from "react";
 import { Formik, Form, Field, ErrorMessage } from "formik";
 import * as Yup from "yup";
 import { Modal } from "@/components/Modal";
-import { Button, Textarea, TextInput } from "@tremor/react";
-
+import { Textarea } from "@/components/ui/textarea";
+import { Button } from "@/components/ui/button";
 import { useInputPrompt } from "../hooks";
 import { EditPromptModalProps } from "../interfaces";
+import { Input } from "@/components/ui/input";
 
 const EditPromptSchema = Yup.object().shape({
   prompt: Yup.string().required("Title is required"),
@@ -33,13 +34,13 @@ const EditPromptModal = ({
 
   if (!promptData)
     return (
-      <Modal onOutsideClick={onClose} width="max-w-xl">
+      <Modal onOutsideClick={onClose} width="w-full max-w-xl">
         <p>Loading...</p>
       </Modal>
     );
 
   return (
-    <Modal onOutsideClick={onClose} width="max-w-xl">
+    <Modal onOutsideClick={onClose} width="w-full max-w-xl">
       <Formik
         initialValues={{
           prompt: promptData.prompt,
@@ -74,7 +75,7 @@ const EditPromptModal = ({
                   Title
                 </label>
                 <Field
-                  as={TextInput}
+                  as={Input}
                   id="prompt"
                   name="prompt"
                   placeholder="Title (e.g. 'Draft email')"
@@ -118,7 +119,6 @@ const EditPromptModal = ({
             <div className="mt-6">
               <Button
                 type="submit"
-                className="w-full"
                 disabled={
                   isSubmitting ||
                   (values.prompt === promptData.prompt &&
diff --git a/web/src/app/admin/prompt-library/promptLibrary.tsx b/web/src/app/admin/prompt-library/promptLibrary.tsx
index a5427bdf04b..c4f535e9b0d 100644
--- a/web/src/app/admin/prompt-library/promptLibrary.tsx
+++ b/web/src/app/admin/prompt-library/promptLibrary.tsx
@@ -8,15 +8,15 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-} from "@tremor/react";
+} from "@/components/ui/table";
 import { FilterDropdown } from "@/components/search/filtering/FilterDropdown";
 import { FiTag } from "react-icons/fi";
 import { PageSelector } from "@/components/PageSelector";
 import { InputPrompt } from "./interfaces";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
+import { TableHeader } from "@/components/ui/table";
 
 const CategoryBubble = ({
   name,
@@ -174,7 +174,7 @@ export const PromptLibraryTable = ({
           icon={<FiTag size={16} />}
           defaultDisplay="All Statuses"
         />
-        <div className="flex flex-wrap pb-4 mt-3">
+        <div className="flex flex-col items-stretch w-full flex-wrap pb-4 mt-3">
           {selectedStatus.map((status) => (
             <CategoryBubble
               key={status}
@@ -186,15 +186,13 @@ export const PromptLibraryTable = ({
       </div>
       <div className="mx-auto overflow-x-auto">
         <Table>
-          <TableHead>
+          <TableHeader>
             <TableRow>
               {columns.map((column) => (
-                <TableHeaderCell key={column.key}>
-                  {column.name}
-                </TableHeaderCell>
+                <TableHead key={column.key}>{column.name}</TableHead>
               ))}
             </TableRow>
-          </TableHead>
+          </TableHeader>
           <TableBody>
             {paginatedPromptLibrary.length > 0 ? (
               paginatedPromptLibrary
diff --git a/web/src/app/admin/prompt-library/promptSection.tsx b/web/src/app/admin/prompt-library/promptSection.tsx
index f719ad500bd..015408ef0f0 100644
--- a/web/src/app/admin/prompt-library/promptSection.tsx
+++ b/web/src/app/admin/prompt-library/promptSection.tsx
@@ -3,7 +3,9 @@
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
+import Text from "@/components/ui/text";
 import { useState } from "react";
 import AddPromptModal from "./modals/AddPromptModal";
 import EditPromptModal from "./modals/EditPromptModal";
@@ -91,7 +93,9 @@ export const PromptSection = ({
 
   return (
     <div
-      className={`w-full ${centering ? "flex-col flex justify-center" : ""} mb-8`}
+      className={`w-full ${
+        centering ? "flex-col flex justify-center" : ""
+      } mb-8`}
     >
       {popup}
 
@@ -124,13 +128,13 @@ export const PromptSection = ({
       <Button
         onClick={() => setNewPrompt(true)}
         className={centering ? "mx-auto" : ""}
-        color="green"
-        size="xs"
+        variant="navigate"
+        size="sm"
       >
         New Prompt
       </Button>
 
-      <Divider />
+      <Separator />
 
       <div>
         <PromptLibraryTable
diff --git a/web/src/app/admin/settings/SettingsForm.tsx b/web/src/app/admin/settings/SettingsForm.tsx
index 9017c920af8..6f8f7dee031 100644
--- a/web/src/app/admin/settings/SettingsForm.tsx
+++ b/web/src/app/admin/settings/SettingsForm.tsx
@@ -2,7 +2,8 @@
 
 import { Label, SubLabel } from "@/components/admin/connectors/Field";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { Title, Button } from "@tremor/react";
+import Title from "@/components/ui/title";
+import { Button } from "@/components/ui/button";
 import { Settings } from "./interfaces";
 import { useRouter } from "next/navigation";
 import { DefaultDropdown, Option } from "@/components/Dropdown";
@@ -271,13 +272,17 @@ export function SettingsForm() {
           />
           <Button
             onClick={handleSetChatRetention}
-            color="green"
-            size="xs"
+            variant="submit"
+            size="sm"
             className="mr-3"
           >
             Set Retention Limit
           </Button>
-          <Button onClick={handleClearChatRetention} color="blue" size="xs">
+          <Button
+            onClick={handleClearChatRetention}
+            variant="default"
+            size="sm"
+          >
             Retain All
           </Button>
         </>
diff --git a/web/src/app/admin/settings/page.tsx b/web/src/app/admin/settings/page.tsx
index 9cb2f630e29..d8104fb568a 100644
--- a/web/src/app/admin/settings/page.tsx
+++ b/web/src/app/admin/settings/page.tsx
@@ -1,7 +1,7 @@
 import { AdminPageTitle } from "@/components/admin/Title";
 
 import { SettingsForm } from "./SettingsForm";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { SettingsIcon } from "@/components/icons/icons";
 
 export default async function Page() {
diff --git a/web/src/app/admin/token-rate-limits/CreateRateLimitModal.tsx b/web/src/app/admin/token-rate-limits/CreateRateLimitModal.tsx
index 085114a836b..16779b552ab 100644
--- a/web/src/app/admin/token-rate-limits/CreateRateLimitModal.tsx
+++ b/web/src/app/admin/token-rate-limits/CreateRateLimitModal.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import * as Yup from "yup";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { useEffect, useState } from "react";
 import { Modal } from "@/components/Modal";
 import { Form, Formik } from "formik";
@@ -72,7 +72,7 @@ export const CreateRateLimitModal = ({
     <Modal
       title={"Create a Token Rate Limit"}
       onOutsideClick={() => setIsOpen(false)}
-      width="w-2/6"
+      width="max-w-2xl w-full"
     >
       <Formik
         initialValues={{
@@ -116,7 +116,7 @@ export const CreateRateLimitModal = ({
         }}
       >
         {({ isSubmitting, values, setFieldValue }) => (
-          <Form>
+          <Form className="overflow-visible px-2">
             {!forSpecificScope && (
               <SelectorFormField
                 name="target_scope"
@@ -158,8 +158,8 @@ export const CreateRateLimitModal = ({
             />
             <Button
               type="submit"
-              size="xs"
-              color="green"
+              variant="submit"
+              size="sm"
               disabled={isSubmitting}
             >
               Create!
diff --git a/web/src/app/admin/token-rate-limits/TokenRateLimitTables.tsx b/web/src/app/admin/token-rate-limits/TokenRateLimitTables.tsx
index ec5fdc60028..5607eb5fce6 100644
--- a/web/src/app/admin/token-rate-limits/TokenRateLimitTables.tsx
+++ b/web/src/app/admin/token-rate-limits/TokenRateLimitTables.tsx
@@ -4,12 +4,10 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Title,
-  Text,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import Title from "@/components/ui/title";
 import { DeleteButton } from "@/components/DeleteButton";
 import { deleteTokenRateLimit, updateTokenRateLimit } from "./lib";
 import { ThreeDotsLoader } from "@/components/Loading";
@@ -17,6 +15,8 @@ import { TokenRateLimitDisplay } from "./types";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR, { mutate } from "swr";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
+import { TableHeader } from "@/components/ui/table";
+import Text from "@/components/ui/text";
 
 type TokenRateLimitTableArgs = {
   tokenRateLimits: TokenRateLimitDisplay[];
@@ -82,19 +82,19 @@ export const TokenRateLimitTable = ({
         <Text className="my-2">{description}</Text>
       )}
       <Table
-        className={`overflow-visible ${!hideHeading && "my-8"} [&_td]:text-center [&_th]:text-center`}
+        className={`overflow-visible ${
+          !hideHeading && "my-8"
+        } [&_td]:text-center [&_th]:text-center`}
       >
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Enabled</TableHeaderCell>
-            {shouldRenderGroupName() && (
-              <TableHeaderCell>Group Name</TableHeaderCell>
-            )}
-            <TableHeaderCell>Time Window (Hours)</TableHeaderCell>
-            <TableHeaderCell>Token Budget (Thousands)</TableHeaderCell>
-            {isAdmin && <TableHeaderCell>Delete</TableHeaderCell>}
+            <TableHead>Enabled</TableHead>
+            {shouldRenderGroupName() && <TableHead>Group Name</TableHead>}
+            <TableHead>Time Window (Hours)</TableHead>
+            <TableHead>Token Budget (Thousands)</TableHead>
+            {isAdmin && <TableHead>Delete</TableHead>}
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {tokenRateLimits.map((tokenRateLimit) => {
             return (
diff --git a/web/src/app/admin/token-rate-limits/page.tsx b/web/src/app/admin/token-rate-limits/page.tsx
index c64a8cc5f1f..4afc43376e3 100644
--- a/web/src/app/admin/token-rate-limits/page.tsx
+++ b/web/src/app/admin/token-rate-limits/page.tsx
@@ -1,15 +1,9 @@
 "use client";
 
 import { AdminPageTitle } from "@/components/admin/Title";
-import {
-  Button,
-  Tab,
-  TabGroup,
-  TabList,
-  TabPanel,
-  TabPanels,
-  Text,
-} from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Tabs, TabsList, TabsTrigger, TabsContent } from "@/components/ui/tabs";
+import Text from "@/components/ui/text";
 import { useState } from "react";
 import { FiGlobe, FiUser, FiUsers } from "react-icons/fi";
 import {
@@ -145,53 +139,63 @@ function Main() {
       </ul>
 
       <Button
-        color="green"
-        size="xs"
-        className="mt-3"
+        variant="navigate"
+        size="sm"
+        className="my-4"
         onClick={() => setModalIsOpen(true)}
       >
         Create a Token Rate Limit
       </Button>
 
       {isPaidEnterpriseFeaturesEnabled && (
-        <TabGroup className="mt-6" index={tabIndex} onIndexChange={setTabIndex}>
-          <TabList variant="line">
-            <Tab icon={FiGlobe}>Global</Tab>
-            <Tab icon={FiUser}>User</Tab>
-            <Tab icon={FiUsers}>User Groups</Tab>
-          </TabList>
-          <TabPanels className="mt-6">
-            <TabPanel>
-              <GenericTokenRateLimitTable
-                fetchUrl={GLOBAL_TOKEN_FETCH_URL}
-                title={"Global Token Rate Limits"}
-                description={GLOBAL_DESCRIPTION}
-              />
-            </TabPanel>
-            <TabPanel>
-              <GenericTokenRateLimitTable
-                fetchUrl={USER_TOKEN_FETCH_URL}
-                title={"User Token Rate Limits"}
-                description={USER_DESCRIPTION}
-              />
-            </TabPanel>
-            <TabPanel>
-              <GenericTokenRateLimitTable
-                fetchUrl={USER_GROUP_FETCH_URL}
-                title={"User Group Token Rate Limits"}
-                description={USER_GROUP_DESCRIPTION}
-                responseMapper={(data: Record<string, TokenRateLimit[]>) =>
-                  Object.entries(data).flatMap(([group_name, elements]) =>
-                    elements.map((element) => ({
-                      ...element,
-                      group_name,
-                    }))
-                  )
-                }
-              />
-            </TabPanel>
-          </TabPanels>
-        </TabGroup>
+        <Tabs
+          value={tabIndex.toString()}
+          onValueChange={(val) => setTabIndex(parseInt(val))}
+        >
+          <TabsList>
+            <TabsTrigger value="0" className="flex items-center gap-2">
+              <FiGlobe />
+              Global
+            </TabsTrigger>
+            <TabsTrigger value="1" className="flex items-center gap-2">
+              <FiUser />
+              User
+            </TabsTrigger>
+            <TabsTrigger value="2" className="flex items-center gap-2">
+              <FiUsers />
+              User Groups
+            </TabsTrigger>
+          </TabsList>
+          <TabsContent value="0">
+            <GenericTokenRateLimitTable
+              fetchUrl={GLOBAL_TOKEN_FETCH_URL}
+              title={"Global Token Rate Limits"}
+              description={GLOBAL_DESCRIPTION}
+            />
+          </TabsContent>
+          <TabsContent value="1">
+            <GenericTokenRateLimitTable
+              fetchUrl={USER_TOKEN_FETCH_URL}
+              title={"User Token Rate Limits"}
+              description={USER_DESCRIPTION}
+            />
+          </TabsContent>
+          <TabsContent value="2">
+            <GenericTokenRateLimitTable
+              fetchUrl={USER_GROUP_FETCH_URL}
+              title={"User Group Token Rate Limits"}
+              description={USER_GROUP_DESCRIPTION}
+              responseMapper={(data: Record<string, TokenRateLimit[]>) =>
+                Object.entries(data).flatMap(([group_name, elements]) =>
+                  elements.map((element) => ({
+                    ...element,
+                    group_name,
+                  }))
+                )
+              }
+            />
+          </TabsContent>
+        </Tabs>
       )}
 
       {!isPaidEnterpriseFeaturesEnabled && (
diff --git a/web/src/app/admin/tools/ToolEditor.tsx b/web/src/app/admin/tools/ToolEditor.tsx
index b6d080ebd15..354701d4c8a 100644
--- a/web/src/app/admin/tools/ToolEditor.tsx
+++ b/web/src/app/admin/tools/ToolEditor.tsx
@@ -13,7 +13,7 @@ import {
 import * as Yup from "yup";
 import { MethodSpec, ToolSnapshot } from "@/lib/tools/interfaces";
 import { TextFormField } from "@/components/admin/connectors/Field";
-import { Button, Divider } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import {
   createCustomTool,
   updateCustomTool,
@@ -23,6 +23,7 @@ import { usePopup } from "@/components/admin/connectors/Popup";
 import debounce from "lodash/debounce";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import Link from "next/link";
+import { Separator } from "@/components/ui/separator";
 
 function parseJsonWithTrailingCommas(jsonString: string) {
   // Regular expression to remove trailing commas before } or ]
@@ -244,7 +245,7 @@ function ToolForm({
                           <Button
                             type="button"
                             onClick={() => arrayHelpers.remove(index)}
-                            color="red"
+                            variant="destructive"
                             size="sm"
                             className="transition-colors duration-200 hover:bg-red-600"
                           >
@@ -259,8 +260,8 @@ function ToolForm({
                 <Button
                   type="button"
                   onClick={() => arrayHelpers.push({ key: "", value: "" })}
-                  color="blue"
-                  size="md"
+                  variant="secondary"
+                  size="sm"
                   className="transition-colors duration-200"
                 >
                   Add New Header
@@ -271,13 +272,13 @@ function ToolForm({
         </div>
       )}
 
-      <Divider />
+      <Separator />
 
       <div className="flex">
         <Button
           className="mx-auto"
-          color="green"
-          size="md"
+          variant="submit"
+          size="sm"
           type="submit"
           disabled={isSubmitting || !!definitionError}
         >
diff --git a/web/src/app/admin/tools/ToolsTable.tsx b/web/src/app/admin/tools/ToolsTable.tsx
index a283718afc6..30bb8b34f60 100644
--- a/web/src/app/admin/tools/ToolsTable.tsx
+++ b/web/src/app/admin/tools/ToolsTable.tsx
@@ -4,16 +4,16 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-} from "@tremor/react";
+} from "@/components/ui/table";
 import { ToolSnapshot } from "@/lib/tools/interfaces";
 import { useRouter } from "next/navigation";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { FiCheckCircle, FiEdit2, FiXCircle } from "react-icons/fi";
 import { TrashIcon } from "@/components/icons/icons";
 import { deleteCustomTool } from "@/lib/tools/edit";
+import { TableHeader } from "@/components/ui/table";
 
 export function ToolsTable({ tools }: { tools: ToolSnapshot[] }) {
   const router = useRouter();
@@ -27,14 +27,14 @@ export function ToolsTable({ tools }: { tools: ToolSnapshot[] }) {
       {popup}
 
       <Table>
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Name</TableHeaderCell>
-            <TableHeaderCell>Description</TableHeaderCell>
-            <TableHeaderCell>Built In?</TableHeaderCell>
-            <TableHeaderCell>Delete</TableHeaderCell>
+            <TableHead>Name</TableHead>
+            <TableHead>Description</TableHead>
+            <TableHead>Built In?</TableHead>
+            <TableHead>Delete</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {sortedTools.map((tool) => (
             <TableRow key={tool.id.toString()}>
diff --git a/web/src/app/admin/tools/edit/[toolId]/DeleteToolButton.tsx b/web/src/app/admin/tools/edit/[toolId]/DeleteToolButton.tsx
index c02e141b54a..1c1c9528304 100644
--- a/web/src/app/admin/tools/edit/[toolId]/DeleteToolButton.tsx
+++ b/web/src/app/admin/tools/edit/[toolId]/DeleteToolButton.tsx
@@ -1,6 +1,6 @@
 "use client";
 
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { FiTrash } from "react-icons/fi";
 import { deleteCustomTool } from "@/lib/tools/edit";
 import { useRouter } from "next/navigation";
@@ -10,8 +10,8 @@ export function DeleteToolButton({ toolId }: { toolId: number }) {
 
   return (
     <Button
-      size="xs"
-      color="red"
+      variant="destructive"
+      size="sm"
       onClick={async () => {
         const response = await deleteCustomTool(toolId);
         if (response.data) {
diff --git a/web/src/app/admin/tools/edit/[toolId]/page.tsx b/web/src/app/admin/tools/edit/[toolId]/page.tsx
index 7a9e86b2fff..44d24ccfb81 100644
--- a/web/src/app/admin/tools/edit/[toolId]/page.tsx
+++ b/web/src/app/admin/tools/edit/[toolId]/page.tsx
@@ -1,5 +1,7 @@
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { Card, Text, Title } from "@tremor/react";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import CardSection from "@/components/admin/CardSection";
 import { ToolEditor } from "@/app/admin/tools/ToolEditor";
 import { fetchToolByIdSS } from "@/lib/tools/fetchTools";
 import { DeleteToolButton } from "./DeleteToolButton";
@@ -25,9 +27,9 @@ export default async function Page({ params }: { params: { toolId: string } }) {
       <div className="w-full my-8">
         <div>
           <div>
-            <Card>
+            <CardSection>
               <ToolEditor tool={tool} />
-            </Card>
+            </CardSection>
 
             <Title className="mt-12">Delete Tool</Title>
             <Text>Click the button below to permanently delete this tool.</Text>
diff --git a/web/src/app/admin/tools/new/page.tsx b/web/src/app/admin/tools/new/page.tsx
index 79042eb670e..9146564e698 100644
--- a/web/src/app/admin/tools/new/page.tsx
+++ b/web/src/app/admin/tools/new/page.tsx
@@ -4,7 +4,7 @@ import { ToolEditor } from "@/app/admin/tools/ToolEditor";
 import { BackButton } from "@/components/BackButton";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { ToolIcon } from "@/components/icons/icons";
-import { Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
 
 export default function NewToolPage() {
   return (
@@ -16,9 +16,9 @@ export default function NewToolPage() {
         icon={<ToolIcon size={32} className="my-auto" />}
       />
 
-      <Card>
+      <CardSection>
         <ToolEditor />
-      </Card>
+      </CardSection>
     </div>
   );
 }
diff --git a/web/src/app/admin/tools/page.tsx b/web/src/app/admin/tools/page.tsx
index a1b20c79dfc..6ccb2480163 100644
--- a/web/src/app/admin/tools/page.tsx
+++ b/web/src/app/admin/tools/page.tsx
@@ -2,7 +2,9 @@ import { ToolsTable } from "./ToolsTable";
 import { ToolSnapshot } from "@/lib/tools/interfaces";
 import { FiPlusSquare } from "react-icons/fi";
 import Link from "next/link";
-import { Divider, Text, Title } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
 import { fetchSS } from "@/lib/utilsSS";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { AdminPageTitle } from "@/components/admin/Title";
@@ -34,7 +36,7 @@ export default async function Page() {
       </Text>
 
       <div>
-        <Divider />
+        <Separator />
 
         <Title>Create a Tool</Title>
         <Link
@@ -59,7 +61,7 @@ export default async function Page() {
           </div>
         </Link>
 
-        <Divider />
+        <Separator />
 
         <Title>Existing Tools</Title>
         <ToolsTable tools={tools} />
diff --git a/web/src/app/admin/users/page.tsx b/web/src/app/admin/users/page.tsx
index 8d34594c881..44397515181 100644
--- a/web/src/app/admin/users/page.tsx
+++ b/web/src/app/admin/users/page.tsx
@@ -6,7 +6,8 @@ import { useState } from "react";
 import { FiPlusSquare } from "react-icons/fi";
 import { Modal } from "@/components/Modal";
 
-import { Button, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 import { LoadingAnimation } from "@/components/Loading";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { usePopup, PopupSpec } from "@/components/admin/connectors/Popup";
diff --git a/web/src/app/assistants/edit/[id]/page.tsx b/web/src/app/assistants/edit/[id]/page.tsx
index caf1d0cbf14..e57a32acc0c 100644
--- a/web/src/app/assistants/edit/[id]/page.tsx
+++ b/web/src/app/assistants/edit/[id]/page.tsx
@@ -1,11 +1,13 @@
 import { ErrorCallout } from "@/components/ErrorCallout";
-import { Card, Text, Title } from "@tremor/react";
+import Text from "@/components/ui/text";
+import CardSection from "@/components/admin/CardSection";
 import { HeaderWrapper } from "@/components/header/HeaderWrapper";
 import { AssistantEditor } from "@/app/admin/assistants/AssistantEditor";
 import { SuccessfulPersonaUpdateRedirectType } from "@/app/admin/assistants/enums";
 import { fetchAssistantEditorInfoSS } from "@/lib/assistants/fetchPersonaEditorInfoSS";
 import { DeletePersonaButton } from "@/app/admin/assistants/[id]/DeletePersonaButton";
 import { LargeBackButton } from "../../LargeBackButton";
+import Title from "@/components/ui/title";
 
 export default async function Page({ params }: { params: { id: string } }) {
   const [values, error] = await fetchAssistantEditorInfoSS(params.id);
@@ -22,14 +24,14 @@ export default async function Page({ params }: { params: { id: string } }) {
       <div className="w-full my-16">
         <div className="px-32">
           <div className="mx-auto container">
-            <Card>
+            <CardSection>
               <AssistantEditor
                 {...values}
                 admin
                 defaultPublic={false}
                 redirectType={SuccessfulPersonaUpdateRedirectType.CHAT}
               />
-            </Card>
+            </CardSection>
             <Title className="mt-12">Delete Assistant</Title>
             <Text>
               Click the button below to permanently delete this assistant.
diff --git a/web/src/app/assistants/gallery/AssistantsGallery.tsx b/web/src/app/assistants/gallery/AssistantsGallery.tsx
index 62b9f3906df..e2d2e17004d 100644
--- a/web/src/app/assistants/gallery/AssistantsGallery.tsx
+++ b/web/src/app/assistants/gallery/AssistantsGallery.tsx
@@ -3,7 +3,7 @@
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
 import { User } from "@/lib/types";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { useState } from "react";
 import { FiList, FiMinus, FiPlus } from "react-icons/fi";
 import { AssistantsPageTitle } from "../AssistantsPageTitle";
@@ -91,7 +91,8 @@ export function AssistantGalleryCard({
                     });
                   }
                 }}
-                size="xs"
+                size="sm"
+                variant="destructive"
               >
                 Deselect
               </Button>
@@ -119,8 +120,8 @@ export function AssistantGalleryCard({
                     });
                   }
                 }}
-                size="xs"
-                color="green"
+                size="sm"
+                variant="submit"
               >
                 Add
               </Button>
diff --git a/web/src/app/assistants/mine/AssistantSharingModal.tsx b/web/src/app/assistants/mine/AssistantSharingModal.tsx
index 96f9aeab923..1a19aac618e 100644
--- a/web/src/app/assistants/mine/AssistantSharingModal.tsx
+++ b/web/src/app/assistants/mine/AssistantSharingModal.tsx
@@ -1,7 +1,7 @@
 import { useState } from "react";
 import { Modal } from "@/components/Modal";
 import { MinimalUserSnapshot, User } from "@/lib/types";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { FiPlus, FiX } from "react-icons/fi";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { SearchMultiSelectDropdown } from "@/components/Dropdown";
@@ -217,8 +217,7 @@ export function AssistantSharingModal({
                 setSelectedUsers([]);
               }}
               size="sm"
-              color="blue"
-              className="w-full"
+              variant="secondary"
             >
               Share with Selected Users
             </Button>
diff --git a/web/src/app/assistants/mine/AssistantsList.tsx b/web/src/app/assistants/mine/AssistantsList.tsx
index 26e081d94a3..0af7a98fd20 100644
--- a/web/src/app/assistants/mine/AssistantsList.tsx
+++ b/web/src/app/assistants/mine/AssistantsList.tsx
@@ -3,7 +3,8 @@
 import React, { Dispatch, SetStateAction, useEffect, useState } from "react";
 import { MinimalUserSnapshot, User } from "@/lib/types";
 import { Persona } from "@/app/admin/assistants/interfaces";
-import { Button, Divider } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
 import {
   FiEdit2,
   FiList,
@@ -130,7 +131,9 @@ function AssistantListItem({
         show={showSharingModal}
       />
       <div
-        className={`rounded-lg px-4 py-6 transition-all duration-900 hover:bg-background-125 ${isDragging && "bg-background-125"}`}
+        className={`rounded-lg px-4 py-6 transition-all duration-900 hover:bg-background-125 ${
+          isDragging && "bg-background-125"
+        }`}
       >
         <div className="flex justify-between items-center">
           <AssistantIcon assistant={assistant} />
@@ -446,7 +449,7 @@ export function AssistantsList() {
 
         {ownedButHiddenAssistants.length > 0 && (
           <>
-            <Divider />
+            <Separator />
 
             <h3 className="text-xl font-bold mb-4">Your Hidden Assistants</h3>
 
diff --git a/web/src/app/assistants/new/page.tsx b/web/src/app/assistants/new/page.tsx
index 6af0018fc65..3e39f30c7e0 100644
--- a/web/src/app/assistants/new/page.tsx
+++ b/web/src/app/assistants/new/page.tsx
@@ -1,4 +1,4 @@
-import { Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
 import { HeaderWrapper } from "@/components/header/HeaderWrapper";
 import { AssistantEditor } from "@/app/admin/assistants/AssistantEditor";
 import { SuccessfulPersonaUpdateRedirectType } from "@/app/admin/assistants/enums";
@@ -21,14 +21,14 @@ export default async function Page() {
       <div className="w-full my-16">
         <div className="px-32">
           <div className="mx-auto container">
-            <Card>
+            <CardSection>
               <AssistantEditor
                 {...values}
                 defaultPublic={false}
                 redirectType={SuccessfulPersonaUpdateRedirectType.CHAT}
                 shouldAddAssistantToUserPreferences={true}
               />
-            </Card>
+            </CardSection>
           </div>
         </div>
       </div>
diff --git a/web/src/app/auth/create-account/page.tsx b/web/src/app/auth/create-account/page.tsx
index b5c340afb6a..b3e82540e75 100644
--- a/web/src/app/auth/create-account/page.tsx
+++ b/web/src/app/auth/create-account/page.tsx
@@ -2,7 +2,7 @@
 
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 import { REGISTRATION_URL } from "@/lib/constants";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import Link from "next/link";
 import { FiLogIn } from "react-icons/fi";
 
@@ -26,7 +26,7 @@ const Page = () => {
             href={`${REGISTRATION_URL}/register`}
             className="w-full max-w-xs"
           >
-            <Button size="lg" icon={FiLogIn} color="indigo" className="w-full">
+            <Button size="lg" icon={FiLogIn} className="w-full">
               Create New Organization
             </Button>
           </Link>
diff --git a/web/src/app/auth/error/page.tsx b/web/src/app/auth/error/page.tsx
index c75e620068e..3bee4984161 100644
--- a/web/src/app/auth/error/page.tsx
+++ b/web/src/app/auth/error/page.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import Link from "next/link";
 import { FiLogIn } from "react-icons/fi";
 
@@ -34,7 +34,7 @@ const Page = () => {
         </div>
 
         <Link href="/auth/login" className="w-full">
-          <Button size="lg" icon={FiLogIn} color="indigo" className="w-full">
+          <Button size="lg" icon={FiLogIn} className="w-full">
             Return to Login Page
           </Button>
         </Link>
diff --git a/web/src/app/auth/login/EmailPasswordForm.tsx b/web/src/app/auth/login/EmailPasswordForm.tsx
index 6862baa600c..334c74d14f7 100644
--- a/web/src/app/auth/login/EmailPasswordForm.tsx
+++ b/web/src/app/auth/login/EmailPasswordForm.tsx
@@ -3,7 +3,7 @@
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { basicLogin, basicSignup } from "@/lib/user";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { Form, Formik } from "formik";
 import { useRouter } from "next/navigation";
 import * as Yup from "yup";
diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index 244d2429eb4..3b5ff2bdf70 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -9,11 +9,13 @@ import {
 import { redirect } from "next/navigation";
 import { SignInButton } from "./SignInButton";
 import { EmailPasswordForm } from "./EmailPasswordForm";
-import { Card, Title, Text } from "@tremor/react";
+import Title from "@/components/ui/title";
+import Text from "@/components/ui/text";
 import Link from "next/link";
 import { LoginText } from "./LoginText";
 import { getSecondsUntilExpiration } from "@/lib/time";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
+import CardSection from "@/components/admin/CardSection";
 
 const Page = async ({
   searchParams,
@@ -113,7 +115,7 @@ const Page = async ({
         )}
 
         {authTypeMetadata?.authType === "basic" && (
-          <Card className="mt-4 w-96">
+          <CardSection className="mt-4 w-96">
             <div className="flex">
               <Title className="mb-2 mx-auto font-bold">
                 <LoginText />
@@ -128,7 +130,7 @@ const Page = async ({
                 </Link>
               </Text>
             </div>
-          </Card>
+          </CardSection>
         )}
       </div>
     </AuthFlowContainer>
diff --git a/web/src/app/auth/signup/page.tsx b/web/src/app/auth/signup/page.tsx
index 5a90535df59..29c2f97ea16 100644
--- a/web/src/app/auth/signup/page.tsx
+++ b/web/src/app/auth/signup/page.tsx
@@ -8,7 +8,7 @@ import {
 } from "@/lib/userSS";
 import { redirect } from "next/navigation";
 import { EmailPasswordForm } from "../login/EmailPasswordForm";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import Link from "next/link";
 import { SignInButton } from "../login/SignInButton";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
diff --git a/web/src/app/auth/verify-email/Verify.tsx b/web/src/app/auth/verify-email/Verify.tsx
index 2b5fd1dcc6e..7456231cc2b 100644
--- a/web/src/app/auth/verify-email/Verify.tsx
+++ b/web/src/app/auth/verify-email/Verify.tsx
@@ -3,7 +3,7 @@
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { useRouter, useSearchParams } from "next/navigation";
 import { useCallback, useEffect, useState } from "react";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { RequestNewVerificationEmail } from "../waiting-on-verification/RequestNewVerificationEmail";
 import { User } from "@/lib/types";
 import { Logo } from "@/components/Logo";
diff --git a/web/src/app/auth/waiting-on-verification/page.tsx b/web/src/app/auth/waiting-on-verification/page.tsx
index b1e1f831470..3cd8035ae17 100644
--- a/web/src/app/auth/waiting-on-verification/page.tsx
+++ b/web/src/app/auth/waiting-on-verification/page.tsx
@@ -6,7 +6,7 @@ import {
 import { redirect } from "next/navigation";
 import { HealthCheckBanner } from "@/components/health/healthcheck";
 import { User } from "@/lib/types";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { RequestNewVerificationEmail } from "./RequestNewVerificationEmail";
 import { Logo } from "@/components/Logo";
 
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index a4fcc00e1d9..57e915423ea 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -103,7 +103,7 @@ import { ApiKeyModal } from "@/components/llm/ApiKeyModal";
 import BlurBackground from "./shared_chat_search/BlurBackground";
 import { NoAssistantModal } from "@/components/modals/NoAssistantModal";
 import { useAssistants } from "@/components/context/AssistantsContext";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 
 const TEMP_USER_MESSAGE_ID = -1;
 const TEMP_ASSISTANT_MESSAGE_ID = -2;
@@ -2029,7 +2029,7 @@ export function ChatPage({
                                     currentPersona.starter_messages.length >
                                       0 && (
                                       <>
-                                        <Divider className="mx-2" />
+                                        <Separator className="mx-2" />
 
                                         {currentPersona.starter_messages
                                           .slice(0, 4)
diff --git a/web/src/app/chat/ChatPopup.tsx b/web/src/app/chat/ChatPopup.tsx
index 664cfffacb0..8fb6b1503bd 100644
--- a/web/src/app/chat/ChatPopup.tsx
+++ b/web/src/app/chat/ChatPopup.tsx
@@ -2,7 +2,7 @@
 
 import { Modal } from "@/components/Modal";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { useContext, useEffect, useState } from "react";
 import ReactMarkdown from "react-markdown";
 import remarkGfm from "remark-gfm";
@@ -71,15 +71,15 @@ export function ChatPopup() {
         <div className="flex w-full justify-center gap-4 mt-4">
           {isConsentScreen && (
             <Button
-              size="xs"
-              color="red"
+              size="sm"
+              variant="destructive"
               onClick={() => setShowConsentError(true)}
             >
               Cancel
             </Button>
           )}
           <Button
-            size="xs"
+            size="sm"
             onClick={() => {
               localStorage.setItem(
                 ALL_USERS_INITIAL_POPUP_FLOW_COMPLETED,
diff --git a/web/src/app/chat/documentSidebar/DocumentSidebar.tsx b/web/src/app/chat/documentSidebar/DocumentSidebar.tsx
index e20fd8e6735..5a6ed956fe6 100644
--- a/web/src/app/chat/documentSidebar/DocumentSidebar.tsx
+++ b/web/src/app/chat/documentSidebar/DocumentSidebar.tsx
@@ -1,10 +1,11 @@
 import { DanswerDocument } from "@/lib/search/interfaces";
-import { Divider, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { ChatDocumentDisplay } from "./ChatDocumentDisplay";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { removeDuplicateDocs } from "@/lib/documentUtils";
 import { Message } from "../interfaces";
 import { ForwardedRef, forwardRef } from "react";
+import { Separator } from "@/components/ui/separator";
 
 interface DocumentSidebarProps {
   closeSidebar: () => void;
@@ -53,7 +54,9 @@ export const DocumentSidebar = forwardRef<HTMLDivElement, DocumentSidebarProps>(
     return (
       <div
         id="danswer-chat-sidebar"
-        className={`fixed inset-0 transition-opacity duration-300 z-50 bg-black/80 ${isOpen ? "opacity-100" : "opacity-0 pointer-events-none"}`}
+        className={`fixed inset-0 transition-opacity duration-300 z-50 bg-black/80 ${
+          isOpen ? "opacity-100" : "opacity-0 pointer-events-none"
+        }`}
         onClick={(e) => {
           if (e.target === e.currentTarget) {
             closeSidebar();
@@ -84,7 +87,7 @@ export const DocumentSidebar = forwardRef<HTMLDivElement, DocumentSidebarProps>(
               </p>
             </div>
 
-            <Divider className="mb-0 mt-4 pb-2" />
+            <Separator className="mb-0 mt-4 pb-2" />
 
             {currentDocuments ? (
               <div className="overflow-y-auto flex-grow dark-scrollbar flex relative flex-col">
diff --git a/web/src/app/chat/input/SelectedFilterDisplay.tsx b/web/src/app/chat/input/SelectedFilterDisplay.tsx
index 56b92d9998f..92d398ce984 100644
--- a/web/src/app/chat/input/SelectedFilterDisplay.tsx
+++ b/web/src/app/chat/input/SelectedFilterDisplay.tsx
@@ -2,7 +2,7 @@ import { SourceIcon } from "@/components/SourceIcon";
 import React from "react";
 import { FiBookmark, FiTag, FiX } from "react-icons/fi";
 import { FilterManager } from "@/lib/hooks";
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 
 const displayTimeRange = (timeRange: DateRangePickerValue) => {
   if (timeRange.selectValue) {
diff --git a/web/src/app/chat/modal/MakePublicAssistantModal.tsx b/web/src/app/chat/modal/MakePublicAssistantModal.tsx
index 757cf060e80..a410453f6b0 100644
--- a/web/src/app/chat/modal/MakePublicAssistantModal.tsx
+++ b/web/src/app/chat/modal/MakePublicAssistantModal.tsx
@@ -1,5 +1,7 @@
 import { Modal } from "@/components/Modal";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
+import Text from "@/components/ui/text";
 
 export function MakePublicAssistantModal({
   isPublic,
@@ -28,7 +30,7 @@ export function MakePublicAssistantModal({
             : " Only you can access this assistant."}
         </Text>
 
-        <Divider />
+        <Separator />
 
         {isPublic ? (
           <div className="space-y-4">
@@ -42,7 +44,7 @@ export function MakePublicAssistantModal({
                 onClose();
               }}
               size="sm"
-              color="red"
+              variant="destructive"
             >
               Make Assistant Private
             </Button>
@@ -60,7 +62,7 @@ export function MakePublicAssistantModal({
                 onClose();
               }}
               size="sm"
-              color="green"
+              variant="submit"
             >
               Make Assistant Public
             </Button>
diff --git a/web/src/app/chat/modal/SetDefaultModelModal.tsx b/web/src/app/chat/modal/SetDefaultModelModal.tsx
index 61190120dc2..27696c46916 100644
--- a/web/src/app/chat/modal/SetDefaultModelModal.tsx
+++ b/web/src/app/chat/modal/SetDefaultModelModal.tsx
@@ -1,6 +1,6 @@
 import { Dispatch, SetStateAction, useEffect, useRef } from "react";
 import { Modal } from "@/components/Modal";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { getDisplayNameForModel, LlmOverride } from "@/lib/hooks";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 
@@ -166,7 +166,9 @@ export function SetDefaultModelModal({
                 <td className="p-2">
                   System default{" "}
                   {defaultProvider?.default_model_name &&
-                    `(${getDisplayNameForModel(defaultProvider?.default_model_name)})`}
+                    `(${getDisplayNameForModel(
+                      defaultProvider?.default_model_name
+                    )})`}
                 </td>
               }
             </div>
diff --git a/web/src/app/chat/modal/ShareChatSessionModal.tsx b/web/src/app/chat/modal/ShareChatSessionModal.tsx
index 4743efdf0dc..ffff8bb9714 100644
--- a/web/src/app/chat/modal/ShareChatSessionModal.tsx
+++ b/web/src/app/chat/modal/ShareChatSessionModal.tsx
@@ -1,14 +1,18 @@
 import { useState } from "react";
 import { Modal } from "@/components/Modal";
-import { Button, Callout, Divider, Text } from "@tremor/react";
-import { Spinner } from "@/components/Spinner";
+import { Button } from "@/components/ui/button";
+import { Callout } from "@/components/ui/callout";
+
+import Text from "@/components/ui/text";
+
 import { ChatSessionSharedStatus } from "../interfaces";
 import { FiCopy } from "react-icons/fi";
 import { CopyButton } from "@/components/CopyButton";
 import { SEARCH_PARAM_NAMES } from "../searchParams";
 import { usePopup } from "@/components/admin/connectors/Popup";
-import { destructureValue, structureValue } from "@/lib/llm/utils";
+import { structureValue } from "@/lib/llm/utils";
 import { LlmOverride } from "@/lib/hooks";
+import { Separator } from "@/components/ui/separator";
 
 function buildShareLink(chatSessionId: string) {
   const baseUrl = `${window.location.protocol}//${window.location.host}`;
@@ -53,7 +57,9 @@ async function generateCloneLink(
       : ""
   }${
     model
-      ? `${message || assistantId ? "&" : "?"}${SEARCH_PARAM_NAMES.STRUCTURED_MODEL}=${encodeURIComponent(model)}`
+      ? `${message || assistantId ? "&" : "?"}${
+          SEARCH_PARAM_NAMES.STRUCTURED_MODEL
+        }=${encodeURIComponent(model)}`
       : ""
   }${message ? `&${SEARCH_PARAM_NAMES.SEND_ON_LOAD}=true` : ""}`;
 }
@@ -126,7 +132,7 @@ export function ShareChatSessionModal({
                   </a>
                 </div>
 
-                <Divider />
+                <Separator />
 
                 <Text className="mb-4">
                   Click the button below to make the chat private again.
@@ -142,15 +148,15 @@ export function ShareChatSessionModal({
                       alert("Failed to delete share link");
                     }
                   }}
-                  size="xs"
-                  color="red"
+                  size="sm"
+                  variant="destructive"
                 >
                   Delete Share Link
                 </Button>
               </div>
             ) : (
               <div>
-                <Callout title="Warning" color="yellow" className="mb-4">
+                <Callout type="warning" title="Warning" className="mb-4">
                   Ensure that all content in the chat is safe to share with the
                   whole organization. The content of the retrieved documents
                   will not be visible, but the names of cited documents as well
@@ -176,8 +182,8 @@ export function ShareChatSessionModal({
                         console.error(e);
                       }
                     }}
-                    size="xs"
-                    color="green"
+                    size="sm"
+                    variant="submit"
                   >
                     Generate and Copy Share Link
                   </Button>
@@ -186,9 +192,9 @@ export function ShareChatSessionModal({
             )}
           </div>
 
-          <Divider className="my-4" />
+          <Separator className="my-4" />
           <div className="mb-4">
-            <Callout title="Clone Chat" color="blue">
+            <Callout type="notice" title="Clone Chat">
               Generate a link to clone this chat session with the current query.
               This allows others to start a new chat with the same initial
               message and settings.
@@ -223,8 +229,8 @@ export function ShareChatSessionModal({
                   alert("Failed to generate or copy link.");
                 }
               }}
-              size="xs"
-              color="blue"
+              size="sm"
+              variant="secondary"
             >
               Generate and Copy Clone Link
             </Button>
diff --git a/web/src/app/chat/modal/configuration/LlmTab.tsx b/web/src/app/chat/modal/configuration/LlmTab.tsx
index 2d05e12b222..4e51a21933e 100644
--- a/web/src/app/chat/modal/configuration/LlmTab.tsx
+++ b/web/src/app/chat/modal/configuration/LlmTab.tsx
@@ -2,7 +2,7 @@ import { useChatContext } from "@/components/context/ChatContext";
 import { LlmOverrideManager } from "@/lib/hooks";
 import React, { forwardRef, useCallback, useState } from "react";
 import { debounce } from "lodash";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { destructureValue } from "@/lib/llm/utils";
 import { updateModelOverrideForChatSession } from "../../lib";
diff --git a/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx b/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
index e633d01ba0b..f0acaa0ace3 100644
--- a/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
+++ b/web/src/app/chat/shared/[chatId]/SharedChatDisplay.tsx
@@ -9,12 +9,14 @@ import {
   processRawChatHistory,
 } from "../../lib";
 import { AIMessage, HumanMessage } from "../../message/Messages";
-import { Button, Callout, Divider } from "@tremor/react";
+import { Callout } from "@/components/ui/callout";
+import { Separator } from "@/components/ui/separator";
 import { useRouter } from "next/navigation";
 import { useContext, useEffect, useState } from "react";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { DanswerInitializingLoader } from "@/components/DanswerInitializingLoader";
 import { Persona } from "@/app/admin/assistants/interfaces";
+import { Button } from "@/components/ui/button";
 
 function BackToDanswerButton() {
   const router = useRouter();
@@ -47,7 +49,7 @@ export function SharedChatDisplay({
     return (
       <div className="min-h-full w-full">
         <div className="mx-auto w-fit pt-8">
-          <Callout color="red" title="Shared Chat Not Found">
+          <Callout type="danger" title="Shared Chat Not Found">
             Did not find a shared chat with the specified ID.
           </Callout>
         </div>
@@ -74,7 +76,7 @@ export function SharedChatDisplay({
                 {humanReadableFormat(chatSession.time_created)}
               </p>
 
-              <Divider />
+              <Separator />
             </div>
             {isReady ? (
               <div className="w-full pb-16">
diff --git a/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx b/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
index 8703beb1d7b..03f15e08038 100644
--- a/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
+++ b/web/src/app/ee/admin/api-key/DanswerApiKeyForm.tsx
@@ -6,7 +6,9 @@ import {
 } from "@/components/admin/connectors/Field";
 import { createApiKey, updateApiKey } from "./lib";
 import { Modal } from "@/components/Modal";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
+import Text from "@/components/ui/text";
 import { UserRole } from "@/lib/types";
 import { APIKey } from "./types";
 
@@ -32,7 +34,7 @@ export const DanswerApiKeyForm = ({
           {isUpdate ? "Update API Key" : "Create a new API Key"}
         </h2>
 
-        <Divider />
+        <Separator />
 
         <Formik
           initialValues={{
@@ -107,8 +109,8 @@ export const DanswerApiKeyForm = ({
 
               <Button
                 type="submit"
-                size="xs"
-                color="green"
+                size="sm"
+                variant="submit"
                 disabled={isSubmitting}
               >
                 {isUpdate ? "Update!" : "Create!"}
diff --git a/web/src/app/ee/admin/api-key/page.tsx b/web/src/app/ee/admin/api-key/page.tsx
index 2b7db0bce24..1420a1c518b 100644
--- a/web/src/app/ee/admin/api-key/page.tsx
+++ b/web/src/app/ee/admin/api-key/page.tsx
@@ -6,18 +6,19 @@ import { KeyIcon } from "@/components/icons/icons";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import useSWR, { mutate } from "swr";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
 import {
-  Button,
-  Divider,
   TableBody,
   TableCell,
   TableHead,
-  TableHeaderCell,
+  TableHeader,
   TableRow,
-  Text,
-  Title,
   Table,
-} from "@tremor/react";
+} from "@/components/ui/table";
+
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { useState } from "react";
 import { DeleteButton } from "@/components/DeleteButton";
@@ -116,8 +117,8 @@ function Main() {
 
   const newApiKeyButton = (
     <Button
-      color="green"
-      size="xs"
+      variant="navigate"
+      size="sm"
       className="mt-3"
       onClick={() => setShowCreateUpdateForm(true)}
     >
@@ -166,19 +167,19 @@ function Main() {
       <Text>{API_KEY_TEXT}</Text>
       {newApiKeyButton}
 
-      <Divider />
+      <Separator />
 
       <Title className="mt-6">Existing API Keys</Title>
       <Table className="overflow-visible">
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Name</TableHeaderCell>
-            <TableHeaderCell>API Key</TableHeaderCell>
-            <TableHeaderCell>Role</TableHeaderCell>
-            <TableHeaderCell>Regenerate</TableHeaderCell>
-            <TableHeaderCell>Delete</TableHeaderCell>
+            <TableHead>Name</TableHead>
+            <TableHead>API Key</TableHead>
+            <TableHead>Role</TableHead>
+            <TableHead>Regenerate</TableHead>
+            <TableHead>Delete</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {apiKeys.map((apiKey) => (
             <TableRow key={apiKey.api_key_id}>
diff --git a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
index 1d761caebc4..1b7b4bdbf00 100644
--- a/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
+++ b/web/src/app/ee/admin/groups/UserGroupCreationForm.tsx
@@ -7,7 +7,8 @@ import { createUserGroup } from "./lib";
 import { UserEditor } from "./UserEditor";
 import { ConnectorEditor } from "./ConnectorEditor";
 import { Modal } from "@/components/Modal";
-import { Button, Divider } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
 
 interface UserGroupCreationFormProps {
   onClose: () => void;
@@ -38,7 +39,7 @@ export const UserGroupCreationForm = ({
           {isUpdate ? "Update a User Group" : "Create a new User Group"}
         </h2>
 
-        <Divider />
+        <Separator />
 
         <Formik
           initialValues={{
@@ -87,7 +88,7 @@ export const UserGroupCreationForm = ({
                   autoCompleteDisabled={true}
                 />
 
-                <Divider />
+                <Separator />
 
                 <h2 className="mb-1 font-medium">
                   Select which private connectors this group has access to:
@@ -105,7 +106,7 @@ export const UserGroupCreationForm = ({
                   }
                 />
 
-                <Divider />
+                <Separator />
 
                 <h2 className="mb-1 font-medium">
                   Select which Users should be a part of this Group.
@@ -127,8 +128,8 @@ export const UserGroupCreationForm = ({
                 <div className="flex">
                   <Button
                     type="submit"
-                    size="xs"
-                    color="green"
+                    size="sm"
+                    variant="submit"
                     disabled={isSubmitting}
                     className="mx-auto w-64"
                   >
diff --git a/web/src/app/ee/admin/groups/UserGroupsTable.tsx b/web/src/app/ee/admin/groups/UserGroupsTable.tsx
index 49a84104a6c..e1197031a6d 100644
--- a/web/src/app/ee/admin/groups/UserGroupsTable.tsx
+++ b/web/src/app/ee/admin/groups/UserGroupsTable.tsx
@@ -4,10 +4,9 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-} from "@tremor/react";
+} from "@/components/ui/table";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
 import { LoadingAnimation } from "@/components/Loading";
 import { BasicTable } from "@/components/admin/connectors/BasicTable";
@@ -19,6 +18,7 @@ import { FiEdit2, FiUser } from "react-icons/fi";
 import { User, UserGroup } from "@/lib/types";
 import Link from "next/link";
 import { DeleteButton } from "@/components/DeleteButton";
+import { TableHeader } from "@/components/ui/table";
 
 const MAX_USERS_TO_DISPLAY = 6;
 
@@ -57,15 +57,15 @@ export const UserGroupsTable = ({
   return (
     <div>
       <Table className="overflow-visible">
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Name</TableHeaderCell>
-            <TableHeaderCell>Connectors</TableHeaderCell>
-            <TableHeaderCell>Users</TableHeaderCell>
-            <TableHeaderCell>Status</TableHeaderCell>
-            <TableHeaderCell>Delete</TableHeaderCell>
+            <TableHead>Name</TableHead>
+            <TableHead>Connectors</TableHead>
+            <TableHead>Users</TableHead>
+            <TableHead>Status</TableHead>
+            <TableHead>Delete</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {userGroups
             .filter((userGroup) => !userGroup.is_up_for_deletion)
diff --git a/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx b/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx
index b49ffdce16e..29b325a42e2 100644
--- a/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx
+++ b/web/src/app/ee/admin/groups/[groupId]/GroupDisplay.tsx
@@ -14,19 +14,24 @@ import {
   USER_ROLE_LABELS,
 } from "@/lib/types";
 import { AddConnectorForm } from "./AddConnectorForm";
+import { Separator } from "@/components/ui/separator";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import Text from "@/components/ui/text";
 import {
   Table,
-  TableHead,
-  TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Divider,
-  Button,
-  Text,
-  Select,
-  SelectItem,
-} from "@tremor/react";
+  TableHead,
+  TableHeader,
+  TableRow,
+} from "@/components/ui/table";
+import { Button } from "@/components/ui/button";
 import { DeleteButton } from "@/components/DeleteButton";
 import { Bubble } from "@/components/Bubble";
 import { BookmarkIcon, RobotIcon } from "@/components/icons/icons";
@@ -96,13 +101,19 @@ const UserRoleDropdown = ({
   if (isEditable) {
     return (
       <div className="w-40">
+        Select group
         <Select
           value={localRole}
           onValueChange={handleChange}
           disabled={isSettingRole}
         >
-          <SelectItem value={UserRole.BASIC}>Basic</SelectItem>
-          <SelectItem value={UserRole.CURATOR}>Curator</SelectItem>
+          <SelectTrigger>
+            <SelectValue placeholder="Select role" />
+          </SelectTrigger>
+          <SelectContent>
+            <SelectItem value={UserRole.BASIC}>Basic</SelectItem>
+            <SelectItem value={UserRole.CURATOR}>Curator</SelectItem>
+          </SelectContent>
         </Select>
       </div>
     );
@@ -151,7 +162,7 @@ export const GroupDisplay = ({
         )}
       </div>
 
-      <Divider />
+      <Separator />
 
       <div className="flex w-full">
         <h2 className="text-xl font-bold">Users</h2>
@@ -161,15 +172,15 @@ export const GroupDisplay = ({
         {userGroup.users.length > 0 ? (
           <>
             <Table className="overflow-visible">
-              <TableHead>
+              <TableHeader>
                 <TableRow>
-                  <TableHeaderCell>Email</TableHeaderCell>
-                  <TableHeaderCell>Role</TableHeaderCell>
-                  <TableHeaderCell className="flex w-full">
+                  <TableHead>Email</TableHead>
+                  <TableHead>Role</TableHead>
+                  <TableHead className="flex w-full">
                     <div className="ml-auto">Remove User</div>
-                  </TableHeaderCell>
+                  </TableHead>
                 </TableRow>
-              </TableHead>
+              </TableHeader>
               <TableBody>
                 {userGroup.users.map((groupMember) => {
                   return (
@@ -247,8 +258,8 @@ export const GroupDisplay = ({
 
       <Button
         className="mt-3"
-        size="xs"
-        color="green"
+        size="sm"
+        variant="submit"
         onClick={() => setAddMemberFormVisible(true)}
         disabled={!userGroup.is_up_to_date}
       >
@@ -267,21 +278,21 @@ export const GroupDisplay = ({
         />
       )}
 
-      <Divider />
+      <Separator />
 
       <h2 className="text-xl font-bold mt-8">Connectors</h2>
       <div className="mt-2">
         {userGroup.cc_pairs.length > 0 ? (
           <>
             <Table className="overflow-visible">
-              <TableHead>
+              <TableHeader>
                 <TableRow>
-                  <TableHeaderCell>Connector</TableHeaderCell>
-                  <TableHeaderCell className="flex w-full">
+                  <TableHead>Connector</TableHead>
+                  <TableHead className="flex w-full">
                     <div className="ml-auto">Remove Connector</div>
-                  </TableHeaderCell>
+                  </TableHead>
                 </TableRow>
-              </TableHead>
+              </TableHeader>
               <TableBody>
                 {userGroup.cc_pairs.map((ccPair) => {
                   return (
@@ -347,8 +358,8 @@ export const GroupDisplay = ({
       <Button
         className="mt-3"
         onClick={() => setAddConnectorFormVisible(true)}
-        size="xs"
-        color="green"
+        size="sm"
+        variant="submit"
         disabled={!userGroup.is_up_to_date}
       >
         Add Connectors
@@ -366,7 +377,7 @@ export const GroupDisplay = ({
         />
       )}
 
-      <Divider />
+      <Separator />
 
       <h2 className="text-xl font-bold mt-8 mb-2">Document Sets</h2>
 
@@ -391,7 +402,7 @@ export const GroupDisplay = ({
         )}
       </div>
 
-      <Divider />
+      <Separator />
 
       <h2 className="text-xl font-bold mt-8 mb-2">Assistants</h2>
 
@@ -416,7 +427,7 @@ export const GroupDisplay = ({
         )}
       </div>
 
-      <Divider />
+      <Separator />
 
       <h2 className="text-xl font-bold mt-8 mb-2">Token Rate Limits</h2>
 
@@ -435,8 +446,8 @@ export const GroupDisplay = ({
 
       {isAdmin && (
         <Button
-          color="green"
-          size="xs"
+          variant="submit"
+          size="sm"
           className="mt-3"
           onClick={() => setAddRateLimitFormVisible(true)}
         >
diff --git a/web/src/app/ee/admin/groups/page.tsx b/web/src/app/ee/admin/groups/page.tsx
index 6ee00255fe7..8db128ecc7f 100644
--- a/web/src/app/ee/admin/groups/page.tsx
+++ b/web/src/app/ee/admin/groups/page.tsx
@@ -12,8 +12,10 @@ import {
   useUsers,
 } from "@/lib/hooks";
 import { AdminPageTitle } from "@/components/admin/Title";
-import { Button, Divider } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+
 import { useUser } from "@/components/user/UserProvider";
+import { Separator } from "@/components/ui/separator";
 
 const Main = () => {
   const { popup, setPopup } = usePopup();
@@ -59,14 +61,17 @@ const Main = () => {
       {popup}
       {isAdmin && (
         <div className="my-3">
-          <Button size="xs" color="green" onClick={() => setShowForm(true)}>
+          <Button
+            size="sm"
+            variant="navigate"
+            onClick={() => setShowForm(true)}
+          >
             Create New User Group
           </Button>
         </div>
       )}
       {data.length > 0 && (
         <div>
-          {isAdmin && <Divider />}
           <UserGroupsTable
             userGroups={data}
             setPopup={setPopup}
diff --git a/web/src/app/ee/admin/performance/DateRangeSelector.tsx b/web/src/app/ee/admin/performance/DateRangeSelector.tsx
index 7f5972e3cd7..5947d1b36cf 100644
--- a/web/src/app/ee/admin/performance/DateRangeSelector.tsx
+++ b/web/src/app/ee/admin/performance/DateRangeSelector.tsx
@@ -1,47 +1,113 @@
+import * as React from "react";
+import { Calendar } from "@/components/ui/calendar";
 import {
-  DateRangePicker,
-  DateRangePickerItem,
-  DateRangePickerValue,
-  Text,
-} from "@tremor/react";
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/components/ui/popover";
+import { Button } from "@/components/ui/button";
+import { cn } from "@/lib/utils";
+import { CalendarIcon } from "lucide-react";
+import { format } from "date-fns";
 import { getXDaysAgo } from "./dateUtils";
+import { Separator } from "@/components/ui/separator";
+import { useState } from "react";
 
 export const THIRTY_DAYS = "30d";
 
+export type DateRangePickerValue = DateRange & {
+  selectValue: string;
+};
+
+export type DateRange =
+  | {
+      from: Date;
+      to: Date;
+    }
+  | undefined;
+
 export function DateRangeSelector({
   value,
   onValueChange,
 }: {
-  value: DateRangePickerValue;
-  onValueChange: (value: DateRangePickerValue) => void;
+  value: DateRange;
+  onValueChange: (value: DateRange) => void;
 }) {
+  const [isOpen, setIsOpen] = useState(false);
+
+  const presets = [
+    {
+      label: "Last 30 days",
+      value: {
+        from: getXDaysAgo(30),
+        to: getXDaysAgo(0),
+      },
+    },
+    {
+      label: "Today",
+      value: {
+        from: getXDaysAgo(1),
+        to: getXDaysAgo(0),
+      },
+    },
+  ];
+
   return (
-    <div>
-      <Text className="my-auto mr-2 font-medium mb-1">Date Range</Text>
-      <DateRangePicker
-        className="max-w-md"
-        value={value}
-        onValueChange={onValueChange}
-        selectPlaceholder="Select"
-        enableClear={false}
-      >
-        <DateRangePickerItem
-          key={THIRTY_DAYS}
-          value={THIRTY_DAYS}
-          from={getXDaysAgo(30)}
-          to={getXDaysAgo(0)}
-        >
-          Last 30 days
-        </DateRangePickerItem>
-        <DateRangePickerItem
-          key="today"
-          value="today"
-          from={getXDaysAgo(1)}
-          to={getXDaysAgo(0)}
-        >
-          Today
-        </DateRangePickerItem>
-      </DateRangePicker>
+    <div className="grid gap-2">
+      <Popover open={isOpen} onOpenChange={setIsOpen}>
+        <PopoverTrigger asChild>
+          <Button
+            variant="outline"
+            className={cn(
+              "w-[300px] justify-start text-left font-normal",
+              !value && "text-muted-foreground"
+            )}
+          >
+            <CalendarIcon className="mr-2 h-4 w-4" />
+            {value?.from ? (
+              value.to ? (
+                <>
+                  {format(value.from, "LLL dd, y")} -{" "}
+                  {format(value.to, "LLL dd, y")}
+                </>
+              ) : (
+                format(value.from, "LLL dd, y")
+              )
+            ) : (
+              <span>Pick a date range</span>
+            )}
+          </Button>
+        </PopoverTrigger>
+        <PopoverContent className="w-auto p-0" align="start">
+          <Calendar
+            initialFocus
+            mode="range"
+            defaultMonth={value?.from}
+            selected={value}
+            onSelect={(range) => {
+              if (range?.from && range?.to) {
+                onValueChange({ from: range.from, to: range.to });
+              }
+            }}
+            numberOfMonths={2}
+          />
+          <div className="border-t p-3">
+            {presets.map((preset) => (
+              <Button
+                key={preset.label}
+                variant="ghost"
+                className="w-full justify-start"
+                onClick={() => {
+                  onValueChange(preset.value);
+                  setIsOpen(false);
+                }}
+              >
+                {preset.label}
+              </Button>
+            ))}
+          </div>
+        </PopoverContent>
+      </Popover>
     </div>
   );
 }
diff --git a/web/src/app/ee/admin/performance/custom-analytics/CustomAnalyticsUpdateForm.tsx b/web/src/app/ee/admin/performance/custom-analytics/CustomAnalyticsUpdateForm.tsx
index c542a69eba1..debd1504462 100644
--- a/web/src/app/ee/admin/performance/custom-analytics/CustomAnalyticsUpdateForm.tsx
+++ b/web/src/app/ee/admin/performance/custom-analytics/CustomAnalyticsUpdateForm.tsx
@@ -3,7 +3,9 @@
 import { Label, SubLabel } from "@/components/admin/connectors/Field";
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
-import { Button, Callout, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Callout } from "@/components/ui/callout";
+import Text from "@/components/ui/text";
 import { useContext, useState } from "react";
 
 export function CustomAnalyticsUpdateForm() {
@@ -17,7 +19,7 @@ export function CustomAnalyticsUpdateForm() {
   const { popup, setPopup } = usePopup();
 
   if (!settings) {
-    return <Callout color="red" title="Failed to fetch settings"></Callout>;
+    return <Callout type="danger" title="Failed to fetch settings"></Callout>;
   }
 
   return (
diff --git a/web/src/app/ee/admin/performance/custom-analytics/page.tsx b/web/src/app/ee/admin/performance/custom-analytics/page.tsx
index b0fc619f24f..d4e1258ea0f 100644
--- a/web/src/app/ee/admin/performance/custom-analytics/page.tsx
+++ b/web/src/app/ee/admin/performance/custom-analytics/page.tsx
@@ -1,7 +1,8 @@
 import { AdminPageTitle } from "@/components/admin/Title";
 import { CUSTOM_ANALYTICS_ENABLED } from "@/lib/constants";
-import { Callout, Text } from "@tremor/react";
+import { Callout } from "@/components/ui/callout";
 import { FiBarChart2 } from "react-icons/fi";
+import Text from "@/components/ui/text";
 import { CustomAnalyticsUpdateForm } from "./CustomAnalyticsUpdateForm";
 
 function Main() {
@@ -9,7 +10,7 @@ function Main() {
     return (
       <div>
         <div className="mt-4">
-          <Callout title="Custom Analytics is not enabled." color="red">
+          <Callout type="danger" title="Custom Analytics is not enabled.">
             To set up custom analytics scripts, please work with the team who
             setup Danswer in your organization to set the{" "}
             <i>CUSTOM_ANALYTICS_SECRET_KEY</i> environment variable.
diff --git a/web/src/app/ee/admin/performance/lib.ts b/web/src/app/ee/admin/performance/lib.ts
index a603166139d..d9068b466ff 100644
--- a/web/src/app/ee/admin/performance/lib.ts
+++ b/web/src/app/ee/admin/performance/lib.ts
@@ -9,13 +9,14 @@ import {
 import { useState } from "react";
 import { buildApiPath } from "@/lib/urlBuilder";
 import { Feedback } from "@/lib/types";
-import { DateRangePickerValue } from "@tremor/react";
+
 import {
   convertDateToEndOfDay,
   convertDateToStartOfDay,
   getXDaysAgo,
 } from "./dateUtils";
 import { THIRTY_DAYS } from "./DateRangeSelector";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 
 export const useTimeRange = () => {
   return useState<DateRangePickerValue>({
diff --git a/web/src/app/ee/admin/performance/query-history/FeedbackBadge.tsx b/web/src/app/ee/admin/performance/query-history/FeedbackBadge.tsx
index ebe75e7a02b..b4b778b06e9 100644
--- a/web/src/app/ee/admin/performance/query-history/FeedbackBadge.tsx
+++ b/web/src/app/ee/admin/performance/query-history/FeedbackBadge.tsx
@@ -1,5 +1,5 @@
+import { Badge } from "@/components/ui/badge";
 import { Feedback } from "@/lib/types";
-import { Badge } from "@tremor/react";
 
 export function FeedbackBadge({
   feedback,
@@ -10,28 +10,28 @@ export function FeedbackBadge({
   switch (feedback) {
     case "like":
       feedbackBadge = (
-        <Badge color="green" className="text-sm">
+        <Badge variant="success" className="text-sm">
           Like
         </Badge>
       );
       break;
     case "dislike":
       feedbackBadge = (
-        <Badge color="red" className="text-sm">
+        <Badge variant="destructive" className="text-sm">
           Dislike
         </Badge>
       );
       break;
     case "mixed":
       feedbackBadge = (
-        <Badge color="purple" className="text-sm">
+        <Badge variant="purple" className="text-sm">
           Mixed
         </Badge>
       );
       break;
     default:
       feedbackBadge = (
-        <Badge color="gray" className="text-sm">
+        <Badge variant="outline" className="text-sm">
           N/A
         </Badge>
       );
diff --git a/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx b/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx
index 43a206bc543..b46d77c27d5 100644
--- a/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx
+++ b/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx
@@ -1,18 +1,22 @@
 import { useQueryHistory } from "../lib";
-
+import { Separator } from "@/components/ui/separator";
 import {
-  Card,
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Text,
-  Divider,
+  TableHeader,
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
+
+import {
   Select,
   SelectItem,
-} from "@tremor/react";
+  SelectValue,
+  SelectTrigger,
+  SelectContent,
+} from "@/components/ui/select";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { ChatSessionMinimal } from "../usage/types";
 import { timestampToReadableDate } from "@/lib/dateUtils";
@@ -24,6 +28,7 @@ import { PageSelector } from "@/components/PageSelector";
 import Link from "next/link";
 import { FeedbackBadge } from "./FeedbackBadge";
 import { DownloadAsCSV } from "./DownloadAsCSV";
+import CardSection from "@/components/admin/CardSection";
 
 const NUM_IN_PAGE = 20;
 
@@ -80,17 +85,30 @@ function SelectFeedbackType({
         <Select
           value={value}
           onValueChange={onValueChange as (value: string) => void}
-          enableClear={false}
         >
-          <SelectItem value="all" icon={FiMinus}>
-            Any
-          </SelectItem>
-          <SelectItem value="like" icon={FiSmile}>
-            Like
-          </SelectItem>
-          <SelectItem value="dislike" icon={FiFrown}>
-            Dislike
-          </SelectItem>
+          <SelectTrigger>
+            <SelectValue placeholder="Select feedback type" />
+          </SelectTrigger>
+          <SelectContent>
+            <SelectItem value="all">
+              <div className="flex items-center gap-2">
+                <FiMinus className="h-4 w-4" />
+                <span>Any</span>
+              </div>
+            </SelectItem>
+            <SelectItem value="like">
+              <div className="flex items-center gap-2">
+                <FiSmile className="h-4 w-4" />
+                <span>Like</span>
+              </div>
+            </SelectItem>
+            <SelectItem value="dislike">
+              <div className="flex items-center gap-2">
+                <FiFrown className="h-4 w-4" />
+                <span>Dislike</span>
+              </div>
+            </SelectItem>
+          </SelectContent>
         </Select>
       </div>
     </div>
@@ -109,7 +127,7 @@ export function QueryHistoryTable() {
   const [page, setPage] = useState(1);
 
   return (
-    <Card className="mt-8">
+    <CardSection className="mt-8">
       {chatSessionData ? (
         <>
           <div className="flex">
@@ -121,24 +139,31 @@ export function QueryHistoryTable() {
 
               <DateRangeSelector
                 value={timeRange}
-                onValueChange={setTimeRange}
+                onValueChange={(value) => {
+                  if (value) {
+                    setTimeRange({
+                      ...value,
+                      selectValue: timeRange.selectValue,
+                    });
+                  }
+                }}
               />
             </div>
 
             <DownloadAsCSV />
           </div>
-          <Divider />
+          <Separator />
           <Table className="mt-5">
-            <TableHead>
+            <TableHeader>
               <TableRow>
-                <TableHeaderCell>First User Message</TableHeaderCell>
-                <TableHeaderCell>First AI Response</TableHeaderCell>
-                <TableHeaderCell>Feedback</TableHeaderCell>
-                <TableHeaderCell>User</TableHeaderCell>
-                <TableHeaderCell>Persona</TableHeaderCell>
-                <TableHeaderCell>Date</TableHeaderCell>
+                <TableHead>First User Message</TableHead>
+                <TableHead>First AI Response</TableHead>
+                <TableHead>Feedback</TableHead>
+                <TableHead>User</TableHead>
+                <TableHead>Persona</TableHead>
+                <TableHead>Date</TableHead>
               </TableRow>
-            </TableHead>
+            </TableHeader>
             <TableBody>
               {chatSessionData
                 .slice(NUM_IN_PAGE * (page - 1), NUM_IN_PAGE * page)
@@ -173,6 +198,6 @@ export function QueryHistoryTable() {
           <ThreeDotsLoader />
         </div>
       )}
-    </Card>
+    </CardSection>
   );
 }
diff --git a/web/src/app/ee/admin/performance/query-history/[id]/page.tsx b/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
index 531387664d0..3f924267cb2 100644
--- a/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
+++ b/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
@@ -1,6 +1,8 @@
 "use client";
 
-import { Bold, Text, Card, Title, Divider } from "@tremor/react";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Separator } from "@/components/ui/separator";
 import { ChatSessionSnapshot, MessageSnapshot } from "../../usage/types";
 import { FiBook } from "react-icons/fi";
 import { timestampToReadableDate } from "@/lib/dateUtils";
@@ -10,17 +12,18 @@ import { errorHandlingFetcher } from "@/lib/fetcher";
 import useSWR from "swr";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { ThreeDotsLoader } from "@/components/Loading";
+import CardSection from "@/components/admin/CardSection";
 
 function MessageDisplay({ message }: { message: MessageSnapshot }) {
   return (
     <div>
-      <Bold className="text-xs mb-1">
+      <p className="text-xs font-bold mb-1">
         {message.message_type === "user" ? "User" : "AI"}
-      </Bold>
+      </p>
       <Text>{message.message}</Text>
       {message.documents.length > 0 && (
         <div className="flex flex-col gap-y-2 mt-2">
-          <Bold className="font-bold text-xs">Reference Documents</Bold>
+          <p className="font-bold text-xs">Reference Documents</p>
           {message.documents.slice(0, 5).map((document) => {
             return (
               <Text className="flex" key={document.document_id}>
@@ -48,14 +51,14 @@ function MessageDisplay({ message }: { message: MessageSnapshot }) {
       )}
       {message.feedback_type && (
         <div className="mt-2">
-          <Bold className="font-bold text-xs">Feedback</Bold>
+          <p className="font-bold text-xs">Feedback</p>
           {message.feedback_text && <Text>{message.feedback_text}</Text>}
           <div className="mt-1">
             <FeedbackBadge feedback={message.feedback_type} />
           </div>
         </div>
       )}
-      <Divider />
+      <Separator />
     </div>
   );
 }
@@ -87,7 +90,7 @@ export default function QueryPage({ params }: { params: { id: string } }) {
     <main className="pt-4 mx-auto container">
       <BackButton />
 
-      <Card className="mt-4">
+      <CardSection className="mt-4">
         <Title>Chat Session Details</Title>
 
         <Text className="flex flex-wrap whitespace-normal mt-1 text-xs">
@@ -97,7 +100,7 @@ export default function QueryPage({ params }: { params: { id: string } }) {
           {chatSessionSnapshot.flow_type}
         </Text>
 
-        <Divider />
+        <Separator />
 
         <div className="flex flex-col">
           {chatSessionSnapshot.messages.map((message) => {
@@ -106,7 +109,7 @@ export default function QueryPage({ params }: { params: { id: string } }) {
             );
           })}
         </div>
-      </Card>
+      </CardSection>
     </main>
   );
 }
diff --git a/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx b/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx
index 9d15abcf545..3e4966b5454 100644
--- a/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx
+++ b/web/src/app/ee/admin/performance/usage/DanswerBotChart.tsx
@@ -1,12 +1,10 @@
 import { ThreeDotsLoader } from "@/components/Loading";
 import { getDatesList, useDanswerBotAnalytics } from "../lib";
-import {
-  AreaChart,
-  Card,
-  Title,
-  Text,
-  DateRangePickerValue,
-} from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import CardSection from "@/components/admin/CardSection";
+import { AreaChartDisplay } from "@/components/ui/areaChart";
 
 export function DanswerBotChart({
   timeRange,
@@ -45,8 +43,8 @@ export function DanswerBotChart({
     );
 
     chart = (
-      <AreaChart
-        className="mt-4 h-80"
+      <AreaChartDisplay
+        className="mt-4"
         data={dateRange.map((dateStr) => {
           const danswerBotAnalyticsForDate =
             dateToDanswerBotAnalytics.get(dateStr);
@@ -69,10 +67,10 @@ export function DanswerBotChart({
   }
 
   return (
-    <Card className="mt-8">
+    <CardSection className="mt-8">
       <Title>Slack Bot</Title>
       <Text>Total Queries vs Auto Resolved</Text>
       {chart}
-    </Card>
+    </CardSection>
   );
 }
diff --git a/web/src/app/ee/admin/performance/usage/FeedbackChart.tsx b/web/src/app/ee/admin/performance/usage/FeedbackChart.tsx
index a466d866e06..c254f1cec5f 100644
--- a/web/src/app/ee/admin/performance/usage/FeedbackChart.tsx
+++ b/web/src/app/ee/admin/performance/usage/FeedbackChart.tsx
@@ -1,12 +1,11 @@
 import { ThreeDotsLoader } from "@/components/Loading";
 import { getDatesList, useQueryAnalytics } from "../lib";
-import {
-  AreaChart,
-  Card,
-  Title,
-  Text,
-  DateRangePickerValue,
-} from "@tremor/react";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
+import CardSection from "@/components/admin/CardSection";
+import { AreaChartDisplay } from "@/components/ui/areaChart";
 
 export function FeedbackChart({
   timeRange,
@@ -44,8 +43,8 @@ export function FeedbackChart({
     );
 
     chart = (
-      <AreaChart
-        className="mt-4 h-80"
+      <AreaChartDisplay
+        className="mt-4"
         data={dateRange.map((dateStr) => {
           const queryAnalyticsForDate = dateToQueryAnalytics.get(dateStr);
           return {
@@ -66,10 +65,10 @@ export function FeedbackChart({
   }
 
   return (
-    <Card className="mt-8">
+    <CardSection className="mt-8">
       <Title>Feedback</Title>
       <Text>Thumbs Up / Thumbs Down over time</Text>
       {chart}
-    </Card>
+    </CardSection>
   );
 }
diff --git a/web/src/app/ee/admin/performance/usage/QueryPerformanceChart.tsx b/web/src/app/ee/admin/performance/usage/QueryPerformanceChart.tsx
index b16e80bf698..4a3aaaf168c 100644
--- a/web/src/app/ee/admin/performance/usage/QueryPerformanceChart.tsx
+++ b/web/src/app/ee/admin/performance/usage/QueryPerformanceChart.tsx
@@ -1,14 +1,12 @@
 "use client";
 
-import {
-  Card,
-  AreaChart,
-  Title,
-  Text,
-  DateRangePickerValue,
-} from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { getDatesList, useQueryAnalytics, useUserAnalytics } from "../lib";
 import { ThreeDotsLoader } from "@/components/Loading";
+import { AreaChartDisplay } from "@/components/ui/areaChart";
+import Title from "@/components/ui/title";
+import Text from "@/components/ui/text";
+import CardSection from "@/components/admin/CardSection";
 
 export function QueryPerformanceChart({
   timeRange,
@@ -62,8 +60,8 @@ export function QueryPerformanceChart({
     );
 
     chart = (
-      <AreaChart
-        className="h-80"
+      <AreaChartDisplay
+        className="mt-4"
         data={dateRange.map((dateStr) => {
           const queryAnalyticsForDate = dateToQueryAnalytics.get(dateStr);
           const userAnalyticsForDate = dateToUserAnalytics.get(dateStr);
@@ -85,10 +83,10 @@ export function QueryPerformanceChart({
   }
 
   return (
-    <Card className="mt-8">
+    <CardSection className="mt-8">
       <Title>Usage</Title>
       <Text>Usage over time</Text>
       {chart}
-    </Card>
+    </CardSection>
   );
 }
diff --git a/web/src/app/ee/admin/performance/usage/UsageReports.tsx b/web/src/app/ee/admin/performance/usage/UsageReports.tsx
index f0516254354..539ff6ee2d7 100644
--- a/web/src/app/ee/admin/performance/usage/UsageReports.tsx
+++ b/web/src/app/ee/admin/performance/usage/UsageReports.tsx
@@ -1,23 +1,20 @@
 "use client";
 
+import { format } from "date-fns";
 import { errorHandlingFetcher } from "@/lib/fetcher";
 
 import { FiDownload, FiDownloadCloud } from "react-icons/fi";
 import {
-  DateRangePicker,
-  DateRangePickerItem,
-  DateRangePickerValue,
-  Divider,
   Table,
   TableBody,
   TableCell,
   TableHead,
-  TableHeaderCell,
+  TableHeader,
   TableRow,
-  Text,
-  Title,
-  Button,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
+import Title from "@/components/ui/title";
+import { Button } from "@/components/ui/button";
 import useSWR from "swr";
 import { useState } from "react";
 import { UsageReport } from "./types";
@@ -26,6 +23,16 @@ import Link from "next/link";
 import { humanReadableFormat, humanReadableFormatWithTime } from "@/lib/time";
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { PageSelector } from "@/components/PageSelector";
+import { Separator } from "@/components/ui/separator";
+import { DateRangePickerValue } from "../DateRangeSelector";
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from "@/components/ui/popover";
+import { CalendarIcon } from "lucide-react";
+import { Calendar } from "@/components/ui/calendar";
+import { cn } from "@/lib/utils";
 
 function GenerateReportInput() {
   const [dateRange, setDateRange] = useState<DateRangePickerValue | undefined>(
@@ -101,41 +108,109 @@ function GenerateReportInput() {
       <Text className="mb-8">
         Generate usage statistics for users in the workspace.
       </Text>
-      <DateRangePicker
-        maxDate={new Date()}
-        defaultValue={{
-          from: undefined,
-          to: undefined,
-          selectValue: "allTime",
-        }}
-        className="mb-3"
-        enableClear={false}
-        selectPlaceholder="Range"
-        value={dateRange}
-        onValueChange={setDateRange}
-      >
-        <DateRangePickerItem key="lastWeek" value="lastWeek" from={lastWeek}>
-          Last 7 days
-        </DateRangePickerItem>
-        <DateRangePickerItem key="lastMonth" value="lastMonth" from={lastMonth}>
-          Last 30 days
-        </DateRangePickerItem>
-        <DateRangePickerItem key="lastYear" value="lastYear" from={lastYear}>
-          Last year
-        </DateRangePickerItem>
-        <DateRangePickerItem
-          key="allTime"
-          value="allTime"
-          from={new Date(1970, 0, 1)}
-        >
-          All time
-        </DateRangePickerItem>
-      </DateRangePicker>
+      <div className="grid gap-2 mb-3">
+        <Popover>
+          <PopoverTrigger asChild>
+            <Button
+              variant="outline"
+              className={cn(
+                "w-[300px] justify-start text-left font-normal",
+                !dateRange && "text-muted-foreground"
+              )}
+            >
+              <CalendarIcon className="mr-2 h-4 w-4" />
+              {dateRange?.from ? (
+                dateRange.to ? (
+                  <>
+                    {format(dateRange.from, "LLL dd, y")} -{" "}
+                    {format(dateRange.to, "LLL dd, y")}
+                  </>
+                ) : (
+                  format(dateRange.from, "LLL dd, y")
+                )
+              ) : (
+                <span>Pick a date range</span>
+              )}
+            </Button>
+          </PopoverTrigger>
+          <PopoverContent className="w-auto p-0" align="start">
+            <Calendar
+              initialFocus
+              mode="range"
+              defaultMonth={dateRange?.from}
+              selected={dateRange}
+              onSelect={(range) =>
+                range?.from &&
+                setDateRange({
+                  from: range.from,
+                  to: range.to ?? range.from,
+                  selectValue: "custom",
+                })
+              }
+              numberOfMonths={2}
+              disabled={(date) => date > new Date()}
+            />
+            <div className="border-t p-3">
+              <Button
+                variant="ghost"
+                className="w-full justify-start"
+                onClick={() => {
+                  setDateRange({
+                    from: lastWeek,
+                    to: new Date(),
+                    selectValue: "lastWeek",
+                  });
+                }}
+              >
+                Last 7 days
+              </Button>
+              <Button
+                variant="ghost"
+                className="w-full justify-start"
+                onClick={() => {
+                  setDateRange({
+                    from: lastMonth,
+                    to: new Date(),
+                    selectValue: "lastMonth",
+                  });
+                }}
+              >
+                Last 30 days
+              </Button>
+              <Button
+                variant="ghost"
+                className="w-full justify-start"
+                onClick={() => {
+                  setDateRange({
+                    from: lastYear,
+                    to: new Date(),
+                    selectValue: "lastYear",
+                  });
+                }}
+              >
+                Last year
+              </Button>
+              <Button
+                variant="ghost"
+                className="w-full justify-start"
+                onClick={() => {
+                  setDateRange({
+                    from: new Date(1970, 0, 1),
+                    to: new Date(),
+                    selectValue: "allTime",
+                  });
+                }}
+              >
+                All time
+              </Button>
+            </div>
+          </PopoverContent>
+        </Popover>
+      </div>
       <Button
         color={"blue"}
         icon={FiDownloadCloud}
-        size="xs"
-        loading={isLoading}
+        size="sm"
         disabled={isLoading}
         onClick={() => requestReport()}
       >
@@ -190,15 +265,15 @@ function UsageReportsTable() {
       ) : (
         <>
           <Table>
-            <TableHead>
+            <TableHeader>
               <TableRow>
-                <TableHeaderCell>Report</TableHeaderCell>
-                <TableHeaderCell>Period</TableHeaderCell>
-                <TableHeaderCell>Generated By</TableHeaderCell>
-                <TableHeaderCell>Time Generated</TableHeaderCell>
-                <TableHeaderCell>Download</TableHeaderCell>
+                <TableHead>Report</TableHead>
+                <TableHead>Period</TableHead>
+                <TableHead>Generated By</TableHead>
+                <TableHead>Time Generated</TableHead>
+                <TableHead>Download</TableHead>
               </TableRow>
-            </TableHead>
+            </TableHeader>
 
             <TableBody>
               {paginatedReports.map((r) => (
@@ -256,7 +331,7 @@ export default function UsageReports() {
   return (
     <div className="mx-auto container">
       <GenerateReportInput />
-      <Divider />
+      <Separator />
       <UsageReportsTable />
     </div>
   );
diff --git a/web/src/app/ee/admin/performance/usage/page.tsx b/web/src/app/ee/admin/performance/usage/page.tsx
index ccb2822e253..e1fffc323a2 100644
--- a/web/src/app/ee/admin/performance/usage/page.tsx
+++ b/web/src/app/ee/admin/performance/usage/page.tsx
@@ -8,25 +8,25 @@ import { useTimeRange } from "../lib";
 import { AdminPageTitle } from "@/components/admin/Title";
 import { FiActivity } from "react-icons/fi";
 import UsageReports from "./UsageReports";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 
 export default function AnalyticsPage() {
   const [timeRange, setTimeRange] = useTimeRange();
 
   return (
     <main className="pt-4 mx-auto container">
-      {/* TODO: remove this `dark` once we have a mode selector */}
       <AdminPageTitle
         title="Usage Statistics"
         icon={<FiActivity size={32} />}
       />
-
-      <DateRangeSelector value={timeRange} onValueChange={setTimeRange} />
-
+      <DateRangeSelector
+        value={timeRange}
+        onValueChange={(value) => setTimeRange(value as any)}
+      />
       <QueryPerformanceChart timeRange={timeRange} />
       <FeedbackChart timeRange={timeRange} />
       <DanswerBotChart timeRange={timeRange} />
-      <Divider />
+      <Separator />
       <UsageReports />
     </main>
   );
diff --git a/web/src/app/ee/admin/standard-answer/StandardAnswerCreationForm.tsx b/web/src/app/ee/admin/standard-answer/StandardAnswerCreationForm.tsx
index 15574701fb4..a1f42d7896d 100644
--- a/web/src/app/ee/admin/standard-answer/StandardAnswerCreationForm.tsx
+++ b/web/src/app/ee/admin/standard-answer/StandardAnswerCreationForm.tsx
@@ -2,7 +2,8 @@
 
 import { usePopup } from "@/components/admin/connectors/Popup";
 import { StandardAnswerCategory, StandardAnswer } from "@/lib/types";
-import { Button, Card } from "@tremor/react";
+import CardSection from "@/components/admin/CardSection";
+import { Button } from "@/components/ui/button";
 import { Form, Formik } from "formik";
 import { useRouter } from "next/navigation";
 import * as Yup from "yup";
@@ -41,7 +42,7 @@ export const StandardAnswerCreationForm = ({
 
   return (
     <div>
-      <Card>
+      <CardSection>
         {popup}
         <Formik
           initialValues={{
@@ -200,9 +201,10 @@ export const StandardAnswerCreationForm = ({
                   }))}
                 />
               </div>
-              <div className="p-4 flex">
+              <div className="py-4 flex">
                 <Button
                   type="submit"
+                  variant="submit"
                   disabled={isSubmitting}
                   className="mx-auto w-64"
                 >
@@ -212,7 +214,7 @@ export const StandardAnswerCreationForm = ({
             </Form>
           )}
         </Formik>
-      </Card>
+      </CardSection>
     </div>
   );
 };
diff --git a/web/src/app/ee/admin/standard-answer/page.tsx b/web/src/app/ee/admin/standard-answer/page.tsx
index 0cff20af848..6bc921fdefa 100644
--- a/web/src/app/ee/admin/standard-answer/page.tsx
+++ b/web/src/app/ee/admin/standard-answer/page.tsx
@@ -6,17 +6,16 @@ import { PopupSpec, usePopup } from "@/components/admin/connectors/Popup";
 import { useStandardAnswers, useStandardAnswerCategories } from "./hooks";
 import { ThreeDotsLoader } from "@/components/Loading";
 import { ErrorCallout } from "@/components/ErrorCallout";
+import { Button } from "@/components/ui/button";
+import { Separator } from "@/components/ui/separator";
 import {
-  Button,
-  Divider,
-  Text,
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-} from "@tremor/react";
+} from "@/components/ui/table";
+
 import Link from "next/link";
 import { StandardAnswer, StandardAnswerCategory } from "@/lib/types";
 import { MagnifyingGlass } from "@phosphor-icons/react";
@@ -28,6 +27,8 @@ import { FilterDropdown } from "@/components/search/filtering/FilterDropdown";
 import { FiTag } from "react-icons/fi";
 import { PageSelector } from "@/components/PageSelector";
 import { CustomCheckbox } from "@/components/CustomCheckbox";
+import Text from "@/components/ui/text";
+import { TableHeader } from "@/components/ui/table";
 
 const NUM_RESULTS_PER_PAGE = 10;
 
@@ -290,16 +291,14 @@ const StandardAnswersTable = ({
         </div>
       </div>
       <div className="mx-auto">
-        <Table>
-          <TableHead>
+        <Table className="w-full flex items-stretch">
+          <TableHeader>
             <TableRow>
               {columns.map((column) => (
-                <TableHeaderCell key={column.key}>
-                  {column.name}
-                </TableHeaderCell>
+                <TableHead key={column.key}>{column.name}</TableHead>
               ))}
             </TableRow>
-          </TableHead>
+          </TableHeader>
 
           <TableBody>
             {paginatedStandardAnswers.length > 0 ? (
@@ -404,12 +403,12 @@ const Main = () => {
       <div className="mb-2"></div>
 
       <Link className="flex mb-3 mt-2 w-fit" href="/admin/standard-answer/new">
-        <Button className="my-auto" color="green" size="xs">
+        <Button className="my-auto" variant="submit" size="sm">
           New Standard Answer
         </Button>
       </Link>
 
-      <Divider />
+      <Separator />
 
       <div>
         <StandardAnswersTable
diff --git a/web/src/app/ee/admin/whitelabeling/WhitelabelingForm.tsx b/web/src/app/ee/admin/whitelabeling/WhitelabelingForm.tsx
index 954d0ab8e49..475c689441a 100644
--- a/web/src/app/ee/admin/whitelabeling/WhitelabelingForm.tsx
+++ b/web/src/app/ee/admin/whitelabeling/WhitelabelingForm.tsx
@@ -12,10 +12,12 @@ import {
   SubLabel,
   TextFormField,
 } from "@/components/admin/connectors/Field";
-import { Button, Divider, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 import { ImageUpload } from "./ImageUpload";
 import { AdvancedOptionsToggle } from "@/components/AdvancedOptionsToggle";
 import Link from "next/link";
+import { Separator } from "@/components/ui/separator";
 
 export function WhitelabelingForm() {
   const router = useRouter();
@@ -151,8 +153,8 @@ export function WhitelabelingForm() {
                 />
 
                 <Button
-                  color="red"
-                  size="xs"
+                  variant="destructive"
+                  size="sm"
                   type="button"
                   className="mb-8"
                   onClick={async () => {
@@ -183,7 +185,7 @@ export function WhitelabelingForm() {
               setSelectedFile={setSelectedLogo}
             />
 
-            <Divider />
+            <Separator />
 
             <AdvancedOptionsToggle
               showAdvancedOptions={showAdvancedOptions}
@@ -217,7 +219,7 @@ export function WhitelabelingForm() {
                   subtext="If enabled, the chat header will be displayed on two lines instead of one."
                 />
 
-                <Divider />
+                <Separator />
 
                 <TextFormField
                   label={
@@ -229,7 +231,9 @@ export function WhitelabelingForm() {
                   subtext={
                     values.enable_consent_screen
                       ? `The title for the consent screen that will be displayed for each user on their initial visit to the application. If left blank, title will default to "Terms of Use".`
-                      : `The title for the popup that will be displayed for each user on their initial visit to the application. If left blank AND Custom Popup Content is specified, will use "Welcome to ${values.application_name || "Danswer"}!".`
+                      : `The title for the popup that will be displayed for each user on their initial visit to the application. If left blank AND Custom Popup Content is specified, will use "Welcome to ${
+                          values.application_name || "Danswer"
+                        }!".`
                   }
                   placeholder={
                     values.enable_consent_screen
@@ -292,8 +296,8 @@ export function WhitelabelingForm() {
                       />
 
                       <Button
-                        color="red"
-                        size="xs"
+                        variant="destructive"
+                        size="sm"
                         type="button"
                         className="mb-8"
                         onClick={async () => {
diff --git a/web/src/app/globals.css b/web/src/app/globals.css
index 145263ee359..b3eaa38a23e 100644
--- a/web/src/app/globals.css
+++ b/web/src/app/globals.css
@@ -4,6 +4,32 @@
 
 @layer base {
   :root {
+    --background: 0 0% 100%;
+    --foreground: 0 0% 3.9%;
+    --card: 0 0% 100%;
+    --card-foreground: 0 0% 3.9%;
+    --popover: 0 0% 100%;
+    --popover-foreground: 0 0% 3.9%;
+    --primary: 0 0% 9%;
+    --primary-foreground: 0 0% 98%;
+    --secondary: 0 0% 96.1%;
+    --secondary-foreground: 0 0% 9%;
+    --muted: 0 0% 96.1%;
+    --muted-foreground: 0 0% 45.1%;
+    --accent: 0 0% 96.1%;
+    --accent-foreground: 0 0% 9%;
+    --destructive: 0 84.2% 60.2%;
+    --destructive-foreground: 0 0% 98%;
+    --border: 0 0% 89.8%;
+    --input: 0 0% 89.8%;
+    --ring: 0 0% 3.9%;
+    --radius: 0.5rem;
+    --chart-1: 12 76% 61%;
+    --chart-2: 173 58% 39%;
+    --chart-3: 197 37% 24%;
+    --chart-4: 43 74% 66%;
+    --chart-5: 27 87% 67%;
+
     --text-50: #fafafa;
     /* neutral-50 - Very light gray */
     --text-100: #f5f5f5;
@@ -189,6 +215,33 @@
     /* gray-50 */
     --dark-tremor-content-inverted: #000000;
     /* black */
+    --radius: 0.5rem;
+  }
+  .dark {
+    --background: 0 0% 3.9%;
+    --foreground: 0 0% 98%;
+    --card: 0 0% 3.9%;
+    --card-foreground: 0 0% 98%;
+    --popover: 0 0% 3.9%;
+    --popover-foreground: 0 0% 98%;
+    --primary: 0 0% 98%;
+    --primary-foreground: 0 0% 9%;
+    --secondary: 0 0% 14.9%;
+    --secondary-foreground: 0 0% 98%;
+    --muted: 0 0% 14.9%;
+    --muted-foreground: 0 0% 63.9%;
+    --accent: 0 0% 14.9%;
+    --accent-foreground: 0 0% 98%;
+    --destructive: 0 62.8% 30.6%;
+    --destructive-foreground: 0 0% 98%;
+    --border: 0 0% 14.9%;
+    --input: 0 0% 14.9%;
+    --ring: 0 0% 83.1%;
+    --chart-1: 220 70% 50%;
+    --chart-2: 160 60% 45%;
+    --chart-3: 30 80% 55%;
+    --chart-4: 280 65% 60%;
+    --chart-5: 340 75% 55%;
   }
 }
 
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index 6fd2799ab85..07592b2fda6 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -12,7 +12,6 @@ import { Metadata } from "next";
 import { buildClientUrl } from "@/lib/utilsSS";
 import { Inter } from "next/font/google";
 import { EnterpriseSettings, GatingType } from "./admin/settings/interfaces";
-import { Card } from "@tremor/react";
 import { HeaderTitle } from "@/components/header/HeaderTitle";
 import { Logo } from "@/components/Logo";
 import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
@@ -20,6 +19,7 @@ import { AppProvider } from "@/components/context/AppProvider";
 import { PHProvider } from "./providers";
 import { default as dynamicImport } from "next/dynamic";
 import { getCurrentUserSS } from "@/lib/userSS";
+import CardSection from "@/components/admin/CardSection";
 
 const PostHogPageView = dynamicImport(() => import("./PostHogPageView"), {
   ssr: false,
@@ -104,7 +104,7 @@ export default async function RootLayout({
           <Logo height={40} width={40} />
         </div>
 
-        <Card className="p-8 max-w-md">
+        <CardSection className="max-w-md">
           <h1 className="text-2xl font-bold mb-4 text-error">Error</h1>
           <p className="text-text-500">
             Your Danswer instance was not configured properly and your settings
@@ -137,7 +137,7 @@ export default async function RootLayout({
             </a>
             .
           </p>
-        </Card>
+        </CardSection>
       </div>
     );
   }
@@ -148,7 +148,7 @@ export default async function RootLayout({
           <HeaderTitle>Danswer</HeaderTitle>
           <Logo height={40} width={40} />
         </div>
-        <Card className="p-8 max-w-md">
+        <CardSection className="w-full max-w-md">
           <h1 className="text-2xl font-bold mb-4 text-error">
             Access Restricted
           </h1>
@@ -165,7 +165,7 @@ export default async function RootLayout({
             billing section. For other users, please reach out to your
             administrator to address this matter.
           </p>
-        </Card>
+        </CardSection>
       </div>
     );
   }
diff --git a/web/src/components/AdvancedOptionsToggle.tsx b/web/src/components/AdvancedOptionsToggle.tsx
index 0be0571ddde..3824930b29d 100644
--- a/web/src/components/AdvancedOptionsToggle.tsx
+++ b/web/src/components/AdvancedOptionsToggle.tsx
@@ -1,5 +1,5 @@
 import React from "react";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { FiChevronDown, FiChevronRight } from "react-icons/fi";
 
 interface AdvancedOptionsToggleProps {
@@ -15,8 +15,8 @@ export function AdvancedOptionsToggle({
     <div>
       <Button
         type="button"
-        variant="light"
-        size="xs"
+        variant="link"
+        size="sm"
         icon={showAdvancedOptions ? FiChevronDown : FiChevronRight}
         onClick={() => setShowAdvancedOptions(!showAdvancedOptions)}
         className="text-xs text-text-950 hover:text-text-500"
diff --git a/web/src/components/DanswerInitializingLoader.tsx b/web/src/components/DanswerInitializingLoader.tsx
index 8e4f3c487fa..f548126f0a6 100644
--- a/web/src/components/DanswerInitializingLoader.tsx
+++ b/web/src/components/DanswerInitializingLoader.tsx
@@ -1,4 +1,3 @@
-import { Bold } from "@tremor/react";
 import { Logo } from "./Logo";
 import { useContext } from "react";
 import { SettingsContext } from "./settings/SettingsProvider";
@@ -9,10 +8,10 @@ export function DanswerInitializingLoader() {
   return (
     <div className="mx-auto my-auto animate-pulse">
       <Logo height={96} width={96} className="mx-auto mb-3" />
-      <Bold>
+      <p className="text-lg font-bold">
         Initializing{" "}
         {settings?.enterpriseSettings?.application_name ?? "Danswer"}
-      </Bold>
+      </p>
     </div>
   );
 }
diff --git a/web/src/components/ErrorCallout.tsx b/web/src/components/ErrorCallout.tsx
index 6c236762a39..497412d033d 100644
--- a/web/src/components/ErrorCallout.tsx
+++ b/web/src/components/ErrorCallout.tsx
@@ -1,5 +1,4 @@
-import { Callout } from "@tremor/react";
-import { FiAlertOctagon } from "react-icons/fi";
+import { Callout } from "@/components/ui/callout";
 
 export function ErrorCallout({
   errorTitle,
@@ -13,8 +12,8 @@ export function ErrorCallout({
       <Callout
         className="mt-4"
         title={errorTitle || "Page not found"}
-        icon={FiAlertOctagon}
-        color="rose"
+        icon="alert"
+        type="danger"
       >
         {errorMsg}
       </Callout>
diff --git a/web/src/components/IsPublicGroupSelector.tsx b/web/src/components/IsPublicGroupSelector.tsx
index adb2496ed8a..6cb953f5bdf 100644
--- a/web/src/components/IsPublicGroupSelector.tsx
+++ b/web/src/components/IsPublicGroupSelector.tsx
@@ -1,8 +1,9 @@
 import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import React, { useState, useEffect } from "react";
 import { FormikProps, FieldArray, ArrayHelpers, ErrorMessage } from "formik";
-import { Text, Divider } from "@tremor/react";
+import Text from "@/components/ui/text";
 import { FiUsers } from "react-icons/fi";
+import { Separator } from "@/components/ui/separator";
 import { UserGroup, UserRole } from "@/lib/types";
 import { useUserGroups } from "@/lib/hooks";
 import { BooleanFormField } from "@/components/admin/connectors/Field";
@@ -73,7 +74,7 @@ export const IsPublicGroupSelector = <T extends IsPublicGroupSelectorFormType>({
 
   return (
     <div>
-      <Divider />
+      <Separator />
       {isAdmin && (
         <>
           <BooleanFormField
@@ -148,7 +149,9 @@ export const IsPublicGroupSelector = <T extends IsPublicGroupSelectorFormType>({
                         w-fit 
                         flex 
                         cursor-pointer 
-                        ${isSelected ? "bg-background-strong" : "hover:bg-hover"}
+                        ${
+                          isSelected ? "bg-background-strong" : "hover:bg-hover"
+                        }
                       `}
                           onClick={() => {
                             if (isSelected) {
diff --git a/web/src/components/Modal.tsx b/web/src/components/Modal.tsx
index 0f354a264f9..4abbfd8b9aa 100644
--- a/web/src/components/Modal.tsx
+++ b/web/src/components/Modal.tsx
@@ -1,5 +1,5 @@
 "use client";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 import { FiX } from "react-icons/fi";
 import { IconProps, XIcon } from "./icons/icons";
 import { useRef } from "react";
@@ -85,7 +85,7 @@ export function Modal({
                   {icon && icon({ size: 30 })}
                 </h2>
               </div>
-              {!hideDividerForTitle && <Divider />}
+              {!hideDividerForTitle && <Separator />}
             </>
           )}
           <div className="max-h-[60vh] overflow-y-scroll">{children}</div>
diff --git a/web/src/components/Status.tsx b/web/src/components/Status.tsx
index a4ab2090d1a..ce2cf134b43 100644
--- a/web/src/components/Status.tsx
+++ b/web/src/components/Status.tsx
@@ -1,7 +1,7 @@
 "use client";
 
 import { ValidStatuses } from "@/lib/types";
-import { Badge } from "@tremor/react";
+import { Badge } from "@/components/ui/badge";
 import {
   FiAlertTriangle,
   FiCheckCircle,
@@ -14,17 +14,15 @@ import { HoverPopup } from "./HoverPopup";
 export function IndexAttemptStatus({
   status,
   errorMsg,
-  size = "md",
 }: {
   status: ValidStatuses | null;
   errorMsg?: string | null;
-  size?: "xs" | "sm" | "md" | "lg";
 }) {
   let badge;
 
   if (status === "failed") {
     const icon = (
-      <Badge size={size} color="red" icon={FiAlertTriangle}>
+      <Badge variant="destructive" icon={FiAlertTriangle}>
         Failed
       </Badge>
     );
@@ -44,7 +42,7 @@ export function IndexAttemptStatus({
     }
   } else if (status === "completed_with_errors") {
     const icon = (
-      <Badge size={size} color="yellow" icon={FiAlertTriangle}>
+      <Badge variant="secondary" icon={FiAlertTriangle}>
         Completed with errors
       </Badge>
     );
@@ -54,7 +52,7 @@ export function IndexAttemptStatus({
         popupContent={
           <div className="w-64 p-2 break-words overflow-hidden whitespace-normal">
             The indexing attempt completed, but some errors were encountered
-            during the run.
+            during the rrun.
             <br />
             <br />
             Click View Errors for more details.
@@ -64,25 +62,25 @@ export function IndexAttemptStatus({
     );
   } else if (status === "success") {
     badge = (
-      <Badge size={size} color="green" icon={FiCheckCircle}>
+      <Badge variant="success" icon={FiCheckCircle}>
         Succeeded
       </Badge>
     );
   } else if (status === "in_progress") {
     badge = (
-      <Badge size={size} color="amber" icon={FiClock}>
+      <Badge variant="in_progress" icon={FiClock}>
         In Progress
       </Badge>
     );
   } else if (status === "not_started") {
     badge = (
-      <Badge size={size} color="fuchsia" icon={FiClock}>
+      <Badge variant="purple" icon={FiClock}>
         Scheduled
       </Badge>
     );
   } else {
     badge = (
-      <Badge size={size} color="gray" icon={FiMinus}>
+      <Badge variant="outline" icon={FiMinus}>
         None
       </Badge>
     );
@@ -106,25 +104,25 @@ export function CCPairStatus({
 
   if (isDeleting) {
     badge = (
-      <Badge size={size} color="red" icon={FiAlertTriangle}>
+      <Badge variant="destructive" icon={FiAlertTriangle}>
         Deleting
       </Badge>
     );
   } else if (disabled) {
     badge = (
-      <Badge size={size} color="yellow" icon={FiPauseCircle}>
+      <Badge variant="paused" icon={FiPauseCircle}>
         Paused
       </Badge>
     );
   } else if (status === "failed") {
     badge = (
-      <Badge size={size} color="red" icon={FiAlertTriangle}>
+      <Badge variant="destructive" icon={FiAlertTriangle}>
         Error
       </Badge>
     );
   } else {
     badge = (
-      <Badge size={size} color="green" icon={FiCheckCircle}>
+      <Badge variant="success" icon={FiCheckCircle}>
         Active
       </Badge>
     );
diff --git a/web/src/components/SwitchModelModal.tsx b/web/src/components/SwitchModelModal.tsx
index d3174275d6f..1008a0ad79b 100644
--- a/web/src/components/SwitchModelModal.tsx
+++ b/web/src/components/SwitchModelModal.tsx
@@ -1,6 +1,7 @@
 "use client";
 
-import { Button, Text } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 import { Modal } from "./Modal";
 import Link from "next/link";
 
@@ -29,7 +30,7 @@ export function SwitchModelModal({
 
         <div className="flex mt-4">
           <Link href="/admin/models/embedding" className="w-fit mx-auto">
-            <Button size="xs">Choose your Embedding Model</Button>
+            <Button size="sm">Choose your Embedding Model</Button>
           </Link>
         </div>
       </div>
diff --git a/web/src/components/admin/CardSection.tsx b/web/src/components/admin/CardSection.tsx
new file mode 100644
index 00000000000..c44088b3999
--- /dev/null
+++ b/web/src/components/admin/CardSection.tsx
@@ -0,0 +1,21 @@
+import { cn } from "@/lib/utils";
+
+// Used for all admin page sections
+export default function CardSection({
+  children,
+  className,
+}: {
+  children: React.ReactNode;
+  className?: string;
+}) {
+  return (
+    <div
+      className={cn(
+        "p-6  shadow-sm rounded-lg bg-white border border-border-strong/80",
+        className
+      )}
+    >
+      {children}
+    </div>
+  );
+}
diff --git a/web/src/components/admin/Title.tsx b/web/src/components/admin/Title.tsx
index 0511a39dfed..b5e69677c84 100644
--- a/web/src/components/admin/Title.tsx
+++ b/web/src/components/admin/Title.tsx
@@ -1,6 +1,6 @@
 "use client";
 import { HealthCheckBanner } from "../health/healthcheck";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 
 export function AdminPageTitle({
   icon,
@@ -24,7 +24,7 @@ export function AdminPageTitle({
         </h1>
         {farRightElement && <div className="ml-auto">{farRightElement}</div>}
       </div>
-      {includeDivider ? <Divider /> : <div className="mb-6" />}
+      {includeDivider ? <Separator /> : <div className="mb-6" />}
     </div>
   );
 }
diff --git a/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx b/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx
index d90fbedbfac..7080f92c82c 100644
--- a/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx
+++ b/web/src/components/admin/connectors/AccessTypeGroupSelector.tsx
@@ -1,9 +1,10 @@
 import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import React, { useState, useEffect } from "react";
 import { FieldArray, ArrayHelpers, ErrorMessage, useField } from "formik";
-import { Text, Divider } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Separator } from "@/components/ui/separator";
 import { FiUsers } from "react-icons/fi";
-import { UserGroup, User, UserRole } from "@/lib/types";
+import { UserGroup, UserRole } from "@/lib/types";
 import { useUserGroups } from "@/lib/hooks";
 import { AccessType } from "@/lib/types";
 import { useUser } from "@/components/user/UserProvider";
@@ -81,7 +82,7 @@ export function AccessTypeGroupSelector({}: {}) {
         userGroups &&
         userGroups?.length > 0 && (
           <>
-            <Divider />
+            <Separator />
             <div className="flex mt-4 gap-x-2 items-center">
               <div className="block font-medium text-base">
                 Assign group access for this Connector
@@ -127,7 +128,11 @@ export function AccessTypeGroupSelector({}: {}) {
                             w-fit 
                             flex 
                             cursor-pointer 
-                            ${isSelected ? "bg-background-strong" : "hover:bg-hover"}
+                            ${
+                              isSelected
+                                ? "bg-background-strong"
+                                : "hover:bg-hover"
+                            }
                         `}
                           onClick={() => {
                             if (isSelected) {
diff --git a/web/src/components/admin/connectors/AutoSyncOptions.tsx b/web/src/components/admin/connectors/AutoSyncOptions.tsx
index dfeb7e661d3..e9b736193e3 100644
--- a/web/src/components/admin/connectors/AutoSyncOptions.tsx
+++ b/web/src/components/admin/connectors/AutoSyncOptions.tsx
@@ -1,6 +1,6 @@
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { ValidAutoSyncSources } from "@/lib/types";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 import { autoSyncConfigBySource } from "@/lib/connectors/AutoSyncOptionFields";
 
 export function AutoSyncOptions({
@@ -16,7 +16,7 @@ export function AutoSyncOptions({
 
   return (
     <div>
-      <Divider />
+      <Separator />
       {Object.entries(autoSyncConfig).map(([key, config]) => (
         <div key={key} className="mb-4">
           <TextFormField
diff --git a/web/src/components/admin/connectors/CredentialForm.tsx b/web/src/components/admin/connectors/CredentialForm.tsx
index 2a4b9a63e4b..1a7636db4d9 100644
--- a/web/src/components/admin/connectors/CredentialForm.tsx
+++ b/web/src/components/admin/connectors/CredentialForm.tsx
@@ -1,11 +1,10 @@
 import React, { useState } from "react";
-import { Formik, Form, FormikHelpers } from "formik";
+import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { Popup } from "./Popup";
 import { ValidSources } from "@/lib/types";
 
 import { createCredential } from "@/lib/credential";
-import { Button } from "@tremor/react";
 import { CredentialBase, Credential } from "@/lib/connectors/credentials";
 
 export async function submitCredential<T>(
diff --git a/web/src/components/admin/connectors/Field.tsx b/web/src/components/admin/connectors/Field.tsx
index d869d75687c..601d4d5ffdd 100644
--- a/web/src/components/admin/connectors/Field.tsx
+++ b/web/src/components/admin/connectors/Field.tsx
@@ -1,4 +1,3 @@
-import { Button } from "@tremor/react";
 import {
   ArrayHelpers,
   ErrorMessage,
@@ -9,7 +8,14 @@ import {
 } from "formik";
 import * as Yup from "yup";
 import { FormBodyBuilder } from "./types";
-import { DefaultDropdown, StringOrNumberOption } from "@/components/Dropdown";
+import { StringOrNumberOption } from "@/components/Dropdown";
+import {
+  Select,
+  SelectItem,
+  SelectContent,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
 import { FiInfo, FiPlus, FiX } from "react-icons/fi";
 import {
   TooltipProvider,
@@ -22,6 +28,7 @@ import { FaMarkdown } from "react-icons/fa";
 import { useState } from "react";
 import remarkGfm from "remark-gfm";
 import { EditIcon } from "@/components/icons/icons";
+import { Button } from "@/components/ui/button";
 
 export function SectionHeader({
   children,
@@ -219,7 +226,7 @@ export function TextFormField({
             ${small && "text-sm"}
             border 
             border-border 
-            rounded-lg
+            rounded-md
             w-full 
             py-2 
             px-3 
@@ -538,8 +545,8 @@ export function TextArrayField<T extends Yup.AnyObject>({
                 arrayHelpers.push("");
               }}
               className="mt-3"
-              color="green"
-              size="xs"
+              variant="update"
+              size="sm"
               type="button"
               icon={FiPlus}
             >
@@ -607,15 +614,32 @@ export function SelectorFormField({
       )}
       {subtext && <SubLabel>{subtext}</SubLabel>}
       <div className="mt-2">
-        <DefaultDropdown
-          options={options}
-          selected={field.value}
-          onSelect={onSelect || ((selected) => setFieldValue(name, selected))}
-          includeDefault={includeDefault}
-          side={side}
-          maxHeight={maxHeight}
+        <Select
+          value={field.value}
+          onValueChange={
+            onSelect || ((selected) => setFieldValue(name, selected))
+          }
           defaultValue={defaultValue}
-        />
+        >
+          <SelectTrigger>
+            <SelectValue
+              placeholder={includeDefault ? "Select..." : undefined}
+            />
+          </SelectTrigger>
+          <SelectContent
+            side={side}
+            className={maxHeight ? `max-h-[${maxHeight}]` : undefined}
+          >
+            {includeDefault && (
+              <SelectItem value="default">Select...</SelectItem>
+            )}
+            {options.map((option) => (
+              <SelectItem key={option.value} value={String(option.value)}>
+                {option.name}
+              </SelectItem>
+            ))}
+          </SelectContent>
+        </Select>
       </div>
 
       <ErrorMessage
diff --git a/web/src/components/admin/users/BulkAdd.tsx b/web/src/components/admin/users/BulkAdd.tsx
index 2a632b824e4..b55630c77a8 100644
--- a/web/src/components/admin/users/BulkAdd.tsx
+++ b/web/src/components/admin/users/BulkAdd.tsx
@@ -2,7 +2,7 @@
 
 import { withFormik, FormikProps, FormikErrors, Form, Field } from "formik";
 
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 
 const WHITESPACE_SPLIT = /\s+/;
 const EMAIL_REGEX = /[^@]+@[^.]+\.[^.]/;
@@ -38,8 +38,8 @@ const AddUserFormRenderer = ({
     )}
     <Button
       className="mx-auto"
-      color="green"
-      size="md"
+      variant="submit"
+      size="sm"
       type="submit"
       disabled={isSubmitting}
     >
diff --git a/web/src/components/admin/users/InvitedUserTable.tsx b/web/src/components/admin/users/InvitedUserTable.tsx
index d8265914794..ac50ac15b56 100644
--- a/web/src/components/admin/users/InvitedUserTable.tsx
+++ b/web/src/components/admin/users/InvitedUserTable.tsx
@@ -1,20 +1,19 @@
 import { PopupSpec } from "@/components/admin/connectors/Popup";
-import { HidableSection } from "@/app/admin/assistants/HidableSection";
 import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Button,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import { Button } from "@/components/ui/button";
 import userMutationFetcher from "@/lib/admin/users/userMutationFetcher";
 import CenteredPageSelector from "./CenteredPageSelector";
 import { type PageSelectorProps } from "@/components/PageSelector";
-import useSWR from "swr";
-import { type User, UserStatus } from "@/lib/types";
+
+import { type User } from "@/lib/types";
 import useSWRMutation from "swr/mutation";
+import { TableHeader } from "@/components/ui/table";
 
 interface Props {
   users: Array<User>;
@@ -70,14 +69,14 @@ const InvitedUserTable = ({
   return (
     <>
       <Table className="overflow-visible">
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell>Email</TableHeaderCell>
-            <TableHeaderCell>
+            <TableHead>Email</TableHead>
+            <TableHead>
               <div className="flex justify-end">Actions</div>
-            </TableHeaderCell>
+            </TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {users.map((user) => (
             <TableRow key={user.email}>
diff --git a/web/src/components/admin/users/SignedUpUserTable.tsx b/web/src/components/admin/users/SignedUpUserTable.tsx
index 9f44012d126..dc7da5113b3 100644
--- a/web/src/components/admin/users/SignedUpUserTable.tsx
+++ b/web/src/components/admin/users/SignedUpUserTable.tsx
@@ -9,17 +9,23 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
   TableBody,
   TableCell,
-  Button,
+} from "@/components/ui/table";
+
+import {
   Select,
+  SelectContent,
   SelectItem,
-} from "@tremor/react";
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { Button } from "@/components/ui/button";
 import { GenericConfirmModal } from "@/components/modals/GenericConfirmModal";
 import { useState } from "react";
 import { usePaidEnterpriseFeaturesEnabled } from "@/components/settings/usePaidEnterpriseFeaturesEnabled";
 import { DeleteEntityModal } from "@/components/modals/DeleteEntityModal";
+import { TableHeader } from "@/components/ui/table";
 
 interface Props {
   users: Array<User>;
@@ -76,34 +82,46 @@ const UserRoleDropdown = ({
         value={user.role}
         onValueChange={handleChange}
         disabled={isSettingRole}
-        className="w-40 mx-auto"
       >
-        {Object.entries(USER_ROLE_LABELS).map(([role, label]) =>
-          !isPaidEnterpriseFeaturesEnabled &&
-          (role === UserRole.CURATOR ||
-            role === UserRole.GLOBAL_CURATOR) ? null : (
-            <SelectItem
-              key={role}
-              value={role}
-              className={
-                role === UserRole.CURATOR ? "opacity-30 cursor-not-allowed" : ""
-              }
-              title={
-                role === UserRole.CURATOR
-                  ? "Curator role must be assigned in the Groups tab"
-                  : ""
-              }
-            >
-              {label}
-            </SelectItem>
-          )
-        )}
+        <SelectTrigger>
+          <SelectValue />
+        </SelectTrigger>
+        <SelectContent>
+          {Object.entries(USER_ROLE_LABELS).map(([role, label]) =>
+            !isPaidEnterpriseFeaturesEnabled &&
+            (role === UserRole.CURATOR ||
+              role === UserRole.GLOBAL_CURATOR) ? null : (
+              <SelectItem
+                key={role}
+                value={role}
+                className={
+                  role === UserRole.CURATOR
+                    ? "opacity-30 cursor-not-allowed"
+                    : ""
+                }
+                title={
+                  role === UserRole.CURATOR
+                    ? "Curator role must be assigned in the Groups tab"
+                    : ""
+                }
+              >
+                {label}
+              </SelectItem>
+            )
+          )}
+        </SelectContent>
       </Select>
       {showConfirmModal && (
         <GenericConfirmModal
           title="Change Curator Role"
-          message={`Warning: Switching roles from Curator to ${USER_ROLE_LABELS[pendingRole as UserRole] ?? USER_ROLE_LABELS[user.role]} will remove their status as individual curators from all groups.`}
-          confirmText={`Switch Role to ${USER_ROLE_LABELS[pendingRole as UserRole] ?? USER_ROLE_LABELS[user.role]}`}
+          message={`Warning: Switching roles from Curator to ${
+            USER_ROLE_LABELS[pendingRole as UserRole] ??
+            USER_ROLE_LABELS[user.role]
+          } will remove their status as individual curators from all groups.`}
+          confirmText={`Switch Role to ${
+            USER_ROLE_LABELS[pendingRole as UserRole] ??
+            USER_ROLE_LABELS[user.role]
+          }`}
           onClose={() => setShowConfirmModal(false)}
           onConfirm={handleConfirm}
         />
@@ -145,7 +163,7 @@ const DeactivaterButton = ({
       className="w-min"
       onClick={() => trigger({ user_email: user.email })}
       disabled={isMutating}
-      size="xs"
+      size="sm"
     >
       {deactivate ? "Deactivate" : "Activate"}
     </Button>
@@ -197,8 +215,8 @@ const DeleteUserButton = ({
         className="w-min"
         onClick={() => setShowDeleteModal(true)}
         disabled={isMutating}
-        size="xs"
-        color="red"
+        size="sm"
+        variant="destructive"
       >
         Delete
       </Button>
@@ -237,23 +255,23 @@ const SignedUpUserTable = ({
           />
         ) : null}
         <Table className="overflow-visible">
-          <TableHead>
+          <TableHeader>
             <TableRow>
-              <TableHeaderCell>Email</TableHeaderCell>
-              <TableHeaderCell className="text-center">Role</TableHeaderCell>
-              <TableHeaderCell className="text-center">Status</TableHeaderCell>
-              <TableHeaderCell>
+              <TableHead>Email</TableHead>
+              <TableHead className="text-center">Role</TableHead>
+              <TableHead className="text-center">Status</TableHead>
+              <TableHead>
                 <div className="flex">
                   <div className="ml-auto">Actions</div>
                 </div>
-              </TableHeaderCell>
+              </TableHead>
             </TableRow>
-          </TableHead>
+          </TableHeader>
           <TableBody>
             {users.map((user) => (
               <TableRow key={user.id}>
                 <TableCell>{user.email}</TableCell>
-                <TableCell>
+                <TableCell className="w-40 ">
                   <UserRoleDropdown
                     user={user}
                     onSuccess={onRoleChangeSuccess}
diff --git a/web/src/components/credentials/CredentialSection.tsx b/web/src/components/credentials/CredentialSection.tsx
index a6002fd9a0d..99c97ad589b 100644
--- a/web/src/components/credentials/CredentialSection.tsx
+++ b/web/src/components/credentials/CredentialSection.tsx
@@ -16,7 +16,7 @@ import { usePopup } from "@/components/admin/connectors/Popup";
 import CreateCredential from "./actions/CreateCredential";
 import { CCPairFullInfo } from "@/app/admin/connector/[ccPairId]/types";
 import ModifyCredential from "./actions/ModifyCredential";
-import { Text } from "@tremor/react";
+import Text from "@/components/ui/text";
 import {
   buildCCPairInfoUrl,
   buildSimilarCredentialInfoURL,
diff --git a/web/src/components/credentials/actions/CreateCredential.tsx b/web/src/components/credentials/actions/CreateCredential.tsx
index 5188f3d02a4..43d24d3a120 100644
--- a/web/src/components/credentials/actions/CreateCredential.tsx
+++ b/web/src/components/credentials/actions/CreateCredential.tsx
@@ -1,5 +1,6 @@
 import React, { useState } from "react";
-import { Button, Card } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import { Card } from "@/components/ui/card";
 import { ValidSources } from "@/lib/types";
 import { FaAccusoft } from "react-icons/fa";
 import { submitCredential } from "@/components/admin/connectors/CredentialForm";
@@ -27,6 +28,7 @@ import {
   IsPublicGroupSelector,
 } from "@/components/IsPublicGroupSelector";
 import { useUser } from "@/components/user/UserProvider";
+import CardSection from "@/components/admin/CardSection";
 
 const CreateButton = ({
   onClick,
@@ -201,7 +203,7 @@ export default function CreateCredential({
               for information on setting up this connector.
             </p>
           )}
-          <Card className="!border-0 mt-4 flex flex-col gap-y-6">
+          <CardSection className="!border-0 mt-4 flex flex-col gap-y-6">
             <TextFormField
               name="name"
               placeholder="(Optional) credential name.."
@@ -254,7 +256,7 @@ export default function CreateCredential({
                 </div>
               </div>
             )}
-          </Card>
+          </CardSection>
           {swapConnector && (
             <div className="flex gap-x-4 w-full mt-8 justify-end">
               <Button
diff --git a/web/src/components/credentials/actions/EditCredential.tsx b/web/src/components/credentials/actions/EditCredential.tsx
index 12b666bba58..cdf44bb67ba 100644
--- a/web/src/components/credentials/actions/EditCredential.tsx
+++ b/web/src/components/credentials/actions/EditCredential.tsx
@@ -1,7 +1,8 @@
 import React from "react";
-import { Button, Text, Card } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
 
-import { FaNewspaper, FaPaperPlane, FaTractor, FaTrash } from "react-icons/fa";
+import { FaNewspaper, FaTrash } from "react-icons/fa";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik, FormikHelpers } from "formik";
 import { PopupSpec } from "@/components/admin/connectors/Popup";
diff --git a/web/src/components/credentials/actions/ModifyCredential.tsx b/web/src/components/credentials/actions/ModifyCredential.tsx
index 01433f33443..328ed00873c 100644
--- a/web/src/components/credentials/actions/ModifyCredential.tsx
+++ b/web/src/components/credentials/actions/ModifyCredential.tsx
@@ -1,6 +1,8 @@
 import React, { useState } from "react";
 import { Modal } from "@/components/Modal";
-import { Button, Text, Badge } from "@tremor/react";
+import { Button } from "@/components/ui/button";
+import Text from "@/components/ui/text";
+import { Badge } from "@/components/ui/badge";
 import { ValidSources } from "@/lib/types";
 import {
   EditIcon,
diff --git a/web/src/components/embedding/CustomEmbeddingModelForm.tsx b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
index 7ba29531863..0c0336e2760 100644
--- a/web/src/components/embedding/CustomEmbeddingModelForm.tsx
+++ b/web/src/components/embedding/CustomEmbeddingModelForm.tsx
@@ -3,7 +3,8 @@ import { Formik, Form } from "formik";
 import * as Yup from "yup";
 import { TextFormField, BooleanFormField } from "../admin/connectors/Field";
 import { Dispatch, SetStateAction } from "react";
-import { Button, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Button } from "@/components/ui/button";
 import { EmbeddingDetails } from "@/app/admin/embeddings/EmbeddingModelSelectionForm";
 
 export function CustomEmbeddingModelForm({
@@ -62,7 +63,9 @@ export function CustomEmbeddingModelForm({
             <TextFormField
               name="model_name"
               label="Model Name:"
-              subtext={`The name of the ${embeddingType === EmbeddingProvider.AZURE ? "Azure" : "LiteLLM"} model`}
+              subtext={`The name of the ${
+                embeddingType === EmbeddingProvider.AZURE ? "Azure" : "LiteLLM"
+              } model`}
               placeholder="e.g. 'all-MiniLM-L6-v2'"
               autoCompleteDisabled={true}
             />
diff --git a/web/src/components/embedding/CustomModelForm.tsx b/web/src/components/embedding/CustomModelForm.tsx
index 95dbac0464a..ee86a1c6e32 100644
--- a/web/src/components/embedding/CustomModelForm.tsx
+++ b/web/src/components/embedding/CustomModelForm.tsx
@@ -2,7 +2,7 @@ import {
   BooleanFormField,
   TextFormField,
 } from "@/components/admin/connectors/Field";
-import { Button } from "@tremor/react";
+import { Button } from "@/components/ui/button";
 import { Form, Formik } from "formik";
 import * as Yup from "yup";
 import { HostedEmbeddingModel } from "./interfaces";
diff --git a/web/src/components/embedding/FailedReIndexAttempts.tsx b/web/src/components/embedding/FailedReIndexAttempts.tsx
index 4ab78af961e..1b34672ca19 100644
--- a/web/src/components/embedding/FailedReIndexAttempts.tsx
+++ b/web/src/components/embedding/FailedReIndexAttempts.tsx
@@ -2,23 +2,20 @@ import { buildCCPairInfoUrl } from "@/app/admin/connector/[ccPairId]/lib";
 import { PageSelector } from "@/components/PageSelector";
 import { IndexAttemptStatus } from "@/components/Status";
 import { deleteCCPair } from "@/lib/documentDeletion";
+import { FailedConnectorIndexingStatus } from "@/lib/types";
+import { Button } from "@/components/ui/button";
 import {
-  ConnectorIndexingStatus,
-  FailedConnectorIndexingStatus,
-} from "@/lib/types";
-import {
-  Button,
   Table,
   TableBody,
   TableCell,
   TableHead,
-  TableHeaderCell,
+  TableHeader,
   TableRow,
-  Text,
-} from "@tremor/react";
+} from "@/components/ui/table";
+import Text from "@/components/ui/text";
 import Link from "next/link";
 import { useState } from "react";
-import { FiLink, FiMaximize2, FiTrash, FiTrash2 } from "react-icons/fi";
+import { FiLink, FiMaximize2, FiTrash } from "react-icons/fi";
 import { mutate } from "swr";
 import { PopupSpec } from "../admin/connectors/Popup";
 
@@ -50,27 +47,19 @@ export function FailedReIndexAttempts({
 
       <div>
         <Table>
-          <TableHead>
+          <TableHeader>
             <TableRow>
-              <TableHeaderCell className="w-1/8 sm:w-1/6">
-                Connector Name
-              </TableHeaderCell>
-              <TableHeaderCell className="w-1/8 sm:w-1/6">
-                Status
-              </TableHeaderCell>
-              <TableHeaderCell className="w-4/8 sm:w-2/6">
-                Error Message
-              </TableHeaderCell>
-              <TableHeaderCell className="w-1/8 sm:w-1/6">
-                Visit Connector
-              </TableHeaderCell>
+              <TableHead className="w-1/8 sm:w-1/6">Connector Name</TableHead>
+              <TableHead className="w-1/8 sm:w-1/6">Status</TableHead>
+              <TableHead className="w-4/8 sm:w-2/6">Error Message</TableHead>
+              <TableHead className="w-1/8 sm:w-1/6">Visit Connector</TableHead>
               {anyDeletable && (
-                <TableHeaderCell className="w-1/8 sm:w-2/6">
+                <TableHead className="w-1/8 sm:w-2/6">
                   Delete Connector
-                </TableHeaderCell>
+                </TableHead>
               )}
             </TableRow>
-          </TableHead>
+          </TableHeader>
           <TableBody>
             {failedIndexingStatuses
               .slice(numToDisplay * (page - 1), numToDisplay * page)
@@ -108,8 +97,8 @@ export function FailedReIndexAttempts({
                     </TableCell>
                     <TableCell>
                       <Button
-                        size="xs"
-                        color="red"
+                        size="sm"
+                        variant="destructive"
                         onClick={() =>
                           deleteCCPair(
                             reindexingProgress.connector_id,
diff --git a/web/src/components/embedding/ModelSelector.tsx b/web/src/components/embedding/ModelSelector.tsx
index 9b9455c07b7..22f5bc9b69e 100644
--- a/web/src/components/embedding/ModelSelector.tsx
+++ b/web/src/components/embedding/ModelSelector.tsx
@@ -8,6 +8,7 @@ import {
   HostedEmbeddingModel,
 } from "./interfaces";
 import { FiExternalLink } from "react-icons/fi";
+import CardSection from "../admin/CardSection";
 
 export function ModelPreview({
   model,
@@ -19,8 +20,10 @@ export function ModelPreview({
   const currentModelCopy = getCurrentModelCopy(model.model_name);
 
   return (
-    <div
-      className={`border border-border rounded shadow-md ${display ? "bg-inverted rounded-lg p-4" : "bg-hover-light p-2"} w-96 flex flex-col`}
+    <CardSection
+      className={`shadow-md ${
+        display ? "bg-inverted rounded-lg p-4" : "bg-hover-light p-2"
+      } w-96 flex flex-col`}
     >
       <div className="font-bold text-lg flex">{model.model_name}</div>
       <div className="text-sm mt-1 mx-1">
@@ -28,7 +31,7 @@ export function ModelPreview({
           currentModelCopy?.description ||
           "Custom model—no description is available."}
       </div>
-    </div>
+    </CardSection>
   );
 }
 
diff --git a/web/src/components/embedding/ReindexingProgressTable.tsx b/web/src/components/embedding/ReindexingProgressTable.tsx
index 3706133d0c7..ad26c595c55 100644
--- a/web/src/components/embedding/ReindexingProgressTable.tsx
+++ b/web/src/components/embedding/ReindexingProgressTable.tsx
@@ -6,10 +6,9 @@ import {
   TableBody,
   TableCell,
   TableHead,
-  TableHeaderCell,
+  TableHeader,
   TableRow,
-  Text,
-} from "@tremor/react";
+} from "@/components/ui/table";
 import Link from "next/link";
 import { useState } from "react";
 import { FiMaximize2 } from "react-icons/fi";
@@ -25,17 +24,13 @@ export function ReindexingProgressTable({
   return (
     <div>
       <Table>
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell className="w-1/7 sm:w-1/5">
-              Connector Name
-            </TableHeaderCell>
-            <TableHeaderCell className="w-3/7 sm:w-1/5">Status</TableHeaderCell>
-            <TableHeaderCell className="w-3/7 sm:w-1/5">
-              Docs Re-Indexed
-            </TableHeaderCell>
+            <TableHead className="w-1/7 sm:w-1/5">Connector Name</TableHead>
+            <TableHead className="w-3/7 sm:w-1/5">Status</TableHead>
+            <TableHead className="w-3/7 sm:w-1/5">Docs Re-Indexed</TableHead>
           </TableRow>
-        </TableHead>
+        </TableHeader>
         <TableBody>
           {reindexingProgress
             .slice(numToDisplay * (page - 1), numToDisplay * page)
diff --git a/web/src/components/initialSetup/welcome/WelcomeModal.tsx b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
index cd3dd8ff268..58efa20b8a2 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModal.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModal.tsx
@@ -1,7 +1,8 @@
 "use client";
 
 import React from "react";
-import { Button, Divider, Text } from "@tremor/react";
+import Text from "@/components/ui/text";
+import { Button } from "@/components/ui/button";
 import { Modal } from "../../Modal";
 import Cookies from "js-cookie";
 import { useRouter } from "next/navigation";
diff --git a/web/src/components/llm/ApiKeyForm.tsx b/web/src/components/llm/ApiKeyForm.tsx
index 276bcc36e6c..053b569f712 100644
--- a/web/src/components/llm/ApiKeyForm.tsx
+++ b/web/src/components/llm/ApiKeyForm.tsx
@@ -1,6 +1,6 @@
 import { PopupSpec } from "../admin/connectors/Popup";
 import { useState } from "react";
-import { TabGroup, TabList, Tab, TabPanels, TabPanel } from "@tremor/react";
+import { Tabs, TabsList, TabsContent, TabsTrigger } from "@/components/ui/tabs";
 import { WellKnownLLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { LLMProviderUpdateForm } from "@/app/admin/configuration/llm/LLMProviderUpdateForm";
 import { CustomLLMProviderUpdateForm } from "@/app/admin/configuration/llm/CustomLLMProviderUpdateForm";
@@ -32,47 +32,53 @@ export const ApiKeyForm = ({
 
   return (
     <div>
-      <TabGroup
-        index={providerNameToIndexMap.get(providerName) || 0}
-        onIndexChange={(index) =>
-          setProviderName(providerIndexToNameMap.get(index) || defaultProvider)
+      <Tabs
+        defaultValue={String(providerNameToIndexMap.get(providerName) || 0)}
+        onValueChange={(value) =>
+          setProviderName(
+            providerIndexToNameMap.get(Number(value)) || defaultProvider
+          )
         }
       >
-        <TabList className="mt-3 mb-4">
-          <>
-            {providerOptions.map((provider) => (
-              <Tab key={provider.name}>
-                {provider.display_name || provider.name}
-              </Tab>
-            ))}
-            <Tab key="custom">Custom</Tab>
-          </>
-        </TabList>
-        <TabPanels>
-          {providerOptions.map((provider) => {
-            return (
-              <TabPanel key={provider.name}>
-                <LLMProviderUpdateForm
-                  hideAdvanced
-                  llmProviderDescriptor={provider}
-                  onClose={() => onSuccess()}
-                  shouldMarkAsDefault
-                  setPopup={setPopup}
-                  hideSuccess={hideSuccess}
-                />
-              </TabPanel>
-            );
-          })}
-          <TabPanel key="custom">
-            <CustomLLMProviderUpdateForm
+        <TabsList className="mt-3 mb-4">
+          {providerOptions.map((provider) => (
+            <TabsTrigger
+              key={provider.name}
+              value={String(providerNameToIndexMap.get(provider.name))}
+            >
+              {provider.display_name || provider.name}
+            </TabsTrigger>
+          ))}
+          <TabsTrigger value={String(providerOptions.length)}>
+            Custom
+          </TabsTrigger>
+        </TabsList>
+
+        {providerOptions.map((provider) => (
+          <TabsContent
+            key={provider.name}
+            value={String(providerNameToIndexMap.get(provider.name))}
+          >
+            <LLMProviderUpdateForm
+              hideAdvanced
+              llmProviderDescriptor={provider}
               onClose={() => onSuccess()}
               shouldMarkAsDefault
               setPopup={setPopup}
               hideSuccess={hideSuccess}
             />
-          </TabPanel>
-        </TabPanels>
-      </TabGroup>
+          </TabsContent>
+        ))}
+
+        <TabsContent value={String(providerOptions.length)}>
+          <CustomLLMProviderUpdateForm
+            onClose={() => onSuccess()}
+            shouldMarkAsDefault
+            setPopup={setPopup}
+            hideSuccess={hideSuccess}
+          />
+        </TabsContent>
+      </Tabs>
     </div>
   );
 };
diff --git a/web/src/components/search/DateRangeSelector.tsx b/web/src/components/search/DateRangeSelector.tsx
index dcbd5c1eb25..0a1368199a3 100644
--- a/web/src/components/search/DateRangeSelector.tsx
+++ b/web/src/components/search/DateRangeSelector.tsx
@@ -1,4 +1,4 @@
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { FiCalendar, FiChevronDown, FiXCircle } from "react-icons/fi";
 import { CustomDropdown } from "../Dropdown";
 import { timeRangeValues } from "@/app/config/timeRange";
diff --git a/web/src/components/search/SearchBar.tsx b/web/src/components/search/SearchBar.tsx
index 30b61b20c7e..6a346eaf7a8 100644
--- a/web/src/components/search/SearchBar.tsx
+++ b/web/src/components/search/SearchBar.tsx
@@ -19,7 +19,7 @@ interface FullSearchBarProps {
 
 import { useRef } from "react";
 import { SendIcon } from "../icons/icons";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 import { CustomTooltip } from "../tooltip/CustomTooltip";
 import KeyboardSymbol from "@/lib/browserUtilities";
 import { HorizontalSourceSelector } from "./filtering/Filters";
@@ -49,7 +49,7 @@ export const AnimatedToggle = ({
             Our most powerful search, have an AI agent guide you to pinpoint
             exactly what you&apos;re looking for.
           </p>
-          <Divider />
+          <Separator />
           <h2 className="text-xl text-text-800 font-bold mb-2">Fast Search</h2>
           <p className="text-text-700 text-sm mb-4">
             Get quality results immediately, best suited for instant access to
diff --git a/web/src/components/search/SearchSection.tsx b/web/src/components/search/SearchSection.tsx
index 656d57f2a1b..00cf9070242 100644
--- a/web/src/components/search/SearchSection.tsx
+++ b/web/src/components/search/SearchSection.tsx
@@ -41,10 +41,9 @@ import { ApiKeyModal } from "../llm/ApiKeyModal";
 import { useSearchContext } from "../context/SearchContext";
 import { useUser } from "../user/UserProvider";
 import UnconfiguredProviderText from "../chat_search/UnconfiguredProviderText";
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { Tag } from "@/lib/types";
 import { isEqual } from "lodash";
-import { WelcomeModal } from "../initialSetup/welcome/WelcomeModalWrapper";
 
 export type searchState =
   | "input"
diff --git a/web/src/components/search/filtering/Filters.tsx b/web/src/components/search/filtering/Filters.tsx
index 46f3e7915ce..2f8a0ff7086 100644
--- a/web/src/components/search/filtering/Filters.tsx
+++ b/web/src/components/search/filtering/Filters.tsx
@@ -19,7 +19,7 @@ import {
   FiX,
 } from "react-icons/fi";
 import { DateRangeSelector } from "../DateRangeSelector";
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { FilterDropdown } from "./FilterDropdown";
 import { listSourceMetadata } from "@/lib/sources";
 import { SourceIcon } from "@/components/SourceIcon";
diff --git a/web/src/components/standardAnswers/StandardAnswerCategoryDropdown.tsx b/web/src/components/standardAnswers/StandardAnswerCategoryDropdown.tsx
index 329da59762b..787a0cf23ac 100644
--- a/web/src/components/standardAnswers/StandardAnswerCategoryDropdown.tsx
+++ b/web/src/components/standardAnswers/StandardAnswerCategoryDropdown.tsx
@@ -5,7 +5,7 @@ import MultiSelectDropdown from "../MultiSelectDropdown";
 import { StandardAnswerCategory } from "@/lib/types";
 import { ErrorCallout } from "../ErrorCallout";
 import { LoadingAnimation } from "../Loading";
-import { Divider } from "@tremor/react";
+import { Separator } from "@/components/ui/separator";
 
 interface StandardAnswerCategoryDropdownFieldProps {
   standardAnswerCategoryResponse: StandardAnswerCategoryResponse;
@@ -65,7 +65,7 @@ export const StandardAnswerCategoryDropdownField: FC<
         </div>
       </div>
 
-      <Divider />
+      <Separator />
     </>
   );
 };
diff --git a/web/src/components/table/DraggableRow.tsx b/web/src/components/table/DraggableRow.tsx
index d859d73323d..07b7db951e2 100644
--- a/web/src/components/table/DraggableRow.tsx
+++ b/web/src/components/table/DraggableRow.tsx
@@ -1,5 +1,5 @@
 import { useSortable } from "@dnd-kit/sortable";
-import { TableCell, TableRow } from "@tremor/react";
+import { TableCell, TableRow } from "@/components/ui/table";
 import { CSS } from "@dnd-kit/utilities";
 import { DragHandle } from "./DragHandle";
 import { Row } from "./interfaces";
diff --git a/web/src/components/table/DraggableTable.tsx b/web/src/components/table/DraggableTable.tsx
index 01ff4dd1def..650265ad339 100644
--- a/web/src/components/table/DraggableTable.tsx
+++ b/web/src/components/table/DraggableTable.tsx
@@ -2,9 +2,9 @@ import {
   Table,
   TableHead,
   TableRow,
-  TableHeaderCell,
+  TableHeader,
   TableBody,
-} from "@tremor/react";
+} from "@/components/ui/table";
 import React, { useMemo, useState } from "react";
 import {
   closestCenter,
@@ -88,14 +88,14 @@ export function DraggableTable({
       modifiers={[restrictToVerticalAxis]}
     >
       <Table className="overflow-y-visible">
-        <TableHead>
+        <TableHeader>
           <TableRow>
-            <TableHeaderCell></TableHeaderCell>
+            <TableHead></TableHead>
             {headers.map((header, ind) => (
-              <TableHeaderCell key={ind}>{header}</TableHeaderCell>
+              <TableHead key={ind}>{header}</TableHead>
             ))}
           </TableRow>
-        </TableHead>
+        </TableHeader>
 
         <TableBody>
           <SortableContext items={items} strategy={verticalListSortingStrategy}>
diff --git a/web/src/components/table/DraggableTableBody.tsx b/web/src/components/table/DraggableTableBody.tsx
index 097f46803c8..e71cd2c7bf3 100644
--- a/web/src/components/table/DraggableTableBody.tsx
+++ b/web/src/components/table/DraggableTableBody.tsx
@@ -18,7 +18,7 @@ import {
   SortableContext,
   verticalListSortingStrategy,
 } from "@dnd-kit/sortable";
-import { TableBody } from "@tremor/react";
+import { TableBody } from "@/components/ui/table";
 import { DraggableRow } from "./DraggableRow";
 import { Row } from "./interfaces";
 
diff --git a/web/src/components/table/StaticRow.tsx b/web/src/components/table/StaticRow.tsx
index 4dee4d040da..9c0c69da70f 100644
--- a/web/src/components/table/StaticRow.tsx
+++ b/web/src/components/table/StaticRow.tsx
@@ -1,4 +1,4 @@
-import { TableCell, TableRow } from "@tremor/react";
+import { TableCell, TableRow } from "@/components/ui/table";
 import { DragHandle } from "./DragHandle";
 import { Row } from "./interfaces";
 
diff --git a/web/src/components/ui/alert.tsx b/web/src/components/ui/alert.tsx
new file mode 100644
index 00000000000..7ec0f0bacf0
--- /dev/null
+++ b/web/src/components/ui/alert.tsx
@@ -0,0 +1,60 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "@/lib/utils";
+
+const alertVariants = cva(
+  "relative w-full rounded-lg border border-neutral-200 p-4 [&>svg~*]:pl-7 [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-neutral-950 dark:border-neutral-800 dark:[&>svg]:text-neutral-50",
+  {
+    variants: {
+      variant: {
+        default:
+          "bg-white text-neutral-950 dark:bg-neutral-950 dark:text-neutral-50",
+        destructive:
+          "border-red-500/50 text-red-500 dark:border-red-500 [&>svg]:text-red-500 dark:border-red-900/50 dark:text-red-900 dark:dark:border-red-900 dark:[&>svg]:text-red-900",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+);
+
+const Alert = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement> & VariantProps<typeof alertVariants>
+>(({ className, variant, ...props }, ref) => (
+  <div
+    ref={ref}
+    role="alert"
+    className={cn(alertVariants({ variant }), className)}
+    {...props}
+  />
+));
+Alert.displayName = "Alert";
+
+const AlertTitle = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLHeadingElement>
+>(({ className, ...props }, ref) => (
+  <h5
+    ref={ref}
+    className={cn("mb-1 font-medium leading-none tracking-tight", className)}
+    {...props}
+  />
+));
+AlertTitle.displayName = "AlertTitle";
+
+const AlertDescription = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLParagraphElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("text-sm [&_p]:leading-relaxed", className)}
+    {...props}
+  />
+));
+AlertDescription.displayName = "AlertDescription";
+
+export { Alert, AlertTitle, AlertDescription };
diff --git a/web/src/components/ui/areaChart.tsx b/web/src/components/ui/areaChart.tsx
new file mode 100644
index 00000000000..a2c2045596a
--- /dev/null
+++ b/web/src/components/ui/areaChart.tsx
@@ -0,0 +1,129 @@
+"use client";
+
+import React from "react";
+import {
+  Area,
+  AreaChart as ReChartsAreaChart,
+  CartesianGrid,
+  ResponsiveContainer,
+  Tooltip,
+  XAxis,
+  YAxis,
+} from "recharts";
+
+import {
+  Card,
+  CardContent,
+  CardDescription,
+  CardHeader,
+  CardTitle,
+} from "@/components/ui/card";
+
+interface AreaChartProps {
+  data?: any[];
+  categories?: string[];
+  index?: string;
+  colors?: string[];
+  valueFormatter?: (value: number) => string;
+  startEndOnly?: boolean;
+  showXAxis?: boolean;
+  showYAxis?: boolean;
+  yAxisWidth?: number;
+  showAnimation?: boolean;
+  showTooltip?: boolean;
+  showLegend?: boolean;
+  showGridLines?: boolean;
+  showGradient?: boolean;
+  autoMinValue?: boolean;
+  minValue?: number;
+  maxValue?: number;
+  connectNulls?: boolean;
+  allowDecimals?: boolean;
+  className?: string;
+  title?: string;
+  description?: string;
+}
+
+export function AreaChartDisplay({
+  data = [],
+  categories = [],
+  index,
+  colors = ["indigo", "fuchsia"],
+  valueFormatter = (number: number) =>
+    `${Intl.NumberFormat("us").format(number).toString()}`,
+  startEndOnly = false,
+  showXAxis = true,
+  showYAxis = true,
+  yAxisWidth = 56,
+  showAnimation = true,
+  showTooltip = true,
+  showLegend = false,
+  showGridLines = true,
+  showGradient = true,
+  autoMinValue = false,
+  minValue,
+  maxValue,
+  connectNulls = false,
+  allowDecimals = true,
+  className,
+  title,
+  description,
+}: AreaChartProps) {
+  return (
+    <Card className={className}>
+      <CardHeader>
+        {title && <CardTitle>{title}</CardTitle>}
+        {description && <CardDescription>{description}</CardDescription>}
+      </CardHeader>
+      <CardContent>
+        <div className="h-[350px] w-full">
+          <ResponsiveContainer width="100%" height="100%">
+            <ReChartsAreaChart
+              data={data}
+              margin={{
+                top: 10,
+                right: 30,
+                left: 0,
+                bottom: 0,
+              }}
+            >
+              {showGridLines && <CartesianGrid strokeDasharray="3 3" />}
+              {showXAxis && (
+                <XAxis
+                  dataKey={index}
+                  tickLine={false}
+                  axisLine={false}
+                  tickMargin={8}
+                  tickFormatter={(value) => value.slice(0, 3)}
+                />
+              )}
+              {showYAxis && (
+                <YAxis
+                  width={yAxisWidth}
+                  tickLine={false}
+                  axisLine={false}
+                  tickFormatter={valueFormatter}
+                  allowDecimals={allowDecimals}
+                />
+              )}
+              {showTooltip && <Tooltip />}
+              {categories.map((category, ind) => (
+                <Area
+                  key={category}
+                  type="monotone"
+                  dataKey={category}
+                  stackId="1"
+                  stroke={colors[ind % colors.length]}
+                  fill={colors[ind % colors.length]}
+                  fillOpacity={0.3}
+                  isAnimationActive={showAnimation}
+                  connectNulls={connectNulls}
+                />
+              ))}
+            </ReChartsAreaChart>
+          </ResponsiveContainer>
+        </div>
+      </CardContent>
+    </Card>
+  );
+}
diff --git a/web/src/components/ui/badge.tsx b/web/src/components/ui/badge.tsx
new file mode 100644
index 00000000000..b0f230c632f
--- /dev/null
+++ b/web/src/components/ui/badge.tsx
@@ -0,0 +1,75 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "@/lib/utils";
+
+const badgeVariants = cva(
+  "inline-flex items-center rounded-full border border-neutral-200 px-2.5 py-0.5 text-xs font-normal transition-colors focus:outline-none focus:ring-2 focus:ring-neutral-950 focus:ring-offset-2 dark:border-neutral-800 dark:focus:ring-neutral-300",
+  {
+    variants: {
+      variant: {
+        orange:
+          "border-orange-200 bg-orange-50 text-orange-600 hover:bg-orange-75 dark:bg-orange-900 dark:text-neutral-50 dark:hover:bg-orange-850",
+        paused:
+          "border-amber-200 bg-amber-50 text-amber-600 hover:bg-amber-75 dark:bg-amber-900 dark:text-neutral-50 dark:hover:bg-amber-850",
+        in_progress:
+          "border-blue-200 bg-blue-50 text-blue-600 hover:bg-blue-75 dark:bg-blue-900 dark:text-neutral-50 dark:hover:bg-blue-850",
+
+        purple:
+          "border-purple-400 bg-purple-50 text-purple-600 hover:bg-purple-75 dark:bg-purple-900 dark:text-neutral-50 dark:hover:bg-purple-850",
+        success:
+          "border-green-200 bg-emerald-50 text-green-600 hover:bg-emerald-75 dark:bg-green-900 dark:text-neutral-50 dark:hover:bg-green-850",
+        default:
+          "border-neutral-200 bg-neutral-50 text-neutral-600 hover:bg-neutral-75 dark:bg-neutral-900 dark:text-neutral-50 dark:hover:bg-neutral-850",
+        secondary:
+          "border-neutral-200 bg-neutral-50 text-neutral-600 hover:bg-neutral-75 dark:bg-neutral-900 dark:text-neutral-50 dark:hover:bg-neutral-850",
+        destructive:
+          "border-red-200 bg-red-50 text-red-600 hover:bg-red-75 dark:bg-red-900 dark:text-neutral-50 dark:hover:bg-red-850",
+        outline:
+          "border-neutral-200 bg-neutral-50 text-neutral-600 hover:bg-neutral-75 dark:bg-neutral-900 dark:text-neutral-50 dark:hover:bg-neutral-850",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  }
+);
+
+export interface BadgeProps
+  extends React.HTMLAttributes<HTMLDivElement>,
+    VariantProps<typeof badgeVariants> {}
+function Badge({
+  className,
+  variant,
+  color,
+  icon: Icon,
+  size = "sm",
+  circle,
+  ...props
+}: BadgeProps & {
+  icon?: React.ElementType;
+  size?: "sm" | "md";
+  circle?: boolean;
+}) {
+  const sizeClasses = {
+    sm: "px-2.5 py-0.5 text-xs",
+    md: "px-3 py-1 text-sm",
+  };
+
+  return (
+    <div
+      className={cn(badgeVariants({ variant }), sizeClasses[size], className)}
+      {...props}
+    >
+      {Icon && (
+        <Icon className={cn("mr-1", size === "sm" ? "h-3 w-3" : "h-4 w-4")} />
+      )}
+      {circle && (
+        <div className="h-2.5 w-2.5 mr-2 rounded-full bg-current opacity-80" />
+      )}
+      {props.children}
+    </div>
+  );
+}
+
+export { Badge, badgeVariants };
diff --git a/web/src/components/ui/button.tsx b/web/src/components/ui/button.tsx
new file mode 100644
index 00000000000..91cffa3a926
--- /dev/null
+++ b/web/src/components/ui/button.tsx
@@ -0,0 +1,132 @@
+import * as React from "react";
+import { Slot } from "@radix-ui/react-slot";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "@/lib/utils";
+
+const buttonVariants = cva(
+  "inline-flex cursor-pointer items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium ring-offset-white transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-950 focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0 dark:ring-offset-neutral-950 dark:focus-visible:ring-neutral-300",
+  {
+    variants: {
+      variant: {
+        success:
+          "bg-green-100 text-green-600 hover:bg-green-500/90 dark:bg-blue-500 dark:text-neutral-50 dark:hover:bg-green-900/90",
+        "success-reverse":
+          "bg-green-500 text-green-100 hover:bg-green-600/90 dark:bg-neutral-50 dark:text-blue-500 dark:hover:bg-green-100/90",
+
+        default:
+          "bg-neutral-900 border-border text-neutral-50 hover:bg-neutral-900/90 dark:bg-neutral-50 dark:text-neutral-900 dark:hover:bg-neutral-50/90",
+        "default-reverse":
+          "bg-neutral-50 border-border text-neutral-900 hover:bg-neutral-50/90 dark:bg-neutral-900 dark:text-neutral-50 dark:hover:bg-neutral-900/90",
+        destructive:
+          "bg-red-500 text-neutral-50 hover:bg-red-500/90 dark:bg-red-900 dark:text-neutral-50 dark:hover:bg-red-900/90",
+        "destructive-reverse":
+          "bg-neutral-50 text-red-500 hover:bg-neutral-50/90 dark:bg-neutral-50 dark:text-red-900 dark:hover:bg-neutral-50/90",
+        outline:
+          "border border-neutral-300 bg-white hover:bg-neutral-50 hover:text-neutral-900 dark:border-neutral-800 dark:bg-neutral-950 dark:hover:bg-neutral-800 dark:hover:text-neutral-50",
+        "outline-reverse":
+          "border border-neutral-300 bg-neutral-900 hover:bg-neutral-800 hover:text-neutral-50 dark:border-neutral-800 dark:bg-white dark:hover:bg-neutral-50 dark:hover:text-neutral-900",
+        secondary:
+          "bg-neutral-100 text-neutral-900 hover:bg-neutral-100/80 dark:bg-neutral-800 dark:text-neutral-50 dark:hover:bg-neutral-800/80",
+        "secondary-reverse":
+          "bg-neutral-900 text-neutral-100 hover:bg-neutral-900/80 dark:bg-neutral-50 dark:text-neutral-800 dark:hover:bg-neutral-50/80",
+        ghost:
+          "hover:bg-neutral-100 hover:text-neutral-900 dark:hover:bg-neutral-800 dark:hover:text-neutral-50",
+        "ghost-reverse":
+          "hover:bg-neutral-800 hover:text-neutral-50 dark:hover:bg-neutral-100 dark:hover:text-neutral-900",
+        link: "text-neutral-900 underline-offset-4 hover:underline dark:text-neutral-50",
+        "link-reverse":
+          "text-neutral-50 underline-offset-4 hover:underline dark:text-neutral-900",
+        submit:
+          "bg-green-500 text-green-100 hover:bg-green-600/90 dark:bg-neutral-50 dark:text-blue-500 dark:hover:bg-green-100/90",
+
+        // "bg-blue-600 text-neutral-50 hover:bg-blue-600/80 dark:bg-blue-600 dark:text-neutral-50 dark:hover:bg-blue-600/90",
+        "submit-reverse":
+          "bg-neutral-50 text-blue-600 hover:bg-neutral-50/80 dark:bg-neutral-50 dark:text-blue-600 dark:hover:bg-neutral-50/90",
+        navigate:
+          "bg-blue-500 text-white hover:bg-blue-600 dark:bg-blue-600 dark:hover:bg-blue-700",
+        "navigate-reverse":
+          "bg-white text-blue-500 hover:bg-blue-50 dark:bg-blue-100 dark:hover:bg-blue-200",
+        update:
+          "border border-neutral-300 bg-neutral-100 text-neutral-900 hover:bg-neutral-100/80 dark:bg-neutral-800 dark:text-neutral-50 dark:hover:bg-neutral-800/80",
+        "update-reverse":
+          "bg-neutral-900 text-neutral-100 hover:bg-neutral-900/80 dark:bg-neutral-50 dark:text-neutral-800 dark:hover:bg-neutral-50/80",
+        next: "bg-neutral-700 text-neutral-50 hover:bg-neutral-700/90 dark:bg-neutral-600 dark:text-neutral-50 dark:hover:bg-neutral-600/90",
+        "next-reverse":
+          "bg-neutral-50 text-neutral-700 hover:bg-neutral-50/90 dark:bg-neutral-50 dark:text-neutral-600 dark:hover:bg-neutral-50/90",
+      },
+      size: {
+        default: "h-10 px-4 py-2",
+        sm: "h-9 rounded-md px-3",
+        lg: "h-11 rounded-md px-8",
+        icon: "h-10 w-10",
+      },
+      reverse: {
+        true: "",
+        false: "",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+      size: "default",
+    },
+  }
+);
+
+export interface ButtonProps
+  extends React.ButtonHTMLAttributes<HTMLButtonElement>,
+    VariantProps<typeof buttonVariants> {
+  asChild?: boolean;
+  icon?: React.ElementType;
+  tooltip?: string;
+  reverse?: boolean;
+}
+
+const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
+  (
+    {
+      className,
+      variant,
+      size = "sm",
+      asChild = false,
+      icon: Icon,
+      tooltip,
+      ...props
+    },
+    ref
+  ) => {
+    const Comp = asChild ? Slot : "button";
+    const button = (
+      <Comp
+        className={cn(
+          buttonVariants({
+            variant,
+            size,
+            className,
+          })
+        )}
+        ref={ref}
+        {...props}
+      >
+        {Icon && <Icon />}
+        {props.children}
+      </Comp>
+    );
+
+    if (tooltip) {
+      return (
+        <div className="relative group">
+          {button}
+          <div className="absolute bottom-full left-1/2 transform -translate-x-1/2 mb-2 px-2 py-1 bg-neutral-800 text-white text-sm rounded opacity-0 group-hover:opacity-100 transition-opacity whitespace-nowrap">
+            {tooltip}
+          </div>
+        </div>
+      );
+    }
+
+    return button;
+  }
+);
+Button.displayName = "Button";
+
+export { Button, buttonVariants };
diff --git a/web/src/components/ui/calendar.tsx b/web/src/components/ui/calendar.tsx
new file mode 100644
index 00000000000..233055c6b56
--- /dev/null
+++ b/web/src/components/ui/calendar.tsx
@@ -0,0 +1,67 @@
+"use client";
+
+import * as React from "react";
+import { ChevronLeft, ChevronRight } from "lucide-react";
+import { DayPicker } from "react-day-picker";
+
+import { cn } from "@/lib/utils";
+import { buttonVariants } from "@/components/ui/button";
+
+export type CalendarProps = React.ComponentProps<typeof DayPicker>;
+
+function Calendar({
+  className,
+  classNames,
+  showOutsideDays = true,
+  ...props
+}: CalendarProps) {
+  return (
+    <DayPicker
+      showOutsideDays={showOutsideDays}
+      className={cn("p-3", className)}
+      classNames={{
+        months: "flex flex-col sm:flex-row space-y-4 sm:space-x-4 sm:space-y-0",
+        month: "space-y-4",
+        caption: "flex justify-center pt-1 relative items-center",
+        caption_label: "text-sm font-medium",
+        nav: "space-x-1 flex items-center",
+        nav_button: cn(
+          buttonVariants({ variant: "outline" }),
+          "h-7 w-7 bg-transparent p-0 opacity-50 hover:opacity-100"
+        ),
+        nav_button_previous: "absolute left-1",
+        nav_button_next: "absolute right-1",
+        table: "w-full border-collapse space-y-1",
+        head_row: "flex",
+        head_cell:
+          "text-neutral-500 rounded-md w-9 font-normal text-[0.8rem] dark:text-neutral-400",
+        row: "flex w-full mt-2",
+        cell: "h-9 w-9 text-center text-sm p-0 relative [&:has([aria-selected].day-range-end)]:rounded-r-md [&:has([aria-selected].day-outside)]:bg-neutral-100/50 [&:has([aria-selected])]:bg-neutral-100 first:[&:has([aria-selected])]:rounded-l-md last:[&:has([aria-selected])]:rounded-r-md focus-within:relative focus-within:z-20 dark:[&:has([aria-selected].day-outside)]:bg-neutral-800/50 dark:[&:has([aria-selected])]:bg-neutral-800",
+        day: cn(
+          buttonVariants({ variant: "ghost" }),
+          "h-9 w-9 p-0 font-normal aria-selected:opacity-100"
+        ),
+        day_range_end: "day-range-end",
+        day_selected:
+          "bg-neutral-900 text-neutral-50 hover:bg-neutral-900 hover:text-neutral-50 focus:bg-neutral-900 focus:text-neutral-50 dark:bg-neutral-50 dark:text-neutral-900 dark:hover:bg-neutral-50 dark:hover:text-neutral-900 dark:focus:bg-neutral-50 dark:focus:text-neutral-900",
+        day_today:
+          "bg-neutral-100 text-neutral-900 dark:bg-neutral-800 dark:text-neutral-50",
+        day_outside:
+          "day-outside text-neutral-500 opacity-50 aria-selected:bg-neutral-100/50 aria-selected:text-neutral-500 aria-selected:opacity-30 dark:text-neutral-400 dark:aria-selected:bg-neutral-800/50 dark:aria-selected:text-neutral-400",
+        day_disabled: "text-neutral-500 opacity-50 dark:text-neutral-400",
+        day_range_middle:
+          "aria-selected:bg-neutral-100 aria-selected:text-neutral-900 dark:aria-selected:bg-neutral-800 dark:aria-selected:text-neutral-50",
+        day_hidden: "invisible",
+        ...classNames,
+      }}
+      components={{
+        IconLeft: ({ ...props }) => <ChevronLeft className="h-4 w-4" />,
+        IconRight: ({ ...props }) => <ChevronRight className="h-4 w-4" />,
+      }}
+      {...props}
+    />
+  );
+}
+Calendar.displayName = "Calendar";
+
+export { Calendar };
diff --git a/web/src/components/ui/callout.tsx b/web/src/components/ui/callout.tsx
new file mode 100644
index 00000000000..99dbc31c900
--- /dev/null
+++ b/web/src/components/ui/callout.tsx
@@ -0,0 +1,38 @@
+import { cn } from "@/lib/utils";
+
+interface CalloutProps {
+  icon?: string;
+  children?: React.ReactNode;
+  type?: "default" | "warning" | "danger" | "notice";
+  className?: string;
+}
+
+export function Callout({
+  children,
+  icon,
+  type = "default",
+  title,
+  className,
+  ...props
+}: CalloutProps & { title?: string }) {
+  return (
+    <div
+      className={cn(
+        "my-6 flex items-start rounded-md border border-l-4 p-4",
+        className,
+        {
+          "border-rose-300 bg-rose-50": type === "danger",
+          "border-amber-300 bg-amber-50": type === "warning",
+          "border-sky-300 bg-sky-50": type === "notice",
+        }
+      )}
+      {...props}
+    >
+      {icon && <span className="mr-4 text-2xl">{icon}</span>}
+      <div>
+        {title && <div className="font-medium mb-1">{title}</div>}
+        {children}
+      </div>
+    </div>
+  );
+}
diff --git a/web/src/components/ui/card.tsx b/web/src/components/ui/card.tsx
new file mode 100644
index 00000000000..46a47e15c62
--- /dev/null
+++ b/web/src/components/ui/card.tsx
@@ -0,0 +1,86 @@
+import * as React from "react";
+
+import { cn } from "@/lib/utils";
+
+const Card = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn(
+      "rounded-lg border border-neutral-200 bg-white text-neutral-950 shadow-sm dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-50",
+      className
+    )}
+    {...props}
+  />
+));
+Card.displayName = "Card";
+
+const CardHeader = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("flex flex-col space-y-1.5 p-6", className)}
+    {...props}
+  />
+));
+CardHeader.displayName = "CardHeader";
+
+const CardTitle = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLHeadingElement>
+>(({ className, ...props }, ref) => (
+  <h3
+    ref={ref}
+    className={cn(
+      "text-2xl font-semibold leading-none tracking-tight",
+      className
+    )}
+    {...props}
+  />
+));
+CardTitle.displayName = "CardTitle";
+
+const CardDescription = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLParagraphElement>
+>(({ className, ...props }, ref) => (
+  <p
+    ref={ref}
+    className={cn("text-sm text-neutral-500 dark:text-neutral-400", className)}
+    {...props}
+  />
+));
+CardDescription.displayName = "CardDescription";
+
+const CardContent = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
+));
+CardContent.displayName = "CardContent";
+
+const CardFooter = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("flex items-center p-6 pt-0", className)}
+    {...props}
+  />
+));
+CardFooter.displayName = "CardFooter";
+
+export {
+  Card,
+  CardHeader,
+  CardFooter,
+  CardTitle,
+  CardDescription,
+  CardContent,
+};
diff --git a/web/src/components/ui/chart.tsx b/web/src/components/ui/chart.tsx
new file mode 100644
index 00000000000..deb443f624b
--- /dev/null
+++ b/web/src/components/ui/chart.tsx
@@ -0,0 +1,365 @@
+"use client";
+
+import * as React from "react";
+import * as RechartsPrimitive from "recharts";
+
+import { cn } from "@/lib/utils";
+
+// Format: { THEME_NAME: CSS_SELECTOR }
+const THEMES = { light: "", dark: ".dark" } as const;
+
+export type ChartConfig = {
+  [k in string]: {
+    label?: React.ReactNode;
+    icon?: React.ComponentType;
+  } & (
+    | { color?: string; theme?: never }
+    | { color?: never; theme: Record<keyof typeof THEMES, string> }
+  );
+};
+
+type ChartContextProps = {
+  config: ChartConfig;
+};
+
+const ChartContext = React.createContext<ChartContextProps | null>(null);
+
+function useChart() {
+  const context = React.useContext(ChartContext);
+
+  if (!context) {
+    throw new Error("useChart must be used within a <ChartContainer />");
+  }
+
+  return context;
+}
+
+const ChartContainer = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<"div"> & {
+    config: ChartConfig;
+    children: React.ComponentProps<
+      typeof RechartsPrimitive.ResponsiveContainer
+    >["children"];
+  }
+>(({ id, className, children, config, ...props }, ref) => {
+  const uniqueId = React.useId();
+  const chartId = `chart-${id || uniqueId.replace(/:/g, "")}`;
+
+  return (
+    <ChartContext.Provider value={{ config }}>
+      <div
+        data-chart={chartId}
+        ref={ref}
+        className={cn(
+          "flex aspect-video justify-center text-xs [&_.recharts-cartesian-axis-tick_text]:fill-muted-foreground [&_.recharts-cartesian-grid_line[stroke='#ccc']]:stroke-border/50 [&_.recharts-curve.recharts-tooltip-cursor]:stroke-border [&_.recharts-dot[stroke='#fff']]:stroke-transparent [&_.recharts-layer]:outline-none [&_.recharts-polar-grid_[stroke='#ccc']]:stroke-border [&_.recharts-radial-bar-background-sector]:fill-muted [&_.recharts-rectangle.recharts-tooltip-cursor]:fill-muted [&_.recharts-reference-line_[stroke='#ccc']]:stroke-border [&_.recharts-sector[stroke='#fff']]:stroke-transparent [&_.recharts-sector]:outline-none [&_.recharts-surface]:outline-none",
+          className
+        )}
+        {...props}
+      >
+        <ChartStyle id={chartId} config={config} />
+        <RechartsPrimitive.ResponsiveContainer>
+          {children}
+        </RechartsPrimitive.ResponsiveContainer>
+      </div>
+    </ChartContext.Provider>
+  );
+});
+ChartContainer.displayName = "Chart";
+
+const ChartStyle = ({ id, config }: { id: string; config: ChartConfig }) => {
+  const colorConfig = Object.entries(config).filter(
+    ([_, config]) => config.theme || config.color
+  );
+
+  if (!colorConfig.length) {
+    return null;
+  }
+
+  return (
+    <style
+      dangerouslySetInnerHTML={{
+        __html: Object.entries(THEMES)
+          .map(
+            ([theme, prefix]) => `
+${prefix} [data-chart=${id}] {
+${colorConfig
+  .map(([key, itemConfig]) => {
+    const color =
+      itemConfig.theme?.[theme as keyof typeof itemConfig.theme] ||
+      itemConfig.color;
+    return color ? `  --color-${key}: ${color};` : null;
+  })
+  .join("\n")}
+}
+`
+          )
+          .join("\n"),
+      }}
+    />
+  );
+};
+
+const ChartTooltip = RechartsPrimitive.Tooltip;
+
+const ChartTooltipContent = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<typeof RechartsPrimitive.Tooltip> &
+    React.ComponentProps<"div"> & {
+      hideLabel?: boolean;
+      hideIndicator?: boolean;
+      indicator?: "line" | "dot" | "dashed";
+      nameKey?: string;
+      labelKey?: string;
+    }
+>(
+  (
+    {
+      active,
+      payload,
+      className,
+      indicator = "dot",
+      hideLabel = false,
+      hideIndicator = false,
+      label,
+      labelFormatter,
+      labelClassName,
+      formatter,
+      color,
+      nameKey,
+      labelKey,
+    },
+    ref
+  ) => {
+    const { config } = useChart();
+
+    const tooltipLabel = React.useMemo(() => {
+      if (hideLabel || !payload?.length) {
+        return null;
+      }
+
+      const [item] = payload;
+      const key = `${labelKey || item.dataKey || item.name || "value"}`;
+      const itemConfig = getPayloadConfigFromPayload(config, item, key);
+      const value =
+        !labelKey && typeof label === "string"
+          ? config[label as keyof typeof config]?.label || label
+          : itemConfig?.label;
+
+      if (labelFormatter) {
+        return (
+          <div className={cn("font-medium", labelClassName)}>
+            {labelFormatter(value, payload)}
+          </div>
+        );
+      }
+
+      if (!value) {
+        return null;
+      }
+
+      return <div className={cn("font-medium", labelClassName)}>{value}</div>;
+    }, [
+      label,
+      labelFormatter,
+      payload,
+      hideLabel,
+      labelClassName,
+      config,
+      labelKey,
+    ]);
+
+    if (!active || !payload?.length) {
+      return null;
+    }
+
+    const nestLabel = payload.length === 1 && indicator !== "dot";
+
+    return (
+      <div
+        ref={ref}
+        className={cn(
+          "grid min-w-[8rem] items-start gap-1.5 rounded-lg border border-neutral-200 border-neutral-200/50 bg-white px-2.5 py-1.5 text-xs shadow-xl dark:border-neutral-800 dark:border-neutral-800/50 dark:bg-neutral-950",
+          className
+        )}
+      >
+        {!nestLabel ? tooltipLabel : null}
+        <div className="grid gap-1.5">
+          {payload.map((item, index) => {
+            const key = `${nameKey || item.name || item.dataKey || "value"}`;
+            const itemConfig = getPayloadConfigFromPayload(config, item, key);
+            const indicatorColor = color || item.payload.fill || item.color;
+
+            return (
+              <div
+                key={item.dataKey}
+                className={cn(
+                  "flex w-full flex-wrap items-stretch gap-2 [&>svg]:h-2.5 [&>svg]:w-2.5 [&>svg]:text-neutral-500 dark:[&>svg]:text-neutral-400",
+                  indicator === "dot" && "items-center"
+                )}
+              >
+                {formatter && item?.value !== undefined && item.name ? (
+                  formatter(item.value, item.name, item, index, item.payload)
+                ) : (
+                  <>
+                    {itemConfig?.icon ? (
+                      <itemConfig.icon />
+                    ) : (
+                      !hideIndicator && (
+                        <div
+                          className={cn(
+                            "shrink-0 rounded-[2px] border-[--color-border] bg-[--color-bg]",
+                            {
+                              "h-2.5 w-2.5": indicator === "dot",
+                              "w-1": indicator === "line",
+                              "w-0 border-[1.5px] border-dashed bg-transparent":
+                                indicator === "dashed",
+                              "my-0.5": nestLabel && indicator === "dashed",
+                            }
+                          )}
+                          style={
+                            {
+                              "--color-bg": indicatorColor,
+                              "--color-border": indicatorColor,
+                            } as React.CSSProperties
+                          }
+                        />
+                      )
+                    )}
+                    <div
+                      className={cn(
+                        "flex flex-1 justify-between leading-none",
+                        nestLabel ? "items-end" : "items-center"
+                      )}
+                    >
+                      <div className="grid gap-1.5">
+                        {nestLabel ? tooltipLabel : null}
+                        <span className="text-neutral-500 dark:text-neutral-400">
+                          {itemConfig?.label || item.name}
+                        </span>
+                      </div>
+                      {item.value && (
+                        <span className="font-mono font-medium tabular-nums text-neutral-950 dark:text-neutral-50">
+                          {item.value.toLocaleString()}
+                        </span>
+                      )}
+                    </div>
+                  </>
+                )}
+              </div>
+            );
+          })}
+        </div>
+      </div>
+    );
+  }
+);
+ChartTooltipContent.displayName = "ChartTooltip";
+
+const ChartLegend = RechartsPrimitive.Legend;
+
+const ChartLegendContent = React.forwardRef<
+  HTMLDivElement,
+  React.ComponentProps<"div"> &
+    Pick<RechartsPrimitive.LegendProps, "payload" | "verticalAlign"> & {
+      hideIcon?: boolean;
+      nameKey?: string;
+    }
+>(
+  (
+    { className, hideIcon = false, payload, verticalAlign = "bottom", nameKey },
+    ref
+  ) => {
+    const { config } = useChart();
+
+    if (!payload?.length) {
+      return null;
+    }
+
+    return (
+      <div
+        ref={ref}
+        className={cn(
+          "flex items-center justify-center gap-4",
+          verticalAlign === "top" ? "pb-3" : "pt-3",
+          className
+        )}
+      >
+        {payload.map((item) => {
+          const key = `${nameKey || item.dataKey || "value"}`;
+          const itemConfig = getPayloadConfigFromPayload(config, item, key);
+
+          return (
+            <div
+              key={item.value}
+              className={cn(
+                "flex items-center gap-1.5 [&>svg]:h-3 [&>svg]:w-3 [&>svg]:text-neutral-500 dark:[&>svg]:text-neutral-400"
+              )}
+            >
+              {itemConfig?.icon && !hideIcon ? (
+                <itemConfig.icon />
+              ) : (
+                <div
+                  className="h-2 w-2 shrink-0 rounded-[2px]"
+                  style={{
+                    backgroundColor: item.color,
+                  }}
+                />
+              )}
+              {itemConfig?.label}
+            </div>
+          );
+        })}
+      </div>
+    );
+  }
+);
+ChartLegendContent.displayName = "ChartLegend";
+
+// Helper to extract item config from a payload.
+function getPayloadConfigFromPayload(
+  config: ChartConfig,
+  payload: unknown,
+  key: string
+) {
+  if (typeof payload !== "object" || payload === null) {
+    return undefined;
+  }
+
+  const payloadPayload =
+    "payload" in payload &&
+    typeof payload.payload === "object" &&
+    payload.payload !== null
+      ? payload.payload
+      : undefined;
+
+  let configLabelKey: string = key;
+
+  if (
+    key in payload &&
+    typeof payload[key as keyof typeof payload] === "string"
+  ) {
+    configLabelKey = payload[key as keyof typeof payload] as string;
+  } else if (
+    payloadPayload &&
+    key in payloadPayload &&
+    typeof payloadPayload[key as keyof typeof payloadPayload] === "string"
+  ) {
+    configLabelKey = payloadPayload[
+      key as keyof typeof payloadPayload
+    ] as string;
+  }
+
+  return configLabelKey in config
+    ? config[configLabelKey]
+    : config[key as keyof typeof config];
+}
+
+export {
+  ChartContainer,
+  ChartTooltip,
+  ChartTooltipContent,
+  ChartLegend,
+  ChartLegendContent,
+  ChartStyle,
+};
diff --git a/web/src/components/ui/input.tsx b/web/src/components/ui/input.tsx
new file mode 100644
index 00000000000..12f1c7dff96
--- /dev/null
+++ b/web/src/components/ui/input.tsx
@@ -0,0 +1,25 @@
+import * as React from "react";
+
+import { cn } from "@/lib/utils";
+
+export interface InputProps
+  extends React.InputHTMLAttributes<HTMLInputElement> {}
+
+const Input = React.forwardRef<HTMLInputElement, InputProps>(
+  ({ className, type, ...props }, ref) => {
+    return (
+      <input
+        type={type}
+        className={cn(
+          "flex h-10 w-full rounded-md border border-neutral-200 bg-white px-3 py-2 text-sm ring-offset-white file:border-0 file:bg-transparent file:text-sm file:font-medium file:text-neutral-950 placeholder:text-neutral-500 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-950 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-neutral-800 dark:bg-neutral-950 dark:ring-offset-neutral-950 dark:file:text-neutral-50 dark:placeholder:text-neutral-400 dark:focus-visible:ring-neutral-300",
+          className
+        )}
+        ref={ref}
+        {...props}
+      />
+    );
+  }
+);
+Input.displayName = "Input";
+
+export { Input };
diff --git a/web/src/components/ui/popover.tsx b/web/src/components/ui/popover.tsx
new file mode 100644
index 00000000000..10c3830324d
--- /dev/null
+++ b/web/src/components/ui/popover.tsx
@@ -0,0 +1,31 @@
+"use client";
+
+import * as React from "react";
+import * as PopoverPrimitive from "@radix-ui/react-popover";
+
+import { cn } from "@/lib/utils";
+
+const Popover = PopoverPrimitive.Root;
+
+const PopoverTrigger = PopoverPrimitive.Trigger;
+
+const PopoverContent = React.forwardRef<
+  React.ElementRef<typeof PopoverPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof PopoverPrimitive.Content>
+>(({ className, align = "center", sideOffset = 4, ...props }, ref) => (
+  <PopoverPrimitive.Portal>
+    <PopoverPrimitive.Content
+      ref={ref}
+      align={align}
+      sideOffset={sideOffset}
+      className={cn(
+        "z-50 w-72 rounded-md border border-neutral-200 bg-white p-4 text-neutral-950 shadow-md outline-none data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-50",
+        className
+      )}
+      {...props}
+    />
+  </PopoverPrimitive.Portal>
+));
+PopoverContent.displayName = PopoverPrimitive.Content.displayName;
+
+export { Popover, PopoverTrigger, PopoverContent };
diff --git a/web/src/components/ui/select.tsx b/web/src/components/ui/select.tsx
new file mode 100644
index 00000000000..8bbcc971748
--- /dev/null
+++ b/web/src/components/ui/select.tsx
@@ -0,0 +1,165 @@
+"use client";
+
+import * as React from "react";
+import * as SelectPrimitive from "@radix-ui/react-select";
+import { Check, ChevronDown, ChevronUp } from "lucide-react";
+
+import { cn } from "@/lib/utils";
+
+const Select = SelectPrimitive.Root;
+
+const SelectGroup = SelectPrimitive.Group;
+
+const SelectValue = SelectPrimitive.Value;
+
+const SelectTrigger = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Trigger>
+>(({ className, children, ...props }, ref) => (
+  <SelectPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "flex h-10 w-full items-center justify-between rounded-md border border-neutral-200 bg-white px-3 py-2 text-sm ring-offset-white placeholder:text-neutral-500 focus:outline-none focus:ring-2 focus:ring-neutral-950 focus:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1 dark:border-neutral-800 dark:bg-neutral-950 dark:ring-offset-neutral-950 dark:placeholder:text-neutral-400 dark:focus:ring-neutral-300",
+      className
+    )}
+    {...props}
+  >
+    {children}
+    <SelectPrimitive.Icon asChild>
+      <ChevronDown className="h-4 w-4 opacity-50" />
+    </SelectPrimitive.Icon>
+  </SelectPrimitive.Trigger>
+));
+SelectTrigger.displayName = SelectPrimitive.Trigger.displayName;
+
+const SelectScrollUpButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollUpButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollUpButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollUpButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className
+    )}
+    {...props}
+  >
+    <ChevronUp className="h-4 w-4" />
+  </SelectPrimitive.ScrollUpButton>
+));
+SelectScrollUpButton.displayName = SelectPrimitive.ScrollUpButton.displayName;
+
+const SelectScrollDownButton = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.ScrollDownButton>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.ScrollDownButton>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.ScrollDownButton
+    ref={ref}
+    className={cn(
+      "flex cursor-default items-center justify-center py-1",
+      className
+    )}
+    {...props}
+  >
+    <ChevronDown className="h-4 w-4" />
+  </SelectPrimitive.ScrollDownButton>
+));
+SelectScrollDownButton.displayName =
+  SelectPrimitive.ScrollDownButton.displayName;
+
+const SelectContent = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
+>(({ className, children, position = "popper", ...props }, ref) => (
+  <SelectPrimitive.Portal>
+    <SelectPrimitive.Content
+      ref={ref}
+      className={cn(
+        "relative z-50 max-h-96 min-w-[8rem] overflow-hidden rounded-md border border-neutral-200 bg-white text-neutral-950 shadow-md data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-50",
+        position === "popper" &&
+          "data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
+        className
+      )}
+      position={position}
+      {...props}
+    >
+      <SelectScrollUpButton />
+      <SelectPrimitive.Viewport
+        className={cn(
+          "p-1",
+          position === "popper" &&
+            "h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)]"
+        )}
+      >
+        {children}
+      </SelectPrimitive.Viewport>
+      <SelectScrollDownButton />
+    </SelectPrimitive.Content>
+  </SelectPrimitive.Portal>
+));
+SelectContent.displayName = SelectPrimitive.Content.displayName;
+
+const SelectLabel = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Label>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Label>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Label
+    ref={ref}
+    className={cn("py-1.5 pl-8 pr-2 text-sm font-semibold", className)}
+    {...props}
+  />
+));
+SelectLabel.displayName = SelectPrimitive.Label.displayName;
+
+const SelectItem = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Item>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item> & {
+    icon?: React.ElementType;
+  }
+>(({ className, children, icon: Icon, ...props }, ref) => (
+  <SelectPrimitive.Item
+    ref={ref}
+    className={cn(
+      "relative flex w-full cursor-default select-none items-center rounded-sm py-1.5 pl-8 pr-2 text-sm outline-none focus:bg-neutral-100 focus:text-neutral-900 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 dark:focus:bg-neutral-800 dark:focus:text-neutral-50",
+      className
+    )}
+    {...props}
+  >
+    <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+      <SelectPrimitive.ItemIndicator>
+        {Icon ? <Icon className="h-4 w-4" /> : <Check className="h-4 w-4" />}
+      </SelectPrimitive.ItemIndicator>
+    </span>
+
+    <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+  </SelectPrimitive.Item>
+));
+SelectItem.displayName = SelectPrimitive.Item.displayName;
+
+const SelectSeparator = React.forwardRef<
+  React.ElementRef<typeof SelectPrimitive.Separator>,
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Separator>
+>(({ className, ...props }, ref) => (
+  <SelectPrimitive.Separator
+    ref={ref}
+    className={cn(
+      "-mx-1 my-1 h-px bg-neutral-100 dark:bg-neutral-800",
+      className
+    )}
+    {...props}
+  />
+));
+SelectSeparator.displayName = SelectPrimitive.Separator.displayName;
+
+export {
+  Select,
+  SelectGroup,
+  SelectValue,
+  SelectTrigger,
+  SelectContent,
+  SelectLabel,
+  SelectItem,
+  SelectSeparator,
+  SelectScrollUpButton,
+  SelectScrollDownButton,
+};
diff --git a/web/src/components/ui/separator.tsx b/web/src/components/ui/separator.tsx
new file mode 100644
index 00000000000..51765db8402
--- /dev/null
+++ b/web/src/components/ui/separator.tsx
@@ -0,0 +1,31 @@
+"use client";
+
+import * as React from "react";
+import * as SeparatorPrimitive from "@radix-ui/react-separator";
+
+import { cn } from "@/lib/utils";
+
+const Separator = React.forwardRef<
+  React.ElementRef<typeof SeparatorPrimitive.Root>,
+  React.ComponentPropsWithoutRef<typeof SeparatorPrimitive.Root>
+>(
+  (
+    { className, orientation = "horizontal", decorative = true, ...props },
+    ref
+  ) => (
+    <SeparatorPrimitive.Root
+      ref={ref}
+      decorative={decorative}
+      orientation={orientation}
+      className={cn(
+        "shrink-0 my-4 bg-neutral-200 dark:bg-neutral-800",
+        orientation === "horizontal" ? "h-[1px] w-full" : "h-full w-[1px]",
+        className
+      )}
+      {...props}
+    />
+  )
+);
+Separator.displayName = SeparatorPrimitive.Root.displayName;
+
+export { Separator };
diff --git a/web/src/components/ui/table.tsx b/web/src/components/ui/table.tsx
new file mode 100644
index 00000000000..25756a80cc4
--- /dev/null
+++ b/web/src/components/ui/table.tsx
@@ -0,0 +1,120 @@
+import * as React from "react";
+
+import { cn } from "@/lib/utils";
+
+const Table = React.forwardRef<
+  HTMLTableElement,
+  React.HTMLAttributes<HTMLTableElement>
+>(({ className, ...props }, ref) => (
+  <div className="relative w-full overflow-auto">
+    <table
+      ref={ref}
+      className={cn("w-full caption-bottom text-sm", className)}
+      {...props}
+    />
+  </div>
+));
+Table.displayName = "Table";
+
+const TableHeader = React.forwardRef<
+  HTMLTableSectionElement,
+  React.HTMLAttributes<HTMLTableSectionElement>
+>(({ className, ...props }, ref) => (
+  <thead ref={ref} className={cn("[&_tr]:border-b", className)} {...props} />
+));
+TableHeader.displayName = "TableHeader";
+
+const TableBody = React.forwardRef<
+  HTMLTableSectionElement,
+  React.HTMLAttributes<HTMLTableSectionElement>
+>(({ className, ...props }, ref) => (
+  <tbody
+    ref={ref}
+    className={cn("[&_tr:last-child]:border-0", className)}
+    {...props}
+  />
+));
+TableBody.displayName = "TableBody";
+
+const TableFooter = React.forwardRef<
+  HTMLTableSectionElement,
+  React.HTMLAttributes<HTMLTableSectionElement>
+>(({ className, ...props }, ref) => (
+  <tfoot
+    ref={ref}
+    className={cn(
+      "border-t bg-neutral-100/50 font-medium [&>tr]:last:border-b-0 dark:bg-neutral-800/50",
+      className
+    )}
+    {...props}
+  />
+));
+TableFooter.displayName = "TableFooter";
+
+const TableRow = React.forwardRef<
+  HTMLTableRowElement,
+  React.HTMLAttributes<HTMLTableRowElement>
+>(({ className, ...props }, ref) => (
+  <tr
+    ref={ref}
+    className={cn(
+      "border-b transition-colors hover:bg-neutral-100/50 data-[state=selected]:bg-neutral-100 dark:hover:bg-neutral-800/50 dark:data-[state=selected]:bg-neutral-800",
+      className
+    )}
+    {...props}
+  />
+));
+TableRow.displayName = "TableRow";
+
+const TableHead = React.forwardRef<
+  HTMLTableCellElement,
+  React.ThHTMLAttributes<HTMLTableCellElement>
+>(({ className, ...props }, ref) => (
+  <th
+    ref={ref}
+    className={cn(
+      "h-12 px-4 text-left align-middle font-medium text-neutral-800 [&:has([role=checkbox])]:pr-0 dark:text-neutral-400",
+      className
+    )}
+    {...props}
+  />
+));
+TableHead.displayName = "TableHead";
+
+const TableCell = React.forwardRef<
+  HTMLTableCellElement,
+  React.TdHTMLAttributes<HTMLTableCellElement>
+>(({ className, ...props }, ref) => (
+  <td
+    ref={ref}
+    className={cn("p-4 align-middle [&:has([role=checkbox])]:pr-0", className)}
+    {...props}
+  />
+));
+TableCell.displayName = "TableCell";
+
+const TableCaption = React.forwardRef<
+  HTMLTableCaptionElement,
+  React.HTMLAttributes<HTMLTableCaptionElement>
+>(({ className, ...props }, ref) => (
+  <caption
+    ref={ref}
+    className={cn(
+      "mt-4 text-sm text-neutral-500 dark:text-neutral-400",
+      className
+    )}
+    {...props}
+  />
+));
+TableCaption.displayName = "TableCaption";
+
+export {
+  Table,
+  TableHeader,
+  TableBody,
+  TableFooter,
+  TableHead,
+  TableRow,
+  TableCell,
+  TableCaption,
+};
diff --git a/web/src/components/ui/tabs.tsx b/web/src/components/ui/tabs.tsx
new file mode 100644
index 00000000000..35a7d2569f9
--- /dev/null
+++ b/web/src/components/ui/tabs.tsx
@@ -0,0 +1,55 @@
+"use client";
+
+import * as React from "react";
+import * as TabsPrimitive from "@radix-ui/react-tabs";
+
+import { cn } from "@/lib/utils";
+
+const Tabs = TabsPrimitive.Root;
+
+const TabsList = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.List>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.List
+    ref={ref}
+    className={cn(
+      "inline-flex h-10 items-center justify-center rounded-md bg-neutral-100 p-1 text-neutral-500 dark:bg-neutral-800 dark:text-neutral-400",
+      className
+    )}
+    {...props}
+  />
+));
+TabsList.displayName = TabsPrimitive.List.displayName;
+
+const TabsTrigger = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.Trigger>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.Trigger
+    ref={ref}
+    className={cn(
+      "inline-flex items-center justify-center whitespace-nowrap rounded-sm px-3 py-1.5 text-sm font-medium ring-offset-white transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-950 focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 data-[state=active]:bg-white data-[state=active]:text-neutral-950 data-[state=active]:shadow-sm dark:ring-offset-neutral-950 dark:focus-visible:ring-neutral-300 dark:data-[state=active]:bg-neutral-950 dark:data-[state=active]:text-neutral-50",
+      className
+    )}
+    {...props}
+  />
+));
+TabsTrigger.displayName = TabsPrimitive.Trigger.displayName;
+
+const TabsContent = React.forwardRef<
+  React.ElementRef<typeof TabsPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
+>(({ className, ...props }, ref) => (
+  <TabsPrimitive.Content
+    ref={ref}
+    className={cn(
+      "mt-2 ring-offset-white focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-950 focus-visible:ring-offset-2 dark:ring-offset-neutral-950 dark:focus-visible:ring-neutral-300",
+      className
+    )}
+    {...props}
+  />
+));
+TabsContent.displayName = TabsPrimitive.Content.displayName;
+
+export { Tabs, TabsList, TabsTrigger, TabsContent };
diff --git a/web/src/components/ui/text.tsx b/web/src/components/ui/text.tsx
new file mode 100644
index 00000000000..e6544c5c074
--- /dev/null
+++ b/web/src/components/ui/text.tsx
@@ -0,0 +1,11 @@
+import { cn } from "@/lib/utils";
+
+export default function Text({
+  children,
+  className,
+}: {
+  children: React.ReactNode;
+  className?: string;
+}) {
+  return <p className={cn("text-sm", className)}>{children}</p>;
+}
diff --git a/web/src/components/ui/textarea.tsx b/web/src/components/ui/textarea.tsx
new file mode 100644
index 00000000000..014590b0bd0
--- /dev/null
+++ b/web/src/components/ui/textarea.tsx
@@ -0,0 +1,24 @@
+import * as React from "react";
+
+import { cn } from "@/lib/utils";
+
+export interface TextareaProps
+  extends React.TextareaHTMLAttributes<HTMLTextAreaElement> {}
+
+const Textarea = React.forwardRef<HTMLTextAreaElement, TextareaProps>(
+  ({ className, ...props }, ref) => {
+    return (
+      <textarea
+        className={cn(
+          "flex min-h-[80px] w-full rounded-md border border-neutral-200 bg-white px-3 py-2 text-sm ring-offset-white placeholder:text-neutral-500 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-neutral-950 focus-visible:ring-offset-2 disabled:cursor-not-allowed disabled:opacity-50 dark:border-neutral-800 dark:bg-neutral-950 dark:ring-offset-neutral-950 dark:placeholder:text-neutral-400 dark:focus-visible:ring-neutral-300",
+          className
+        )}
+        ref={ref}
+        {...props}
+      />
+    );
+  }
+);
+Textarea.displayName = "Textarea";
+
+export { Textarea };
diff --git a/web/src/components/ui/title.tsx b/web/src/components/ui/title.tsx
new file mode 100644
index 00000000000..e93e75d350c
--- /dev/null
+++ b/web/src/components/ui/title.tsx
@@ -0,0 +1,15 @@
+import { cn } from "@/lib/utils";
+
+export default function Title({
+  children,
+  className,
+}: {
+  children: React.ReactNode;
+  className?: string;
+}) {
+  return (
+    <h1 className={cn("text-lg text-text-800 font-medium", className)}>
+      {children}
+    </h1>
+  );
+}
diff --git a/web/src/lib/hooks.ts b/web/src/lib/hooks.ts
index c5a11b320d4..a1c4a0ddb1a 100644
--- a/web/src/lib/hooks.ts
+++ b/web/src/lib/hooks.ts
@@ -8,7 +8,7 @@ import {
 import useSWR, { mutate, useSWRConfig } from "swr";
 import { errorHandlingFetcher } from "./fetcher";
 import { useContext, useEffect, useState } from "react";
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { SourceMetadata } from "./search/interfaces";
 import { destructureValue } from "./llm/utils";
 import { ChatSession } from "@/app/chat/interfaces";
diff --git a/web/src/lib/search/interfaces.ts b/web/src/lib/search/interfaces.ts
index 1129bc67205..a24cc222652 100644
--- a/web/src/lib/search/interfaces.ts
+++ b/web/src/lib/search/interfaces.ts
@@ -1,4 +1,4 @@
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 import { Tag, ValidSources } from "../types";
 import { Persona } from "@/app/admin/assistants/interfaces";
 
diff --git a/web/src/lib/search/utils.ts b/web/src/lib/search/utils.ts
index 509c72826a2..b6b1c5675e3 100644
--- a/web/src/lib/search/utils.ts
+++ b/web/src/lib/search/utils.ts
@@ -1,6 +1,6 @@
 import { Tag } from "../types";
 import { Filters, SourceMetadata } from "./interfaces";
-import { DateRangePickerValue } from "@tremor/react";
+import { DateRangePickerValue } from "@/app/ee/admin/performance/DateRangeSelector";
 
 export const buildFilters = (
   sources: SourceMetadata[],
diff --git a/web/src/lib/utils.ts b/web/src/lib/utils.ts
new file mode 100644
index 00000000000..a5ef193506d
--- /dev/null
+++ b/web/src/lib/utils.ts
@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx";
+import { twMerge } from "tailwind-merge";
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs));
+}

From 12c82de78f99479835f2f56c693dc4eae76fef9d Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 1 Nov 2024 17:10:59 -0700
Subject: [PATCH 326/376] experimental github action to scan licenses

---
 .github/workflows/nightly-scan-licenses.yml | 74 +++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 .github/workflows/nightly-scan-licenses.yml

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
new file mode 100644
index 00000000000..c55bd3028bc
--- /dev/null
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -0,0 +1,74 @@
+# Scan for problematic software licenses
+
+# trivy has their own rate limiting issues causing this action to flake
+# we worked around it by hardcoding to different db repos in env
+# can re-enable when they figure it out
+# https://github.com/aquasecurity/trivy/discussions/7538
+# https://github.com/aquasecurity/trivy-action/issues/389
+
+name: 'Nightly - Scan licenses'
+on:
+  schedule:
+    - cron: '0 14 * * *'  # Runs every day at 6 AM PST / 7 AM PDT / 2 PM UTC
+  workflow_dispatch:  # Allows manual triggering
+  
+jobs:
+  scan-licenses:
+    # See https://runs-on.com/runners/linux/
+    runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
+
+    steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to Docker Hub
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_TOKEN }}
+
+    # Backend
+    - name: Pull backend docker image
+      run: docker pull danswer/danswer-backend:latest
+
+    - name: Run Trivy vulnerability scanner on backend
+      uses: aquasecurity/trivy-action@master
+      env:
+        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+      with:
+        image-ref: danswer/danswer-backend:latest
+        scanners: license
+        severity: HIGH,CRITICAL
+        trivyignores: ./backend/.trivyignore
+        exit-code: 1
+
+    # Web server
+    - name: Pull web server docker image
+      run: docker pull danswer/danswer-web-server:latest
+          
+    - name: Run Trivy vulnerability scanner on web server
+      uses: aquasecurity/trivy-action@master
+      env:
+        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+      with:
+        image-ref: danswer/danswer-web-server:latest
+        scanners: license
+        severity: HIGH,CRITICAL
+        exit-code: 1
+
+    # Model server
+    - name: Pull model server docker image
+      run: docker pull danswer/danswer-model-server:latest
+
+    - name: Run Trivy vulnerability scanner
+      uses: aquasecurity/trivy-action@master
+      env:
+        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
+        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
+      with:
+        image-ref: danswer/danswer-model-server:latest
+        scanners: license
+        severity: HIGH,CRITICAL
+        exit-code: 1
\ No newline at end of file

From 5e050f830599e29e8c489ed0c865adee7aafc45f Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 1 Nov 2024 17:16:28 -0700
Subject: [PATCH 327/376] we didn't checkout the code, no trivy ignore

---
 .github/workflows/nightly-scan-licenses.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index c55bd3028bc..093ffd39c7f 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -40,7 +40,6 @@ jobs:
         image-ref: danswer/danswer-backend:latest
         scanners: license
         severity: HIGH,CRITICAL
-        trivyignores: ./backend/.trivyignore
         exit-code: 1
 
     # Web server

From 5439c333136e87ec1aa91e7304a2bf8f27a303aa Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Fri, 1 Nov 2024 17:24:41 -0700
Subject: [PATCH 328/376] don't scan the os packages

---
 .github/workflows/nightly-scan-licenses.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index 093ffd39c7f..4f3aee3e0a6 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -40,6 +40,7 @@ jobs:
         image-ref: danswer/danswer-backend:latest
         scanners: license
         severity: HIGH,CRITICAL
+        vuln-type: library
         exit-code: 1
 
     # Web server
@@ -55,6 +56,7 @@ jobs:
         image-ref: danswer/danswer-web-server:latest
         scanners: license
         severity: HIGH,CRITICAL
+        vuln-type: library
         exit-code: 1
 
     # Model server
@@ -70,4 +72,5 @@ jobs:
         image-ref: danswer/danswer-model-server:latest
         scanners: license
         severity: HIGH,CRITICAL
+        vuln-type: library
         exit-code: 1
\ No newline at end of file

From f98110611108a9270770c710a4447af70a08f17b Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Fri, 1 Nov 2024 19:27:03 -0700
Subject: [PATCH 329/376] Update connector.py

---
 backend/danswer/connectors/freshdesk/connector.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index a98ca34bdd2..bc0b1acab77 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -129,10 +129,10 @@ def load_credentials(self, credentials: dict[str, str | int]) -> None:
     def _fetch_tickets(
         self, start: datetime | None = None, end: datetime | None = None
     ) -> Iterator[List[dict]]:
-        """ 
+        """
         'end' is not currently used, so we may double fetch tickets created after the indexing
         starts but before the actual call is made.
-        
+
         To use 'end' would require us to use the search endpoint but it has limitations,
         namely having to fetch all IDs and then individually fetch each ticket because there is no
         'include' field available for this endpoint:

From 4fc8a35220999e9b3e2aeaff3b4e259dc2cbfcb0 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Fri, 1 Nov 2024 21:59:23 -0700
Subject: [PATCH 330/376] try repo level scan

---
 .github/workflows/nightly-scan-licenses.yml | 72 +++++----------------
 1 file changed, 16 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index 4f3aee3e0a6..a5629ec3026 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -18,59 +18,19 @@ jobs:
     runs-on: [runs-on,runner=2cpu-linux-x64,"run-id=${{ github.run_id }}"]
 
     steps:
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v3
-
-    - name: Login to Docker Hub
-      uses: docker/login-action@v3
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_TOKEN }}
-
-    # Backend
-    - name: Pull backend docker image
-      run: docker pull danswer/danswer-backend:latest
-
-    - name: Run Trivy vulnerability scanner on backend
-      uses: aquasecurity/trivy-action@master
-      env:
-        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
-        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
-      with:
-        image-ref: danswer/danswer-backend:latest
-        scanners: license
-        severity: HIGH,CRITICAL
-        vuln-type: library
-        exit-code: 1
-
-    # Web server
-    - name: Pull web server docker image
-      run: docker pull danswer/danswer-web-server:latest
-          
-    - name: Run Trivy vulnerability scanner on web server
-      uses: aquasecurity/trivy-action@master
-      env:
-        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
-        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
-      with:
-        image-ref: danswer/danswer-web-server:latest
-        scanners: license
-        severity: HIGH,CRITICAL
-        vuln-type: library
-        exit-code: 1
-
-    # Model server
-    - name: Pull model server docker image
-      run: docker pull danswer/danswer-model-server:latest
-
-    - name: Run Trivy vulnerability scanner
-      uses: aquasecurity/trivy-action@master
-      env:
-        TRIVY_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-db:2'
-        TRIVY_JAVA_DB_REPOSITORY: 'public.ecr.aws/aquasecurity/trivy-java-db:1'
-      with:
-        image-ref: danswer/danswer-model-server:latest
-        scanners: license
-        severity: HIGH,CRITICAL
-        vuln-type: library
-        exit-code: 1
\ No newline at end of file
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@0.28.0
+        with:
+          scan-type: fs
+          scanners: license
+          format: sarif
+          output: trivy-results.sarif
+          severity: HIGH,CRITICAL
+
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: trivy-results.sarif
\ No newline at end of file

From ea2da63cf256c9d7fc1ecb16177c9e871c908340 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Fri, 1 Nov 2024 22:09:06 -0700
Subject: [PATCH 331/376] try installing npm deps

---
 .github/workflows/nightly-scan-licenses.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index a5629ec3026..a2911856cd2 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -21,6 +21,10 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Install npm dependencies
+        working-directory: ./web
+        run: npm ci
+        
       - name: Run Trivy vulnerability scanner in repo mode
         uses: aquasecurity/trivy-action@0.28.0
         with:
@@ -33,4 +37,4 @@ jobs:
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/upload-sarif@v3
         with:
-          sarif_file: trivy-results.sarif
\ No newline at end of file
+          sarif_file: trivy-results.sarif

From 678ed2385383399c6ea844fc85a17cf040139139 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Fri, 1 Nov 2024 22:34:41 -0700
Subject: [PATCH 332/376] codel permissions?

---
 .github/workflows/nightly-scan-licenses.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index a2911856cd2..e4189f9288f 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -11,6 +11,11 @@ on:
   schedule:
     - cron: '0 14 * * *'  # Runs every day at 6 AM PST / 7 AM PDT / 2 PM UTC
   workflow_dispatch:  # Allows manual triggering
+
+permissions:
+  actions: read
+  contents: read
+  security-events: write
   
 jobs:
   scan-licenses:

From d3062cacea40894fe3de9cf8cc0de43b425a5d52 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Sat, 2 Nov 2024 00:01:55 -0700
Subject: [PATCH 333/376] manual only for now

---
 .github/workflows/nightly-scan-licenses.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index e4189f9288f..323f5e47539 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -8,8 +8,8 @@
 
 name: 'Nightly - Scan licenses'
 on:
-  schedule:
-    - cron: '0 14 * * *'  # Runs every day at 6 AM PST / 7 AM PDT / 2 PM UTC
+#   schedule:
+#     - cron: '0 14 * * *'  # Runs every day at 6 AM PST / 7 AM PDT / 2 PM UTC
   workflow_dispatch:  # Allows manual triggering
 
 permissions:

From 40beda30a4878a0fb3efa78a68f31a91d41b503a Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Sat, 2 Nov 2024 02:20:58 -0700
Subject: [PATCH 334/376] try pip-license-checker

---
 .github/workflows/nightly-scan-licenses.yml | 44 +++++++++++++++++----
 1 file changed, 37 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index 323f5e47539..772c0724d03 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -25,7 +25,36 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
-
+        
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: |
+            backend/requirements/default.txt
+            backend/requirements/dev.txt
+            backend/requirements/model_server.txt
+      
+      - name: Get explicit and transitive dependencies
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
+          pip install --retries 5 --timeout 30 -r backend/requirements/model_server.txt
+          pip freeze > requirements-all.txt
+                    
+      - name: Check python
+        id: license_check_report
+        uses: pilosus/action-pip-license-checker@v2
+        with:
+          requirements: 'requirements-all.txt'
+          fail: 'Copyleft'
+          exclude: '(?i)^(pylint|aio[-_]*).*'
+      - name: Print report
+        if: ${{ always() }}
+        run: echo "${{ steps.license_check_report.outputs.report }}"
+      
       - name: Install npm dependencies
         working-directory: ./web
         run: npm ci
@@ -35,11 +64,12 @@ jobs:
         with:
           scan-type: fs
           scanners: license
-          format: sarif
-          output: trivy-results.sarif
+          format: table
+#           format: sarif
+#           output: trivy-results.sarif
           severity: HIGH,CRITICAL
 
-      - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v3
-        with:
-          sarif_file: trivy-results.sarif
+#       - name: Upload Trivy scan results to GitHub Security tab
+#         uses: github/codeql-action/upload-sarif@v3
+#         with:
+#           sarif_file: trivy-results.sarif

From a1ae22ef4ac48e15217d770cbe98ec3c14709c52 Mon Sep 17 00:00:00 2001
From: Richard Kuo <rkuo@rkuo.com>
Date: Sat, 2 Nov 2024 02:23:08 -0700
Subject: [PATCH 335/376] fix run key

---
 .github/workflows/nightly-scan-licenses.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-scan-licenses.yml b/.github/workflows/nightly-scan-licenses.yml
index 772c0724d03..9aa7030e0b9 100644
--- a/.github/workflows/nightly-scan-licenses.yml
+++ b/.github/workflows/nightly-scan-licenses.yml
@@ -37,7 +37,7 @@ jobs:
             backend/requirements/model_server.txt
       
       - name: Get explicit and transitive dependencies
-      - run: |
+        run: |
           python -m pip install --upgrade pip
           pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
           pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
@@ -51,6 +51,7 @@ jobs:
           requirements: 'requirements-all.txt'
           fail: 'Copyleft'
           exclude: '(?i)^(pylint|aio[-_]*).*'
+          
       - name: Print report
         if: ${{ always() }}
         run: echo "${{ steps.license_check_report.outputs.report }}"

From 013292a0e3158fa6bb13c14b37fe639e74a63638 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 08:15:36 -0700
Subject: [PATCH 336/376] mypy fixes

---
 .../danswer/connectors/freshdesk/connector.py | 22 +++++++++++++------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index bc0b1acab77..b1f1dc1d2df 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -1,13 +1,21 @@
-import requests
 import json
-from datetime import datetime, timezone
-from typing import List, Iterator
-from danswer.file_processing.html_utils import parse_html_page_basic
+from collections.abc import Iterator
+from datetime import datetime
+from datetime import timezone
+from typing import List
+
+import requests
+
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
-from danswer.connectors.interfaces import GenerateDocumentsOutput, SecondsSinceUnixEpoch
-from danswer.connectors.interfaces import PollConnector, LoadConnector
-from danswer.connectors.models import ConnectorMissingCredentialError, Document, Section
+from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import LoadConnector
+from danswer.connectors.interfaces import PollConnector
+from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.models import ConnectorMissingCredentialError
+from danswer.connectors.models import Document
+from danswer.connectors.models import Section
+from danswer.file_processing.html_utils import parse_html_page_basic
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()

From 6f440d126aa32a07b12676b82191ab2c733b1509 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 08:17:53 -0700
Subject: [PATCH 337/376] more mypy fixes

---
 backend/danswer/connectors/factory.py | 2 +-
 backend/danswer/db/models.py          | 2 +-
 backend/danswer/file_store/models.py  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/backend/danswer/connectors/factory.py b/backend/danswer/connectors/factory.py
index a22c320458b..a484266eaf1 100644
--- a/backend/danswer/connectors/factory.py
+++ b/backend/danswer/connectors/factory.py
@@ -16,6 +16,7 @@
 from danswer.connectors.document360.connector import Document360Connector
 from danswer.connectors.dropbox.connector import DropboxConnector
 from danswer.connectors.file.connector import LocalFileConnector
+from danswer.connectors.freshdesk.connector import FreshdeskConnector
 from danswer.connectors.github.connector import GithubConnector
 from danswer.connectors.gitlab.connector import GitlabConnector
 from danswer.connectors.gmail.connector import GmailConnector
@@ -46,7 +47,6 @@
 from danswer.connectors.xenforo.connector import XenforoConnector
 from danswer.connectors.zendesk.connector import ZendeskConnector
 from danswer.connectors.zulip.connector import ZulipConnector
-from danswer.connectors.freshdesk.connector import FreshdeskConnector
 from danswer.db.credentials import backend_update_credential_json
 from danswer.db.models import Credential
 
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index 0ebd7910280..bee69353437 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -3,7 +3,7 @@
 from enum import Enum as PyEnum
 from typing import Any
 from typing import Literal
-from typing_extensions import NotRequired
+from typing import NotRequired
 from typing import Optional
 from uuid import uuid4
 from typing_extensions import TypedDict  # noreorder
diff --git a/backend/danswer/file_store/models.py b/backend/danswer/file_store/models.py
index 0a3b513bd6a..d944a2fd270 100644
--- a/backend/danswer/file_store/models.py
+++ b/backend/danswer/file_store/models.py
@@ -1,6 +1,6 @@
 import base64
 from enum import Enum
-from typing_extensions import NotRequired
+from typing import NotRequired
 from typing_extensions import TypedDict  # noreorder
 
 from pydantic import BaseModel

From 4a6b8db65f1e51caab7f4c4233b721fe52583cd4 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 08:20:08 -0700
Subject: [PATCH 338/376] out of scope

---
 backend/model_server/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/model_server/main.py b/backend/model_server/main.py
index 50cf9f8b9a5..5505bbc8cdb 100644
--- a/backend/model_server/main.py
+++ b/backend/model_server/main.py
@@ -28,7 +28,7 @@
 os.environ["HF_HUB_DISABLE_TELEMETRY"] = "1"
 
 HF_CACHE_PATH = Path("/root/.cache/huggingface/")
-TEMP_HF_CACHE_PATH = Path.home() / ".cache" / "temp_huggingface"
+TEMP_HF_CACHE_PATH = Path("/root/.cache/temp_huggingface/")
 
 transformer_logging.set_verbosity_error()
 

From d7bcd32d9aabf578a86a9d19df2e3bdffad8a582 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 08:21:33 -0700
Subject: [PATCH 339/376] out of scope

---
 web/src/app/api/[...path]/route.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/web/src/app/api/[...path]/route.ts b/web/src/app/api/[...path]/route.ts
index 8c3d4522be4..6ca13aba146 100644
--- a/web/src/app/api/[...path]/route.ts
+++ b/web/src/app/api/[...path]/route.ts
@@ -75,13 +75,9 @@ async function handleRequest(request: NextRequest, path: string[]) {
       backendUrl.searchParams.append(key, value);
     });
 
-    // Create a new headers object, omitting the 'connection' header
-    const headers = new Headers(request.headers);
-    headers.delete('connection');
-
     const response = await fetch(backendUrl, {
       method: request.method,
-      headers: headers,
+      headers: request.headers,
       body: request.body,
       signal: request.signal,
       // @ts-ignore

From 5e444d43f9e3a9cadc80184ee78247c16a89d497 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 09:01:15 -0700
Subject: [PATCH 340/376] Cleaned up connector

---
 .../danswer/connectors/freshdesk/connector.py | 138 ++++++++++--------
 1 file changed, 75 insertions(+), 63 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index b1f1dc1d2df..448c3d95cd9 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -21,76 +21,84 @@
 logger = setup_logger()
 
 
+_TICKET_FIELDS_TO_INCLUDE = {
+    "fr_escalated",
+    "spam",
+    "priority",
+    "source",
+    "status",
+    "type",
+    "is_escalated",
+    "tags",
+    "nr_due_by",
+    "nr_escalated",
+    "cc_emails",
+    "fwd_emails",
+    "reply_cc_emails",
+    "ticket_cc_emails",
+    "support_email",
+    "to_emails",
+}
+
+_SOURCE_TYPES = {
+    "1": "Email",
+    "2": "Portal",
+    "3": "Phone",
+    "7": "Chat",
+    "9": "Feedback Widget",
+    "10": "Outbound Email",
+}
+
+_PRIORITY_TYPES = {"1": "low", "2": "medium", "3": "high", "4": "urgent"}
+
+_STATUS_TYPES = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"}
+
+
 def _create_metadata_from_ticket(ticket: dict) -> dict:
-    included_fields = {
-        "fr_escalated",
-        "spam",
-        "priority",
-        "source",
-        "status",
-        "type",
-        "is_escalated",
-        "tags",
-        "nr_due_by",
-        "nr_escalated",
-        "cc_emails",
-        "fwd_emails",
-        "reply_cc_emails",
-        "ticket_cc_emails",
-        "support_email",
-        "to_emails",
-    }
-
-    metadata = {}
-    email_data = {}
+    metadata: dict[str, str | list[str]] = {}
+    # Combine all emails into a list so there are no repeated emails
+    email_data: set[str] = set()
 
     for key, value in ticket.items():
-        if (
-            key in included_fields
-            and value is not None
-            and value != []
-            and value != {}
-            and value != "[]"
-            and value != ""
-        ):
-            value_to_str = (
-                [str(item) for item in value] if isinstance(value, List) else str(value)
-            )
-            if "email" in key:
-                email_data[key] = value_to_str
+        # Skip fields that aren't useful for embedding
+        if key not in _TICKET_FIELDS_TO_INCLUDE:
+            continue
+
+        # Skip empty fields
+        if not value or value == "[]":
+            continue
+
+        # Convert strings or lists to strings
+        stringified_value: str | list[str]
+        if isinstance(value, list):
+            stringified_value = [str(item) for item in value]
+        else:
+            stringified_value = str(value)
+
+        if "email" in key:
+            if isinstance(stringified_value, list):
+                email_data.update(stringified_value)
             else:
-                metadata[key] = value_to_str
+                email_data.add(stringified_value)
+        else:
+            metadata[key] = stringified_value
 
     if email_data:
-        metadata["email_data"] = str(email_data)
-
-    # Convert source to human-parsable string
-    source_types = {
-        "1": "Email",
-        "2": "Portal",
-        "3": "Phone",
-        "7": "Chat",
-        "9": "Feedback Widget",
-        "10": "Outbound Email",
-    }
-    if ticket.get("source"):
-        metadata["source"] = source_types.get(
-            str(ticket.get("source")), "Unknown Source Type"
-        )
+        metadata["emails"] = list(email_data)
 
-    # Convert priority to human-parsable string
-    priority_types = {"1": "low", "2": "medium", "3": "high", "4": "urgent"}
-    if ticket.get("priority"):
-        metadata["priority"] = priority_types.get(
-            str(ticket.get("priority")), "Unknown Priority"
+    # Convert source numbers to human-parsable string
+    if source_number := ticket.get("source"):
+        metadata["source"] = _SOURCE_TYPES.get(
+            str(source_number), "Unknown Source Type"
         )
 
+    # Convert priority numbers to human-parsable string
+    if priority_number := ticket.get("priority"):
+        metadata["priority"] = _PRIORITY_TYPES.get(priority_number, "Unknown Priority")
+
     # Convert status to human-parsable string
-    status_types = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"}
-    if ticket.get("status"):
-        metadata["status"] = status_types.get(
-            str(ticket.get("status")), "Unknown Status"
-        )
+    if status_number := ticket.get("status"):
+        metadata["status"] = _STATUS_TYPES.get(str(status_number), "Unknown Status")
 
     due_by = datetime.fromisoformat(ticket["due_by"].replace("Z", "+00:00"))
     metadata["overdue"] = str(datetime.now(timezone.utc) > due_by)
@@ -99,17 +107,21 @@ def _create_metadata_from_ticket(ticket: dict) -> dict:
 
 
 def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
+    # Use the ticket description as the text
+    text = f"Ticket description: {parse_html_page_basic(ticket.get('description_text', ''))}"
+    metadata = _create_metadata_from_ticket(ticket)
+
     return Document(
         id=str(ticket["id"]),
         sections=[
             Section(
                 link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
-                text=f"description: {parse_html_page_basic(ticket.get('description_text', ''))}",
+                text=text,
             )
         ],
         source=DocumentSource.FRESHDESK,
         semantic_identifier=ticket["subject"],
-        metadata=_create_metadata_from_ticket(ticket),
+        metadata=metadata,
         doc_updated_at=datetime.fromisoformat(
             ticket["updated_at"].replace("Z", "+00:00")
         ),
@@ -146,7 +158,7 @@ def _fetch_tickets(
         'include' field available for this endpoint:
         https://developers.freshdesk.com/api/#filter_tickets
         """
-        if any(attr is None for attr in [self.api_key, self.domain, self.password]):
+        if self.api_key is None or self.domain is None or self.password is None:
             raise ConnectorMissingCredentialError("freshdesk")
 
         base_url = f"https://{self.domain}.freshdesk.com/api/v2/tickets"

From d652cb3141fff4c213d3bb9872fffd839612438a Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 09:03:42 -0700
Subject: [PATCH 341/376] renamed variables

---
 .../danswer/connectors/freshdesk/connector.py    | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 448c3d95cd9..28fa20ec8c2 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -40,7 +40,7 @@
     "to_emails",
 }
 
-_SOURCE_TYPES = {
+_SOURCE_NUMBER_TYPE_MAP = {
     "1": "Email",
     "2": "Portal",
     "3": "Phone",
@@ -49,9 +49,9 @@
     "10": "Outbound Email",
 }
 
-_PRIORITY_TYPES = {"1": "low", "2": "medium", "3": "high", "4": "urgent"}
+_PRIORITY_NUMBER_TYPE_MAP = {"1": "low", "2": "medium", "3": "high", "4": "urgent"}
 
-_STATUS_TYPES = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"}
+_STATUS_NUMBER_TYPE_MAP = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"}
 
 
 def _create_metadata_from_ticket(ticket: dict) -> dict:
@@ -88,17 +88,21 @@ def _create_metadata_from_ticket(ticket: dict) -> dict:
 
     # Convert source numbers to human-parsable string
     if source_number := ticket.get("source"):
-        metadata["source"] = _SOURCE_TYPES.get(
+        metadata["source"] = _SOURCE_NUMBER_TYPE_MAP.get(
             str(source_number), "Unknown Source Type"
         )
 
     # Convert priority numbers to human-parsable string
     if priority_number := ticket.get("priority"):
-        metadata["priority"] = _PRIORITY_TYPES.get(priority_number, "Unknown Priority")
+        metadata["priority"] = _PRIORITY_NUMBER_TYPE_MAP.get(
+            priority_number, "Unknown Priority"
+        )
 
     # Convert status to human-parsable string
     if status_number := ticket.get("status"):
-        metadata["status"] = _STATUS_TYPES.get(str(status_number), "Unknown Status")
+        metadata["status"] = _STATUS_NUMBER_TYPE_MAP.get(
+            str(status_number), "Unknown Status"
+        )
 
     due_by = datetime.fromisoformat(ticket["due_by"].replace("Z", "+00:00"))
     metadata["overdue"] = str(datetime.now(timezone.utc) > due_by)

From 6e8c88ed71ac0e3ea4df7b2f4f842fbe10721667 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 09:05:24 -0700
Subject: [PATCH 342/376] made id more unique

---
 backend/danswer/connectors/freshdesk/connector.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 28fa20ec8c2..30bfee662c7 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -20,6 +20,8 @@
 
 logger = setup_logger()
 
+_FRESHDESK_ID_PREFIX = "FRESHDESK_"
+
 
 _TICKET_FIELDS_TO_INCLUDE = {
     "fr_escalated",
@@ -116,7 +118,7 @@ def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     metadata = _create_metadata_from_ticket(ticket)
 
     return Document(
-        id=str(ticket["id"]),
+        id=_FRESHDESK_ID_PREFIX + str(ticket["id"]),
         sections=[
             Section(
                 link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",

From 6b4143cc30256177203650854d64f5d8f71a8213 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 09:08:26 -0700
Subject: [PATCH 343/376] ID fix

---
 backend/danswer/connectors/freshdesk/connector.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 30bfee662c7..7154e62bf9f 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -117,11 +117,14 @@ def _create_doc_from_ticket(ticket: dict, domain: str) -> Document:
     text = f"Ticket description: {parse_html_page_basic(ticket.get('description_text', ''))}"
     metadata = _create_metadata_from_ticket(ticket)
 
+    # This is also used in the ID because it is more unique than the just the ticket ID
+    link = f"https://{domain}.freshdesk.com/helpdesk/tickets/{ticket['id']}"
+
     return Document(
-        id=_FRESHDESK_ID_PREFIX + str(ticket["id"]),
+        id=_FRESHDESK_ID_PREFIX + link,
         sections=[
             Section(
-                link=f"https://{domain}.freshdesk.com/helpdesk/tickets/{int(ticket['id'])}",
+                link=link,
                 text=text,
             )
         ],

From 83c299ebc8e183c9da63bf33b1ef94723f92ff6b Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 09:09:46 -0700
Subject: [PATCH 344/376] troll logger statement

---
 backend/danswer/connectors/freshdesk/connector.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 7154e62bf9f..04033b05f3f 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -208,7 +208,6 @@ def _process_tickets(
 
         for ticket_batch in self._fetch_tickets(start, end):
             for ticket in ticket_batch:
-                logger.info(_create_doc_from_ticket(ticket, self.domain))
                 doc_batch.append(_create_doc_from_ticket(ticket, self.domain))
 
                 if len(doc_batch) >= self.batch_size:

From e4bb14d4e14178700d3dfc20e47f477d0cb9af0a Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sat, 2 Nov 2024 10:29:23 -0700
Subject: [PATCH 345/376] Super user (#2944)

* add super user

* nits
---
 backend/danswer/auth/users.py                 |  10 +-
 backend/danswer/configs/app_configs.py        |   4 +
 backend/danswer/connectors/file/connector.py  |   2 +-
 backend/danswer/danswerbot/slack/listener.py  |   2 +-
 backend/danswer/db/engine.py                  |   2 +-
 backend/danswer/key_value_store/store.py      |   2 +-
 backend/danswer/server/auth_check.py          |   2 +
 backend/danswer/server/manage/models.py       |   3 +
 backend/danswer/server/manage/users.py        |   3 +
 .../server/query_and_chat/token_limit.py      |   2 +-
 backend/danswer/utils/logger.py               |   2 +-
 backend/ee/danswer/auth/users.py              |  20 +++
 .../danswer/background/celery/apps/primary.py |   2 +-
 backend/ee/danswer/db/api_key.py              |   2 +-
 backend/ee/danswer/db/query_history.py        |   1 +
 .../server/middleware/tenant_tracking.py      |  12 +-
 backend/ee/danswer/server/tenants/api.py      |  36 ++++-
 backend/ee/danswer/server/tenants/models.py   |   4 +
 backend/shared_configs/configs.py             |   5 -
 backend/shared_configs/contextvars.py         |   8 ++
 web/src/app/auth/impersonate/page.tsx         | 132 ++++++++++++++++++
 web/src/app/auth/login/LoginText.tsx          |   5 -
 web/src/components/auth/AuthFlowContainer.tsx |   2 +-
 web/src/components/user/UserProvider.tsx      |   2 +
 web/src/lib/types.ts                          |   1 +
 25 files changed, 237 insertions(+), 29 deletions(-)
 create mode 100644 backend/shared_configs/contextvars.py
 create mode 100644 web/src/app/auth/impersonate/page.tsx

diff --git a/backend/danswer/auth/users.py b/backend/danswer/auth/users.py
index aa5054983fa..0cb4ae2326c 100644
--- a/backend/danswer/auth/users.py
+++ b/backend/danswer/auth/users.py
@@ -93,9 +93,9 @@
 from danswer.utils.telemetry import optional_telemetry
 from danswer.utils.telemetry import RecordType
 from danswer.utils.variable_functionality import fetch_versioned_implementation
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 logger = setup_logger()
@@ -510,19 +510,23 @@ async def get_user_manager(
 
 # This strategy is used to add tenant_id to the JWT token
 class TenantAwareJWTStrategy(JWTStrategy):
-    async def write_token(self, user: User) -> str:
+    async def _create_token_data(self, user: User, impersonate: bool = False) -> dict:
         tenant_id = get_tenant_id_for_email(user.email)
         data = {
             "sub": str(user.id),
             "aud": self.token_audience,
             "tenant_id": tenant_id,
         }
+        return data
+
+    async def write_token(self, user: User) -> str:
+        data = await self._create_token_data(user)
         return generate_jwt(
             data, self.encode_key, self.lifetime_seconds, algorithm=self.algorithm
         )
 
 
-def get_jwt_strategy() -> JWTStrategy:
+def get_jwt_strategy() -> TenantAwareJWTStrategy:
     return TenantAwareJWTStrategy(
         secret=USER_AUTH_SECRET,
         lifetime_seconds=SESSION_EXPIRE_TIME_SECONDS,
diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index fb6b4996363..b76ad6042d1 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -478,3 +478,7 @@
 
 # JWT configuration
 JWT_ALGORITHM = "HS256"
+
+# Super Users
+SUPER_USERS = json.loads(os.environ.get("SUPER_USERS", '["pablo@danswer.ai"]'))
+SUPER_CLOUD_API_KEY = os.environ.get("SUPER_CLOUD_API_KEY", "api_key")
diff --git a/backend/danswer/connectors/file/connector.py b/backend/danswer/connectors/file/connector.py
index d07a224478e..13744f02b2f 100644
--- a/backend/danswer/connectors/file/connector.py
+++ b/backend/danswer/connectors/file/connector.py
@@ -27,8 +27,8 @@
 from danswer.file_processing.extract_file_text import read_text_file
 from danswer.file_store.file_store import get_default_file_store
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
diff --git a/backend/danswer/danswerbot/slack/listener.py b/backend/danswer/danswerbot/slack/listener.py
index a40dbe9a9b9..2078d621325 100644
--- a/backend/danswer/danswerbot/slack/listener.py
+++ b/backend/danswer/danswerbot/slack/listener.py
@@ -57,10 +57,10 @@
 from danswer.server.manage.models import SlackBotTokens
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MODEL_SERVER_HOST
 from shared_configs.configs import MODEL_SERVER_PORT
 from shared_configs.configs import SLACK_CHANNEL_ID
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index a1fbbddc65f..9dc6024ab8c 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -37,10 +37,10 @@
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.configs.constants import POSTGRES_UNKNOWN_APP_NAME
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import TENANT_ID_PREFIX
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
diff --git a/backend/danswer/key_value_store/store.py b/backend/danswer/key_value_store/store.py
index 5fec7dea40c..8e60dab88b1 100644
--- a/backend/danswer/key_value_store/store.py
+++ b/backend/danswer/key_value_store/store.py
@@ -16,9 +16,9 @@
 from danswer.key_value_store.interface import KvKeyNotFoundError
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
diff --git a/backend/danswer/server/auth_check.py b/backend/danswer/server/auth_check.py
index 69aede4241f..4300bc464cb 100644
--- a/backend/danswer/server/auth_check.py
+++ b/backend/danswer/server/auth_check.py
@@ -10,6 +10,7 @@
 from danswer.auth.users import current_user_with_expired_token
 from danswer.configs.app_configs import APP_API_PREFIX
 from danswer.server.danswer_api.ingestion import api_key_dep
+from ee.danswer.auth.users import current_cloud_superuser
 from ee.danswer.server.tenants.access import control_plane_dep
 
 
@@ -100,6 +101,7 @@ def check_router_auth(
                     or depends_fn == api_key_dep
                     or depends_fn == current_user_with_expired_token
                     or depends_fn == control_plane_dep
+                    or depends_fn == current_cloud_superuser
                 ):
                     found_auth = True
                     break
diff --git a/backend/danswer/server/manage/models.py b/backend/danswer/server/manage/models.py
index 6e021b562ff..e24b96e9a1e 100644
--- a/backend/danswer/server/manage/models.py
+++ b/backend/danswer/server/manage/models.py
@@ -57,6 +57,7 @@ class UserInfo(BaseModel):
     oidc_expiry: datetime | None = None
     current_token_created_at: datetime | None = None
     current_token_expiry_length: int | None = None
+    is_cloud_superuser: bool = False
     organization_name: str | None = None
 
     @classmethod
@@ -65,6 +66,7 @@ def from_model(
         user: User,
         current_token_created_at: datetime | None = None,
         expiry_length: int | None = None,
+        is_cloud_superuser: bool = False,
         organization_name: str | None = None,
     ) -> "UserInfo":
         return cls(
@@ -90,6 +92,7 @@ def from_model(
             oidc_expiry=user.oidc_expiry if TRACK_EXTERNAL_IDP_EXPIRY else None,
             current_token_created_at=current_token_created_at,
             current_token_expiry_length=expiry_length,
+            is_cloud_superuser=is_cloud_superuser,
         )
 
 
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index f0675c39282..9e385908fa4 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -35,6 +35,7 @@
 from danswer.configs.app_configs import AUTH_TYPE
 from danswer.configs.app_configs import ENABLE_EMAIL_INVITES
 from danswer.configs.app_configs import SESSION_EXPIRE_TIME_SECONDS
+from danswer.configs.app_configs import SUPER_USERS
 from danswer.configs.app_configs import VALID_EMAIL_DOMAINS
 from danswer.configs.constants import AuthType
 from danswer.db.auth import get_total_users_count
@@ -476,6 +477,7 @@ def verify_user_logged_in(
     # NOTE: this does not use `current_user` / `current_admin_user` because we don't want
     # to enforce user verification here - the frontend always wants to get the info about
     # the current user regardless of if they are currently verified
+
     if user is None:
         # if auth type is disabled, return a dummy user with preferences from
         # the key-value store
@@ -502,6 +504,7 @@ def verify_user_logged_in(
         user,
         current_token_created_at=token_created_at,
         expiry_length=SESSION_EXPIRE_TIME_SECONDS,
+        is_cloud_superuser=user.email in SUPER_USERS,
         organization_name=organization_name,
     )
 
diff --git a/backend/danswer/server/query_and_chat/token_limit.py b/backend/danswer/server/query_and_chat/token_limit.py
index ec94e2ece4d..d439e15a379 100644
--- a/backend/danswer/server/query_and_chat/token_limit.py
+++ b/backend/danswer/server/query_and_chat/token_limit.py
@@ -21,7 +21,7 @@
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from ee.danswer.db.token_limit import fetch_all_global_token_rate_limits
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 logger = setup_logger()
diff --git a/backend/danswer/utils/logger.py b/backend/danswer/utils/logger.py
index bc872b0c423..bd784513898 100644
--- a/backend/danswer/utils/logger.py
+++ b/backend/danswer/utils/logger.py
@@ -5,7 +5,6 @@
 from logging.handlers import RotatingFileHandler
 from typing import Any
 
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import DEV_LOGGING_ENABLED
 from shared_configs.configs import LOG_FILE_NAME
 from shared_configs.configs import LOG_LEVEL
@@ -13,6 +12,7 @@
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import SLACK_CHANNEL_ID
 from shared_configs.configs import TENANT_ID_PREFIX
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 logging.addLevelName(logging.INFO + 5, "NOTICE")
diff --git a/backend/ee/danswer/auth/users.py b/backend/ee/danswer/auth/users.py
index 18dff6ab064..1ad384555c1 100644
--- a/backend/ee/danswer/auth/users.py
+++ b/backend/ee/danswer/auth/users.py
@@ -1,9 +1,13 @@
 from fastapi import Depends
 from fastapi import HTTPException
 from fastapi import Request
+from fastapi import status
 from sqlalchemy.orm import Session
 
+from danswer.auth.users import current_admin_user
 from danswer.configs.app_configs import AUTH_TYPE
+from danswer.configs.app_configs import SUPER_CLOUD_API_KEY
+from danswer.configs.app_configs import SUPER_USERS
 from danswer.configs.constants import AuthType
 from danswer.db.engine import get_session
 from danswer.db.models import User
@@ -68,3 +72,19 @@ def get_default_admin_user_emails_() -> list[str]:
     if seed_config and seed_config.admin_user_emails:
         return seed_config.admin_user_emails
     return []
+
+
+async def current_cloud_superuser(
+    request: Request,
+    user: User | None = Depends(current_admin_user),
+) -> User | None:
+    api_key = request.headers.get("Authorization", "").replace("Bearer ", "")
+    if api_key != SUPER_CLOUD_API_KEY:
+        raise HTTPException(status_code=401, detail="Invalid API key")
+
+    if user and user.email not in SUPER_USERS:
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Access denied. User must be a cloud superuser to perform this action.",
+        )
+    return user
diff --git a/backend/ee/danswer/background/celery/apps/primary.py b/backend/ee/danswer/background/celery/apps/primary.py
index b9929068688..fecc21b58ef 100644
--- a/backend/ee/danswer/background/celery/apps/primary.py
+++ b/backend/ee/danswer/background/celery/apps/primary.py
@@ -28,8 +28,8 @@
     run_external_group_permission_sync,
 )
 from ee.danswer.server.reporting.usage_export_generation import create_new_usage_report
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()
 
diff --git a/backend/ee/danswer/db/api_key.py b/backend/ee/danswer/db/api_key.py
index 4cc0774f129..db876067bd1 100644
--- a/backend/ee/danswer/db/api_key.py
+++ b/backend/ee/danswer/db/api_key.py
@@ -15,8 +15,8 @@
 from ee.danswer.auth.api_key import generate_api_key
 from ee.danswer.auth.api_key import hash_api_key
 from ee.danswer.server.api_key.models import APIKeyArgs
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 def get_api_key_email_pattern() -> str:
diff --git a/backend/ee/danswer/db/query_history.py b/backend/ee/danswer/db/query_history.py
index 868afef23ce..b6a79cb7727 100644
--- a/backend/ee/danswer/db/query_history.py
+++ b/backend/ee/danswer/db/query_history.py
@@ -29,6 +29,7 @@ def fetch_chat_sessions_eagerly_by_time(
     filters: list[ColumnElement | BinaryExpression] = [
         ChatSession.time_created.between(start, end)
     ]
+
     if initial_id:
         filters.append(ChatSession.id < initial_id)
     subquery = (
diff --git a/backend/ee/danswer/server/middleware/tenant_tracking.py b/backend/ee/danswer/server/middleware/tenant_tracking.py
index f9fe75425e0..20c0ba0afe2 100644
--- a/backend/ee/danswer/server/middleware/tenant_tracking.py
+++ b/backend/ee/danswer/server/middleware/tenant_tracking.py
@@ -11,9 +11,9 @@
 from danswer.configs.app_configs import USER_AUTH_SECRET
 from danswer.db.engine import is_valid_schema_name
 from ee.danswer.auth.api_key import extract_tenant_from_api_key_header
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 def add_tenant_id_middleware(app: FastAPI, logger: logging.LoggerAdapter) -> None:
@@ -22,11 +22,11 @@ async def set_tenant_id(
         request: Request, call_next: Callable[[Request], Awaitable[Response]]
     ) -> Response:
         try:
-            tenant_id = POSTGRES_DEFAULT_SCHEMA
-
-            if MULTI_TENANT:
-                tenant_id = _get_tenant_id_from_request(request, logger)
-
+            tenant_id = (
+                _get_tenant_id_from_request(request, logger)
+                if MULTI_TENANT
+                else POSTGRES_DEFAULT_SCHEMA
+            )
             CURRENT_TENANT_ID_CONTEXTVAR.set(tenant_id)
             return await call_next(request)
 
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index 66485975f31..05d66b4c582 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -2,29 +2,36 @@
 from fastapi import APIRouter
 from fastapi import Depends
 from fastapi import HTTPException
+from fastapi import Response
 
+from danswer.auth.users import auth_backend
 from danswer.auth.users import current_admin_user
+from danswer.auth.users import get_jwt_strategy
+from danswer.auth.users import get_tenant_id_for_email
 from danswer.auth.users import User
 from danswer.configs.app_configs import WEB_DOMAIN
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.notification import create_notification
+from danswer.db.users import get_user_by_email
 from danswer.server.settings.store import load_settings
 from danswer.server.settings.store import store_settings
 from danswer.setup import setup_danswer
 from danswer.utils.logger import setup_logger
+from ee.danswer.auth.users import current_cloud_superuser
 from ee.danswer.configs.app_configs import STRIPE_SECRET_KEY
 from ee.danswer.server.tenants.access import control_plane_dep
 from ee.danswer.server.tenants.billing import fetch_billing_information
 from ee.danswer.server.tenants.billing import fetch_tenant_stripe_information
 from ee.danswer.server.tenants.models import BillingInformation
 from ee.danswer.server.tenants.models import CreateTenantRequest
+from ee.danswer.server.tenants.models import ImpersonateRequest
 from ee.danswer.server.tenants.models import ProductGatingRequest
 from ee.danswer.server.tenants.provisioning import add_users_to_tenant
 from ee.danswer.server.tenants.provisioning import ensure_schema_exists
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
 from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
-from shared_configs.configs import CURRENT_TENANT_ID_CONTEXTVAR
 from shared_configs.configs import MULTI_TENANT
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 stripe.api_key = STRIPE_SECRET_KEY
@@ -132,3 +139,30 @@ async def create_customer_portal_session(_: User = Depends(current_admin_user))
     except Exception as e:
         logger.exception("Failed to create customer portal session")
         raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post("/impersonate")
+async def impersonate_user(
+    impersonate_request: ImpersonateRequest,
+    _: User = Depends(current_cloud_superuser),
+) -> Response:
+    """Allows a cloud superuser to impersonate another user by generating an impersonation JWT token"""
+    tenant_id = get_tenant_id_for_email(impersonate_request.email)
+
+    with get_session_with_tenant(tenant_id) as tenant_session:
+        user_to_impersonate = get_user_by_email(
+            impersonate_request.email, tenant_session
+        )
+        if user_to_impersonate is None:
+            raise HTTPException(status_code=404, detail="User not found")
+        token = await get_jwt_strategy().write_token(user_to_impersonate)
+
+    response = await auth_backend.transport.get_login_response(token)
+    response.set_cookie(
+        key="fastapiusersauth",
+        value=token,
+        httponly=True,
+        secure=True,
+        samesite="lax",
+    )
+    return response
diff --git a/backend/ee/danswer/server/tenants/models.py b/backend/ee/danswer/server/tenants/models.py
index 30f656c0824..2c1fdbecdb3 100644
--- a/backend/ee/danswer/server/tenants/models.py
+++ b/backend/ee/danswer/server/tenants/models.py
@@ -29,3 +29,7 @@ class BillingInformation(BaseModel):
 
 class CheckoutSessionCreationResponse(BaseModel):
     id: str
+
+
+class ImpersonateRequest(BaseModel):
+    email: str
diff --git a/backend/shared_configs/configs.py b/backend/shared_configs/configs.py
index 5f25deedfca..d4378251aa5 100644
--- a/backend/shared_configs/configs.py
+++ b/backend/shared_configs/configs.py
@@ -1,4 +1,3 @@
-import contextvars
 import os
 from typing import List
 from urllib.parse import urlparse
@@ -134,10 +133,6 @@ def validate_cors_origin(origin: str) -> None:
 
 POSTGRES_DEFAULT_SCHEMA = os.environ.get("POSTGRES_DEFAULT_SCHEMA") or "public"
 
-CURRENT_TENANT_ID_CONTEXTVAR = contextvars.ContextVar(
-    "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
-)
-
 # Prefix used for all tenant ids
 TENANT_ID_PREFIX = "tenant_"
 
diff --git a/backend/shared_configs/contextvars.py b/backend/shared_configs/contextvars.py
new file mode 100644
index 00000000000..df66b141c6e
--- /dev/null
+++ b/backend/shared_configs/contextvars.py
@@ -0,0 +1,8 @@
+import contextvars
+
+from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+
+# Context variable for the current tenant id
+CURRENT_TENANT_ID_CONTEXTVAR = contextvars.ContextVar(
+    "current_tenant_id", default=POSTGRES_DEFAULT_SCHEMA
+)
diff --git a/web/src/app/auth/impersonate/page.tsx b/web/src/app/auth/impersonate/page.tsx
new file mode 100644
index 00000000000..1a2c77d2cdb
--- /dev/null
+++ b/web/src/app/auth/impersonate/page.tsx
@@ -0,0 +1,132 @@
+"use client";
+import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
+import { HealthCheckBanner } from "@/components/health/healthcheck";
+import { useUser } from "@/components/user/UserProvider";
+import { redirect, useRouter } from "next/navigation";
+import { Formik, Form, Field } from "formik";
+import * as Yup from "yup";
+import { usePopup } from "@/components/admin/connectors/Popup";
+
+const ImpersonateSchema = Yup.object().shape({
+  email: Yup.string().email("Invalid email").required("Required"),
+  apiKey: Yup.string().required("Required"),
+});
+
+export default function ImpersonatePage() {
+  const router = useRouter();
+  const { user, isLoadingUser, isCloudSuperuser } = useUser();
+  const { popup, setPopup } = usePopup();
+
+  if (isLoadingUser) {
+    return null;
+  }
+
+  if (!user) {
+    redirect("/auth/login");
+  }
+
+  if (!isCloudSuperuser) {
+    redirect("/search");
+  }
+
+  const handleImpersonate = async (values: {
+    email: string;
+    apiKey: string;
+  }) => {
+    try {
+      const response = await fetch("/api/tenants/impersonate", {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${values.apiKey}`,
+        },
+        body: JSON.stringify({ email: values.email }),
+        credentials: "same-origin",
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json();
+        setPopup({
+          message: errorData.detail || "Failed to impersonate user",
+          type: "error",
+        });
+      } else {
+        router.push("/search");
+      }
+    } catch (error) {
+      setPopup({
+        message:
+          error instanceof Error ? error.message : "Failed to impersonate user",
+        type: "error",
+      });
+    }
+  };
+
+  return (
+    <AuthFlowContainer>
+      {popup}
+      <div className="absolute top-10x w-full">
+        <HealthCheckBanner />
+      </div>
+
+      <div className="flex flex-col w-full justify-center">
+        <h2 className="text-center text-xl text-strong font-bold mb-8">
+          Impersonate User
+        </h2>
+
+        <Formik
+          initialValues={{ email: "", apiKey: "" }}
+          validationSchema={ImpersonateSchema}
+          onSubmit={handleImpersonate}
+        >
+          {({ errors, touched }) => (
+            <Form className="flex flex-col items-stretch gap-y-2">
+              <div className="relative">
+                <Field
+                  type="email"
+                  name="email"
+                  placeholder="Enter user email to impersonate"
+                  className="w-full px-4 py-3 border border-border rounded-lg bg-input focus:outline-none focus:ring-2 focus:ring-primary transition-all duration-200"
+                />
+                <div className="h-8">
+                  {errors.email && touched.email && (
+                    <div className="text-red-500 text-sm mt-1">
+                      {errors.email}
+                    </div>
+                  )}
+                </div>
+              </div>
+
+              <div className="relative">
+                <Field
+                  type="password"
+                  name="apiKey"
+                  placeholder="Enter API Key"
+                  className="w-full px-4 py-3 border border-border rounded-lg bg-input focus:outline-none focus:ring-2 focus:ring-primary transition-all duration-200"
+                />
+                <div className="h-8">
+                  {errors.apiKey && touched.apiKey && (
+                    <div className="text-red-500 text-sm mt-1">
+                      {errors.apiKey}
+                    </div>
+                  )}
+                </div>
+              </div>
+
+              <button
+                type="submit"
+                className="w-full py-3 bg-accent text-white rounded-lg hover:bg-accent/90 transition-colors"
+              >
+                Impersonate User
+              </button>
+            </Form>
+          )}
+        </Formik>
+
+        <div className="text-sm text-text-500 mt-4 text-center px-4 rounded-md">
+          Note: This feature is only available for @danswer.ai administrators
+        </div>
+      </div>
+    </AuthFlowContainer>
+  );
+}
diff --git a/web/src/app/auth/login/LoginText.tsx b/web/src/app/auth/login/LoginText.tsx
index e31aeb81321..7b5eb97fb8e 100644
--- a/web/src/app/auth/login/LoginText.tsx
+++ b/web/src/app/auth/login/LoginText.tsx
@@ -5,11 +5,6 @@ import { SettingsContext } from "@/components/settings/SettingsProvider";
 
 export const LoginText = () => {
   const settings = useContext(SettingsContext);
-
-  // if (!settings) {
-  //   throw new Error("SettingsContext is not available");
-  // }
-
   return (
     <>
       Log In to{" "}
diff --git a/web/src/components/auth/AuthFlowContainer.tsx b/web/src/components/auth/AuthFlowContainer.tsx
index 3be441a0a7b..c6790028088 100644
--- a/web/src/components/auth/AuthFlowContainer.tsx
+++ b/web/src/components/auth/AuthFlowContainer.tsx
@@ -7,7 +7,7 @@ export default function AuthFlowContainer({
 }) {
   return (
     <div className="flex flex-col items-center justify-center min-h-screen bg-background">
-      <div className="w-full max-w-md bg-black p-8 mx-4 gap-y-4 bg-white flex items-center flex-col rounded-xl shadow-lg border border-bacgkround-100">
+      <div className="w-full max-w-md bg-black pt-8 pb-4 px-8 mx-4 gap-y-4 bg-white flex items-center flex-col rounded-xl shadow-lg border border-bacgkround-100">
         <Logo width={70} height={70} />
         {children}
       </div>
diff --git a/web/src/components/user/UserProvider.tsx b/web/src/components/user/UserProvider.tsx
index 5a3e7cbc099..48ea8826f22 100644
--- a/web/src/components/user/UserProvider.tsx
+++ b/web/src/components/user/UserProvider.tsx
@@ -11,6 +11,7 @@ interface UserContextType {
   isAdmin: boolean;
   isCurator: boolean;
   refreshUser: () => Promise<void>;
+  isCloudSuperuser: boolean;
 }
 
 const UserContext = createContext<UserContextType | undefined>(undefined);
@@ -67,6 +68,7 @@ export function UserProvider({
         refreshUser,
         isAdmin: upToDateUser?.role === UserRole.ADMIN,
         isCurator: upToDateUser?.role === UserRole.CURATOR,
+        isCloudSuperuser: upToDateUser?.is_cloud_superuser ?? false,
       }}
     >
       {children}
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index 25d98a1df38..f532a09ee31 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -42,6 +42,7 @@ export interface User {
   current_token_created_at?: Date;
   current_token_expiry_length?: number;
   oidc_expiry?: Date;
+  is_cloud_superuser?: boolean;
   organization_name: string | null;
 }
 

From 5f5cc9a724f42ea6dbf3585fa84d6b83d4f284d9 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Sat, 2 Nov 2024 12:53:04 -0700
Subject: [PATCH 346/376] Feature/redis connector refactor (#2992)

* refactor RedisConnectorDeletion into RedisConnector

* refactor redis stop and deletion

* port pruning

* nest pruning

* port deletion

* port indexing

* refactor into individual files

* refactor redis connector index  to take search settings at init

* move back to debug level log

* refactor doc set and user group (mostly)

* mypy fixes
---
 .../background/celery/apps/app_base.py        |  29 +-
 .../danswer/background/celery/apps/primary.py |  59 +-
 .../danswer/background/celery/celery_redis.py | 558 ------------------
 .../danswer/background/celery/celery_utils.py |  13 +-
 .../celery/tasks/connector_deletion/tasks.py  |  45 +-
 .../background/celery/tasks/indexing/tasks.py | 118 ++--
 .../background/celery/tasks/pruning/tasks.py  |  57 +-
 .../shared/RedisConnectorDeletionFenceData.py |   8 -
 .../shared/RedisConnectorIndexingFenceData.py |  10 -
 .../background/celery/tasks/vespa/tasks.py    | 234 +++-----
 backend/danswer/redis/redis_connector.py      |  72 +++
 .../redis/redis_connector_credential_pair.py  |  97 +++
 .../danswer/redis/redis_connector_delete.py   | 145 +++++
 .../danswer/redis/redis_connector_index.py    | 146 +++++
 .../danswer/redis/redis_connector_prune.py    | 171 ++++++
 backend/danswer/redis/redis_connector_stop.py |  34 ++
 backend/danswer/redis/redis_document_set.py   |  99 ++++
 backend/danswer/redis/redis_object_helper.py  |  91 +++
 backend/danswer/redis/redis_usergroup.py      | 112 ++++
 backend/danswer/server/documents/cc_pair.py   |  18 +-
 backend/danswer/server/documents/connector.py |  11 +-
 .../background/celery/tasks/vespa/tasks.py    |  20 +-
 .../common_utils/managers/cc_pair.py          |   1 +
 23 files changed, 1205 insertions(+), 943 deletions(-)
 delete mode 100644 backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py
 delete mode 100644 backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py
 create mode 100644 backend/danswer/redis/redis_connector.py
 create mode 100644 backend/danswer/redis/redis_connector_credential_pair.py
 create mode 100644 backend/danswer/redis/redis_connector_delete.py
 create mode 100644 backend/danswer/redis/redis_connector_index.py
 create mode 100644 backend/danswer/redis/redis_connector_prune.py
 create mode 100644 backend/danswer/redis/redis_connector_stop.py
 create mode 100644 backend/danswer/redis/redis_document_set.py
 create mode 100644 backend/danswer/redis/redis_object_helper.py
 create mode 100644 backend/danswer/redis/redis_usergroup.py

diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
index 5105b755f57..8649227ea36 100644
--- a/backend/danswer/background/celery/apps/app_base.py
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -14,15 +14,16 @@
 
 from danswer.background.celery.apps.task_formatters import CeleryTaskColoredFormatter
 from danswer.background.celery.apps.task_formatters import CeleryTaskPlainFormatter
-from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorPruning
-from danswer.background.celery.celery_redis import RedisDocumentSet
-from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.db.engine import get_all_tenant_ids
+from danswer.redis.redis_connector import RedisConnector
+from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
+from danswer.redis.redis_connector_delete import RedisConnectorDelete
+from danswer.redis.redis_connector_prune import RedisConnectorPrune
+from danswer.redis.redis_document_set import RedisDocumentSet
 from danswer.redis.redis_pool import get_redis_client
+from danswer.redis.redis_usergroup import RedisUserGroup
 from danswer.utils.logger import ColoredFormatter
 from danswer.utils.logger import PlainFormatter
 from danswer.utils.logger import setup_logger
@@ -108,29 +109,27 @@ def on_task_postrun(
     if task_id.startswith(RedisDocumentSet.PREFIX):
         document_set_id = RedisDocumentSet.get_id_from_task_id(task_id)
         if document_set_id is not None:
-            rds = RedisDocumentSet(int(document_set_id))
+            rds = RedisDocumentSet(tenant_id, int(document_set_id))
             r.srem(rds.taskset_key, task_id)
         return
 
     if task_id.startswith(RedisUserGroup.PREFIX):
         usergroup_id = RedisUserGroup.get_id_from_task_id(task_id)
         if usergroup_id is not None:
-            rug = RedisUserGroup(int(usergroup_id))
+            rug = RedisUserGroup(tenant_id, int(usergroup_id))
             r.srem(rug.taskset_key, task_id)
         return
 
-    if task_id.startswith(RedisConnectorDeletion.PREFIX):
-        cc_pair_id = RedisConnectorDeletion.get_id_from_task_id(task_id)
+    if task_id.startswith(RedisConnectorDelete.PREFIX):
+        cc_pair_id = RedisConnector.get_id_from_task_id(task_id)
         if cc_pair_id is not None:
-            rcd = RedisConnectorDeletion(int(cc_pair_id))
-            r.srem(rcd.taskset_key, task_id)
+            RedisConnectorDelete.remove_from_taskset(int(cc_pair_id), task_id, r)
         return
 
-    if task_id.startswith(RedisConnectorPruning.SUBTASK_PREFIX):
-        cc_pair_id = RedisConnectorPruning.get_id_from_task_id(task_id)
+    if task_id.startswith(RedisConnectorPrune.SUBTASK_PREFIX):
+        cc_pair_id = RedisConnector.get_id_from_task_id(task_id)
         if cc_pair_id is not None:
-            rcp = RedisConnectorPruning(int(cc_pair_id))
-            r.srem(rcp.taskset_key, task_id)
+            RedisConnectorPrune.remove_from_taskset(int(cc_pair_id), task_id, r)
         return
 
 
diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 33719608b52..5de692bd842 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -13,20 +13,20 @@
 
 import danswer.background.celery.apps.app_base as app_base
 from danswer.background.celery.apps.app_base import task_logger
-from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.celery_redis import RedisConnectorPruning
-from danswer.background.celery.celery_redis import RedisConnectorStop
-from danswer.background.celery.celery_redis import RedisDocumentSet
-from danswer.background.celery.celery_redis import RedisUserGroup
 from danswer.background.celery.celery_utils import celery_is_worker_primary
 from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
 from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import SqlEngine
+from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
+from danswer.redis.redis_connector_delete import RedisConnectorDelete
+from danswer.redis.redis_connector_index import RedisConnectorIndex
+from danswer.redis.redis_connector_prune import RedisConnectorPrune
+from danswer.redis.redis_connector_stop import RedisConnectorStop
+from danswer.redis.redis_document_set import RedisDocumentSet
 from danswer.redis.redis_pool import get_redis_client
+from danswer.redis.redis_usergroup import RedisUserGroup
 from danswer.utils.logger import setup_logger
 
 
@@ -120,50 +120,17 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
         r.delete(RedisConnectorCredentialPair.get_taskset_key())
         r.delete(RedisConnectorCredentialPair.get_fence_key())
 
-        for key in r.scan_iter(RedisDocumentSet.TASKSET_PREFIX + "*"):
-            r.delete(key)
+        RedisDocumentSet.reset_all(r)
 
-        for key in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
-            r.delete(key)
+        RedisUserGroup.reset_all(r)
 
-        for key in r.scan_iter(RedisUserGroup.TASKSET_PREFIX + "*"):
-            r.delete(key)
+        RedisConnectorDelete.reset_all(r)
 
-        for key in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
-            r.delete(key)
+        RedisConnectorPrune.reset_all(r)
 
-        for key in r.scan_iter(RedisConnectorDeletion.TASKSET_PREFIX + "*"):
-            r.delete(key)
+        RedisConnectorIndex.reset_all(r)
 
-        for key in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorPruning.TASKSET_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorPruning.GENERATOR_COMPLETE_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorPruning.GENERATOR_PROGRESS_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorIndexing.TASKSET_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_COMPLETE_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorIndexing.GENERATOR_PROGRESS_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
-            r.delete(key)
-
-        for key in r.scan_iter(RedisConnectorStop.FENCE_PREFIX + "*"):
-            r.delete(key)
+        RedisConnectorStop.reset_all(r)
 
 
 @worker_ready.connect
diff --git a/backend/danswer/background/celery/celery_redis.py b/backend/danswer/background/celery/celery_redis.py
index e412b0bf733..3e205d71ded 100644
--- a/backend/danswer/background/celery/celery_redis.py
+++ b/backend/danswer/background/celery/celery_redis.py
@@ -1,568 +1,10 @@
 # These are helper objects for tracking the keys we need to write in redis
-import time
-from abc import ABC
-from abc import abstractmethod
 from typing import cast
-from uuid import uuid4
 
-import redis
-from celery import Celery
 from redis import Redis
-from sqlalchemy.orm import Session
 
 from danswer.background.celery.configs.base import CELERY_SEPARATOR
-from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerCeleryPriority
-from danswer.configs.constants import DanswerCeleryQueues
-from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
-from danswer.db.document import construct_document_select_for_connector_credential_pair
-from danswer.db.document import (
-    construct_document_select_for_connector_credential_pair_by_needs_sync,
-)
-from danswer.db.document_set import construct_document_select_by_docset
-from danswer.utils.variable_functionality import fetch_versioned_implementation
-from danswer.utils.variable_functionality import global_version
-
-
-class RedisObjectHelper(ABC):
-    PREFIX = "base"
-    FENCE_PREFIX = PREFIX + "_fence"
-    TASKSET_PREFIX = PREFIX + "_taskset"
-
-    def __init__(self, id: str):
-        self._id: str = id
-
-    @property
-    def task_id_prefix(self) -> str:
-        return f"{self.PREFIX}_{self._id}"
-
-    @property
-    def fence_key(self) -> str:
-        # example: documentset_fence_1
-        return f"{self.FENCE_PREFIX}_{self._id}"
-
-    @property
-    def taskset_key(self) -> str:
-        # example: documentset_taskset_1
-        return f"{self.TASKSET_PREFIX}_{self._id}"
-
-    @staticmethod
-    def get_id_from_fence_key(key: str) -> str | None:
-        """
-        Extracts the object ID from a fence key in the format `PREFIX_fence_X`.
-
-        Args:
-            key (str): The fence key string.
-
-        Returns:
-            Optional[int]: The extracted ID if the key is in the correct format, otherwise None.
-        """
-        parts = key.split("_")
-        if len(parts) != 3:
-            return None
-
-        object_id = parts[2]
-        return object_id
-
-    @staticmethod
-    def get_id_from_task_id(task_id: str) -> str | None:
-        """
-        Extracts the object ID from a task ID string.
-
-        This method assumes the task ID is formatted as `prefix_objectid_suffix`, where:
-        - `prefix` is an arbitrary string (e.g., the name of the task or entity),
-        - `objectid` is the ID you want to extract,
-        - `suffix` is another arbitrary string (e.g., a UUID).
-
-        Example:
-            If the input `task_id` is `documentset_1_cbfdc96a-80ca-4312-a242-0bb68da3c1dc`,
-            this method will return the string `"1"`.
-
-        Args:
-            task_id (str): The task ID string from which to extract the object ID.
-
-        Returns:
-            str | None: The extracted object ID if the task ID is in the correct format, otherwise None.
-        """
-        # example: task_id=documentset_1_cbfdc96a-80ca-4312-a242-0bb68da3c1dc
-        parts = task_id.split("_")
-        if len(parts) != 3:
-            return None
-
-        object_id = parts[1]
-        return object_id
-
-    @abstractmethod
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock,
-        tenant_id: str | None,
-    ) -> int | None:
-        pass
-
-
-class RedisDocumentSet(RedisObjectHelper):
-    PREFIX = "documentset"
-    FENCE_PREFIX = PREFIX + "_fence"
-    TASKSET_PREFIX = PREFIX + "_taskset"
-
-    def __init__(self, id: int) -> None:
-        super().__init__(str(id))
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock,
-        tenant_id: str | None,
-    ) -> int | None:
-        last_lock_time = time.monotonic()
-
-        async_results = []
-        stmt = construct_document_select_by_docset(int(self._id), current_only=False)
-        for doc in db_session.scalars(stmt).yield_per(1):
-            current_time = time.monotonic()
-            if current_time - last_lock_time >= (
-                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
-            ):
-                lock.reacquire()
-                last_lock_time = current_time
-
-            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # we prefix the task id so it's easier to keep track of who created the task
-            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
-
-            # add to the set BEFORE creating the task.
-            redis_client.sadd(self.taskset_key, custom_task_id)
-
-            result = celery_app.send_task(
-                "vespa_metadata_sync_task",
-                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
-                queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
-                task_id=custom_task_id,
-                priority=DanswerCeleryPriority.LOW,
-            )
-
-            async_results.append(result)
-
-        return len(async_results)
-
-
-class RedisUserGroup(RedisObjectHelper):
-    PREFIX = "usergroup"
-    FENCE_PREFIX = PREFIX + "_fence"
-    TASKSET_PREFIX = PREFIX + "_taskset"
-
-    def __init__(self, id: int) -> None:
-        super().__init__(str(id))
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock,
-        tenant_id: str | None,
-    ) -> int | None:
-        last_lock_time = time.monotonic()
-
-        async_results = []
-
-        if not global_version.is_ee_version():
-            return 0
-
-        try:
-            construct_document_select_by_usergroup = fetch_versioned_implementation(
-                "danswer.db.user_group",
-                "construct_document_select_by_usergroup",
-            )
-        except ModuleNotFoundError:
-            return 0
-
-        stmt = construct_document_select_by_usergroup(int(self._id))
-        for doc in db_session.scalars(stmt).yield_per(1):
-            current_time = time.monotonic()
-            if current_time - last_lock_time >= (
-                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
-            ):
-                lock.reacquire()
-                last_lock_time = current_time
-
-            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # we prefix the task id so it's easier to keep track of who created the task
-            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
-
-            # add to the set BEFORE creating the task.
-            redis_client.sadd(self.taskset_key, custom_task_id)
-
-            result = celery_app.send_task(
-                "vespa_metadata_sync_task",
-                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
-                queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
-                task_id=custom_task_id,
-                priority=DanswerCeleryPriority.LOW,
-            )
-
-            async_results.append(result)
-
-        return len(async_results)
-
-
-class RedisConnectorCredentialPair(RedisObjectHelper):
-    """This class is used to scan documents by cc_pair in the db and collect them into
-    a unified set for syncing.
-
-    It differs from the other redis helpers in that the taskset used spans
-    all connectors and is not per connector."""
-
-    PREFIX = "connectorsync"
-    FENCE_PREFIX = PREFIX + "_fence"
-    TASKSET_PREFIX = PREFIX + "_taskset"
-
-    def __init__(self, id: int) -> None:
-        super().__init__(str(id))
-
-    @classmethod
-    def get_fence_key(cls) -> str:
-        return RedisConnectorCredentialPair.FENCE_PREFIX
-
-    @classmethod
-    def get_taskset_key(cls) -> str:
-        return RedisConnectorCredentialPair.TASKSET_PREFIX
-
-    @property
-    def taskset_key(self) -> str:
-        """Notice that this is intentionally reusing the same taskset for all
-        connector syncs"""
-        # example: connector_taskset
-        return f"{self.TASKSET_PREFIX}"
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock,
-        tenant_id: str | None,
-    ) -> int | None:
-        last_lock_time = time.monotonic()
-
-        async_results = []
-        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
-        if not cc_pair:
-            return None
-
-        stmt = construct_document_select_for_connector_credential_pair_by_needs_sync(
-            cc_pair.connector_id, cc_pair.credential_id
-        )
-        for doc in db_session.scalars(stmt).yield_per(1):
-            current_time = time.monotonic()
-            if current_time - last_lock_time >= (
-                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
-            ):
-                lock.reacquire()
-                last_lock_time = current_time
-
-            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # we prefix the task id so it's easier to keep track of who created the task
-            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
-
-            # add to the tracking taskset in redis BEFORE creating the celery task.
-            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
-            redis_client.sadd(
-                RedisConnectorCredentialPair.get_taskset_key(), custom_task_id
-            )
-
-            # Priority on sync's triggered by new indexing should be medium
-            result = celery_app.send_task(
-                "vespa_metadata_sync_task",
-                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
-                queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
-                task_id=custom_task_id,
-                priority=DanswerCeleryPriority.MEDIUM,
-            )
-
-            async_results.append(result)
-
-        return len(async_results)
-
-
-class RedisConnectorDeletion(RedisObjectHelper):
-    PREFIX = "connectordeletion"
-    FENCE_PREFIX = PREFIX + "_fence"
-    TASKSET_PREFIX = PREFIX + "_taskset"
-
-    def __init__(self, id: int) -> None:
-        super().__init__(str(id))
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock,
-        tenant_id: str | None,
-    ) -> int | None:
-        """Returns None if the cc_pair doesn't exist.
-        Otherwise, returns an int with the number of generated tasks."""
-        last_lock_time = time.monotonic()
-
-        async_results = []
-        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
-        if not cc_pair:
-            return None
-
-        stmt = construct_document_select_for_connector_credential_pair(
-            cc_pair.connector_id, cc_pair.credential_id
-        )
-        for doc in db_session.scalars(stmt).yield_per(1):
-            current_time = time.monotonic()
-            if current_time - last_lock_time >= (
-                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
-            ):
-                lock.reacquire()
-                last_lock_time = current_time
-
-            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # the actual redis key is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # we prefix the task id so it's easier to keep track of who created the task
-            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
-
-            # add to the tracking taskset in redis BEFORE creating the celery task.
-            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
-            redis_client.sadd(self.taskset_key, custom_task_id)
-
-            # Priority on sync's triggered by new indexing should be medium
-            result = celery_app.send_task(
-                "document_by_cc_pair_cleanup_task",
-                kwargs=dict(
-                    document_id=doc.id,
-                    connector_id=cc_pair.connector_id,
-                    credential_id=cc_pair.credential_id,
-                    tenant_id=tenant_id,
-                ),
-                queue=DanswerCeleryQueues.CONNECTOR_DELETION,
-                task_id=custom_task_id,
-                priority=DanswerCeleryPriority.MEDIUM,
-            )
-
-            async_results.append(result)
-
-        return len(async_results)
-
-
-class RedisConnectorPruning(RedisObjectHelper):
-    """Celery will kick off a long running generator task to crawl the connector and
-    find any missing docs, which will each then get a new cleanup task. The progress of
-    those tasks will then be monitored to completion.
-
-    Example rough happy path order:
-    Check connectorpruning_fence_1
-    Send generator task with id connectorpruning+generator_1_{uuid}
-
-    generator runs connector with callbacks that increment connectorpruning_generator_progress_1
-    generator creates many subtasks with id connectorpruning+sub_1_{uuid}
-      in taskset connectorpruning_taskset_1
-    on completion, generator sets connectorpruning_generator_complete_1
-
-    celery postrun removes subtasks from taskset
-    monitor beat task cleans up when taskset reaches 0 items
-    """
-
-    PREFIX = "connectorpruning"
-    FENCE_PREFIX = PREFIX + "_fence"  # a fence for the entire pruning process
-    GENERATOR_TASK_PREFIX = PREFIX + "+generator"
-
-    TASKSET_PREFIX = PREFIX + "_taskset"  # stores a list of prune tasks id's
-    SUBTASK_PREFIX = PREFIX + "+sub"
-
-    GENERATOR_PROGRESS_PREFIX = (
-        PREFIX + "_generator_progress"
-    )  # a signal that contains generator progress
-    GENERATOR_COMPLETE_PREFIX = (
-        PREFIX + "_generator_complete"
-    )  # a signal that the generator has finished
-
-    def __init__(self, id: int) -> None:
-        super().__init__(str(id))
-        self.documents_to_prune: set[str] = set()
-
-    @property
-    def generator_task_id_prefix(self) -> str:
-        return f"{self.GENERATOR_TASK_PREFIX}_{self._id}"
-
-    @property
-    def generator_progress_key(self) -> str:
-        # example: connectorpruning_generator_progress_1
-        return f"{self.GENERATOR_PROGRESS_PREFIX}_{self._id}"
-
-    @property
-    def generator_complete_key(self) -> str:
-        # example: connectorpruning_generator_complete_1
-        return f"{self.GENERATOR_COMPLETE_PREFIX}_{self._id}"
-
-    @property
-    def subtask_id_prefix(self) -> str:
-        return f"{self.SUBTASK_PREFIX}_{self._id}"
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock | None,
-        tenant_id: str | None,
-    ) -> int | None:
-        last_lock_time = time.monotonic()
-
-        async_results = []
-        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
-        if not cc_pair:
-            return None
-
-        for doc_id in self.documents_to_prune:
-            current_time = time.monotonic()
-            if lock and current_time - last_lock_time >= (
-                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
-            ):
-                lock.reacquire()
-                last_lock_time = current_time
-
-            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # the actual redis key is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            # we prefix the task id so it's easier to keep track of who created the task
-            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
-            custom_task_id = f"{self.subtask_id_prefix}_{uuid4()}"
-
-            # add to the tracking taskset in redis BEFORE creating the celery task.
-            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
-            redis_client.sadd(self.taskset_key, custom_task_id)
-
-            # Priority on sync's triggered by new indexing should be medium
-            result = celery_app.send_task(
-                "document_by_cc_pair_cleanup_task",
-                kwargs=dict(
-                    document_id=doc_id,
-                    connector_id=cc_pair.connector_id,
-                    credential_id=cc_pair.credential_id,
-                    tenant_id=tenant_id,
-                ),
-                queue=DanswerCeleryQueues.CONNECTOR_DELETION,
-                task_id=custom_task_id,
-                priority=DanswerCeleryPriority.MEDIUM,
-            )
-
-            async_results.append(result)
-
-        return len(async_results)
-
-    def is_pruning(self, redis_client: Redis) -> bool:
-        """A single example of a helper method being refactored into the redis helper"""
-        if redis_client.exists(self.fence_key):
-            return True
-
-        return False
-
-
-class RedisConnectorIndexing(RedisObjectHelper):
-    """Celery will kick off a long running indexing task to crawl the connector and
-    find any new or updated docs docs, which will each then get a new sync task or be
-    indexed inline.
-
-    ID should be a concatenation of cc_pair_id and search_setting_id, delimited by "/".
-    e.g. "2/5"
-    """
-
-    PREFIX = "connectorindexing"
-    FENCE_PREFIX = PREFIX + "_fence"  # a fence for the entire indexing process
-    GENERATOR_TASK_PREFIX = PREFIX + "+generator"
-
-    TASKSET_PREFIX = PREFIX + "_taskset"  # stores a list of prune tasks id's
-    SUBTASK_PREFIX = PREFIX + "+sub"
-
-    GENERATOR_LOCK_PREFIX = "da_lock:indexing"
-    GENERATOR_PROGRESS_PREFIX = (
-        PREFIX + "_generator_progress"
-    )  # a signal that contains generator progress
-    GENERATOR_COMPLETE_PREFIX = (
-        PREFIX + "_generator_complete"
-    )  # a signal that the generator has finished
-
-    def __init__(self, cc_pair_id: int, search_settings_id: int) -> None:
-        super().__init__(f"{cc_pair_id}/{search_settings_id}")
-
-    @property
-    def generator_lock_key(self) -> str:
-        return f"{self.GENERATOR_LOCK_PREFIX}_{self._id}"
-
-    @property
-    def generator_task_id_prefix(self) -> str:
-        return f"{self.GENERATOR_TASK_PREFIX}_{self._id}"
-
-    @property
-    def generator_progress_key(self) -> str:
-        # example: connectorpruning_generator_progress_1
-        return f"{self.GENERATOR_PROGRESS_PREFIX}_{self._id}"
-
-    @property
-    def generator_complete_key(self) -> str:
-        # example: connectorpruning_generator_complete_1
-        return f"{self.GENERATOR_COMPLETE_PREFIX}_{self._id}"
-
-    @property
-    def subtask_id_prefix(self) -> str:
-        return f"{self.SUBTASK_PREFIX}_{self._id}"
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock | None,
-        tenant_id: str | None,
-    ) -> int | None:
-        return None
-
-    def is_indexing(self, redis_client: Redis) -> bool:
-        """A single example of a helper method being refactored into the redis helper"""
-        if redis_client.exists(self.fence_key):
-            return True
-
-        return False
-
-
-class RedisConnectorStop(RedisObjectHelper):
-    """Used to signal any running tasks for a connector to stop. We should refactor
-    connector related redis helpers into a single class.
-    """
-
-    PREFIX = "connectorstop"
-    FENCE_PREFIX = PREFIX + "_fence"  # a fence for the entire indexing process
-    TASKSET_PREFIX = PREFIX + "_taskset"  # stores a list of prune tasks id's
-
-    def __init__(self, id: int) -> None:
-        super().__init__(str(id))
-
-    def generate_tasks(
-        self,
-        celery_app: Celery,
-        db_session: Session,
-        redis_client: Redis,
-        lock: redis.lock.Lock | None,
-        tenant_id: str | None,
-    ) -> int | None:
-        return None
 
 
 def celery_get_queue_length(queue: str, r: Redis) -> int:
diff --git a/backend/danswer/background/celery/celery_utils.py b/backend/danswer/background/celery/celery_utils.py
index 18038a349de..d0df7af02d7 100644
--- a/backend/danswer/background/celery/celery_utils.py
+++ b/backend/danswer/background/celery/celery_utils.py
@@ -4,7 +4,6 @@
 
 from sqlalchemy.orm import Session
 
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
 from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
 from danswer.configs.app_configs import MAX_PRUNING_DOCUMENT_RETRIEVAL_PER_MINUTE
 from danswer.connectors.cross_connector_utils.rate_limit_wrapper import (
@@ -18,7 +17,7 @@
 from danswer.db.connector_credential_pair import get_connector_credential_pair
 from danswer.db.enums import TaskStatus
 from danswer.db.models import TaskQueueState
-from danswer.redis.redis_pool import get_redis_client
+from danswer.redis.redis_connector import RedisConnector
 from danswer.server.documents.models import DeletionAttemptSnapshot
 from danswer.utils.logger import setup_logger
 
@@ -41,14 +40,14 @@ def _get_deletion_status(
     if not cc_pair:
         return None
 
-    rcd = RedisConnectorDeletion(cc_pair.id)
-
-    r = get_redis_client(tenant_id=tenant_id)
-    if not r.exists(rcd.fence_key):
+    redis_connector = RedisConnector(tenant_id, cc_pair.id)
+    if not redis_connector.delete.fenced:
         return None
 
     return TaskQueueState(
-        task_id="", task_name=rcd.fence_key, status=TaskStatus.STARTED
+        task_id="",
+        task_name=redis_connector.delete.fence_key,
+        status=TaskStatus.STARTED,
     )
 
 
diff --git a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
index 0b8c98c18c6..360481015bb 100644
--- a/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
+++ b/backend/danswer/background/celery/tasks/connector_deletion/tasks.py
@@ -10,13 +10,6 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.apps.app_base import task_logger
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.celery_redis import RedisConnectorPruning
-from danswer.background.celery.celery_redis import RedisConnectorStop
-from danswer.background.celery.tasks.shared.RedisConnectorDeletionFenceData import (
-    RedisConnectorDeletionFenceData,
-)
 from danswer.configs.app_configs import JOB_TIMEOUT
 from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
@@ -25,6 +18,8 @@
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.search_settings import get_all_search_settings
+from danswer.redis.redis_connector import RedisConnector
+from danswer.redis.redis_connector_delete import RedisConnectorDeletionFenceData
 from danswer.redis.redis_pool import get_redis_client
 
 
@@ -62,7 +57,7 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> N
         # try running cleanup on the cc_pair_ids
         for cc_pair_id in cc_pair_ids:
             with get_session_with_tenant(tenant_id) as db_session:
-                rcs = RedisConnectorStop(cc_pair_id)
+                redis_connector = RedisConnector(tenant_id, cc_pair_id)
                 try:
                     try_generate_document_cc_pair_cleanup_tasks(
                         self.app, cc_pair_id, db_session, r, lock_beat, tenant_id
@@ -71,10 +66,10 @@ def check_for_connector_deletion_task(self: Task, *, tenant_id: str | None) -> N
                     # this means we wanted to start deleting but dependent tasks were running
                     # Leave a stop signal to clear indexing and pruning tasks more quickly
                     task_logger.info(str(e))
-                    r.set(rcs.fence_key, cc_pair_id)
+                    redis_connector.stop.set_fence(True)
                 else:
                     # clear the stop signal if it exists ... no longer needed
-                    r.delete(rcs.fence_key)
+                    redis_connector.stop.set_fence(False)
 
     except SoftTimeLimitExceeded:
         task_logger.info(
@@ -106,10 +101,10 @@ def try_generate_document_cc_pair_cleanup_tasks(
 
     lock_beat.reacquire()
 
-    rcd = RedisConnectorDeletion(cc_pair_id)
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
 
     # don't generate sync tasks if tasks are still pending
-    if r.exists(rcd.fence_key):
+    if redis_connector.delete.fenced:
         return None
 
     # we need to load the state of the object inside the fence
@@ -123,47 +118,49 @@ def try_generate_document_cc_pair_cleanup_tasks(
         return None
 
     # set a basic fence to start
-    fence_value = RedisConnectorDeletionFenceData(
+    fence_payload = RedisConnectorDeletionFenceData(
         num_tasks=None,
         submitted=datetime.now(timezone.utc),
     )
-    r.set(rcd.fence_key, fence_value.model_dump_json())
+
+    redis_connector.delete.set_fence(fence_payload)
 
     try:
         # do not proceed if connector indexing or connector pruning are running
         search_settings_list = get_all_search_settings(db_session)
         for search_settings in search_settings_list:
-            rci = RedisConnectorIndexing(cc_pair_id, search_settings.id)
-            if r.get(rci.fence_key):
+            redis_connector_index = redis_connector.new_index(search_settings.id)
+            if redis_connector_index.fenced:
                 raise TaskDependencyError(
                     f"Connector deletion - Delayed (indexing in progress): "
                     f"cc_pair={cc_pair_id} "
                     f"search_settings={search_settings.id}"
                 )
 
-        rcp = RedisConnectorPruning(cc_pair_id)
-        if r.get(rcp.fence_key):
+        if redis_connector.prune.fenced:
             raise TaskDependencyError(
                 f"Connector deletion - Delayed (pruning in progress): "
                 f"cc_pair={cc_pair_id}"
             )
 
         # add tasks to celery and build up the task set to monitor in redis
-        r.delete(rcd.taskset_key)
+        redis_connector.delete.taskset_clear()
 
         # Add all documents that need to be updated into the queue
         task_logger.info(
             f"RedisConnectorDeletion.generate_tasks starting. cc_pair={cc_pair_id}"
         )
-        tasks_generated = rcd.generate_tasks(app, db_session, r, lock_beat, tenant_id)
+        tasks_generated = redis_connector.delete.generate_tasks(
+            app, db_session, lock_beat
+        )
         if tasks_generated is None:
             raise ValueError("RedisConnectorDeletion.generate_tasks returned None")
     except TaskDependencyError:
-        r.delete(rcd.fence_key)
+        redis_connector.delete.set_fence(None)
         raise
     except Exception:
         task_logger.exception("Unexpected exception")
-        r.delete(rcd.fence_key)
+        redis_connector.delete.set_fence(None)
         return None
     else:
         # Currently we are allowing the sync to proceed with 0 tasks.
@@ -178,7 +175,7 @@ def try_generate_document_cc_pair_cleanup_tasks(
         )
 
         # set this only after all tasks have been added
-        fence_value.num_tasks = tasks_generated
-        r.set(rcd.fence_key, fence_value.model_dump_json())
+        fence_payload.num_tasks = tasks_generated
+        redis_connector.delete.set_fence(fence_payload)
 
     return tasks_generated
diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 175aef54055..7d2e6e0b3dd 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -2,8 +2,6 @@
 from datetime import timezone
 from http import HTTPStatus
 from time import sleep
-from typing import cast
-from uuid import uuid4
 
 import redis
 from celery import Celery
@@ -14,12 +12,6 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.apps.app_base import task_logger
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.celery_redis import RedisConnectorStop
-from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
-    RedisConnectorIndexingFenceData,
-)
 from danswer.background.indexing.job_client import SimpleJobClient
 from danswer.background.indexing.run_indexing import run_indexing_entrypoint
 from danswer.background.indexing.run_indexing import RunIndexingCallbackInterface
@@ -50,6 +42,8 @@
 from danswer.db.swap_index import check_index_swap
 from danswer.natural_language_processing.search_nlp_models import EmbeddingModel
 from danswer.natural_language_processing.search_nlp_models import warm_up_bi_encoder
+from danswer.redis.redis_connector import RedisConnector
+from danswer.redis.redis_connector_index import RedisConnectorIndexingFenceData
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import global_version
@@ -129,6 +123,7 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                 cc_pair_ids.append(cc_pair_entry.id)
 
         for cc_pair_id in cc_pair_ids:
+            redis_connector = RedisConnector(tenant_id, cc_pair_id)
             with get_session_with_tenant(tenant_id) as db_session:
                 # Get the primary search settings
                 primary_search_settings = get_current_search_settings(db_session)
@@ -141,10 +136,10 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     search_settings.append(secondary_search_settings)
 
                 for search_settings_instance in search_settings:
-                    rci = RedisConnectorIndexing(
-                        cc_pair_id, search_settings_instance.id
+                    redis_connector_index = redis_connector.new_index(
+                        search_settings_instance.id
                     )
-                    if r.exists(rci.fence_key):
+                    if redis_connector_index.fenced:
                         continue
 
                     cc_pair = get_connector_credential_pair_from_id(
@@ -307,15 +302,15 @@ def try_creating_indexing_task(
         return None
 
     try:
-        rci = RedisConnectorIndexing(cc_pair.id, search_settings.id)
+        redis_connector = RedisConnector(tenant_id, cc_pair.id)
+        redis_connector_index = redis_connector.new_index(search_settings.id)
 
         # skip if already indexing
-        if r.exists(rci.fence_key):
+        if redis_connector_index.fenced:
             return None
 
         # skip indexing if the cc_pair is deleting
-        rcd = RedisConnectorDeletion(cc_pair.id)
-        if r.exists(rcd.fence_key):
+        if redis_connector.delete.fenced:
             return None
 
         db_session.refresh(cc_pair)
@@ -323,19 +318,17 @@ def try_creating_indexing_task(
             return None
 
         # add a long running generator task to the queue
-        r.delete(rci.generator_complete_key)
-        r.delete(rci.taskset_key)
-
-        custom_task_id = f"{rci.generator_task_id_prefix}_{uuid4()}"
+        redis_connector_index.generator_clear()
 
         # set a basic fence to start
-        fence_value = RedisConnectorIndexingFenceData(
+        payload = RedisConnectorIndexingFenceData(
             index_attempt_id=None,
             started=None,
             submitted=datetime.now(timezone.utc),
             celery_task_id=None,
         )
-        r.set(rci.fence_key, fence_value.model_dump_json())
+
+        redis_connector_index.set_fence(payload)
 
         # create the index attempt for tracking purposes
         # code elsewhere checks for index attempts without an associated redis key
@@ -348,6 +341,8 @@ def try_creating_indexing_task(
             db_session=db_session,
         )
 
+        custom_task_id = redis_connector_index.generate_generator_task_id()
+
         result = celery_app.send_task(
             "connector_indexing_proxy_task",
             kwargs=dict(
@@ -364,11 +359,12 @@ def try_creating_indexing_task(
             raise RuntimeError("send_task for connector_indexing_proxy_task failed.")
 
         # now fill out the fence with the rest of the data
-        fence_value.index_attempt_id = index_attempt_id
-        fence_value.celery_task_id = result.id
-        r.set(rci.fence_key, fence_value.model_dump_json())
+        payload.index_attempt_id = index_attempt_id
+        payload.celery_task_id = result.id
+        redis_connector_index.set_fence(payload)
+
     except Exception:
-        r.delete(rci.fence_key)
+        redis_connector_index.set_fence(payload)
         task_logger.exception(
             f"Unexpected exception: "
             f"tenant={tenant_id} "
@@ -494,61 +490,52 @@ def connector_indexing_task(
     )
 
     attempt = None
-    n_final_progress = 0
+    n_final_progress: int | None = None
+
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
+    redis_connector_index = redis_connector.new_index(search_settings_id)
 
     r = get_redis_client(tenant_id=tenant_id)
 
-    rcd = RedisConnectorDeletion(cc_pair_id)
-    if r.exists(rcd.fence_key):
+    if redis_connector.delete.fenced:
         raise RuntimeError(
             f"Indexing will not start because connector deletion is in progress: "
             f"cc_pair={cc_pair_id} "
-            f"fence={rcd.fence_key}"
+            f"fence={redis_connector.delete.fence_key}"
         )
 
-    rcs = RedisConnectorStop(cc_pair_id)
-    if r.exists(rcs.fence_key):
+    if redis_connector.stop.fenced:
         raise RuntimeError(
             f"Indexing will not start because a connector stop signal was detected: "
             f"cc_pair={cc_pair_id} "
-            f"fence={rcs.fence_key}"
+            f"fence={redis_connector.stop.fence_key}"
         )
 
-    rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
-
     while True:
-        # read related data and evaluate/print task progress
-        fence_value = cast(bytes, r.get(rci.fence_key))
-        if fence_value is None:
+        # wait for the fence to come up
+        if not redis_connector_index.fenced:
             raise ValueError(
-                f"connector_indexing_task: fence_value not found: fence={rci.fence_key}"
+                f"connector_indexing_task - fence not found: fence={redis_connector_index.fence_key}"
             )
 
-        try:
-            fence_json = fence_value.decode("utf-8")
-            fence_data = RedisConnectorIndexingFenceData.model_validate_json(
-                cast(str, fence_json)
-            )
-        except ValueError:
-            logger.exception(
-                f"connector_indexing_task: fence_data not decodeable: fence={rci.fence_key}"
-            )
-            raise
+        payload = redis_connector_index.payload
+        if not payload:
+            raise ValueError("connector_indexing_task: payload invalid or not found")
 
-        if fence_data.index_attempt_id is None or fence_data.celery_task_id is None:
+        if payload.index_attempt_id is None or payload.celery_task_id is None:
             logger.info(
-                f"connector_indexing_task - Waiting for fence: fence={rci.fence_key}"
+                f"connector_indexing_task - Waiting for fence: fence={redis_connector_index.fence_key}"
             )
             sleep(1)
             continue
 
         logger.info(
-            f"connector_indexing_task - Fence found, continuing...: fence={rci.fence_key}"
+            f"connector_indexing_task - Fence found, continuing...: fence={redis_connector_index.fence_key}"
         )
         break
 
     lock = r.lock(
-        rci.generator_lock_key,
+        redis_connector_index.generator_lock_key,
         timeout=CELERY_INDEXING_LOCK_TIMEOUT,
     )
 
@@ -558,11 +545,10 @@ def connector_indexing_task(
             f"Indexing task already running, exiting...: "
             f"cc_pair={cc_pair_id} search_settings={search_settings_id}"
         )
-        # r.set(rci.generator_complete_key, HTTPStatus.CONFLICT.value)
         return None
 
-    fence_data.started = datetime.now(timezone.utc)
-    r.set(rci.fence_key, fence_data.model_dump_json())
+    payload.started = datetime.now(timezone.utc)
+    redis_connector_index.set_fence(payload)
 
     try:
         with get_session_with_tenant(tenant_id) as db_session:
@@ -590,11 +576,12 @@ def connector_indexing_task(
                     f"Credential not found: cc_pair={cc_pair_id} credential={cc_pair.credential_id}"
                 )
 
-            rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
-
             # define a callback class
             callback = RunIndexingCallback(
-                rcs.fence_key, rci.generator_progress_key, lock, r
+                redis_connector.stop.fence_key,
+                redis_connector_index.generator_progress_key,
+                lock,
+                r,
             )
 
             logger.info(
@@ -613,14 +600,8 @@ def connector_indexing_task(
             )
 
             # get back the total number of indexed docs and return it
-            generator_progress_value = r.get(rci.generator_progress_key)
-            if generator_progress_value is not None:
-                try:
-                    n_final_progress = int(cast(int, generator_progress_value))
-                except ValueError:
-                    pass
-
-            r.set(rci.generator_complete_key, HTTPStatus.OK.value)
+            n_final_progress = redis_connector_index.get_progress()
+            redis_connector_index.set_generator_complete(HTTPStatus.OK.value)
     except Exception as e:
         logger.exception(
             f"Indexing spawned task failed: attempt={index_attempt_id} "
@@ -632,10 +613,7 @@ def connector_indexing_task(
             with get_session_with_tenant(tenant_id) as db_session:
                 mark_attempt_failed(attempt, db_session, failure_reason=str(e))
 
-        r.delete(rci.generator_lock_key)
-        r.delete(rci.generator_progress_key)
-        r.delete(rci.taskset_key)
-        r.delete(rci.fence_key)
+        redis_connector_index.reset()
         raise e
     finally:
         if lock.owned():
diff --git a/backend/danswer/background/celery/tasks/pruning/tasks.py b/backend/danswer/background/celery/tasks/pruning/tasks.py
index d20430df1f8..af80e6b886c 100644
--- a/backend/danswer/background/celery/tasks/pruning/tasks.py
+++ b/backend/danswer/background/celery/tasks/pruning/tasks.py
@@ -11,9 +11,6 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.apps.app_base import task_logger
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorPruning
-from danswer.background.celery.celery_redis import RedisConnectorStop
 from danswer.background.celery.celery_utils import extract_ids_from_runnable_connector
 from danswer.background.celery.tasks.indexing.tasks import RunIndexingCallback
 from danswer.configs.app_configs import ALLOW_SIMULTANEOUS_PRUNING
@@ -33,6 +30,7 @@
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.enums import ConnectorCredentialPairStatus
 from danswer.db.models import ConnectorCredentialPair
+from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_pool import get_redis_client
 from danswer.utils.logger import pruning_ctx
 from danswer.utils.logger import setup_logger
@@ -147,8 +145,11 @@ def try_creating_prune_generator_task(
     is used to trigger prunes immediately, e.g. via the web ui.
     """
 
+    redis_connector = RedisConnector(tenant_id, cc_pair.id)
+
     if not ALLOW_SIMULTANEOUS_PRUNING:
-        for key in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+        count = redis_connector.prune.get_active_task_count()
+        if count > 0:
             return None
 
     LOCK_TIMEOUT = 30
@@ -165,15 +166,10 @@ def try_creating_prune_generator_task(
         return None
 
     try:
-        rcp = RedisConnectorPruning(cc_pair.id)
-
-        # skip pruning if already pruning
-        if r.exists(rcp.fence_key):
+        if redis_connector.prune.fenced:  # skip pruning if already pruning
             return None
 
-        # skip pruning if the cc_pair is deleting
-        rcd = RedisConnectorDeletion(cc_pair.id)
-        if r.exists(rcd.fence_key):
+        if redis_connector.delete.fenced:  # skip pruning if the cc_pair is deleting
             return None
 
         db_session.refresh(cc_pair)
@@ -181,10 +177,10 @@ def try_creating_prune_generator_task(
             return None
 
         # add a long running generator task to the queue
-        r.delete(rcp.generator_complete_key)
-        r.delete(rcp.taskset_key)
+        redis_connector.prune.generator_clear()
+        redis_connector.prune.taskset_clear()
 
-        custom_task_id = f"{rcp.generator_task_id_prefix}_{uuid4()}"
+        custom_task_id = f"{redis_connector.prune.generator_task_key}_{uuid4()}"
 
         celery_app.send_task(
             "connector_pruning_generator_task",
@@ -200,7 +196,7 @@ def try_creating_prune_generator_task(
         )
 
         # set this only after all tasks have been added
-        r.set(rcp.fence_key, 1)
+        redis_connector.prune.set_fence(True)
     except Exception:
         task_logger.exception(f"Unexpected exception: cc_pair={cc_pair.id}")
         return None
@@ -235,12 +231,12 @@ def connector_pruning_generator_task(
     pruning_ctx_dict["request_id"] = self.request.id
     pruning_ctx.set(pruning_ctx_dict)
 
-    rcp = RedisConnectorPruning(cc_pair_id)
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
 
     r = get_redis_client(tenant_id=tenant_id)
 
     lock = r.lock(
-        DanswerRedisLocks.PRUNING_LOCK_PREFIX + f"_{rcp._id}",
+        DanswerRedisLocks.PRUNING_LOCK_PREFIX + f"_{redis_connector.id}",
         timeout=CELERY_PRUNING_LOCK_TIMEOUT,
     )
 
@@ -273,10 +269,11 @@ def connector_pruning_generator_task(
                 cc_pair.credential,
             )
 
-            rcs = RedisConnectorStop(cc_pair_id)
-
             callback = RunIndexingCallback(
-                rcs.fence_key, rcp.generator_progress_key, lock, r
+                redis_connector.stop.fence_key,
+                redis_connector.prune.generator_progress_key,
+                lock,
+                r,
             )
             # a list of docs in the source
             all_connector_doc_ids: set[str] = extract_ids_from_runnable_connector(
@@ -303,31 +300,29 @@ def connector_pruning_generator_task(
                 f"doc_source={cc_pair.connector.source}"
             )
 
-            rcp.documents_to_prune = set(doc_ids_to_remove)
-
             task_logger.info(
-                f"RedisConnectorPruning.generate_tasks starting. cc_pair={cc_pair.id}"
+                f"RedisConnector.prune.generate_tasks starting. cc_pair={cc_pair_id}"
             )
-            tasks_generated = rcp.generate_tasks(
-                self.app, db_session, r, None, tenant_id
+            tasks_generated = redis_connector.prune.generate_tasks(
+                set(doc_ids_to_remove), self.app, db_session, None
             )
             if tasks_generated is None:
                 return None
 
             task_logger.info(
-                f"RedisConnectorPruning.generate_tasks finished. "
-                f"cc_pair={cc_pair.id} tasks_generated={tasks_generated}"
+                f"RedisConnector.prune.generate_tasks finished. "
+                f"cc_pair={cc_pair_id} tasks_generated={tasks_generated}"
             )
 
-            r.set(rcp.generator_complete_key, tasks_generated)
+            redis_connector.prune.generator_complete = tasks_generated
     except Exception as e:
         task_logger.exception(
             f"Failed to run pruning: cc_pair={cc_pair_id} connector={connector_id}"
         )
 
-        r.delete(rcp.generator_progress_key)
-        r.delete(rcp.taskset_key)
-        r.delete(rcp.fence_key)
+        redis_connector.prune.generator_clear()
+        redis_connector.prune.taskset_clear()
+        redis_connector.prune.set_fence(False)
         raise e
     finally:
         if lock.owned():
diff --git a/backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py b/backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py
deleted file mode 100644
index 1c664d14b4b..00000000000
--- a/backend/danswer/background/celery/tasks/shared/RedisConnectorDeletionFenceData.py
+++ /dev/null
@@ -1,8 +0,0 @@
-from datetime import datetime
-
-from pydantic import BaseModel
-
-
-class RedisConnectorDeletionFenceData(BaseModel):
-    num_tasks: int | None
-    submitted: datetime
diff --git a/backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py b/backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py
deleted file mode 100644
index 224571a4231..00000000000
--- a/backend/danswer/background/celery/tasks/shared/RedisConnectorIndexingFenceData.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from datetime import datetime
-
-from pydantic import BaseModel
-
-
-class RedisConnectorIndexingFenceData(BaseModel):
-    index_attempt_id: int | None
-    started: datetime | None
-    submitted: datetime
-    celery_task_id: str | None
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index b058a97e1bc..944750c7a78 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -19,18 +19,6 @@
 from danswer.access.access import get_access_for_document
 from danswer.background.celery.apps.app_base import task_logger
 from danswer.background.celery.celery_redis import celery_get_queue_length
-from danswer.background.celery.celery_redis import RedisConnectorCredentialPair
-from danswer.background.celery.celery_redis import RedisConnectorDeletion
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.celery_redis import RedisConnectorPruning
-from danswer.background.celery.celery_redis import RedisDocumentSet
-from danswer.background.celery.celery_redis import RedisUserGroup
-from danswer.background.celery.tasks.shared.RedisConnectorDeletionFenceData import (
-    RedisConnectorDeletionFenceData,
-)
-from danswer.background.celery.tasks.shared.RedisConnectorIndexingFenceData import (
-    RedisConnectorIndexingFenceData,
-)
 from danswer.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex
 from danswer.background.celery.tasks.shared.tasks import LIGHT_SOFT_TIME_LIMIT
 from danswer.background.celery.tasks.shared.tasks import LIGHT_TIME_LIMIT
@@ -67,7 +55,14 @@
 from danswer.document_index.document_index_utils import get_both_index_names
 from danswer.document_index.factory import get_default_document_index
 from danswer.document_index.interfaces import VespaDocumentFields
+from danswer.redis.redis_connector import RedisConnector
+from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
+from danswer.redis.redis_connector_delete import RedisConnectorDelete
+from danswer.redis.redis_connector_index import RedisConnectorIndex
+from danswer.redis.redis_connector_prune import RedisConnectorPrune
+from danswer.redis.redis_document_set import RedisDocumentSet
 from danswer.redis.redis_pool import get_redis_client
+from danswer.redis.redis_usergroup import RedisUserGroup
 from danswer.utils.logger import setup_logger
 from danswer.utils.variable_functionality import fetch_versioned_implementation
 from danswer.utils.variable_functionality import (
@@ -192,7 +187,7 @@ def try_generate_stale_document_sync_tasks(
     total_tasks_generated = 0
     cc_pairs = get_connector_credential_pairs(db_session)
     for cc_pair in cc_pairs:
-        rc = RedisConnectorCredentialPair(cc_pair.id)
+        rc = RedisConnectorCredentialPair(tenant_id, cc_pair.id)
         tasks_generated = rc.generate_tasks(
             celery_app, db_session, r, lock_beat, tenant_id
         )
@@ -228,10 +223,10 @@ def try_generate_document_set_sync_tasks(
 ) -> int | None:
     lock_beat.reacquire()
 
-    rds = RedisDocumentSet(document_set_id)
+    rds = RedisDocumentSet(tenant_id, document_set_id)
 
     # don't generate document set sync tasks if tasks are still pending
-    if r.exists(rds.fence_key):
+    if rds.fenced:
         return None
 
     # don't generate sync tasks if we're up to date
@@ -269,7 +264,7 @@ def try_generate_document_set_sync_tasks(
     )
 
     # set this only after all tasks have been added
-    r.set(rds.fence_key, tasks_generated)
+    rds.set_fence(tasks_generated)
     return tasks_generated
 
 
@@ -283,10 +278,9 @@ def try_generate_user_group_sync_tasks(
 ) -> int | None:
     lock_beat.reacquire()
 
-    rug = RedisUserGroup(usergroup_id)
-
-    # don't generate sync tasks if tasks are still pending
-    if r.exists(rug.fence_key):
+    rug = RedisUserGroup(tenant_id, usergroup_id)
+    if rug.fenced:
+        # don't generate sync tasks if tasks are still pending
         return None
 
     # race condition with the monitor/cleanup function if we use a cached result!
@@ -326,7 +320,7 @@ def try_generate_user_group_sync_tasks(
     )
 
     # set this only after all tasks have been added
-    r.set(rug.fence_key, tasks_generated)
+    rug.set_fence(tasks_generated)
     return tasks_generated
 
 
@@ -352,7 +346,7 @@ def monitor_connector_taskset(r: Redis) -> None:
 
 
 def monitor_document_set_taskset(
-    key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
     document_set_id_str = RedisDocumentSet.get_id_from_fence_key(fence_key)
@@ -362,16 +356,12 @@ def monitor_document_set_taskset(
 
     document_set_id = int(document_set_id_str)
 
-    rds = RedisDocumentSet(document_set_id)
-
-    fence_value = r.get(rds.fence_key)
-    if fence_value is None:
+    rds = RedisDocumentSet(tenant_id, document_set_id)
+    if not rds.fenced:
         return
 
-    try:
-        initial_count = int(cast(int, fence_value))
-    except ValueError:
-        task_logger.error("The value is not an integer.")
+    initial_count = rds.payload
+    if initial_count is None:
         return
 
     count = cast(int, r.scard(rds.taskset_key))
@@ -399,48 +389,38 @@ def monitor_document_set_taskset(
                 f"Successfully synced document set: document_set={document_set_id}"
             )
 
-    r.delete(rds.taskset_key)
-    r.delete(rds.fence_key)
+    rds.reset()
 
 
 def monitor_connector_deletion_taskset(
-    key_bytes: bytes, r: Redis, tenant_id: str | None
+    tenant_id: str | None, key_bytes: bytes, r: Redis
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
-    cc_pair_id_str = RedisConnectorDeletion.get_id_from_fence_key(fence_key)
+    cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
     if cc_pair_id_str is None:
         task_logger.warning(f"could not parse cc_pair_id from {fence_key}")
         return
 
     cc_pair_id = int(cc_pair_id_str)
 
-    rcd = RedisConnectorDeletion(cc_pair_id)
-
-    # read related data and evaluate/print task progress
-    fence_value = cast(bytes, r.get(rcd.fence_key))
-    if fence_value is None:
-        return
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
 
-    try:
-        fence_json = fence_value.decode("utf-8")
-        fence_data = RedisConnectorDeletionFenceData.model_validate_json(
-            cast(str, fence_json)
-        )
-    except ValueError:
-        task_logger.exception(
-            "monitor_ccpair_indexing_taskset: fence_data not decodeable."
+    fence_data = redis_connector.delete.payload
+    if not fence_data:
+        task_logger.warning(
+            f"Connector deletion - fence payload invalid: cc_pair={cc_pair_id}"
         )
-        raise
+        return
 
-    # the fence is setting up but isn't ready yet
     if fence_data.num_tasks is None:
+        # the fence is setting up but isn't ready yet
         return
 
-    count = cast(int, r.scard(rcd.taskset_key))
+    remaining = redis_connector.delete.get_remaining()
     task_logger.info(
-        f"Connector deletion progress: cc_pair={cc_pair_id} remaining={count} initial={fence_data.num_tasks}"
+        f"Connector deletion progress: cc_pair={cc_pair_id} remaining={remaining} initial={fence_data.num_tasks}"
     )
-    if count > 0:
+    if remaining > 0:
         return
 
     with get_session_with_tenant(tenant_id) as db_session:
@@ -524,15 +504,15 @@ def monitor_connector_deletion_taskset(
         f"docs_deleted={fence_data.num_tasks}"
     )
 
-    r.delete(rcd.taskset_key)
-    r.delete(rcd.fence_key)
+    redis_connector.delete.taskset_clear()
+    redis_connector.delete.set_fence(None)
 
 
 def monitor_ccpair_pruning_taskset(
-    key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     fence_key = key_bytes.decode("utf-8")
-    cc_pair_id_str = RedisConnectorPruning.get_id_from_fence_key(fence_key)
+    cc_pair_id_str = RedisConnector.get_id_from_fence_key(fence_key)
     if cc_pair_id_str is None:
         task_logger.warning(
             f"monitor_ccpair_pruning_taskset: could not parse cc_pair_id from {fence_key}"
@@ -541,46 +521,37 @@ def monitor_ccpair_pruning_taskset(
 
     cc_pair_id = int(cc_pair_id_str)
 
-    rcp = RedisConnectorPruning(cc_pair_id)
-
-    fence_value = r.get(rcp.fence_key)
-    if fence_value is None:
-        return
-
-    generator_value = r.get(rcp.generator_complete_key)
-    if generator_value is None:
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
+    if not redis_connector.prune.fenced:
         return
 
-    try:
-        initial_count = int(cast(int, generator_value))
-    except ValueError:
-        task_logger.error("The value is not an integer.")
+    initial = redis_connector.prune.generator_complete
+    if initial is None:
         return
 
-    count = cast(int, r.scard(rcp.taskset_key))
+    remaining = redis_connector.prune.get_remaining()
     task_logger.info(
-        f"Connector pruning progress: cc_pair_id={cc_pair_id} remaining={count} initial={initial_count}"
+        f"Connector pruning progress: cc_pair={cc_pair_id} remaining={remaining} initial={initial}"
     )
-    if count > 0:
+    if remaining > 0:
         return
 
     mark_ccpair_as_pruned(int(cc_pair_id), db_session)
     task_logger.info(
-        f"Successfully pruned connector credential pair. cc_pair_id={cc_pair_id}"
+        f"Successfully pruned connector credential pair. cc_pair={cc_pair_id}"
     )
 
-    r.delete(rcp.taskset_key)
-    r.delete(rcp.generator_progress_key)
-    r.delete(rcp.generator_complete_key)
-    r.delete(rcp.fence_key)
+    redis_connector.prune.taskset_clear()
+    redis_connector.prune.generator_clear()
+    redis_connector.prune.set_fence(False)
 
 
 def monitor_ccpair_indexing_taskset(
-    key_bytes: bytes, r: Redis, db_session: Session
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
 ) -> None:
     # if the fence doesn't exist, there's nothing to do
     fence_key = key_bytes.decode("utf-8")
-    composite_id = RedisConnectorIndexing.get_id_from_fence_key(fence_key)
+    composite_id = RedisConnector.get_id_from_fence_key(fence_key)
     if composite_id is None:
         task_logger.warning(
             f"monitor_ccpair_indexing_taskset: could not parse composite_id from {fence_key}"
@@ -595,53 +566,37 @@ def monitor_ccpair_indexing_taskset(
     cc_pair_id = int(parts[0])
     search_settings_id = int(parts[1])
 
-    rci = RedisConnectorIndexing(cc_pair_id, search_settings_id)
-
-    # read related data and evaluate/print task progress
-    fence_value = cast(bytes, r.get(rci.fence_key))
-    if fence_value is None:
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
+    redis_connector_index = redis_connector.new_index(search_settings_id)
+    if not redis_connector_index.fenced:
         return
 
-    try:
-        fence_json = fence_value.decode("utf-8")
-        fence_data = RedisConnectorIndexingFenceData.model_validate_json(
-            cast(str, fence_json)
-        )
-    except ValueError:
-        task_logger.exception(
-            "monitor_ccpair_indexing_taskset: fence_data not decodeable."
-        )
-        raise
-
-    elapsed_submitted = datetime.now(timezone.utc) - fence_data.submitted
+    payload = redis_connector_index.payload
+    if not payload:
+        return
 
-    generator_progress_value = r.get(rci.generator_progress_key)
-    if generator_progress_value is not None:
-        try:
-            progress_count = int(cast(int, generator_progress_value))
+    elapsed_submitted = datetime.now(timezone.utc) - payload.submitted
 
-            task_logger.info(
-                f"Connector indexing progress: cc_pair_id={cc_pair_id} "
-                f"search_settings_id={search_settings_id} "
-                f"progress={progress_count} "
-                f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
-            )
-        except ValueError:
-            task_logger.error(
-                "monitor_ccpair_indexing_taskset: generator_progress_value is not an integer."
-            )
+    progress = redis_connector_index.get_progress()
+    if progress is not None:
+        task_logger.info(
+            f"Connector indexing progress: cc_pair_id={cc_pair_id} "
+            f"search_settings_id={search_settings_id} "
+            f"progress={progress} "
+            f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
+        )
 
-    if fence_data.index_attempt_id is None or fence_data.celery_task_id is None:
+    if payload.index_attempt_id is None or payload.celery_task_id is None:
         # the task is still setting up
         return
 
     # Read result state BEFORE generator_complete_key to avoid a race condition
     # never use any blocking methods on the result from inside a task!
-    result: AsyncResult = AsyncResult(fence_data.celery_task_id)
+    result: AsyncResult = AsyncResult(payload.celery_task_id)
     result_state = result.state
 
-    generator_complete_value = r.get(rci.generator_complete_key)
-    if generator_complete_value is None:
+    status_int = redis_connector_index.get_completion()
+    if status_int is None:
         if result_state in READY_STATES:
             # IF the task state is READY, THEN generator_complete should be set
             # if it isn't, then the worker crashed
@@ -652,7 +607,7 @@ def monitor_ccpair_indexing_taskset(
                 f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
             )
 
-            index_attempt = get_index_attempt(db_session, fence_data.index_attempt_id)
+            index_attempt = get_index_attempt(db_session, payload.index_attempt_id)
             if index_attempt:
                 mark_attempt_failed(
                     index_attempt=index_attempt,
@@ -660,22 +615,10 @@ def monitor_ccpair_indexing_taskset(
                     failure_reason="Connector indexing aborted or exceptioned.",
                 )
 
-            r.delete(rci.generator_lock_key)
-            r.delete(rci.taskset_key)
-            r.delete(rci.generator_progress_key)
-            r.delete(rci.generator_complete_key)
-            r.delete(rci.fence_key)
+            redis_connector_index.reset()
         return
 
-    status_enum = HTTPStatus.INTERNAL_SERVER_ERROR
-    try:
-        status_value = int(cast(int, generator_complete_value))
-        status_enum = HTTPStatus(status_value)
-    except ValueError:
-        task_logger.error(
-            f"monitor_ccpair_indexing_taskset: "
-            f"generator_complete_value=f{generator_complete_value} could not be parsed."
-        )
+    status_enum = HTTPStatus(status_int)
 
     task_logger.info(
         f"Connector indexing finished: cc_pair_id={cc_pair_id} "
@@ -684,11 +627,7 @@ def monitor_ccpair_indexing_taskset(
         f"elapsed_submitted={elapsed_submitted.total_seconds():.2f}"
     )
 
-    r.delete(rci.generator_lock_key)
-    r.delete(rci.taskset_key)
-    r.delete(rci.generator_progress_key)
-    r.delete(rci.generator_complete_key)
-    r.delete(rci.fence_key)
+    redis_connector_index.reset()
 
 
 @shared_task(name="monitor_vespa_sync", soft_time_limit=300, bind=True)
@@ -700,7 +639,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
     This task lock timeout is CELERY_METADATA_SYNC_BEAT_LOCK_TIMEOUT seconds, so don't
     do anything too expensive in this function!
 
-    Returns True if the task actually did work, False
+    Returns True if the task actually did work, False if it exited early to prevent overlap
     """
     r = get_redis_client(tenant_id=tenant_id)
 
@@ -751,11 +690,12 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
 
             for a in attempts:
                 # if attempts exist in the db but we don't detect them in redis, mark them as failed
-                rci = RedisConnectorIndexing(
-                    a.connector_credential_pair_id, a.search_settings_id
-                )
                 failure_reason = f"Unknown index attempt {a.id}. Might be left over from a process restart."
-                if not r.exists(rci.fence_key):
+                if not r.exists(
+                    RedisConnectorIndex.fence_key_with_ids(
+                        a.connector_credential_pair_id, a.search_settings_id
+                    )
+                ):
                     mark_attempt_failed(a, db_session, failure_reason=failure_reason)
 
         lock_beat.reacquire()
@@ -763,15 +703,15 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
             monitor_connector_taskset(r)
 
         lock_beat.reacquire()
-        for key_bytes in r.scan_iter(RedisConnectorDeletion.FENCE_PREFIX + "*"):
+        for key_bytes in r.scan_iter(RedisConnectorDelete.FENCE_PREFIX + "*"):
             lock_beat.reacquire()
-            monitor_connector_deletion_taskset(key_bytes, r, tenant_id)
+            monitor_connector_deletion_taskset(tenant_id, key_bytes, r)
 
         lock_beat.reacquire()
         for key_bytes in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
             lock_beat.reacquire()
             with get_session_with_tenant(tenant_id) as db_session:
-                monitor_document_set_taskset(key_bytes, r, db_session)
+                monitor_document_set_taskset(tenant_id, key_bytes, r, db_session)
 
         lock_beat.reacquire()
         for key_bytes in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
@@ -782,19 +722,19 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
                 noop_fallback,
             )
             with get_session_with_tenant(tenant_id) as db_session:
-                monitor_usergroup_taskset(key_bytes, r, db_session)
+                monitor_usergroup_taskset(tenant_id, key_bytes, r, db_session)
 
         lock_beat.reacquire()
-        for key_bytes in r.scan_iter(RedisConnectorPruning.FENCE_PREFIX + "*"):
+        for key_bytes in r.scan_iter(RedisConnectorPrune.FENCE_PREFIX + "*"):
             lock_beat.reacquire()
             with get_session_with_tenant(tenant_id) as db_session:
-                monitor_ccpair_pruning_taskset(key_bytes, r, db_session)
+                monitor_ccpair_pruning_taskset(tenant_id, key_bytes, r, db_session)
 
         lock_beat.reacquire()
-        for key_bytes in r.scan_iter(RedisConnectorIndexing.FENCE_PREFIX + "*"):
+        for key_bytes in r.scan_iter(RedisConnectorIndex.FENCE_PREFIX + "*"):
             lock_beat.reacquire()
             with get_session_with_tenant(tenant_id) as db_session:
-                monitor_ccpair_indexing_taskset(key_bytes, r, db_session)
+                monitor_ccpair_indexing_taskset(tenant_id, key_bytes, r, db_session)
 
         # uncomment for debugging if needed
         # r_celery = celery_app.broker_connection().channel().client
diff --git a/backend/danswer/redis/redis_connector.py b/backend/danswer/redis/redis_connector.py
new file mode 100644
index 00000000000..df61f986ede
--- /dev/null
+++ b/backend/danswer/redis/redis_connector.py
@@ -0,0 +1,72 @@
+import redis
+
+from danswer.redis.redis_connector_delete import RedisConnectorDelete
+from danswer.redis.redis_connector_index import RedisConnectorIndex
+from danswer.redis.redis_connector_prune import RedisConnectorPrune
+from danswer.redis.redis_connector_stop import RedisConnectorStop
+from danswer.redis.redis_pool import get_redis_client
+
+
+class RedisConnector:
+    """Composes several classes to simplify interacting with a connector and its
+    associated background tasks / associated redis interactions."""
+
+    def __init__(self, tenant_id: str | None, id: int) -> None:
+        self.tenant_id: str | None = tenant_id
+        self.id: int = id
+        self.redis: redis.Redis = get_redis_client(tenant_id=tenant_id)
+
+        self.stop = RedisConnectorStop(tenant_id, id, self.redis)
+        self.prune = RedisConnectorPrune(tenant_id, id, self.redis)
+        self.delete = RedisConnectorDelete(tenant_id, id, self.redis)
+
+    def new_index(self, search_settings_id: int) -> RedisConnectorIndex:
+        return RedisConnectorIndex(
+            self.tenant_id, self.id, search_settings_id, self.redis
+        )
+
+    @staticmethod
+    def get_id_from_fence_key(key: str) -> str | None:
+        """
+        Extracts the object ID from a fence key in the format `PREFIX_fence_X`.
+
+        Args:
+            key (str): The fence key string.
+
+        Returns:
+            Optional[int]: The extracted ID if the key is in the correct format, otherwise None.
+        """
+        parts = key.split("_")
+        if len(parts) != 3:
+            return None
+
+        object_id = parts[2]
+        return object_id
+
+    @staticmethod
+    def get_id_from_task_id(task_id: str) -> str | None:
+        """
+        Extracts the object ID from a task ID string.
+
+        This method assumes the task ID is formatted as `prefix_objectid_suffix`, where:
+        - `prefix` is an arbitrary string (e.g., the name of the task or entity),
+        - `objectid` is the ID you want to extract,
+        - `suffix` is another arbitrary string (e.g., a UUID).
+
+        Example:
+            If the input `task_id` is `documentset_1_cbfdc96a-80ca-4312-a242-0bb68da3c1dc`,
+            this method will return the string `"1"`.
+
+        Args:
+            task_id (str): The task ID string from which to extract the object ID.
+
+        Returns:
+            str | None: The extracted object ID if the task ID is in the correct format, otherwise None.
+        """
+        # example: task_id=documentset_1_cbfdc96a-80ca-4312-a242-0bb68da3c1dc
+        parts = task_id.split("_")
+        if len(parts) != 3:
+            return None
+
+        object_id = parts[1]
+        return object_id
diff --git a/backend/danswer/redis/redis_connector_credential_pair.py b/backend/danswer/redis/redis_connector_credential_pair.py
new file mode 100644
index 00000000000..bbad3700111
--- /dev/null
+++ b/backend/danswer/redis/redis_connector_credential_pair.py
@@ -0,0 +1,97 @@
+import time
+from uuid import uuid4
+
+import redis
+from celery import Celery
+from redis import Redis
+from sqlalchemy.orm import Session
+
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
+from danswer.db.document import (
+    construct_document_select_for_connector_credential_pair_by_needs_sync,
+)
+from danswer.redis.redis_object_helper import RedisObjectHelper
+
+
+class RedisConnectorCredentialPair(RedisObjectHelper):
+    """This class is used to scan documents by cc_pair in the db and collect them into
+    a unified set for syncing.
+
+    It differs from the other redis helpers in that the taskset used spans
+    all connectors and is not per connector."""
+
+    PREFIX = "connectorsync"
+    FENCE_PREFIX = PREFIX + "_fence"
+    TASKSET_PREFIX = PREFIX + "_taskset"
+
+    def __init__(self, tenant_id: str | None, id: int) -> None:
+        super().__init__(tenant_id, str(id))
+
+    @classmethod
+    def get_fence_key(cls) -> str:
+        return RedisConnectorCredentialPair.FENCE_PREFIX
+
+    @classmethod
+    def get_taskset_key(cls) -> str:
+        return RedisConnectorCredentialPair.TASKSET_PREFIX
+
+    @property
+    def taskset_key(self) -> str:
+        """Notice that this is intentionally reusing the same taskset for all
+        connector syncs"""
+        # example: connector_taskset
+        return f"{self.TASKSET_PREFIX}"
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock,
+        tenant_id: str | None,
+    ) -> int | None:
+        last_lock_time = time.monotonic()
+
+        async_results = []
+        cc_pair = get_connector_credential_pair_from_id(int(self._id), db_session)
+        if not cc_pair:
+            return None
+
+        stmt = construct_document_select_for_connector_credential_pair_by_needs_sync(
+            cc_pair.connector_id, cc_pair.credential_id
+        )
+        for doc in db_session.scalars(stmt).yield_per(1):
+            current_time = time.monotonic()
+            if current_time - last_lock_time >= (
+                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
+            ):
+                lock.reacquire()
+                last_lock_time = current_time
+
+            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # we prefix the task id so it's easier to keep track of who created the task
+            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
+
+            # add to the tracking taskset in redis BEFORE creating the celery task.
+            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
+            redis_client.sadd(
+                RedisConnectorCredentialPair.get_taskset_key(), custom_task_id
+            )
+
+            # Priority on sync's triggered by new indexing should be medium
+            result = celery_app.send_task(
+                "vespa_metadata_sync_task",
+                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
+                queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
+                task_id=custom_task_id,
+                priority=DanswerCeleryPriority.MEDIUM,
+            )
+
+            async_results.append(result)
+
+        return len(async_results)
diff --git a/backend/danswer/redis/redis_connector_delete.py b/backend/danswer/redis/redis_connector_delete.py
new file mode 100644
index 00000000000..ba250c5b0f7
--- /dev/null
+++ b/backend/danswer/redis/redis_connector_delete.py
@@ -0,0 +1,145 @@
+import time
+from datetime import datetime
+from typing import cast
+from uuid import uuid4
+
+import redis
+from celery import Celery
+from pydantic import BaseModel
+from sqlalchemy.orm import Session
+
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
+from danswer.db.document import construct_document_select_for_connector_credential_pair
+
+
+class RedisConnectorDeletionFenceData(BaseModel):
+    num_tasks: int | None
+    submitted: datetime
+
+
+class RedisConnectorDelete:
+    """Manages interactions with redis for deletion tasks. Should only be accessed
+    through RedisConnector."""
+
+    PREFIX = "connectordeletion"
+    FENCE_PREFIX = f"{PREFIX}_fence"  # "connectordeletion_fence"
+    TASKSET_PREFIX = f"{PREFIX}_taskset"  # "connectordeletion_taskset"
+
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
+        self.id = id
+        self.redis = redis
+
+        self.fence_key: str = f"{self.FENCE_PREFIX}_{id}"
+        self.taskset_key = f"{self.TASKSET_PREFIX}_{id}"
+
+    def taskset_clear(self) -> None:
+        self.redis.delete(self.taskset_key)
+
+    def get_remaining(self) -> int:
+        # todo: move into fence
+        remaining = cast(int, self.redis.scard(self.taskset_key))
+        return remaining
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    @property
+    def payload(self) -> RedisConnectorDeletionFenceData | None:
+        # read related data and evaluate/print task progress
+        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
+        if fence_bytes is None:
+            return None
+
+        fence_str = fence_bytes.decode("utf-8")
+        payload = RedisConnectorDeletionFenceData.model_validate_json(
+            cast(str, fence_str)
+        )
+
+        return payload
+
+    def set_fence(self, payload: RedisConnectorDeletionFenceData | None) -> None:
+        if not payload:
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, payload.model_dump_json())
+
+    def _generate_task_id(self) -> str:
+        # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+        # we prefix the task id so it's easier to keep track of who created the task
+        # aka "connectordeletion_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+
+        return f"{self.PREFIX}_{self.id}_{uuid4()}"
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        lock: redis.lock.Lock,
+    ) -> int | None:
+        """Returns None if the cc_pair doesn't exist.
+        Otherwise, returns an int with the number of generated tasks."""
+        last_lock_time = time.monotonic()
+
+        async_results = []
+        cc_pair = get_connector_credential_pair_from_id(int(self.id), db_session)
+        if not cc_pair:
+            return None
+
+        stmt = construct_document_select_for_connector_credential_pair(
+            cc_pair.connector_id, cc_pair.credential_id
+        )
+        for doc in db_session.scalars(stmt).yield_per(1):
+            current_time = time.monotonic()
+            if current_time - last_lock_time >= (
+                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
+            ):
+                lock.reacquire()
+                last_lock_time = current_time
+
+            custom_task_id = self._generate_task_id()
+
+            # add to the tracking taskset in redis BEFORE creating the celery task.
+            # note that for the moment we are using a single taskset key, not differentiated by cc_pair id
+            self.redis.sadd(self.taskset_key, custom_task_id)
+
+            # Priority on sync's triggered by new indexing should be medium
+            result = celery_app.send_task(
+                "document_by_cc_pair_cleanup_task",
+                kwargs=dict(
+                    document_id=doc.id,
+                    connector_id=cc_pair.connector_id,
+                    credential_id=cc_pair.credential_id,
+                    tenant_id=self.tenant_id,
+                ),
+                queue=DanswerCeleryQueues.CONNECTOR_DELETION,
+                task_id=custom_task_id,
+                priority=DanswerCeleryPriority.MEDIUM,
+            )
+
+            async_results.append(result)
+
+        return len(async_results)
+
+    @staticmethod
+    def remove_from_taskset(id: int, task_id: str, r: redis.Redis) -> None:
+        taskset_key = f"{RedisConnectorDelete.TASKSET_PREFIX}_{id}"
+        r.srem(taskset_key, task_id)
+        return
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        """Deletes all redis values for all connectors"""
+        for key in r.scan_iter(RedisConnectorDelete.TASKSET_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorDelete.FENCE_PREFIX + "*"):
+            r.delete(key)
diff --git a/backend/danswer/redis/redis_connector_index.py b/backend/danswer/redis/redis_connector_index.py
new file mode 100644
index 00000000000..3883ddceaa3
--- /dev/null
+++ b/backend/danswer/redis/redis_connector_index.py
@@ -0,0 +1,146 @@
+from datetime import datetime
+from typing import cast
+from uuid import uuid4
+
+import redis
+from pydantic import BaseModel
+
+
+class RedisConnectorIndexingFenceData(BaseModel):
+    index_attempt_id: int | None
+    started: datetime | None
+    submitted: datetime
+    celery_task_id: str | None
+
+
+class RedisConnectorIndex:
+    """Manages interactions with redis for indexing tasks. Should only be accessed
+    through RedisConnector."""
+
+    PREFIX = "connectorindexing"
+    FENCE_PREFIX = f"{PREFIX}_fence"  # "connectorindexing_fence"
+    GENERATOR_TASK_PREFIX = PREFIX + "+generator"  # "connectorindexing+generator_fence"
+    GENERATOR_PROGRESS_PREFIX = (
+        PREFIX + "_generator_progress"
+    )  # connectorindexing_generator_progress
+    GENERATOR_COMPLETE_PREFIX = (
+        PREFIX + "_generator_complete"
+    )  # connectorindexing_generator_complete
+
+    GENERATOR_LOCK_PREFIX = "da_lock:indexing"
+
+    def __init__(
+        self,
+        tenant_id: str | None,
+        id: int,
+        search_settings_id: int,
+        redis: redis.Redis,
+    ) -> None:
+        self.tenant_id: str | None = tenant_id
+        self.id = id
+        self.search_settings_id = search_settings_id
+        self.redis = redis
+
+        self.fence_key: str = f"{self.FENCE_PREFIX}_{id}/{search_settings_id}"
+        self.generator_progress_key = (
+            f"{self.GENERATOR_PROGRESS_PREFIX}_{id}/{search_settings_id}"
+        )
+        self.generator_complete_key = (
+            f"{self.GENERATOR_COMPLETE_PREFIX}_{id}/{search_settings_id}"
+        )
+        self.generator_lock_key = (
+            f"{self.GENERATOR_LOCK_PREFIX}_{id}/{search_settings_id}"
+        )
+
+    @classmethod
+    def fence_key_with_ids(cls, cc_pair_id: int, search_settings_id: int) -> str:
+        return f"{cls.FENCE_PREFIX}_{cc_pair_id}/{search_settings_id}"
+
+    def generate_generator_task_id(self) -> str:
+        # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+        # we prefix the task id so it's easier to keep track of who created the task
+        # aka "connectorindexing+generator_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+
+        return f"{self.GENERATOR_TASK_PREFIX}_{self.id}/{self.search_settings_id}_{uuid4()}"
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    @property
+    def payload(self) -> RedisConnectorIndexingFenceData | None:
+        # read related data and evaluate/print task progress
+        fence_bytes = cast(bytes, self.redis.get(self.fence_key))
+        if fence_bytes is None:
+            return None
+
+        fence_str = fence_bytes.decode("utf-8")
+        payload = RedisConnectorIndexingFenceData.model_validate_json(
+            cast(str, fence_str)
+        )
+
+        return payload
+
+    def set_fence(
+        self,
+        payload: RedisConnectorIndexingFenceData | None,
+    ) -> None:
+        if not payload:
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, payload.model_dump_json())
+
+    def set_generator_complete(self, payload: int | None) -> None:
+        if not payload:
+            self.redis.delete(self.generator_complete_key)
+            return
+
+        self.redis.set(self.generator_complete_key, payload)
+
+    def generator_clear(self) -> None:
+        self.redis.delete(self.generator_progress_key)
+        self.redis.delete(self.generator_complete_key)
+
+    def get_progress(self) -> int | None:
+        """Returns None if the key doesn't exist. The"""
+        # TODO: move into fence?
+        bytes = self.redis.get(self.generator_progress_key)
+        if bytes is None:
+            return None
+
+        progress = int(cast(int, bytes))
+        return progress
+
+    def get_completion(self) -> int | None:
+        # TODO: move into fence?
+        bytes = self.redis.get(self.generator_complete_key)
+        if bytes is None:
+            return None
+
+        status = int(cast(int, bytes))
+        return status
+
+    def reset(self) -> None:
+        self.redis.delete(self.generator_lock_key)
+        self.redis.delete(self.generator_progress_key)
+        self.redis.delete(self.generator_complete_key)
+        self.redis.delete(self.fence_key)
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        """Deletes all redis values for all connectors"""
+        for key in r.scan_iter(RedisConnectorIndex.GENERATOR_LOCK_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorIndex.GENERATOR_COMPLETE_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorIndex.GENERATOR_PROGRESS_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorIndex.FENCE_PREFIX + "*"):
+            r.delete(key)
diff --git a/backend/danswer/redis/redis_connector_prune.py b/backend/danswer/redis/redis_connector_prune.py
new file mode 100644
index 00000000000..8892c12b647
--- /dev/null
+++ b/backend/danswer/redis/redis_connector_prune.py
@@ -0,0 +1,171 @@
+import time
+from typing import cast
+from uuid import uuid4
+
+import redis
+from celery import Celery
+from sqlalchemy.orm import Session
+
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.db.connector_credential_pair import get_connector_credential_pair_from_id
+
+
+class RedisConnectorPrune:
+    """Manages interactions with redis for pruning tasks. Should only be accessed
+    through RedisConnector."""
+
+    PREFIX = "connectorpruning"
+
+    FENCE_PREFIX = f"{PREFIX}_fence"
+
+    # phase 1 - geneartor task and progress signals
+    GENERATORTASK_PREFIX = f"{PREFIX}+generator"  # connectorpruning+generator
+    GENERATOR_PROGRESS_PREFIX = (
+        PREFIX + "_generator_progress"
+    )  # connectorpruning_generator_progress
+    GENERATOR_COMPLETE_PREFIX = (
+        PREFIX + "_generator_complete"
+    )  # connectorpruning_generator_complete
+
+    TASKSET_PREFIX = f"{PREFIX}_taskset"  # connectorpruning_taskset
+    SUBTASK_PREFIX = f"{PREFIX}+sub"  # connectorpruning+sub
+
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
+        self.id = id
+        self.redis = redis
+
+        self.fence_key: str = f"{self.FENCE_PREFIX}_{id}"
+        self.generator_task_key = f"{self.GENERATORTASK_PREFIX}_{id}"
+        self.generator_progress_key = f"{self.GENERATOR_PROGRESS_PREFIX}_{id}"
+        self.generator_complete_key = f"{self.GENERATOR_COMPLETE_PREFIX}_{id}"
+
+        self.taskset_key = f"{self.TASKSET_PREFIX}_{id}"
+
+        self.subtask_prefix: str = f"{self.SUBTASK_PREFIX}_{id}"
+
+    def taskset_clear(self) -> None:
+        self.redis.delete(self.taskset_key)
+
+    def generator_clear(self) -> None:
+        self.redis.delete(self.generator_progress_key)
+        self.redis.delete(self.generator_complete_key)
+
+    def get_remaining(self) -> int:
+        # todo: move into fence
+        remaining = cast(int, self.redis.scard(self.taskset_key))
+        return remaining
+
+    def get_active_task_count(self) -> int:
+        """Count of active pruning tasks"""
+        count = 0
+        for key in self.redis.scan_iter(RedisConnectorPrune.FENCE_PREFIX + "*"):
+            count += 1
+        return count
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    def set_fence(self, value: bool) -> None:
+        if not value:
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, 0)
+
+    @property
+    def generator_complete(self) -> int | None:
+        """the fence payload is an int representing the starting number of
+        pruning tasks to be processed ... just after the generator completes."""
+        fence_bytes = self.redis.get(self.generator_complete_key)
+        if fence_bytes is None:
+            return None
+
+        fence_int = cast(int, fence_bytes)
+        return fence_int
+
+    @generator_complete.setter
+    def generator_complete(self, payload: int | None) -> None:
+        """Set the payload to an int to set the fence, otherwise if None it will
+        be deleted"""
+        if payload is None:
+            self.redis.delete(self.generator_complete_key)
+            return
+
+        self.redis.set(self.generator_complete_key, payload)
+
+    def generate_tasks(
+        self,
+        documents_to_prune: set[str],
+        celery_app: Celery,
+        db_session: Session,
+        lock: redis.lock.Lock | None,
+    ) -> int | None:
+        last_lock_time = time.monotonic()
+
+        async_results = []
+        cc_pair = get_connector_credential_pair_from_id(int(self.id), db_session)
+        if not cc_pair:
+            return None
+
+        for doc_id in documents_to_prune:
+            current_time = time.monotonic()
+            if lock and current_time - last_lock_time >= (
+                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
+            ):
+                lock.reacquire()
+                last_lock_time = current_time
+
+            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # the actual redis key is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # we prefix the task id so it's easier to keep track of who created the task
+            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            custom_task_id = f"{self.subtask_prefix}_{uuid4()}"
+
+            # add to the tracking taskset in redis BEFORE creating the celery task.
+            self.redis.sadd(self.taskset_key, custom_task_id)
+
+            # Priority on sync's triggered by new indexing should be medium
+            result = celery_app.send_task(
+                "document_by_cc_pair_cleanup_task",
+                kwargs=dict(
+                    document_id=doc_id,
+                    connector_id=cc_pair.connector_id,
+                    credential_id=cc_pair.credential_id,
+                    tenant_id=self.tenant_id,
+                ),
+                queue=DanswerCeleryQueues.CONNECTOR_DELETION,
+                task_id=custom_task_id,
+                priority=DanswerCeleryPriority.MEDIUM,
+            )
+
+            async_results.append(result)
+
+        return len(async_results)
+
+    @staticmethod
+    def remove_from_taskset(id: int, task_id: str, r: redis.Redis) -> None:
+        taskset_key = f"{RedisConnectorPrune.TASKSET_PREFIX}_{id}"
+        r.srem(taskset_key, task_id)
+        return
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        """Deletes all redis values for all connectors"""
+        for key in r.scan_iter(RedisConnectorPrune.TASKSET_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorPrune.GENERATOR_COMPLETE_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorPrune.GENERATOR_PROGRESS_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisConnectorPrune.FENCE_PREFIX + "*"):
+            r.delete(key)
diff --git a/backend/danswer/redis/redis_connector_stop.py b/backend/danswer/redis/redis_connector_stop.py
new file mode 100644
index 00000000000..c65c57ff7f4
--- /dev/null
+++ b/backend/danswer/redis/redis_connector_stop.py
@@ -0,0 +1,34 @@
+import redis
+
+
+class RedisConnectorStop:
+    """Manages interactions with redis for stop signaling. Should only be accessed
+    through RedisConnector."""
+
+    FENCE_PREFIX = "connectorstop_fence"
+
+    def __init__(self, tenant_id: str | None, id: int, redis: redis.Redis) -> None:
+        self.tenant_id: str | None = tenant_id
+        self.id: int = id
+        self.redis = redis
+
+        self.fence_key: str = f"{self.FENCE_PREFIX}_{id}"
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    def set_fence(self, value: bool) -> None:
+        if not value:
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, 0)
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        for key in r.scan_iter(RedisConnectorStop.FENCE_PREFIX + "*"):
+            r.delete(key)
diff --git a/backend/danswer/redis/redis_document_set.py b/backend/danswer/redis/redis_document_set.py
new file mode 100644
index 00000000000..102e910feec
--- /dev/null
+++ b/backend/danswer/redis/redis_document_set.py
@@ -0,0 +1,99 @@
+import time
+from typing import cast
+from uuid import uuid4
+
+import redis
+from celery import Celery
+from redis import Redis
+from sqlalchemy.orm import Session
+
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.db.document_set import construct_document_select_by_docset
+from danswer.redis.redis_object_helper import RedisObjectHelper
+
+
+class RedisDocumentSet(RedisObjectHelper):
+    PREFIX = "documentset"
+    FENCE_PREFIX = PREFIX + "_fence"
+    TASKSET_PREFIX = PREFIX + "_taskset"
+
+    def __init__(self, tenant_id: str | None, id: int) -> None:
+        super().__init__(tenant_id, str(id))
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    def set_fence(self, payload: int | None) -> None:
+        if payload is None:
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, payload)
+
+    @property
+    def payload(self) -> int | None:
+        bytes = self.redis.get(self.fence_key)
+        if bytes is None:
+            return None
+
+        progress = int(cast(int, bytes))
+        return progress
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock,
+        tenant_id: str | None,
+    ) -> int | None:
+        last_lock_time = time.monotonic()
+
+        async_results = []
+        stmt = construct_document_select_by_docset(int(self._id), current_only=False)
+        for doc in db_session.scalars(stmt).yield_per(1):
+            current_time = time.monotonic()
+            if current_time - last_lock_time >= (
+                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
+            ):
+                lock.reacquire()
+                last_lock_time = current_time
+
+            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # we prefix the task id so it's easier to keep track of who created the task
+            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
+
+            # add to the set BEFORE creating the task.
+            redis_client.sadd(self.taskset_key, custom_task_id)
+
+            result = celery_app.send_task(
+                "vespa_metadata_sync_task",
+                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
+                queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
+                task_id=custom_task_id,
+                priority=DanswerCeleryPriority.LOW,
+            )
+
+            async_results.append(result)
+
+        return len(async_results)
+
+    def reset(self) -> None:
+        self.redis.delete(self.taskset_key)
+        self.redis.delete(self.fence_key)
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        for key in r.scan_iter(RedisDocumentSet.TASKSET_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisDocumentSet.FENCE_PREFIX + "*"):
+            r.delete(key)
diff --git a/backend/danswer/redis/redis_object_helper.py b/backend/danswer/redis/redis_object_helper.py
new file mode 100644
index 00000000000..629f15e6058
--- /dev/null
+++ b/backend/danswer/redis/redis_object_helper.py
@@ -0,0 +1,91 @@
+from abc import ABC
+from abc import abstractmethod
+
+import redis
+from celery import Celery
+from redis import Redis
+from sqlalchemy.orm import Session
+
+from danswer.redis.redis_pool import get_redis_client
+
+
+class RedisObjectHelper(ABC):
+    PREFIX = "base"
+    FENCE_PREFIX = PREFIX + "_fence"
+    TASKSET_PREFIX = PREFIX + "_taskset"
+
+    def __init__(self, tenant_id: str | None, id: str):
+        self._tenant_id: str | None = tenant_id
+        self._id: str = id
+        self.redis = get_redis_client(tenant_id=tenant_id)
+
+    @property
+    def task_id_prefix(self) -> str:
+        return f"{self.PREFIX}_{self._id}"
+
+    @property
+    def fence_key(self) -> str:
+        # example: documentset_fence_1
+        return f"{self.FENCE_PREFIX}_{self._id}"
+
+    @property
+    def taskset_key(self) -> str:
+        # example: documentset_taskset_1
+        return f"{self.TASKSET_PREFIX}_{self._id}"
+
+    @staticmethod
+    def get_id_from_fence_key(key: str) -> str | None:
+        """
+        Extracts the object ID from a fence key in the format `PREFIX_fence_X`.
+
+        Args:
+            key (str): The fence key string.
+
+        Returns:
+            Optional[int]: The extracted ID if the key is in the correct format, otherwise None.
+        """
+        parts = key.split("_")
+        if len(parts) != 3:
+            return None
+
+        object_id = parts[2]
+        return object_id
+
+    @staticmethod
+    def get_id_from_task_id(task_id: str) -> str | None:
+        """
+        Extracts the object ID from a task ID string.
+
+        This method assumes the task ID is formatted as `prefix_objectid_suffix`, where:
+        - `prefix` is an arbitrary string (e.g., the name of the task or entity),
+        - `objectid` is the ID you want to extract,
+        - `suffix` is another arbitrary string (e.g., a UUID).
+
+        Example:
+            If the input `task_id` is `documentset_1_cbfdc96a-80ca-4312-a242-0bb68da3c1dc`,
+            this method will return the string `"1"`.
+
+        Args:
+            task_id (str): The task ID string from which to extract the object ID.
+
+        Returns:
+            str | None: The extracted object ID if the task ID is in the correct format, otherwise None.
+        """
+        # example: task_id=documentset_1_cbfdc96a-80ca-4312-a242-0bb68da3c1dc
+        parts = task_id.split("_")
+        if len(parts) != 3:
+            return None
+
+        object_id = parts[1]
+        return object_id
+
+    @abstractmethod
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock,
+        tenant_id: str | None,
+    ) -> int | None:
+        pass
diff --git a/backend/danswer/redis/redis_usergroup.py b/backend/danswer/redis/redis_usergroup.py
new file mode 100644
index 00000000000..53d2d4fc0a9
--- /dev/null
+++ b/backend/danswer/redis/redis_usergroup.py
@@ -0,0 +1,112 @@
+import time
+from typing import cast
+from uuid import uuid4
+
+import redis
+from celery import Celery
+from redis import Redis
+from sqlalchemy.orm import Session
+
+from danswer.configs.constants import CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT
+from danswer.configs.constants import DanswerCeleryPriority
+from danswer.configs.constants import DanswerCeleryQueues
+from danswer.redis.redis_object_helper import RedisObjectHelper
+from danswer.utils.variable_functionality import fetch_versioned_implementation
+from danswer.utils.variable_functionality import global_version
+
+
+class RedisUserGroup(RedisObjectHelper):
+    PREFIX = "usergroup"
+    FENCE_PREFIX = PREFIX + "_fence"
+    TASKSET_PREFIX = PREFIX + "_taskset"
+
+    def __init__(self, tenant_id: str | None, id: int) -> None:
+        super().__init__(tenant_id, str(id))
+
+    @property
+    def fenced(self) -> bool:
+        if self.redis.exists(self.fence_key):
+            return True
+
+        return False
+
+    def set_fence(self, payload: int | None) -> None:
+        if payload is None:
+            self.redis.delete(self.fence_key)
+            return
+
+        self.redis.set(self.fence_key, payload)
+
+    @property
+    def payload(self) -> int | None:
+        bytes = self.redis.get(self.fence_key)
+        if bytes is None:
+            return None
+
+        progress = int(cast(int, bytes))
+        return progress
+
+    def generate_tasks(
+        self,
+        celery_app: Celery,
+        db_session: Session,
+        redis_client: Redis,
+        lock: redis.lock.Lock,
+        tenant_id: str | None,
+    ) -> int | None:
+        last_lock_time = time.monotonic()
+
+        async_results = []
+
+        if not global_version.is_ee_version():
+            return 0
+
+        try:
+            construct_document_select_by_usergroup = fetch_versioned_implementation(
+                "danswer.db.user_group",
+                "construct_document_select_by_usergroup",
+            )
+        except ModuleNotFoundError:
+            return 0
+
+        stmt = construct_document_select_by_usergroup(int(self._id))
+        for doc in db_session.scalars(stmt).yield_per(1):
+            current_time = time.monotonic()
+            if current_time - last_lock_time >= (
+                CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT / 4
+            ):
+                lock.reacquire()
+                last_lock_time = current_time
+
+            # celery's default task id format is "dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # the key for the result is "celery-task-meta-dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            # we prefix the task id so it's easier to keep track of who created the task
+            # aka "documentset_1_6dd32ded3-00aa-4884-8b21-42f8332e7fac"
+            custom_task_id = f"{self.task_id_prefix}_{uuid4()}"
+
+            # add to the set BEFORE creating the task.
+            redis_client.sadd(self.taskset_key, custom_task_id)
+
+            result = celery_app.send_task(
+                "vespa_metadata_sync_task",
+                kwargs=dict(document_id=doc.id, tenant_id=tenant_id),
+                queue=DanswerCeleryQueues.VESPA_METADATA_SYNC,
+                task_id=custom_task_id,
+                priority=DanswerCeleryPriority.LOW,
+            )
+
+            async_results.append(result)
+
+        return len(async_results)
+
+    def reset(self) -> None:
+        self.redis.delete(self.taskset_key)
+        self.redis.delete(self.fence_key)
+
+    @staticmethod
+    def reset_all(r: redis.Redis) -> None:
+        for key in r.scan_iter(RedisUserGroup.TASKSET_PREFIX + "*"):
+            r.delete(key)
+
+        for key in r.scan_iter(RedisUserGroup.FENCE_PREFIX + "*"):
+            r.delete(key)
diff --git a/backend/danswer/server/documents/cc_pair.py b/backend/danswer/server/documents/cc_pair.py
index ddc084498ba..68b48b85b0f 100644
--- a/backend/danswer/server/documents/cc_pair.py
+++ b/backend/danswer/server/documents/cc_pair.py
@@ -11,8 +11,6 @@
 
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
-from danswer.background.celery.celery_redis import RedisConnectorPruning
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
 from danswer.background.celery.tasks.pruning.tasks import (
     try_creating_prune_generator_task,
@@ -39,6 +37,7 @@
 from danswer.db.search_settings import get_current_search_settings
 from danswer.db.tasks import check_task_is_live_and_not_timed_out
 from danswer.db.tasks import get_latest_task
+from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import CCPairFullInfo
 from danswer.server.documents.models import CCStatusUpdateRequest
@@ -97,8 +96,6 @@ def get_cc_pair_full_info(
     db_session: Session = Depends(get_session),
     tenant_id: str | None = Depends(get_current_tenant_id),
 ) -> CCPairFullInfo:
-    r = get_redis_client(tenant_id=tenant_id)
-
     cc_pair = get_connector_credential_pair_from_id(
         cc_pair_id, db_session, user, get_editable=False
     )
@@ -134,9 +131,9 @@ def get_cc_pair_full_info(
     )
 
     search_settings = get_current_search_settings(db_session)
-    rci = RedisConnectorIndexing(
-        cc_pair_id=cc_pair_id, search_settings_id=search_settings.id
-    )
+
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
+    redis_connector_index = redis_connector.new_index(search_settings.id)
 
     return CCPairFullInfo.from_models(
         cc_pair_model=cc_pair,
@@ -153,7 +150,7 @@ def get_cc_pair_full_info(
         ),
         num_docs_indexed=documents_indexed,
         is_editable_for_current_user=is_editable_for_current_user,
-        indexing=rci.is_indexing(r),
+        indexing=redis_connector_index.fenced,
     )
 
 
@@ -263,8 +260,9 @@ def prune_cc_pair(
         )
 
     r = get_redis_client(tenant_id=tenant_id)
-    rcp = RedisConnectorPruning(cc_pair_id)
-    if rcp.is_pruning(r):
+
+    redis_connector = RedisConnector(tenant_id, cc_pair_id)
+    if redis_connector.prune.fenced:
         raise HTTPException(
             status_code=HTTPStatus.CONFLICT,
             detail="Pruning task already in progress.",
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index b0866a826c1..1933503ea5b 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -16,7 +16,6 @@
 from danswer.auth.users import current_admin_user
 from danswer.auth.users import current_curator_or_admin_user
 from danswer.auth.users import current_user
-from danswer.background.celery.celery_redis import RedisConnectorIndexing
 from danswer.background.celery.celery_utils import get_deletion_attempt_snapshot
 from danswer.background.celery.tasks.indexing.tasks import try_creating_indexing_task
 from danswer.background.celery.versioned_apps.primary import app as primary_app
@@ -85,6 +84,7 @@
 from danswer.db.search_settings import get_secondary_search_settings
 from danswer.file_store.file_store import get_default_file_store
 from danswer.key_value_store.interface import KvKeyNotFoundError
+from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_pool import get_redis_client
 from danswer.server.documents.models import AuthStatus
 from danswer.server.documents.models import AuthUrl
@@ -486,12 +486,10 @@ def get_connector_indexing_status(
 ) -> list[ConnectorIndexingStatus]:
     indexing_statuses: list[ConnectorIndexingStatus] = []
 
-    r = get_redis_client(tenant_id=tenant_id)
-
     # NOTE: If the connector is deleting behind the scenes,
     # accessing cc_pairs can be inconsistent and members like
     # connector or credential may be None.
-    # Additional checks are done to make sure the connector and credential still exists.
+    # Additional checks are done to make sure the connector and credential still exist.
     # TODO: make this one query ... possibly eager load or wrap in a read transaction
     # to avoid the complexity of trying to error check throughout the function
     cc_pairs = get_connector_credential_pairs(
@@ -558,8 +556,9 @@ def get_connector_indexing_status(
 
         in_progress = False
         if search_settings:
-            rci = RedisConnectorIndexing(cc_pair.id, search_settings.id)
-            if r.exists(rci.fence_key):
+            redis_connector = RedisConnector(tenant_id, cc_pair.id)
+            redis_connector_index = redis_connector.new_index(search_settings.id)
+            if redis_connector_index.fenced:
                 in_progress = True
 
         latest_index_attempt = cc_pair_to_latest_index_attempt.get(
diff --git a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
index a2b45324d46..84d67f0a3ac 100644
--- a/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/ee/danswer/background/celery/tasks/vespa/tasks.py
@@ -4,7 +4,7 @@
 from sqlalchemy.orm import Session
 
 from danswer.background.celery.apps.app_base import task_logger
-from danswer.background.celery.celery_redis import RedisUserGroup
+from danswer.redis.redis_usergroup import RedisUserGroup
 from danswer.utils.logger import setup_logger
 from ee.danswer.db.user_group import delete_user_group
 from ee.danswer.db.user_group import fetch_user_group
@@ -13,7 +13,9 @@
 logger = setup_logger()
 
 
-def monitor_usergroup_taskset(key_bytes: bytes, r: Redis, db_session: Session) -> None:
+def monitor_usergroup_taskset(
+    tenant_id: str | None, key_bytes: bytes, r: Redis, db_session: Session
+) -> None:
     """This function is likely to move in the worker refactor happening next."""
     fence_key = key_bytes.decode("utf-8")
     usergroup_id_str = RedisUserGroup.get_id_from_fence_key(fence_key)
@@ -27,15 +29,12 @@ def monitor_usergroup_taskset(key_bytes: bytes, r: Redis, db_session: Session) -
         task_logger.exception(f"usergroup_id ({usergroup_id_str}) is not an integer!")
         raise
 
-    rug = RedisUserGroup(usergroup_id)
-    fence_value = r.get(rug.fence_key)
-    if fence_value is None:
+    rug = RedisUserGroup(tenant_id, usergroup_id)
+    if not rug.fenced:
         return
 
-    try:
-        initial_count = int(cast(int, fence_value))
-    except ValueError:
-        task_logger.error("The value is not an integer.")
+    initial_count = rug.payload
+    if initial_count is None:
         return
 
     count = cast(int, r.scard(rug.taskset_key))
@@ -54,5 +53,4 @@ def monitor_usergroup_taskset(key_bytes: bytes, r: Redis, db_session: Session) -
             mark_user_group_as_synced(db_session=db_session, user_group=user_group)
             task_logger.info(f"Synced usergroup. id='{usergroup_id}'")
 
-    r.delete(rug.taskset_key)
-    r.delete(rug.fence_key)
+    rug.reset()
diff --git a/backend/tests/integration/common_utils/managers/cc_pair.py b/backend/tests/integration/common_utils/managers/cc_pair.py
index 94114298507..68469d144b9 100644
--- a/backend/tests/integration/common_utils/managers/cc_pair.py
+++ b/backend/tests/integration/common_utils/managers/cc_pair.py
@@ -308,6 +308,7 @@ def wait_for_prune(
         while True:
             last_pruned = CCPairManager.last_pruned(cc_pair, user_performing_action)
             if last_pruned and last_pruned > after:
+                print(f"Pruning complete: cc_pair={cc_pair.id}")
                 break
 
             elapsed = time.monotonic() - start

From 8bcf80aa768c61619f8e940eecf9e56fda303bc7 Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Sat, 2 Nov 2024 18:05:06 -0700
Subject: [PATCH 347/376] minor nit

---
 .../danswer/connectors/freshdesk/connector.py | 32 ++++++++++++-------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/backend/danswer/connectors/freshdesk/connector.py b/backend/danswer/connectors/freshdesk/connector.py
index 04033b05f3f..db2a293e95f 100644
--- a/backend/danswer/connectors/freshdesk/connector.py
+++ b/backend/danswer/connectors/freshdesk/connector.py
@@ -42,18 +42,28 @@
     "to_emails",
 }
 
-_SOURCE_NUMBER_TYPE_MAP = {
-    "1": "Email",
-    "2": "Portal",
-    "3": "Phone",
-    "7": "Chat",
-    "9": "Feedback Widget",
-    "10": "Outbound Email",
+_SOURCE_NUMBER_TYPE_MAP: dict[int, str] = {
+    1: "Email",
+    2: "Portal",
+    3: "Phone",
+    7: "Chat",
+    9: "Feedback Widget",
+    10: "Outbound Email",
 }
 
-_PRIORITY_NUMBER_TYPE_MAP = {"1": "low", "2": "medium", "3": "high", "4": "urgent"}
+_PRIORITY_NUMBER_TYPE_MAP: dict[int, str] = {
+    1: "low",
+    2: "medium",
+    3: "high",
+    4: "urgent",
+}
 
-_STATUS_NUMBER_TYPE_MAP = {"2": "open", "3": "pending", "4": "resolved", "5": "closed"}
+_STATUS_NUMBER_TYPE_MAP: dict[int, str] = {
+    2: "open",
+    3: "pending",
+    4: "resolved",
+    5: "closed",
+}
 
 
 def _create_metadata_from_ticket(ticket: dict) -> dict:
@@ -91,7 +101,7 @@ def _create_metadata_from_ticket(ticket: dict) -> dict:
     # Convert source numbers to human-parsable string
     if source_number := ticket.get("source"):
         metadata["source"] = _SOURCE_NUMBER_TYPE_MAP.get(
-            str(source_number), "Unknown Source Type"
+            source_number, "Unknown Source Type"
         )
 
     # Convert priority numbers to human-parsable string
@@ -103,7 +113,7 @@ def _create_metadata_from_ticket(ticket: dict) -> dict:
     # Convert status to human-parsable string
     if status_number := ticket.get("status"):
         metadata["status"] = _STATUS_NUMBER_TYPE_MAP.get(
-            str(status_number), "Unknown Status"
+            status_number, "Unknown Status"
         )
 
     due_by = datetime.fromisoformat(ticket["due_by"].replace("Z", "+00:00"))

From 938d5788b6c1ce1c2ff707ce4ba732ce4c6f250d Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Sat, 2 Nov 2024 19:56:23 -0700
Subject: [PATCH 348/376] Upgrade to latest NextJS + switch to turbopack
 (#3027)

* Upgrade to NextJS 15 + use turbopacK

* Remove unintended change

* Update nextjs version

* Remove override

* Upgrade react

* Fix charts

* Style

* Style

* Fix prettier

* slight modification

---------

Co-authored-by: pablodanswer <pablo@danswer.ai>
---
 web/next.config.js                            |    1 -
 web/package-lock.json                         | 2611 ++++++++---------
 web/package.json                              |    9 +-
 web/public/Wikipedia.svg                      |  536 +++-
 web/src/app/admin/assistants/[id]/page.tsx    |    3 +-
 web/src/app/admin/bot/[id]/page.tsx           |    3 +-
 .../app/admin/connector/[ccPairId]/page.tsx   |    5 +-
 .../[connector]/auth/callback/route.ts        |    5 +-
 .../app/admin/connectors/[connector]/page.tsx |    7 +-
 web/src/app/admin/documents/explorer/page.tsx |    7 +-
 .../documents/sets/[documentSetId]/page.tsx   |    8 +-
 web/src/app/admin/indexing/[id]/page.tsx      |    4 +-
 .../status/CCPairIndexingStatusTable.tsx      |   91 +-
 .../app/admin/tools/edit/[toolId]/page.tsx    |    5 +-
 web/src/app/api/[...path]/route.ts            |   21 +-
 web/src/app/assistants/edit/[id]/page.tsx     |    3 +-
 web/src/app/assistants/gallery/page.tsx       |   13 +-
 web/src/app/assistants/mine/page.tsx          |   13 +-
 web/src/app/auth/login/page.tsx               |    7 +-
 web/src/app/chat/page.tsx                     |   14 +-
 web/src/app/chat/shared/[chatId]/page.tsx     |    6 +-
 .../app/ee/admin/groups/[groupId]/page.tsx    |    4 +-
 .../performance/query-history/[id]/page.tsx   |    4 +-
 .../ee/admin/standard-answer/[id]/page.tsx    |    3 +-
 web/src/app/layout.tsx                        |   11 +-
 web/src/app/prompts/page.tsx                  |    7 +-
 web/src/app/search/page.tsx                   |   26 +-
 web/src/components/context/AppProvider.tsx    |    1 -
 web/src/components/icons/icons.tsx            |    8 -
 .../welcome/WelcomeModalWrapper.tsx           |   18 +-
 web/src/lib/chat/fetchChatData.ts             |   14 +-
 web/src/lib/chat/fetchSomeChatData.ts         |   11 +-
 web/src/lib/userSS.ts                         |    2 +-
 web/src/lib/utilsSS.ts                        |    4 +-
 web/src/middleware.ts                         |   33 +-
 web/tailwind.config.js                        |   15 +-
 36 files changed, 2002 insertions(+), 1531 deletions(-)

diff --git a/web/next.config.js b/web/next.config.js
index 3b4c7bbd379..7e001f764ba 100644
--- a/web/next.config.js
+++ b/web/next.config.js
@@ -10,7 +10,6 @@ const { withSentryConfig } = require("@sentry/nextjs");
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: "standalone",
-  swcMinify: true,
   publicRuntimeConfig: {
     version,
   },
diff --git a/web/package-lock.json b/web/package-lock.json
index abf460023fc..2b02ef81659 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -11,6 +11,8 @@
         "@dnd-kit/core": "^6.1.0",
         "@dnd-kit/modifiers": "^7.0.0",
         "@dnd-kit/sortable": "^8.0.0",
+        "@dnd-kit/utilities": "^3.2.2",
+        "@headlessui/react": "^2.2.0",
         "@headlessui/tailwindcss": "^0.2.1",
         "@phosphor-icons/react": "^2.0.8",
         "@radix-ui/react-dialog": "^1.0.5",
@@ -38,7 +40,7 @@
         "lodash": "^4.17.21",
         "lucide-react": "^0.454.0",
         "mdast-util-find-and-replace": "^3.0.1",
-        "next": "^14.2.3",
+        "next": "^15.0.2",
         "npm": "^10.8.0",
         "postcss": "^8.4.31",
         "posthog-js": "^1.176.0",
@@ -167,11 +169,11 @@
       }
     },
     "node_modules/@babel/helper-annotate-as-pure": {
-      "version": "7.22.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.22.5.tgz",
-      "integrity": "sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.25.9.tgz",
+      "integrity": "sha512-gv7320KBUFJz1RnylIg5WWYPRXKZ884AGkYpgpWW02TH66Dl+HaC1t1CKd0z3R4b6hdYEcmrNZHUmfCP+1u3/g==",
       "dependencies": {
-        "@babel/types": "^7.22.5"
+        "@babel/types": "^7.25.9"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -269,9 +271,9 @@
       }
     },
     "node_modules/@babel/helper-plugin-utils": {
-      "version": "7.24.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.24.5.tgz",
-      "integrity": "sha512-xjNLDopRzW2o6ba0gKbkZq5YWEBaK3PCyTOY1K2P/O07LGMhMqlMXPxwN4S5/RhWuCobT8z0jrlKGlYmeR1OhQ==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.25.9.tgz",
+      "integrity": "sha512-kSMlyUVdWe25rEsRGviIgOWnoT/nfABVWlqt9N19/dIPWViAOW2s9wznP5tURbs/IDuNk4gPy3YdYRgH3uxhBw==",
       "engines": {
         "node": ">=6.9.0"
       }
@@ -299,17 +301,17 @@
       }
     },
     "node_modules/@babel/helper-string-parser": {
-      "version": "7.24.1",
-      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.1.tgz",
-      "integrity": "sha512-2ofRCjnnA9y+wk8b9IAREroeUP02KHp431N2mhKniy2yKIDKpbrHv9eXwm8cBeWQYcJmzv5qKCu65P47eCF7CQ==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz",
+      "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==",
       "engines": {
         "node": ">=6.9.0"
       }
     },
     "node_modules/@babel/helper-validator-identifier": {
-      "version": "7.24.5",
-      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.5.tgz",
-      "integrity": "sha512-3q93SSKX2TWCG30M2G2kwaKeTYgEUp5Snjuj8qm729SObL6nbtUldAi37qbxkD5gg3xnBio+f9nqpSepGZMvxA==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz",
+      "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==",
       "engines": {
         "node": ">=6.9.0"
       }
@@ -425,11 +427,11 @@
       }
     },
     "node_modules/@babel/plugin-syntax-jsx": {
-      "version": "7.24.1",
-      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.24.1.tgz",
-      "integrity": "sha512-2eCtxZXf+kbkMIsXS4poTvT4Yu5rXiRa+9xGVT56raghjmBTKMpFNc9R4IDiB4emao9eO22Ox7CxuJG7BgExqA==",
+      "version": "7.25.9",
+      "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.25.9.tgz",
+      "integrity": "sha512-ld6oezHQMZsZfp6pWtbjaNDF2tiiCYYDqQszHt5VV437lewP9aSi2Of99CK0D0XB21k7FLgnLcmQKyKzynfeAA==",
       "dependencies": {
-        "@babel/helper-plugin-utils": "^7.24.0"
+        "@babel/helper-plugin-utils": "^7.25.9"
       },
       "engines": {
         "node": ">=6.9.0"
@@ -491,29 +493,17 @@
       }
     },
     "node_modules/@babel/types": {
-      "version": "7.24.5",
-      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.5.tgz",
-      "integrity": "sha512-6mQNsaLeXTw0nxYUYu+NSa4Hx4BlF1x1x8/PMFbiR+GBSr+2DkECc69b8hgy2frEodNcvPffeH8YfWd3LI6jhQ==",
+      "version": "7.26.0",
+      "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz",
+      "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==",
       "dependencies": {
-        "@babel/helper-string-parser": "^7.24.1",
-        "@babel/helper-validator-identifier": "^7.24.5",
-        "to-fast-properties": "^2.0.0"
+        "@babel/helper-string-parser": "^7.25.9",
+        "@babel/helper-validator-identifier": "^7.25.9"
       },
       "engines": {
         "node": ">=6.9.0"
       }
     },
-    "node_modules/@dnd-kit/accessibility": {
-      "version": "3.1.0",
-      "resolved": "https://registry.npmjs.org/@dnd-kit/accessibility/-/accessibility-3.1.0.tgz",
-      "integrity": "sha512-ea7IkhKvlJUv9iSHJOnxinBcoOI3ppGnnL+VDJ75O45Nss6HtZd8IdN8touXPDtASfeI2T2LImb8VOZcL47wjQ==",
-      "dependencies": {
-        "tslib": "^2.0.0"
-      },
-      "peerDependencies": {
-        "react": ">=16.8.0"
-      }
-    },
     "node_modules/@dnd-kit/core": {
       "version": "6.1.0",
       "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.1.0.tgz",
@@ -528,6 +518,17 @@
         "react-dom": ">=16.8.0"
       }
     },
+    "node_modules/@dnd-kit/core/node_modules/@dnd-kit/accessibility": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/@dnd-kit/accessibility/-/accessibility-3.1.0.tgz",
+      "integrity": "sha512-ea7IkhKvlJUv9iSHJOnxinBcoOI3ppGnnL+VDJ75O45Nss6HtZd8IdN8touXPDtASfeI2T2LImb8VOZcL47wjQ==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0"
+      }
+    },
     "node_modules/@dnd-kit/modifiers": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/@dnd-kit/modifiers/-/modifiers-7.0.0.tgz",
@@ -575,15 +576,15 @@
       }
     },
     "node_modules/@emotion/babel-plugin": {
-      "version": "11.11.0",
-      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz",
-      "integrity": "sha512-m4HEDZleaaCH+XgDDsPF15Ht6wTLsgDTeR3WYj9Q/k76JtWhrJjcP4+/XlG8LGT/Rol9qUfOIztXeA84ATpqPQ==",
+      "version": "11.12.0",
+      "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.12.0.tgz",
+      "integrity": "sha512-y2WQb+oP8Jqvvclh8Q55gLUyb7UFvgv7eJfsj7td5TToBrIUtPay2kMrZi4xjq9qw2vD0ZR5fSho0yqoFgX7Rw==",
       "dependencies": {
         "@babel/helper-module-imports": "^7.16.7",
         "@babel/runtime": "^7.18.3",
-        "@emotion/hash": "^0.9.1",
-        "@emotion/memoize": "^0.8.1",
-        "@emotion/serialize": "^1.1.2",
+        "@emotion/hash": "^0.9.2",
+        "@emotion/memoize": "^0.9.0",
+        "@emotion/serialize": "^1.2.0",
         "babel-plugin-macros": "^3.1.0",
         "convert-source-map": "^1.5.0",
         "escape-string-regexp": "^4.0.0",
@@ -598,79 +599,56 @@
       "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A=="
     },
     "node_modules/@emotion/cache": {
-      "version": "11.11.0",
-      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.11.0.tgz",
-      "integrity": "sha512-P34z9ssTCBi3e9EI1ZsWpNHcfY1r09ZO0rZbRO2ob3ZQMnFI35jB536qoXbkdesr5EUhYi22anuEJuyxifaqAQ==",
-      "dependencies": {
-        "@emotion/memoize": "^0.8.1",
-        "@emotion/sheet": "^1.2.2",
-        "@emotion/utils": "^1.2.1",
-        "@emotion/weak-memoize": "^0.3.1",
+      "version": "11.13.1",
+      "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.13.1.tgz",
+      "integrity": "sha512-iqouYkuEblRcXmylXIwwOodiEK5Ifl7JcX7o6V4jI3iW4mLXX3dmt5xwBtIkJiQEXFAI+pC8X0i67yiPkH9Ucw==",
+      "dependencies": {
+        "@emotion/memoize": "^0.9.0",
+        "@emotion/sheet": "^1.4.0",
+        "@emotion/utils": "^1.4.0",
+        "@emotion/weak-memoize": "^0.4.0",
         "stylis": "4.2.0"
       }
     },
     "node_modules/@emotion/hash": {
-      "version": "0.9.1",
-      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.1.tgz",
-      "integrity": "sha512-gJB6HLm5rYwSLI6PQa+X1t5CFGrv1J1TWG+sOyMCeKz2ojaj6Fnl/rZEspogG+cvqbt4AE/2eIyD2QfLKTBNlQ=="
+      "version": "0.9.2",
+      "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz",
+      "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g=="
     },
     "node_modules/@emotion/is-prop-valid": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.2.2.tgz",
-      "integrity": "sha512-uNsoYd37AFmaCdXlg6EYD1KaPOaRWRByMCYzbKUX4+hhMfrxdVSelShywL4JVaAeM/eHUOSprYBQls+/neX3pw==",
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.3.1.tgz",
+      "integrity": "sha512-/ACwoqx7XQi9knQs/G0qKvv5teDMhD7bXYns9N/wM8ah8iNb8jZ2uNO0YOgiq2o2poIvVtJS2YALasQuMSQ7Kw==",
       "dependencies": {
-        "@emotion/memoize": "^0.8.1"
+        "@emotion/memoize": "^0.9.0"
       }
     },
     "node_modules/@emotion/memoize": {
-      "version": "0.8.1",
-      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.8.1.tgz",
-      "integrity": "sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA=="
-    },
-    "node_modules/@emotion/react": {
-      "version": "11.11.4",
-      "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.11.4.tgz",
-      "integrity": "sha512-t8AjMlF0gHpvvxk5mAtCqR4vmxiGHCeJBaQO6gncUSdklELOgtwjerNY2yuJNfwnc6vi16U/+uMF+afIawJ9iw==",
-      "dependencies": {
-        "@babel/runtime": "^7.18.3",
-        "@emotion/babel-plugin": "^11.11.0",
-        "@emotion/cache": "^11.11.0",
-        "@emotion/serialize": "^1.1.3",
-        "@emotion/use-insertion-effect-with-fallbacks": "^1.0.1",
-        "@emotion/utils": "^1.2.1",
-        "@emotion/weak-memoize": "^0.3.1",
-        "hoist-non-react-statics": "^3.3.1"
-      },
-      "peerDependencies": {
-        "react": ">=16.8.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
+      "version": "0.9.0",
+      "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz",
+      "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ=="
     },
     "node_modules/@emotion/serialize": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.1.4.tgz",
-      "integrity": "sha512-RIN04MBT8g+FnDwgvIUi8czvr1LU1alUMI05LekWB5DGyTm8cCBMCRpq3GqaiyEDRptEXOyXnvZ58GZYu4kBxQ==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.3.2.tgz",
+      "integrity": "sha512-grVnMvVPK9yUVE6rkKfAJlYZgo0cu3l9iMC77V7DW6E1DUIrU68pSEXRmFZFOFB1QFo57TncmOcvcbMDWsL4yA==",
       "dependencies": {
-        "@emotion/hash": "^0.9.1",
-        "@emotion/memoize": "^0.8.1",
-        "@emotion/unitless": "^0.8.1",
-        "@emotion/utils": "^1.2.1",
+        "@emotion/hash": "^0.9.2",
+        "@emotion/memoize": "^0.9.0",
+        "@emotion/unitless": "^0.10.0",
+        "@emotion/utils": "^1.4.1",
         "csstype": "^3.0.2"
       }
     },
     "node_modules/@emotion/serialize/node_modules/@emotion/unitless": {
-      "version": "0.8.1",
-      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.8.1.tgz",
-      "integrity": "sha512-KOEGMu6dmJZtpadb476IsZBclKvILjopjUii3V+7MnXIQCYh8W3NgNcgwo21n9LXZX6EDIKvqfjYxXebDwxKmQ=="
+      "version": "0.10.0",
+      "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.10.0.tgz",
+      "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg=="
     },
     "node_modules/@emotion/sheet": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.2.2.tgz",
-      "integrity": "sha512-0QBtGvaqtWi+nx6doRwDdBIzhNdZrXUppvTM4dtZZWEGTXL/XE/yJxLMGlDT1Gt+UHH5IX1n+jkXyytE/av7OA=="
+      "version": "1.4.0",
+      "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz",
+      "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg=="
     },
     "node_modules/@emotion/stylis": {
       "version": "0.8.5",
@@ -682,23 +660,15 @@
       "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.7.5.tgz",
       "integrity": "sha512-OWORNpfjMsSSUBVrRBVGECkhWcULOAJz9ZW8uK9qgxD+87M7jHRcvh/A96XXNhXTLmKcoYSQtBEX7lHMO7YRwg=="
     },
-    "node_modules/@emotion/use-insertion-effect-with-fallbacks": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.0.1.tgz",
-      "integrity": "sha512-jT/qyKZ9rzLErtrjGgdkMBn2OP8wl0G3sQlBb3YPryvKHsjvINUhVaPFfP+fpBcOkmrVOVEEHQFJ7nbj2TH2gw==",
-      "peerDependencies": {
-        "react": ">=16.8.0"
-      }
-    },
     "node_modules/@emotion/utils": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.2.1.tgz",
-      "integrity": "sha512-Y2tGf3I+XVnajdItskUCn6LX+VUDmP6lTL4fcqsXAv43dnlbZiuW4MWQW38rW/BVWSE7Q/7+XQocmpnRYILUmg=="
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.4.1.tgz",
+      "integrity": "sha512-BymCXzCG3r72VKJxaYVwOXATqXIZ85cuvg0YOUDxMGNrKc1DJRZk8MgV5wyXRyEayIMd4FuXJIUgTBXvDNW5cA=="
     },
     "node_modules/@emotion/weak-memoize": {
-      "version": "0.3.1",
-      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz",
-      "integrity": "sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww=="
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz",
+      "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg=="
     },
     "node_modules/@eslint-community/eslint-utils": {
       "version": "4.4.0",
@@ -774,9 +744,9 @@
       }
     },
     "node_modules/@floating-ui/react-dom": {
-      "version": "2.0.9",
-      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.0.9.tgz",
-      "integrity": "sha512-q0umO0+LQK4+p6aGyvzASqKbKOJcAHJ7ycE9CuUvfx3s9zTHWmGJTPOIlM/hmSBfUfg/XfY5YhLBLR/LHwShQQ==",
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.2.tgz",
+      "integrity": "sha512-06okr5cgPzMNBy+Ycse2A6udMi4bqwW/zgBF/rwjcNqWkyr82Mcg8b0vjX8OJpZFy/FKjJmw6wV7t44kK6kW7A==",
       "dependencies": {
         "@floating-ui/dom": "^1.0.0"
       },
@@ -786,9 +756,182 @@
       }
     },
     "node_modules/@floating-ui/utils": {
-      "version": "0.2.2",
-      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.2.tgz",
-      "integrity": "sha512-J4yDIIthosAsRZ5CPYP/jQvUAQtlZTTD/4suA08/FEnlxqW3sKS9iAhgsa9VYLZ6vDHn/ixJgIqRQPotoBjxIw=="
+      "version": "0.2.8",
+      "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.8.tgz",
+      "integrity": "sha512-kym7SodPp8/wloecOpcmSnWJsK7M0E5Wg8UcFA+uO4B9s5d0ywXOEro/8HM9x0rW+TljRzul/14UYz3TleT3ig=="
+    },
+    "node_modules/@headlessui/react": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/@headlessui/react/-/react-2.2.0.tgz",
+      "integrity": "sha512-RzCEg+LXsuI7mHiSomsu/gBJSjpupm6A1qIZ5sWjd7JhARNlMiSA4kKfJpCKwU9tE+zMRterhhrP74PvfJrpXQ==",
+      "dependencies": {
+        "@floating-ui/react": "^0.26.16",
+        "@react-aria/focus": "^3.17.1",
+        "@react-aria/interactions": "^3.21.3",
+        "@tanstack/react-virtual": "^3.8.1"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "react": "^18 || ^19 || ^19.0.0-rc",
+        "react-dom": "^18 || ^19 || ^19.0.0-rc"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@floating-ui/react": {
+      "version": "0.26.27",
+      "resolved": "https://registry.npmjs.org/@floating-ui/react/-/react-0.26.27.tgz",
+      "integrity": "sha512-jLP72x0Kr2CgY6eTYi/ra3VA9LOkTo4C+DUTrbFgFOExKy3omYVmwMjNKqxAHdsnyLS96BIDLcO2SlnsNf8KUQ==",
+      "dependencies": {
+        "@floating-ui/react-dom": "^2.1.2",
+        "@floating-ui/utils": "^0.2.8",
+        "tabbable": "^6.0.0"
+      },
+      "peerDependencies": {
+        "react": ">=16.8.0",
+        "react-dom": ">=16.8.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/focus": {
+      "version": "3.18.4",
+      "resolved": "https://registry.npmjs.org/@react-aria/focus/-/focus-3.18.4.tgz",
+      "integrity": "sha512-91J35077w9UNaMK1cpMUEFRkNNz0uZjnSwiyBCFuRdaVuivO53wNC9XtWSDNDdcO5cGy87vfJRVAiyoCn/mjqA==",
+      "dependencies": {
+        "@react-aria/interactions": "^3.22.4",
+        "@react-aria/utils": "^3.25.3",
+        "@react-types/shared": "^3.25.0",
+        "@swc/helpers": "^0.5.0",
+        "clsx": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/focus/node_modules/@react-aria/utils": {
+      "version": "3.25.3",
+      "resolved": "https://registry.npmjs.org/@react-aria/utils/-/utils-3.25.3.tgz",
+      "integrity": "sha512-PR5H/2vaD8fSq0H/UB9inNbc8KDcVmW6fYAfSWkkn+OAdhTTMVKqXXrZuZBWyFfSD5Ze7VN6acr4hrOQm2bmrA==",
+      "dependencies": {
+        "@react-aria/ssr": "^3.9.6",
+        "@react-stately/utils": "^3.10.4",
+        "@react-types/shared": "^3.25.0",
+        "@swc/helpers": "^0.5.0",
+        "clsx": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/focus/node_modules/@react-aria/utils/node_modules/@react-aria/ssr": {
+      "version": "3.9.6",
+      "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.6.tgz",
+      "integrity": "sha512-iLo82l82ilMiVGy342SELjshuWottlb5+VefO3jOQqQRNYnJBFpUSadswDPbRimSgJUZuFwIEYs6AabkP038fA==",
+      "dependencies": {
+        "@swc/helpers": "^0.5.0"
+      },
+      "engines": {
+        "node": ">= 12"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/focus/node_modules/@react-aria/utils/node_modules/@react-stately/utils": {
+      "version": "3.10.4",
+      "resolved": "https://registry.npmjs.org/@react-stately/utils/-/utils-3.10.4.tgz",
+      "integrity": "sha512-gBEQEIMRh5f60KCm7QKQ2WfvhB2gLUr9b72sqUdIZ2EG+xuPgaIlCBeSicvjmjBvYZwOjoOEnmIkcx2GHp/HWw==",
+      "dependencies": {
+        "@swc/helpers": "^0.5.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/focus/node_modules/@react-types/shared": {
+      "version": "3.25.0",
+      "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.25.0.tgz",
+      "integrity": "sha512-OZSyhzU6vTdW3eV/mz5i6hQwQUhkRs7xwY2d1aqPvTdMe0+2cY7Fwp45PAiwYLEj73i9ro2FxF9qC4DvHGSCgQ==",
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/interactions": {
+      "version": "3.22.4",
+      "resolved": "https://registry.npmjs.org/@react-aria/interactions/-/interactions-3.22.4.tgz",
+      "integrity": "sha512-E0vsgtpItmknq/MJELqYJwib+YN18Qag8nroqwjk1qOnBa9ROIkUhWJerLi1qs5diXq9LHKehZDXRlwPvdEFww==",
+      "dependencies": {
+        "@react-aria/ssr": "^3.9.6",
+        "@react-aria/utils": "^3.25.3",
+        "@react-types/shared": "^3.25.0",
+        "@swc/helpers": "^0.5.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/interactions/node_modules/@react-aria/ssr": {
+      "version": "3.9.6",
+      "resolved": "https://registry.npmjs.org/@react-aria/ssr/-/ssr-3.9.6.tgz",
+      "integrity": "sha512-iLo82l82ilMiVGy342SELjshuWottlb5+VefO3jOQqQRNYnJBFpUSadswDPbRimSgJUZuFwIEYs6AabkP038fA==",
+      "dependencies": {
+        "@swc/helpers": "^0.5.0"
+      },
+      "engines": {
+        "node": ">= 12"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/interactions/node_modules/@react-aria/utils": {
+      "version": "3.25.3",
+      "resolved": "https://registry.npmjs.org/@react-aria/utils/-/utils-3.25.3.tgz",
+      "integrity": "sha512-PR5H/2vaD8fSq0H/UB9inNbc8KDcVmW6fYAfSWkkn+OAdhTTMVKqXXrZuZBWyFfSD5Ze7VN6acr4hrOQm2bmrA==",
+      "dependencies": {
+        "@react-aria/ssr": "^3.9.6",
+        "@react-stately/utils": "^3.10.4",
+        "@react-types/shared": "^3.25.0",
+        "@swc/helpers": "^0.5.0",
+        "clsx": "^2.0.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/interactions/node_modules/@react-aria/utils/node_modules/@react-stately/utils": {
+      "version": "3.10.4",
+      "resolved": "https://registry.npmjs.org/@react-stately/utils/-/utils-3.10.4.tgz",
+      "integrity": "sha512-gBEQEIMRh5f60KCm7QKQ2WfvhB2gLUr9b72sqUdIZ2EG+xuPgaIlCBeSicvjmjBvYZwOjoOEnmIkcx2GHp/HWw==",
+      "dependencies": {
+        "@swc/helpers": "^0.5.0"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@react-aria/interactions/node_modules/@react-types/shared": {
+      "version": "3.25.0",
+      "resolved": "https://registry.npmjs.org/@react-types/shared/-/shared-3.25.0.tgz",
+      "integrity": "sha512-OZSyhzU6vTdW3eV/mz5i6hQwQUhkRs7xwY2d1aqPvTdMe0+2cY7Fwp45PAiwYLEj73i9ro2FxF9qC4DvHGSCgQ==",
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0"
+      }
+    },
+    "node_modules/@headlessui/react/node_modules/@tanstack/react-virtual": {
+      "version": "3.10.8",
+      "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.10.8.tgz",
+      "integrity": "sha512-VbzbVGSsZlQktyLrP5nxE+vE1ZR+U0NFAWPbJLoG2+DKPwd2D7dVICTVIIaYlJqX1ZCEnYDbaOpmMwbsyhBoIA==",
+      "dependencies": {
+        "@tanstack/virtual-core": "3.10.8"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
     },
     "node_modules/@headlessui/tailwindcss": {
       "version": "0.2.1",
@@ -1272,9 +1415,9 @@
       }
     },
     "node_modules/@next/env": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz",
-      "integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA=="
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/env/-/env-15.0.2.tgz",
+      "integrity": "sha512-c0Zr0ModK5OX7D4ZV8Jt/wqoXtitLNPwUfG9zElCZztdaZyNVnN40rDXVZ/+FGuR4CcNV5AEfM6N8f+Ener7Dg=="
     },
     "node_modules/@next/eslint-plugin-next": {
       "version": "14.2.3",
@@ -1286,9 +1429,9 @@
       }
     },
     "node_modules/@next/swc-darwin-arm64": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz",
-      "integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-15.0.2.tgz",
+      "integrity": "sha512-GK+8w88z+AFlmt+ondytZo2xpwlfAR8U6CRwXancHImh6EdGfHMIrTSCcx5sOSBei00GyLVL0ioo1JLKTfprgg==",
       "cpu": [
         "arm64"
       ],
@@ -1301,9 +1444,9 @@
       }
     },
     "node_modules/@next/swc-darwin-x64": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz",
-      "integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-15.0.2.tgz",
+      "integrity": "sha512-KUpBVxIbjzFiUZhiLIpJiBoelqzQtVZbdNNsehhUn36e2YzKHphnK8eTUW1s/4aPy5kH/UTid8IuVbaOpedhpw==",
       "cpu": [
         "x64"
       ],
@@ -1316,9 +1459,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-gnu": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz",
-      "integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-15.0.2.tgz",
+      "integrity": "sha512-9J7TPEcHNAZvwxXRzOtiUvwtTD+fmuY0l7RErf8Yyc7kMpE47MIQakl+3jecmkhOoIyi/Rp+ddq7j4wG6JDskQ==",
       "cpu": [
         "arm64"
       ],
@@ -1331,9 +1474,9 @@
       }
     },
     "node_modules/@next/swc-linux-arm64-musl": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz",
-      "integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-15.0.2.tgz",
+      "integrity": "sha512-BjH4ZSzJIoTTZRh6rG+a/Ry4SW0HlizcPorqNBixBWc3wtQtj4Sn9FnRZe22QqrPnzoaW0ctvSz4FaH4eGKMww==",
       "cpu": [
         "arm64"
       ],
@@ -1346,9 +1489,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-gnu": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz",
-      "integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-15.0.2.tgz",
+      "integrity": "sha512-i3U2TcHgo26sIhcwX/Rshz6avM6nizrZPvrDVDY1bXcLH1ndjbO8zuC7RoHp0NSK7wjJMPYzm7NYL1ksSKFreA==",
       "cpu": [
         "x64"
       ],
@@ -1361,9 +1504,9 @@
       }
     },
     "node_modules/@next/swc-linux-x64-musl": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz",
-      "integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-15.0.2.tgz",
+      "integrity": "sha512-AMfZfSVOIR8fa+TXlAooByEF4OB00wqnms1sJ1v+iu8ivwvtPvnkwdzzFMpsK5jA2S9oNeeQ04egIWVb4QWmtQ==",
       "cpu": [
         "x64"
       ],
@@ -1376,9 +1519,9 @@
       }
     },
     "node_modules/@next/swc-win32-arm64-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz",
-      "integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-15.0.2.tgz",
+      "integrity": "sha512-JkXysDT0/hEY47O+Hvs8PbZAeiCQVxKfGtr4GUpNAhlG2E0Mkjibuo8ryGD29Qb5a3IOnKYNoZlh/MyKd2Nbww==",
       "cpu": [
         "arm64"
       ],
@@ -1390,25 +1533,10 @@
         "node": ">= 10"
       }
     },
-    "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz",
-      "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==",
-      "cpu": [
-        "ia32"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
-    },
     "node_modules/@next/swc-win32-x64-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz",
-      "integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-15.0.2.tgz",
+      "integrity": "sha512-foaUL0NqJY/dX0Pi/UcZm5zsmSk5MtP/gxx3xOPyREkMFN+CTjctPfu3QaqrQHinaKdPnMWPJDKt4VjDfTBe/Q==",
       "cpu": [
         "x64"
       ],
@@ -2011,18 +2139,17 @@
       }
     },
     "node_modules/@radix-ui/react-arrow": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.0.3.tgz",
-      "integrity": "sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.0.tgz",
+      "integrity": "sha512-FmlW1rCg7hBpEBwFbjHwCW6AmWLQM6g/v0Sn8XbP9NvmSZ2San1FpQeyPtufzOMSIx7Y4dzjlHoifhp+7NkZhw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+        "@radix-ui/react-primitive": "2.0.0"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2037,7 +2164,6 @@
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.0.tgz",
       "integrity": "sha512-GZsZslMJEyo1VKm5L1ZJY8tGDxZNPAoUeQUIbKeJfoi7Q4kmig5AsgLMYYuyYbfjd8fBmFORAIwYAkXMnXZgZw==",
-      "license": "MIT",
       "dependencies": {
         "@radix-ui/react-compose-refs": "1.1.0",
         "@radix-ui/react-context": "1.1.0",
@@ -2059,11 +2185,10 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-collection/node_modules/@radix-ui/react-compose-refs": {
+    "node_modules/@radix-ui/react-compose-refs": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
       "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
-      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
         "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
@@ -2074,11 +2199,10 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-collection/node_modules/@radix-ui/react-context": {
+    "node_modules/@radix-ui/react-context": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
       "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
-      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
         "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
@@ -2089,19 +2213,32 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-collection/node_modules/@radix-ui/react-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
-      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.0.5.tgz",
+      "integrity": "sha512-GjWJX/AUpB703eEBanuBnIWdIXg6NvJFCXcNlSZk4xdszCdhrJgBoUd1cGk67vFO+WdA2pfI/plOpqz/5GUP6Q==",
       "dependencies": {
-        "@radix-ui/react-slot": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-focus-guards": "1.0.1",
+        "@radix-ui/react-focus-scope": "1.0.4",
+        "@radix-ui/react-id": "1.0.1",
+        "@radix-ui/react-portal": "1.0.4",
+        "@radix-ui/react-presence": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-slot": "1.0.2",
+        "@radix-ui/react-use-controllable-state": "1.0.1",
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.5.5"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2112,7 +2249,7 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-compose-refs": {
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-compose-refs": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
       "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==",
@@ -2129,7 +2266,7 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-context": {
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-context": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz",
       "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==",
@@ -2146,26 +2283,17 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-dialog": {
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-dismissable-layer": {
       "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.0.5.tgz",
-      "integrity": "sha512-GjWJX/AUpB703eEBanuBnIWdIXg6NvJFCXcNlSZk4xdszCdhrJgBoUd1cGk67vFO+WdA2pfI/plOpqz/5GUP6Q==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
+      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
       "dependencies": {
         "@babel/runtime": "^7.13.10",
         "@radix-ui/primitive": "1.0.1",
         "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-dismissable-layer": "1.0.5",
-        "@radix-ui/react-focus-guards": "1.0.1",
-        "@radix-ui/react-focus-scope": "1.0.4",
-        "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-portal": "1.0.4",
-        "@radix-ui/react-presence": "1.0.1",
         "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-slot": "1.0.2",
-        "@radix-ui/react-use-controllable-state": "1.0.1",
-        "aria-hidden": "^1.1.1",
-        "react-remove-scroll": "2.5.5"
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-escape-keydown": "1.0.3"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -2182,14 +2310,12 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-slot": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
-      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -2201,14 +2327,17 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-direction": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.0.tgz",
-      "integrity": "sha512-BUuBvgThEiAXh2DWu93XsT+a3aWrGqolGlqqw5VU1kG7p/ZH2cuDlM1sRLNnY3QcBS69UIz2mcKhMxDsdewhjg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
+      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2216,37 +2345,10 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-dismissable-layer": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
-      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-escape-keydown": "1.0.3"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-focus-guards": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
-      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.0.1.tgz",
+      "integrity": "sha512-Rect2dWbQ8waGzhMavsIbmSVCgYxkXLxxR3ZvCX79JOglzdEy4JXMb98lq4hPxUbLr77nP0UOGf4rcMU+s1pUA==",
       "dependencies": {
         "@babel/runtime": "^7.13.10"
       },
@@ -2260,7 +2362,7 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-focus-scope": {
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-focus-scope": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.0.4.tgz",
       "integrity": "sha512-sL04Mgvf+FmyvZeYfNu1EPAaaxD+aw7cYeIB9L9Fvq8+urhltTRaEo5ysKOpHuKPclsZcSUMKlN05x4u+CINpA==",
@@ -2285,13 +2387,12 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-id": {
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-focus-scope/node_modules/@radix-ui/react-use-callback-ref": {
       "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
-      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -2303,119 +2404,78 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.2.tgz",
-      "integrity": "sha512-u2HRUyWW+lOiA2g0Le0tMmT55FGOEWHwPFt1EPfbLly7uXQExFo5duNKqG2DzmFXIdqOeNd+TpE8baHWJCyP9w==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-id": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
+      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
       "dependencies": {
-        "@radix-ui/primitive": "1.1.0",
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-context": "1.1.1",
-        "@radix-ui/react-dismissable-layer": "1.1.1",
-        "@radix-ui/react-focus-guards": "1.1.1",
-        "@radix-ui/react-focus-scope": "1.1.0",
-        "@radix-ui/react-id": "1.1.0",
-        "@radix-ui/react-popper": "1.2.0",
-        "@radix-ui/react-portal": "1.1.2",
-        "@radix-ui/react-presence": "1.1.1",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-slot": "1.1.0",
-        "@radix-ui/react-use-controllable-state": "1.1.0",
-        "aria-hidden": "^1.1.1",
-        "react-remove-scroll": "2.6.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/primitive": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
-      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
-      "license": "MIT"
-    },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-arrow": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.0.tgz",
-      "integrity": "sha512-FmlW1rCg7hBpEBwFbjHwCW6AmWLQM6g/v0Sn8XbP9NvmSZ2San1FpQeyPtufzOMSIx7Y4dzjlHoifhp+7NkZhw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-id/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
       "dependencies": {
-        "@radix-ui/react-primitive": "2.0.0"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
-      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
-      "license": "MIT",
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-portal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
+      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-context": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
-      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
-      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-dismissable-layer": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.1.tgz",
-      "integrity": "sha512-QSxg29lfr/xcev6kSz7MAlmDnzbP1eI/Dwn3Tp1ip0KT5CUELsxkekFEMVBEoykI3oV39hKT4TKZzBNMbcTZYQ==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-presence": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
+      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
       "dependencies": {
-        "@radix-ui/primitive": "1.1.0",
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-callback-ref": "1.1.0",
-        "@radix-ui/react-use-escape-keydown": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2426,14 +2486,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-focus-guards": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz",
-      "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-presence/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2441,21 +2503,19 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-focus-scope": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.0.tgz",
-      "integrity": "sha512-200UD8zylvEyL8Bx+z76RJnASR2gRMuxlgFCPAe/Q/679a/r0eK3MBVYMb7vZODZcffZBdob1EGnky78xmVvcA==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-primitive": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
+      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
       "dependencies": {
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-slot": "1.0.2"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2466,17 +2526,18 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-id": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
-      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
       "license": "MIT",
       "dependencies": {
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2484,46 +2545,34 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-popper": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.0.tgz",
-      "integrity": "sha512-ZnRMshKF43aBxVWPWvbj21+7TQCvhuULWJ4gNIKYpRlQt5xGRhLx66tMp8pya2UkGHTSlhpXwmjqltDYHhw7Vg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
+      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
       "dependencies": {
-        "@floating-ui/react-dom": "^2.0.0",
-        "@radix-ui/react-arrow": "1.1.0",
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-context": "1.1.0",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-callback-ref": "1.1.0",
-        "@radix-ui/react-use-layout-effect": "1.1.0",
-        "@radix-ui/react-use-rect": "1.1.0",
-        "@radix-ui/react-use-size": "1.1.0",
-        "@radix-ui/rect": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-context": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
-      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/@radix-ui/react-use-controllable-state/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2531,81 +2580,118 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-portal": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.2.tgz",
-      "integrity": "sha512-WeDYLGPxJb/5EGBoedyJbT0MpoULmwnIPMJMSldkuiMsBAv7N1cRdsTWZWht9vpPOiN3qyiGAtbK2is47/uMFg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/react-remove-scroll": {
+      "version": "2.5.5",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.5.5.tgz",
+      "integrity": "sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==",
       "dependencies": {
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "react-remove-scroll-bar": "^2.3.3",
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.1.0",
+        "use-callback-ref": "^1.3.0",
+        "use-sidecar": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-presence": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.1.tgz",
-      "integrity": "sha512-IeFXVi4YS1K0wVZzXNrbaaUvIJ3qdY+/Ih4eHFhWA9SwGR9UDX7Ck8abvL57C4cv3wwMvUE0OG69Qc3NCcTe/A==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/react-remove-scroll/node_modules/react-remove-scroll-bar": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz",
+      "integrity": "sha512-DtSYaao4mBmX+HDo5YWYdBWQwYIQQshUV/dVxFxK+KM26Wjwp1gZ6rv6OC3oujI6Bfu6Xyg3TwK533AQutsn/g==",
       "dependencies": {
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-dialog/node_modules/react-remove-scroll/node_modules/react-style-singleton": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
+      "integrity": "sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==",
+      "dependencies": {
+        "get-nonce": "^1.0.0",
+        "invariant": "^2.2.4",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
           "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
-      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dialog/node_modules/react-remove-scroll/node_modules/use-callback-ref": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.2.tgz",
+      "integrity": "sha512-elOQwe6Q8gqZgDA8mrh44qRTQqpIHDcZ3hXTLjBe1i4ph8XpNJnO+aQf3NaG+lriLopI4HMx9VjQLfPQ6vhnoA==",
       "dependencies": {
-        "@radix-ui/react-slot": "1.1.0"
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-dialog/node_modules/react-remove-scroll/node_modules/use-sidecar": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.2.tgz",
+      "integrity": "sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==",
+      "dependencies": {
+        "detect-node-es": "^1.1.0",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.9.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
           "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-callback-ref": {
+    "node_modules/@radix-ui/react-direction": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
-      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.0.tgz",
+      "integrity": "sha512-BUuBvgThEiAXh2DWu93XsT+a3aWrGqolGlqqw5VU1kG7p/ZH2cuDlM1sRLNnY3QcBS69UIz2mcKhMxDsdewhjg==",
       "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
@@ -2617,47 +2703,68 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-controllable-state": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
-      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.1.tgz",
+      "integrity": "sha512-QSxg29lfr/xcev6kSz7MAlmDnzbP1eI/Dwn3Tp1ip0KT5CUELsxkekFEMVBEoykI3oV39hKT4TKZzBNMbcTZYQ==",
       "dependencies": {
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-escape-keydown": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-escape-keydown": {
+    "node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/primitive": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.0.tgz",
-      "integrity": "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw==",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA=="
+    },
+    "node_modules/@radix-ui/react-focus-scope": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.0.tgz",
+      "integrity": "sha512-200UD8zylvEyL8Bx+z76RJnASR2gRMuxlgFCPAe/Q/679a/r0eK3MBVYMb7vZODZcffZBdob1EGnky78xmVvcA==",
       "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
         "@radix-ui/react-use-callback-ref": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-layout-effect": {
+    "node_modules/@radix-ui/react-id": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
-      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
+      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+      "dependencies": {
+        "@radix-ui/react-use-layout-effect": "1.1.0"
+      },
       "peerDependencies": {
         "@types/react": "*",
         "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
@@ -2668,32 +2775,54 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-rect": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.0.tgz",
-      "integrity": "sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==",
+    "node_modules/@radix-ui/react-popover": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.2.tgz",
+      "integrity": "sha512-u2HRUyWW+lOiA2g0Le0tMmT55FGOEWHwPFt1EPfbLly7uXQExFo5duNKqG2DzmFXIdqOeNd+TpE8baHWJCyP9w==",
       "license": "MIT",
       "dependencies": {
-        "@radix-ui/rect": "1.1.0"
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.1",
+        "@radix-ui/react-dismissable-layer": "1.1.1",
+        "@radix-ui/react-focus-guards": "1.1.1",
+        "@radix-ui/react-focus-scope": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-popper": "1.2.0",
+        "@radix-ui/react-portal": "1.1.2",
+        "@radix-ui/react-presence": "1.1.1",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-slot": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0",
+        "aria-hidden": "^1.1.1",
+        "react-remove-scroll": "2.6.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-use-size": {
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/primitive": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.0.tgz",
-      "integrity": "sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
       "license": "MIT",
-      "dependencies": {
-        "@radix-ui/react-use-layout-effect": "1.1.0"
-      },
       "peerDependencies": {
         "@types/react": "*",
         "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
@@ -2704,11 +2833,20 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/rect": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.0.tgz",
-      "integrity": "sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg==",
-      "license": "MIT"
+    "node_modules/@radix-ui/react-popover/node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz",
+      "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==",
+      "license": "MIT",
+      "peerDependencies": {
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
     },
     "node_modules/@radix-ui/react-popover/node_modules/react-remove-scroll": {
       "version": "2.6.0",
@@ -2735,120 +2873,83 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-popper": {
-      "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
-      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
+    "node_modules/@radix-ui/react-popover/node_modules/react-remove-scroll/node_modules/react-remove-scroll-bar": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz",
+      "integrity": "sha512-DtSYaao4mBmX+HDo5YWYdBWQwYIQQshUV/dVxFxK+KM26Wjwp1gZ6rv6OC3oujI6Bfu6Xyg3TwK533AQutsn/g==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@floating-ui/react-dom": "^2.0.0",
-        "@radix-ui/react-arrow": "1.0.3",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1",
-        "@radix-ui/react-use-rect": "1.0.1",
-        "@radix-ui/react-use-size": "1.0.1",
-        "@radix-ui/rect": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.0.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-portal": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
-      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-presence": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
-      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
+    "node_modules/@radix-ui/react-popover/node_modules/react-remove-scroll/node_modules/react-style-singleton": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
+      "integrity": "sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "get-nonce": "^1.0.0",
+        "invariant": "^2.2.4",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-primitive": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
-      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
+    "node_modules/@radix-ui/react-popover/node_modules/react-remove-scroll/node_modules/use-callback-ref": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.2.tgz",
+      "integrity": "sha512-elOQwe6Q8gqZgDA8mrh44qRTQqpIHDcZ3hXTLjBe1i4ph8XpNJnO+aQf3NaG+lriLopI4HMx9VjQLfPQ6vhnoA==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-slot": "1.0.2"
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-primitive/node_modules/@radix-ui/react-slot": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
-      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-popover/node_modules/react-remove-scroll/node_modules/use-sidecar": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.2.tgz",
+      "integrity": "sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1"
+        "detect-node-es": "^1.1.0",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "@types/react": "^16.9.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -2856,21 +2957,21 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-roving-focus": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.0.tgz",
-      "integrity": "sha512-EA6AMGeq9AEeQDeSH0aZgG198qkfHSbvWTf1HvoDmOB5bBG/qTxjYMWUKMnYiV6J/iP/J8MEFSuB2zRU2n7ODA==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-popper": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.0.tgz",
+      "integrity": "sha512-ZnRMshKF43aBxVWPWvbj21+7TQCvhuULWJ4gNIKYpRlQt5xGRhLx66tMp8pya2UkGHTSlhpXwmjqltDYHhw7Vg==",
       "dependencies": {
-        "@radix-ui/primitive": "1.1.0",
-        "@radix-ui/react-collection": "1.1.0",
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.1.0",
         "@radix-ui/react-compose-refs": "1.1.0",
         "@radix-ui/react-context": "1.1.0",
-        "@radix-ui/react-direction": "1.1.0",
-        "@radix-ui/react-id": "1.1.0",
         "@radix-ui/react-primitive": "2.0.0",
         "@radix-ui/react-use-callback-ref": "1.1.0",
-        "@radix-ui/react-use-controllable-state": "1.1.0"
+        "@radix-ui/react-use-layout-effect": "1.1.0",
+        "@radix-ui/react-use-rect": "1.1.0",
+        "@radix-ui/react-use-size": "1.1.0",
+        "@radix-ui/rect": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -2887,65 +2988,56 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/primitive": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
-      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
-      "license": "MIT"
-    },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
-      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
-      "license": "MIT",
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+    "node_modules/@radix-ui/react-portal": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.2.tgz",
+      "integrity": "sha512-WeDYLGPxJb/5EGBoedyJbT0MpoULmwnIPMJMSldkuiMsBAv7N1cRdsTWZWht9vpPOiN3qyiGAtbK2is47/uMFg==",
+      "dependencies": {
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-layout-effect": "1.1.0"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-context": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
-      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
-      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-id": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
-      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-presence": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.1.tgz",
+      "integrity": "sha512-IeFXVi4YS1K0wVZzXNrbaaUvIJ3qdY+/Ih4eHFhWA9SwGR9UDX7Ck8abvL57C4cv3wwMvUE0OG69Qc3NCcTe/A==",
       "dependencies": {
+        "@radix-ui/react-compose-refs": "1.1.0",
         "@radix-ui/react-use-layout-effect": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-primitive": {
+    "node_modules/@radix-ui/react-primitive": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
       "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
-      "license": "MIT",
       "dependencies": {
         "@radix-ui/react-slot": "1.1.0"
       },
@@ -2964,53 +3056,40 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
-      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
-      "license": "MIT",
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-use-controllable-state": {
+    "node_modules/@radix-ui/react-roving-focus": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
-      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
-      "license": "MIT",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.0.tgz",
+      "integrity": "sha512-EA6AMGeq9AEeQDeSH0aZgG198qkfHSbvWTf1HvoDmOB5bBG/qTxjYMWUKMnYiV6J/iP/J8MEFSuB2zRU2n7ODA==",
       "dependencies": {
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-collection": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.0",
+        "@radix-ui/react-direction": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-use-callback-ref": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/react-use-layout-effect": {
+    "node_modules/@radix-ui/react-roving-focus/node_modules/@radix-ui/primitive": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
-      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
-      "license": "MIT",
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA=="
     },
     "node_modules/@radix-ui/react-select": {
       "version": "2.1.2",
@@ -3061,33 +3140,25 @@
       "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
       "license": "MIT"
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-arrow": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.0.tgz",
-      "integrity": "sha512-FmlW1rCg7hBpEBwFbjHwCW6AmWLQM6g/v0Sn8XbP9NvmSZ2San1FpQeyPtufzOMSIx7Y4dzjlHoifhp+7NkZhw==",
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
       "license": "MIT",
-      "dependencies": {
-        "@radix-ui/react-primitive": "2.0.0"
-      },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
-      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
+    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-focus-guards": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz",
+      "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==",
       "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
@@ -3099,14 +3170,24 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-context": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
-      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
+    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz",
+      "integrity": "sha512-I2U4JVEsQenxDAKaVa3VZ/JeJZe0/2DxPWL8Tj8yLKctQJQiZM52pn/GWFpSp8dftjM3pSAHVJZscAnC/y+ySQ==",
       "license": "MIT",
+      "dependencies": {
+        "react-remove-scroll-bar": "^2.3.6",
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.1.0",
+        "use-callback-ref": "^1.3.0",
+        "use-sidecar": "^1.1.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3114,41 +3195,83 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-dismissable-layer": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.1.tgz",
-      "integrity": "sha512-QSxg29lfr/xcev6kSz7MAlmDnzbP1eI/Dwn3Tp1ip0KT5CUELsxkekFEMVBEoykI3oV39hKT4TKZzBNMbcTZYQ==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll/node_modules/react-remove-scroll-bar": {
+      "version": "2.3.6",
+      "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz",
+      "integrity": "sha512-DtSYaao4mBmX+HDo5YWYdBWQwYIQQshUV/dVxFxK+KM26Wjwp1gZ6rv6OC3oujI6Bfu6Xyg3TwK533AQutsn/g==",
       "dependencies": {
-        "@radix-ui/primitive": "1.1.0",
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-callback-ref": "1.1.0",
-        "@radix-ui/react-use-escape-keydown": "1.1.0"
+        "react-style-singleton": "^2.2.1",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
       },
       "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll/node_modules/react-style-singleton": {
+      "version": "2.2.1",
+      "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
+      "integrity": "sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==",
+      "dependencies": {
+        "get-nonce": "^1.0.0",
+        "invariant": "^2.2.4",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll/node_modules/use-callback-ref": {
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.2.tgz",
+      "integrity": "sha512-elOQwe6Q8gqZgDA8mrh44qRTQqpIHDcZ3hXTLjBe1i4ph8XpNJnO+aQf3NaG+lriLopI4HMx9VjQLfPQ6vhnoA==",
+      "dependencies": {
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "peerDependencies": {
+        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-focus-guards": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.1.tgz",
-      "integrity": "sha512-pSIwfrT1a6sIoDASCSpFwOasEwKTZWDw/iBdtnqKO7v6FeOzYJ7U53cPzYFVR3geGGXgVHaH+CdngrrAzqUGxg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll/node_modules/use-sidecar": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.2.tgz",
+      "integrity": "sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==",
+      "dependencies": {
+        "detect-node-es": "^1.1.0",
+        "tslib": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
       "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react": "^16.9.0 || ^17.0.0 || ^18.0.0",
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3156,15 +3279,13 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-focus-scope": {
+    "node_modules/@radix-ui/react-separator": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.0.tgz",
-      "integrity": "sha512-200UD8zylvEyL8Bx+z76RJnASR2gRMuxlgFCPAe/Q/679a/r0eK3MBVYMb7vZODZcffZBdob1EGnky78xmVvcA==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.0.tgz",
+      "integrity": "sha512-3uBAs+egzvJBDZAzvb/n4NxxOYpnspmWxO2u5NbZ8Y6FM/NdrGSF9bop3Cf6F6C71z1rTSn8KV0Fo2ZVd79lGA==",
       "license": "MIT",
       "dependencies": {
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@radix-ui/react-primitive": "2.0.0"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -3181,13 +3302,13 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-id": {
+    "node_modules/@radix-ui/react-slot": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
-      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.1.0.tgz",
+      "integrity": "sha512-FUCf5XMfmW4dtYl69pdS4DbxKy8nj4M7SafBgPllysxmdachynNflAdp/gCsnYWNDnge6tI9onzMp5ARYc1KNw==",
       "license": "MIT",
       "dependencies": {
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "@radix-ui/react-compose-refs": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -3199,22 +3320,20 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-popper": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.2.0.tgz",
-      "integrity": "sha512-ZnRMshKF43aBxVWPWvbj21+7TQCvhuULWJ4gNIKYpRlQt5xGRhLx66tMp8pya2UkGHTSlhpXwmjqltDYHhw7Vg==",
+    "node_modules/@radix-ui/react-tabs": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.1.tgz",
+      "integrity": "sha512-3GBUDmP2DvzmtYLMsHmpA1GtR46ZDZ+OreXM/N+kkQJOPIgytFWWTfDQmBQKBvaFS0Vno0FktdbVzN28KGrMdw==",
       "license": "MIT",
       "dependencies": {
-        "@floating-ui/react-dom": "^2.0.0",
-        "@radix-ui/react-arrow": "1.1.0",
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-context": "1.1.0",
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-context": "1.1.1",
+        "@radix-ui/react-direction": "1.1.0",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-presence": "1.1.1",
         "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-callback-ref": "1.1.0",
-        "@radix-ui/react-use-layout-effect": "1.1.0",
-        "@radix-ui/react-use-rect": "1.1.0",
-        "@radix-ui/react-use-size": "1.1.0",
-        "@radix-ui/rect": "1.1.0"
+        "@radix-ui/react-roving-focus": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -3231,10 +3350,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-context": {
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/primitive": {
       "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.0.tgz",
-      "integrity": "sha512-OKrckBy+sMEgYM/sMmqmErVn0kZqrHPJze+Ql3DzYsDDp0hl0L62nx/2122/Bvps1qz645jlcu2tD9lrRSdf8A==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
+    },
+    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
       "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
@@ -3246,20 +3371,30 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-portal": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.2.tgz",
-      "integrity": "sha512-WeDYLGPxJb/5EGBoedyJbT0MpoULmwnIPMJMSldkuiMsBAv7N1cRdsTWZWht9vpPOiN3qyiGAtbK2is47/uMFg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz",
+      "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==",
       "dependencies": {
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-dismissable-layer": "1.0.5",
+        "@radix-ui/react-id": "1.0.1",
+        "@radix-ui/react-popper": "1.1.3",
+        "@radix-ui/react-portal": "1.0.4",
+        "@radix-ui/react-presence": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-slot": "1.0.2",
+        "@radix-ui/react-use-controllable-state": "1.0.1",
+        "@radix-ui/react-visually-hidden": "1.0.3"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3270,37 +3405,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
-      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-compose-refs": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
+      "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==",
       "dependencies": {
-        "@radix-ui/react-slot": "1.1.0"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@babel/runtime": "^7.13.10"
       },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
-      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
-      "license": "MIT",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3308,17 +3422,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-controllable-state": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
-      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-context": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz",
+      "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==",
       "dependencies": {
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3326,50 +3439,43 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-escape-keydown": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.0.tgz",
-      "integrity": "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-dismissable-layer": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
+      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
       "dependencies": {
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/primitive": "1.0.1",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-escape-keydown": "1.0.3"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-layout-effect": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
-      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
-      "license": "MIT",
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
+        },
+        "@types/react-dom": {
           "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-rect": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.0.tgz",
-      "integrity": "sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
       "dependencies": {
-        "@radix-ui/rect": "1.1.0"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3377,17 +3483,17 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-use-size": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.0.tgz",
-      "integrity": "sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-escape-keydown": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
+      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
       "dependencies": {
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3395,53 +3501,34 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/react-visually-hidden": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.1.0.tgz",
-      "integrity": "sha512-N8MDZqtgCgG5S3aV60INAB475osJousYpZ4cTJ2cFbMpdHS5Y6loLTH8LPtkj2QN0x93J30HT/M3qJXM0+lyeQ==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-id": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
+      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
       "dependencies": {
-        "@radix-ui/react-primitive": "2.0.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-select/node_modules/@radix-ui/rect": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.0.tgz",
-      "integrity": "sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg==",
-      "license": "MIT"
-    },
-    "node_modules/@radix-ui/react-select/node_modules/react-remove-scroll": {
-      "version": "2.6.0",
-      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz",
-      "integrity": "sha512-I2U4JVEsQenxDAKaVa3VZ/JeJZe0/2DxPWL8Tj8yLKctQJQiZM52pn/GWFpSp8dftjM3pSAHVJZscAnC/y+ySQ==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-id/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
       "dependencies": {
-        "react-remove-scroll-bar": "^2.3.6",
-        "react-style-singleton": "^2.2.1",
-        "tslib": "^2.1.0",
-        "use-callback-ref": "^1.3.0",
-        "use-sidecar": "^1.1.2"
-      },
-      "engines": {
-        "node": ">=10"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
-        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+        "@types/react": "*",
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3449,19 +3536,28 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-separator": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.0.tgz",
-      "integrity": "sha512-3uBAs+egzvJBDZAzvb/n4NxxOYpnspmWxO2u5NbZ8Y6FM/NdrGSF9bop3Cf6F6C71z1rTSn8KV0Fo2ZVd79lGA==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper": {
+      "version": "1.1.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
+      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
       "dependencies": {
-        "@radix-ui/react-primitive": "2.0.0"
+        "@babel/runtime": "^7.13.10",
+        "@floating-ui/react-dom": "^2.0.0",
+        "@radix-ui/react-arrow": "1.0.3",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-context": "1.0.1",
+        "@radix-ui/react-primitive": "1.0.3",
+        "@radix-ui/react-use-callback-ref": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1",
+        "@radix-ui/react-use-rect": "1.0.1",
+        "@radix-ui/react-use-size": "1.0.1",
+        "@radix-ui/rect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3472,19 +3568,19 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-separator/node_modules/@radix-ui/react-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
-      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-arrow": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.0.3.tgz",
+      "integrity": "sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==",
       "dependencies": {
-        "@radix-ui/react-slot": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3495,17 +3591,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-slot": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.1.0.tgz",
-      "integrity": "sha512-FUCf5XMfmW4dtYl69pdS4DbxKy8nj4M7SafBgPllysxmdachynNflAdp/gCsnYWNDnge6tI9onzMp5ARYc1KNw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
       "dependencies": {
-        "@radix-ui/react-compose-refs": "1.1.0"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3513,14 +3608,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-slot/node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
-      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3528,50 +3625,35 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.1.tgz",
-      "integrity": "sha512-3GBUDmP2DvzmtYLMsHmpA1GtR46ZDZ+OreXM/N+kkQJOPIgytFWWTfDQmBQKBvaFS0Vno0FktdbVzN28KGrMdw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-rect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz",
+      "integrity": "sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==",
       "dependencies": {
-        "@radix-ui/primitive": "1.1.0",
-        "@radix-ui/react-context": "1.1.1",
-        "@radix-ui/react-direction": "1.1.0",
-        "@radix-ui/react-id": "1.1.0",
-        "@radix-ui/react-presence": "1.1.1",
-        "@radix-ui/react-primitive": "2.0.0",
-        "@radix-ui/react-roving-focus": "1.1.0",
-        "@radix-ui/react-use-controllable-state": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/rect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/primitive": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
-      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
-      "license": "MIT"
-    },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.0.tgz",
-      "integrity": "sha512-b4inOtiaOnYf9KWyO3jAeeCG6FeyfY6ldiEPanbUjWd+xIk5wZeHa8yVwmrJ2vderhu/BQvzCrJI0lHd+wIiqw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-size": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.0.1.tgz",
+      "integrity": "sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3579,76 +3661,83 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-context": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
-      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-portal": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
+      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-primitive": "1.0.3"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-id": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.0.tgz",
-      "integrity": "sha512-EJUrI8yYh7WOjNOqpoJaf1jlFIH2LvtgAl+YcFqNCa+4hj64ZXmPkAKOFs/ukjz3byN6bdb/AVUqHkI8/uWWMA==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-presence": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
+      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
       "dependencies": {
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1",
+        "@radix-ui/react-use-layout-effect": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "@types/react-dom": "*",
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
+        },
+        "@types/react-dom": {
+          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-presence": {
-      "version": "1.1.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.1.tgz",
-      "integrity": "sha512-IeFXVi4YS1K0wVZzXNrbaaUvIJ3qdY+/Ih4eHFhWA9SwGR9UDX7Ck8abvL57C4cv3wwMvUE0OG69Qc3NCcTe/A==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-presence/node_modules/@radix-ui/react-use-layout-effect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
+      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
       "dependencies": {
-        "@radix-ui/react-compose-refs": "1.1.0",
-        "@radix-ui/react-use-layout-effect": "1.1.0"
+        "@babel/runtime": "^7.13.10"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-primitive": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.0.0.tgz",
-      "integrity": "sha512-ZSpFm0/uHa8zTvKBDjLFWLo8dkr4MBsiDLz0g3gMUwqgLHz9rTaRRGYDgvZPtBJgYCBKXkS9fzmoySgr8CO6Cw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-primitive": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
+      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
       "dependencies": {
-        "@radix-ui/react-slot": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-slot": "1.0.2"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
-        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0",
+        "react-dom": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3659,14 +3748,18 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
-      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-slot": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
+      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
       "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-compose-refs": "1.0.1"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3674,17 +3767,17 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-use-controllable-state": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
-      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-use-controllable-state": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
+      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
       "dependencies": {
-        "@radix-ui/react-use-callback-ref": "1.1.0"
+        "@babel/runtime": "^7.13.10",
+        "@radix-ui/react-use-callback-ref": "1.0.1"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3692,14 +3785,16 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tabs/node_modules/@radix-ui/react-use-layout-effect": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
-      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-use-controllable-state/node_modules/@radix-ui/react-use-callback-ref": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
+      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
+      "dependencies": {
+        "@babel/runtime": "^7.13.10"
+      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
+        "react": "^16.8 || ^17.0 || ^18.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3707,24 +3802,13 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tooltip": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz",
-      "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-visually-hidden": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.0.3.tgz",
+      "integrity": "sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==",
       "dependencies": {
         "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-dismissable-layer": "1.0.5",
-        "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-popper": "1.1.3",
-        "@radix-ui/react-portal": "1.0.4",
-        "@radix-ui/react-presence": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-slot": "1.0.2",
-        "@radix-ui/react-use-controllable-state": "1.0.1",
-        "@radix-ui/react-visually-hidden": "1.0.3"
+        "@radix-ui/react-primitive": "1.0.3"
       },
       "peerDependencies": {
         "@types/react": "*",
@@ -3741,35 +3825,21 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-slot": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
-      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
-      "license": "MIT",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/rect": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.0.1.tgz",
+      "integrity": "sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
+        "@babel/runtime": "^7.13.10"
       }
     },
     "node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
-      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
+      "integrity": "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3778,16 +3848,15 @@
       }
     },
     "node_modules/@radix-ui/react-use-controllable-state": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
-      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.1.0.tgz",
+      "integrity": "sha512-MtfMVJiSr2NjzS0Aa90NPTnvTSg6C/JLCV7ma0W6+OMV78vd8OyRpID+Ng9LxzsPbLeuBnWBA1Nq30AtBIDChw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
+        "@radix-ui/react-use-callback-ref": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3796,16 +3865,15 @@
       }
     },
     "node_modules/@radix-ui/react-use-escape-keydown": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
-      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.0.tgz",
+      "integrity": "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
+        "@radix-ui/react-use-callback-ref": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3814,15 +3882,12 @@
       }
     },
     "node_modules/@radix-ui/react-use-layout-effect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
-      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.0.tgz",
+      "integrity": "sha512-+FPE0rOdziWSrH9athwI1R0HDVbWlEhd+FR+aSDk4uWGmSJ9Z54sdZVDQPZAinJhJXwfT+qnj969mCsT2gfm5w==",
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3846,16 +3911,15 @@
       }
     },
     "node_modules/@radix-ui/react-use-rect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz",
-      "integrity": "sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.0.tgz",
+      "integrity": "sha512-0Fmkebhr6PiseyZlYAOtLS+nb7jLmpqTrJyv61Pe68MKYW6OWdRE2kI70TaYY27u7H0lajqM3hSMMLFq18Z7nQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/rect": "1.0.1"
+        "@radix-ui/rect": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3864,16 +3928,15 @@
       }
     },
     "node_modules/@radix-ui/react-use-size": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.0.1.tgz",
-      "integrity": "sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.0.tgz",
+      "integrity": "sha512-XW3/vWuIXHa+2Uwcc2ABSfcCledmXhhQPlGbfcRXbiUQI5Icjcg19BGCZVKKInYbvUCut/ufbbLLPFC5cbb1hw==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "@radix-ui/react-use-layout-effect": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3882,18 +3945,17 @@
       }
     },
     "node_modules/@radix-ui/react-visually-hidden": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.0.3.tgz",
-      "integrity": "sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==",
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.1.0.tgz",
+      "integrity": "sha512-N8MDZqtgCgG5S3aV60INAB475osJousYpZ4cTJ2cFbMpdHS5Y6loLTH8LPtkj2QN0x93J30HT/M3qJXM0+lyeQ==",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
+        "@radix-ui/react-primitive": "2.0.0"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3905,12 +3967,9 @@
       }
     },
     "node_modules/@radix-ui/rect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.0.1.tgz",
-      "integrity": "sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      }
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.0.tgz",
+      "integrity": "sha512-A9+lCBZoaMJlVKcRBz2YByCG+Cp2t6nAnMnNba+XiWxnj6r4JUFqfsgwocMBZU9LPtdxC6wB56ySYpc7LQIoJg=="
     },
     "node_modules/@rollup/plugin-commonjs": {
       "version": "26.0.1",
@@ -4221,102 +4280,6 @@
         "node": ">=10"
       }
     },
-    "node_modules/@sentry/cli-linux-arm": {
-      "version": "2.37.0",
-      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm/-/cli-linux-arm-2.37.0.tgz",
-      "integrity": "sha512-Dz0qH4Yt+gGUgoVsqVt72oDj4VQynRF1QB1/Sr8g76Vbi+WxWZmUh0iFwivYVwWxdQGu/OQrE0tx946HToCRyA==",
-      "cpu": [
-        "arm"
-      ],
-      "optional": true,
-      "os": [
-        "linux",
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/@sentry/cli-linux-arm64": {
-      "version": "2.37.0",
-      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-arm64/-/cli-linux-arm64-2.37.0.tgz",
-      "integrity": "sha512-2vzUWHLZ3Ct5gpcIlfd/2Qsha+y9M8LXvbZE26VxzYrIkRoLAWcnClBv8m4XsHLMURYvz3J9QSZHMZHSO7kAzw==",
-      "cpu": [
-        "arm64"
-      ],
-      "optional": true,
-      "os": [
-        "linux",
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/@sentry/cli-linux-i686": {
-      "version": "2.37.0",
-      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-i686/-/cli-linux-i686-2.37.0.tgz",
-      "integrity": "sha512-MHRLGs4t/CQE1pG+mZBQixyWL6xDZfNalCjO8GMcTTbZFm44S3XRHfYJZNVCgdtnUP7b6OHGcu1v3SWE10LcwQ==",
-      "cpu": [
-        "x86",
-        "ia32"
-      ],
-      "optional": true,
-      "os": [
-        "linux",
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/@sentry/cli-linux-x64": {
-      "version": "2.37.0",
-      "resolved": "https://registry.npmjs.org/@sentry/cli-linux-x64/-/cli-linux-x64-2.37.0.tgz",
-      "integrity": "sha512-k76ClefKZaDNJZU/H3mGeR8uAzAGPzDRG/A7grzKfBeyhP3JW09L7Nz9IQcSjCK+xr399qLhM2HFCaPWQ6dlMw==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "linux",
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/@sentry/cli-win32-i686": {
-      "version": "2.37.0",
-      "resolved": "https://registry.npmjs.org/@sentry/cli-win32-i686/-/cli-win32-i686-2.37.0.tgz",
-      "integrity": "sha512-FFyi5RNYQQkEg4GkP2f3BJcgQn0F4fjFDMiWkjCkftNPXQG+HFUEtrGsWr6mnHPdFouwbYg3tEPUWNxAoypvTw==",
-      "cpu": [
-        "x86",
-        "ia32"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=10"
-      }
-    },
-    "node_modules/@sentry/cli-win32-x64": {
-      "version": "2.37.0",
-      "resolved": "https://registry.npmjs.org/@sentry/cli-win32-x64/-/cli-win32-x64-2.37.0.tgz",
-      "integrity": "sha512-nSMj4OcfQmyL+Tu/jWCJwhKCXFsCZW1MUk6wjjQlRt9SDLfgeapaMlK1ZvT1eZv5ZH6bj3qJfefwj4U8160uOA==",
-      "cpu": [
-        "x64"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/@sentry/core": {
       "version": "8.34.0",
       "resolved": "https://registry.npmjs.org/@sentry/core/-/core-8.34.0.tgz",
@@ -4364,6 +4327,24 @@
         }
       }
     },
+    "node_modules/@sentry/nextjs/node_modules/@sentry/react": {
+      "version": "8.34.0",
+      "resolved": "https://registry.npmjs.org/@sentry/react/-/react-8.34.0.tgz",
+      "integrity": "sha512-gIgzhj7h67C+Sdq2ul4fOSK142Gf0uV99bqHRdtIiUlXw9yjzZQY5TKTtzbOaevn7qBJ0xrRKtIRUbOBMl0clw==",
+      "dependencies": {
+        "@sentry/browser": "8.34.0",
+        "@sentry/core": "8.34.0",
+        "@sentry/types": "8.34.0",
+        "@sentry/utils": "8.34.0",
+        "hoist-non-react-statics": "^3.3.2"
+      },
+      "engines": {
+        "node": ">=14.18"
+      },
+      "peerDependencies": {
+        "react": "^16.14.0 || 17.x || 18.x || 19.x"
+      }
+    },
     "node_modules/@sentry/nextjs/node_modules/chalk": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz",
@@ -4441,24 +4422,6 @@
         "@opentelemetry/semantic-conventions": "^1.27.0"
       }
     },
-    "node_modules/@sentry/react": {
-      "version": "8.34.0",
-      "resolved": "https://registry.npmjs.org/@sentry/react/-/react-8.34.0.tgz",
-      "integrity": "sha512-gIgzhj7h67C+Sdq2ul4fOSK142Gf0uV99bqHRdtIiUlXw9yjzZQY5TKTtzbOaevn7qBJ0xrRKtIRUbOBMl0clw==",
-      "dependencies": {
-        "@sentry/browser": "8.34.0",
-        "@sentry/core": "8.34.0",
-        "@sentry/types": "8.34.0",
-        "@sentry/utils": "8.34.0",
-        "hoist-non-react-statics": "^3.3.2"
-      },
-      "engines": {
-        "node": ">=14.18"
-      },
-      "peerDependencies": {
-        "react": "^16.14.0 || 17.x || 18.x || 19.x"
-      }
-    },
     "node_modules/@sentry/types": {
       "version": "8.34.0",
       "resolved": "https://registry.npmjs.org/@sentry/types/-/types-8.34.0.tgz",
@@ -4521,11 +4484,10 @@
       "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="
     },
     "node_modules/@swc/helpers": {
-      "version": "0.5.5",
-      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz",
-      "integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==",
+      "version": "0.5.13",
+      "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.13.tgz",
+      "integrity": "sha512-UoKGxQ3r5kYI9dALKJapMmuK+1zWM/H17Z1+iwnNmzcJRnfFuevZs375TA5rW31pu4BS4NoSy1fRsexDXfWn5w==",
       "dependencies": {
-        "@swc/counter": "^0.1.3",
         "tslib": "^2.4.0"
       }
     },
@@ -4544,6 +4506,15 @@
         "tailwindcss": ">=3.0.0 || insiders"
       }
     },
+    "node_modules/@tanstack/virtual-core": {
+      "version": "3.10.8",
+      "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.10.8.tgz",
+      "integrity": "sha512-PBu00mtt95jbKFi6Llk9aik8bnR3tR/oQP1o3TSi+iG//+Q2RTIzCEgKkHG8BB86kxMNW6O8wku+Lmi+QFR6jA==",
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      }
+    },
     "node_modules/@types/connect": {
       "version": "3.4.36",
       "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz",
@@ -5459,21 +5430,6 @@
         "npm": ">=6"
       }
     },
-    "node_modules/babel-plugin-styled-components": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/babel-plugin-styled-components/-/babel-plugin-styled-components-2.1.4.tgz",
-      "integrity": "sha512-Xgp9g+A/cG47sUyRwwYxGM4bR/jDRg5N6it/8+HxCnbT5XNKSKDT9xm4oag/osgqjC2It/vH0yXsomOG6k558g==",
-      "dependencies": {
-        "@babel/helper-annotate-as-pure": "^7.22.5",
-        "@babel/helper-module-imports": "^7.22.5",
-        "@babel/plugin-syntax-jsx": "^7.22.5",
-        "lodash": "^4.17.21",
-        "picomatch": "^2.3.1"
-      },
-      "peerDependencies": {
-        "styled-components": ">= 2"
-      }
-    },
     "node_modules/bail": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz",
@@ -9322,11 +9278,11 @@
       ]
     },
     "node_modules/micromatch": {
-      "version": "4.0.5",
-      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz",
-      "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==",
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz",
+      "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==",
       "dependencies": {
-        "braces": "^3.0.2",
+        "braces": "^3.0.3",
         "picomatch": "^2.3.1"
       },
       "engines": {
@@ -9433,40 +9389,41 @@
       "peer": true
     },
     "node_modules/next": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz",
-      "integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==",
+      "version": "15.0.2",
+      "resolved": "https://registry.npmjs.org/next/-/next-15.0.2.tgz",
+      "integrity": "sha512-rxIWHcAu4gGSDmwsELXacqAPUk+j8dV/A9cDF5fsiCMpkBDYkO2AEaL1dfD+nNmDiU6QMCFN8Q30VEKapT9UHQ==",
       "dependencies": {
-        "@next/env": "14.2.3",
-        "@swc/helpers": "0.5.5",
+        "@next/env": "15.0.2",
+        "@swc/counter": "0.1.3",
+        "@swc/helpers": "0.5.13",
         "busboy": "1.6.0",
         "caniuse-lite": "^1.0.30001579",
-        "graceful-fs": "^4.2.11",
         "postcss": "8.4.31",
-        "styled-jsx": "5.1.1"
+        "styled-jsx": "5.1.6"
       },
       "bin": {
         "next": "dist/bin/next"
       },
       "engines": {
-        "node": ">=18.17.0"
+        "node": ">=18.18.0"
       },
       "optionalDependencies": {
-        "@next/swc-darwin-arm64": "14.2.3",
-        "@next/swc-darwin-x64": "14.2.3",
-        "@next/swc-linux-arm64-gnu": "14.2.3",
-        "@next/swc-linux-arm64-musl": "14.2.3",
-        "@next/swc-linux-x64-gnu": "14.2.3",
-        "@next/swc-linux-x64-musl": "14.2.3",
-        "@next/swc-win32-arm64-msvc": "14.2.3",
-        "@next/swc-win32-ia32-msvc": "14.2.3",
-        "@next/swc-win32-x64-msvc": "14.2.3"
+        "@next/swc-darwin-arm64": "15.0.2",
+        "@next/swc-darwin-x64": "15.0.2",
+        "@next/swc-linux-arm64-gnu": "15.0.2",
+        "@next/swc-linux-arm64-musl": "15.0.2",
+        "@next/swc-linux-x64-gnu": "15.0.2",
+        "@next/swc-linux-x64-musl": "15.0.2",
+        "@next/swc-win32-arm64-msvc": "15.0.2",
+        "@next/swc-win32-x64-msvc": "15.0.2",
+        "sharp": "^0.33.5"
       },
       "peerDependencies": {
         "@opentelemetry/api": "^1.1.0",
         "@playwright/test": "^1.41.2",
-        "react": "^18.2.0",
-        "react-dom": "^18.2.0",
+        "babel-plugin-react-compiler": "*",
+        "react": "^18.2.0 || 19.0.0-rc-02c0e824-20241028",
+        "react-dom": "^18.2.0 || 19.0.0-rc-02c0e824-20241028",
         "sass": "^1.3.0"
       },
       "peerDependenciesMeta": {
@@ -9476,6 +9433,9 @@
         "@playwright/test": {
           "optional": true
         },
+        "babel-plugin-react-compiler": {
+          "optional": true
+        },
         "sass": {
           "optional": true
         }
@@ -12713,33 +12673,91 @@
       "resolved": "https://registry.npmjs.org/react-icons/-/react-icons-4.12.0.tgz",
       "integrity": "sha512-IBaDuHiShdZqmfc/TwHu6+d6k2ltNCf3AszxNmjJc1KUfXdEeRJOKyNvLmAHaarhzGmTSVygNdyu8/opXv2gaw==",
       "peerDependencies": {
-        "react": "*"
+        "react": "*"
+      }
+    },
+    "node_modules/react-is": {
+      "version": "19.0.0-rc-fb9a90fa48-20240614",
+      "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.0.0-rc-fb9a90fa48-20240614.tgz",
+      "integrity": "sha512-60qI7v1B9RhmZwjTCnAgzcuABOQsIH20vTbETQPaze96s1lY2lSawv9dvXAfF8Z1MIqOppWSKLNOshF0WsZ3OA=="
+    },
+    "node_modules/react-loader-spinner": {
+      "version": "5.4.5",
+      "resolved": "https://registry.npmjs.org/react-loader-spinner/-/react-loader-spinner-5.4.5.tgz",
+      "integrity": "sha512-32f+sb/v2tnNfyvnCCOS4fpyVHsGXjSyNo6QLniHcaj1XjKLxx14L2z0h6szRugOL8IEJ+53GPwNAdbkDqmy4g==",
+      "dependencies": {
+        "react-is": "^18.2.0",
+        "styled-components": "^5.3.5",
+        "styled-tools": "^1.7.2"
+      },
+      "peerDependencies": {
+        "react": "^16.0.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
+    "node_modules/react-loader-spinner/node_modules/has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/react-loader-spinner/node_modules/styled-components": {
+      "version": "5.3.11",
+      "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-5.3.11.tgz",
+      "integrity": "sha512-uuzIIfnVkagcVHv9nE0VPlHPSCmXIUGKfJ42LNjxCCTDTL5sgnJ8Z7GZBq0EnLYGln77tPpEpExt2+qa+cZqSw==",
+      "dependencies": {
+        "@babel/helper-module-imports": "^7.0.0",
+        "@babel/traverse": "^7.4.5",
+        "@emotion/is-prop-valid": "^1.1.0",
+        "@emotion/stylis": "^0.8.4",
+        "@emotion/unitless": "^0.7.4",
+        "babel-plugin-styled-components": ">= 1.12.0",
+        "css-to-react-native": "^3.0.0",
+        "hoist-non-react-statics": "^3.0.0",
+        "shallowequal": "^1.1.0",
+        "supports-color": "^5.5.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/styled-components"
+      },
+      "peerDependencies": {
+        "react": ">= 16.8.0",
+        "react-dom": ">= 16.8.0",
+        "react-is": ">= 16.8.0"
+      }
+    },
+    "node_modules/react-loader-spinner/node_modules/styled-components/node_modules/babel-plugin-styled-components": {
+      "version": "2.1.4",
+      "resolved": "https://registry.npmjs.org/babel-plugin-styled-components/-/babel-plugin-styled-components-2.1.4.tgz",
+      "integrity": "sha512-Xgp9g+A/cG47sUyRwwYxGM4bR/jDRg5N6it/8+HxCnbT5XNKSKDT9xm4oag/osgqjC2It/vH0yXsomOG6k558g==",
+      "dependencies": {
+        "@babel/helper-annotate-as-pure": "^7.22.5",
+        "@babel/helper-module-imports": "^7.22.5",
+        "@babel/plugin-syntax-jsx": "^7.22.5",
+        "lodash": "^4.17.21",
+        "picomatch": "^2.3.1"
+      },
+      "peerDependencies": {
+        "styled-components": ">= 2"
       }
     },
-    "node_modules/react-is": {
-      "version": "16.13.1",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
-      "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="
-    },
-    "node_modules/react-loader-spinner": {
-      "version": "5.4.5",
-      "resolved": "https://registry.npmjs.org/react-loader-spinner/-/react-loader-spinner-5.4.5.tgz",
-      "integrity": "sha512-32f+sb/v2tnNfyvnCCOS4fpyVHsGXjSyNo6QLniHcaj1XjKLxx14L2z0h6szRugOL8IEJ+53GPwNAdbkDqmy4g==",
+    "node_modules/react-loader-spinner/node_modules/supports-color": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
       "dependencies": {
-        "react-is": "^18.2.0",
-        "styled-components": "^5.3.5",
-        "styled-tools": "^1.7.2"
+        "has-flag": "^3.0.0"
       },
-      "peerDependencies": {
-        "react": "^16.0.0 || ^17.0.0 || ^18.0.0",
-        "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0"
+      "engines": {
+        "node": ">=4"
       }
     },
-    "node_modules/react-loader-spinner/node_modules/react-is": {
-      "version": "18.3.1",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
-      "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="
-    },
     "node_modules/react-markdown": {
       "version": "9.0.1",
       "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-9.0.1.tgz",
@@ -12765,51 +12783,6 @@
         "react": ">=18"
       }
     },
-    "node_modules/react-remove-scroll": {
-      "version": "2.5.5",
-      "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.5.5.tgz",
-      "integrity": "sha512-ImKhrzJJsyXJfBZ4bzu8Bwpka14c/fQt0k+cyFp/PBhTfyDnU5hjOtM4AG/0AMyy8oKzOTR0lDgJIM7pYXI0kw==",
-      "dependencies": {
-        "react-remove-scroll-bar": "^2.3.3",
-        "react-style-singleton": "^2.2.1",
-        "tslib": "^2.1.0",
-        "use-callback-ref": "^1.3.0",
-        "use-sidecar": "^1.1.2"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "peerDependencies": {
-        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/react-remove-scroll-bar": {
-      "version": "2.3.6",
-      "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz",
-      "integrity": "sha512-DtSYaao4mBmX+HDo5YWYdBWQwYIQQshUV/dVxFxK+KM26Wjwp1gZ6rv6OC3oujI6Bfu6Xyg3TwK533AQutsn/g==",
-      "dependencies": {
-        "react-style-singleton": "^2.2.1",
-        "tslib": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "peerDependencies": {
-        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
     "node_modules/react-select": {
       "version": "5.8.0",
       "resolved": "https://registry.npmjs.org/react-select/-/react-select-5.8.0.tgz",
@@ -12830,35 +12803,22 @@
         "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
       }
     },
-    "node_modules/react-smooth": {
-      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.1.tgz",
-      "integrity": "sha512-OE4hm7XqR0jNOq3Qmk9mFLyd6p2+j6bvbPJ7qlB7+oo0eNcL2l7WQzG6MBnT3EXY6xzkLMUBec3AfewJdA0J8w==",
-      "dependencies": {
-        "fast-equals": "^5.0.1",
-        "prop-types": "^15.8.1",
-        "react-transition-group": "^4.4.5"
-      },
-      "peerDependencies": {
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      }
-    },
-    "node_modules/react-style-singleton": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.1.tgz",
-      "integrity": "sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g==",
+    "node_modules/react-select/node_modules/@emotion/react": {
+      "version": "11.13.3",
+      "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.13.3.tgz",
+      "integrity": "sha512-lIsdU6JNrmYfJ5EbUCf4xW1ovy5wKQ2CkPRM4xogziOxH1nXxBSjpC9YqbFAP7circxMfYp+6x676BqWcEiixg==",
       "dependencies": {
-        "get-nonce": "^1.0.0",
-        "invariant": "^2.2.4",
-        "tslib": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
+        "@babel/runtime": "^7.18.3",
+        "@emotion/babel-plugin": "^11.12.0",
+        "@emotion/cache": "^11.13.0",
+        "@emotion/serialize": "^1.3.1",
+        "@emotion/use-insertion-effect-with-fallbacks": "^1.1.0",
+        "@emotion/utils": "^1.4.0",
+        "@emotion/weak-memoize": "^0.4.0",
+        "hoist-non-react-statics": "^3.3.1"
       },
       "peerDependencies": {
-        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+        "react": ">=16.8.0"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -12866,7 +12826,15 @@
         }
       }
     },
-    "node_modules/react-transition-group": {
+    "node_modules/react-select/node_modules/@emotion/react/node_modules/@emotion/use-insertion-effect-with-fallbacks": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.1.0.tgz",
+      "integrity": "sha512-+wBOcIV5snwGgI2ya3u99D7/FJquOIniQT1IKyDsBmEgwvpxMNeS65Oib7OnE2d2aY+3BU4OiH+0Wchf8yk3Hw==",
+      "peerDependencies": {
+        "react": ">=16.8.0"
+      }
+    },
+    "node_modules/react-select/node_modules/react-transition-group": {
       "version": "4.4.5",
       "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
       "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
@@ -12881,6 +12849,19 @@
         "react-dom": ">=16.6.0"
       }
     },
+    "node_modules/react-select/node_modules/use-isomorphic-layout-effect": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz",
+      "integrity": "sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==",
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "@types/react": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
@@ -12931,11 +12912,34 @@
         "decimal.js-light": "^2.4.1"
       }
     },
-    "node_modules/recharts/node_modules/react-is": {
-      "version": "18.3.1",
-      "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
-      "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==",
-      "license": "MIT"
+    "node_modules/recharts/node_modules/react-smooth": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/react-smooth/-/react-smooth-4.0.1.tgz",
+      "integrity": "sha512-OE4hm7XqR0jNOq3Qmk9mFLyd6p2+j6bvbPJ7qlB7+oo0eNcL2l7WQzG6MBnT3EXY6xzkLMUBec3AfewJdA0J8w==",
+      "dependencies": {
+        "fast-equals": "^5.0.1",
+        "prop-types": "^15.8.1",
+        "react-transition-group": "^4.4.5"
+      },
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0",
+        "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
+    "node_modules/recharts/node_modules/react-smooth/node_modules/react-transition-group": {
+      "version": "4.4.5",
+      "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz",
+      "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==",
+      "dependencies": {
+        "@babel/runtime": "^7.5.5",
+        "dom-helpers": "^5.0.1",
+        "loose-envify": "^1.4.0",
+        "prop-types": "^15.6.2"
+      },
+      "peerDependencies": {
+        "react": ">=16.6.0",
+        "react-dom": ">=16.6.0"
+      }
     },
     "node_modules/reflect.getprototypeof": {
       "version": "1.0.6",
@@ -13752,58 +13756,10 @@
         "inline-style-parser": "0.2.3"
       }
     },
-    "node_modules/styled-components": {
-      "version": "5.3.11",
-      "resolved": "https://registry.npmjs.org/styled-components/-/styled-components-5.3.11.tgz",
-      "integrity": "sha512-uuzIIfnVkagcVHv9nE0VPlHPSCmXIUGKfJ42LNjxCCTDTL5sgnJ8Z7GZBq0EnLYGln77tPpEpExt2+qa+cZqSw==",
-      "dependencies": {
-        "@babel/helper-module-imports": "^7.0.0",
-        "@babel/traverse": "^7.4.5",
-        "@emotion/is-prop-valid": "^1.1.0",
-        "@emotion/stylis": "^0.8.4",
-        "@emotion/unitless": "^0.7.4",
-        "babel-plugin-styled-components": ">= 1.12.0",
-        "css-to-react-native": "^3.0.0",
-        "hoist-non-react-statics": "^3.0.0",
-        "shallowequal": "^1.1.0",
-        "supports-color": "^5.5.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "funding": {
-        "type": "opencollective",
-        "url": "https://opencollective.com/styled-components"
-      },
-      "peerDependencies": {
-        "react": ">= 16.8.0",
-        "react-dom": ">= 16.8.0",
-        "react-is": ">= 16.8.0"
-      }
-    },
-    "node_modules/styled-components/node_modules/has-flag": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
-      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
-      "engines": {
-        "node": ">=4"
-      }
-    },
-    "node_modules/styled-components/node_modules/supports-color": {
-      "version": "5.5.0",
-      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
-      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
-      "dependencies": {
-        "has-flag": "^3.0.0"
-      },
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/styled-jsx": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz",
-      "integrity": "sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==",
+      "version": "5.1.6",
+      "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz",
+      "integrity": "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==",
       "dependencies": {
         "client-only": "0.0.1"
       },
@@ -13811,7 +13767,7 @@
         "node": ">= 12.0.0"
       },
       "peerDependencies": {
-        "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0"
+        "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0"
       },
       "peerDependenciesMeta": {
         "@babel/core": {
@@ -13887,6 +13843,19 @@
         "react": "^16.11.0 || ^17.0.0 || ^18.0.0"
       }
     },
+    "node_modules/swr/node_modules/use-sync-external-store": {
+      "version": "1.2.2",
+      "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.2.tgz",
+      "integrity": "sha512-PElTlVMwpblvbNqQ82d2n6RjStvdSoNe9FG28kNfz3WiXilJm4DdNkEzRhCZuIDwY8U08WVihhGR5iRqAwfDiw==",
+      "peerDependencies": {
+        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
+      }
+    },
+    "node_modules/tabbable": {
+      "version": "6.2.0",
+      "resolved": "https://registry.npmjs.org/tabbable/-/tabbable-6.2.0.tgz",
+      "integrity": "sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew=="
+    },
     "node_modules/tailwind-merge": {
       "version": "2.5.4",
       "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-2.5.4.tgz",
@@ -14060,14 +14029,6 @@
       "resolved": "https://registry.npmjs.org/tiny-warning/-/tiny-warning-1.0.3.tgz",
       "integrity": "sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA=="
     },
-    "node_modules/to-fast-properties": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
-      "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==",
-      "engines": {
-        "node": ">=4"
-      }
-    },
     "node_modules/to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -14429,68 +14390,6 @@
         "punycode": "^2.1.0"
       }
     },
-    "node_modules/use-callback-ref": {
-      "version": "1.3.2",
-      "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.2.tgz",
-      "integrity": "sha512-elOQwe6Q8gqZgDA8mrh44qRTQqpIHDcZ3hXTLjBe1i4ph8XpNJnO+aQf3NaG+lriLopI4HMx9VjQLfPQ6vhnoA==",
-      "dependencies": {
-        "tslib": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "peerDependencies": {
-        "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0",
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/use-isomorphic-layout-effect": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/use-isomorphic-layout-effect/-/use-isomorphic-layout-effect-1.1.2.tgz",
-      "integrity": "sha512-49L8yCO3iGT/ZF9QttjwLF/ZD9Iwto5LnH5LmEdk/6cFmXddqi2ulF0edxTwjj+7mqvpVVGQWvbXZdn32wRSHA==",
-      "peerDependencies": {
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/use-sidecar": {
-      "version": "1.1.2",
-      "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.2.tgz",
-      "integrity": "sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw==",
-      "dependencies": {
-        "detect-node-es": "^1.1.0",
-        "tslib": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=10"
-      },
-      "peerDependencies": {
-        "@types/react": "^16.9.0 || ^17.0.0 || ^18.0.0",
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/use-sync-external-store": {
-      "version": "1.2.2",
-      "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.2.tgz",
-      "integrity": "sha512-PElTlVMwpblvbNqQ82d2n6RjStvdSoNe9FG28kNfz3WiXilJm4DdNkEzRhCZuIDwY8U08WVihhGR5iRqAwfDiw==",
-      "peerDependencies": {
-        "react": "^16.8.0 || ^17.0.0 || ^18.0.0"
-      }
-    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
diff --git a/web/package.json b/web/package.json
index ea6201642a8..6fb1a3b2903 100644
--- a/web/package.json
+++ b/web/package.json
@@ -3,7 +3,7 @@
   "version": "0.2-dev",
   "private": true,
   "scripts": {
-    "dev": "next dev",
+    "dev": "next dev --turbopack",
     "build": "next build",
     "start": "next start",
     "lint": "next lint"
@@ -12,6 +12,8 @@
     "@dnd-kit/core": "^6.1.0",
     "@dnd-kit/modifiers": "^7.0.0",
     "@dnd-kit/sortable": "^8.0.0",
+    "@dnd-kit/utilities": "^3.2.2",
+    "@headlessui/react": "^2.2.0",
     "@headlessui/tailwindcss": "^0.2.1",
     "@phosphor-icons/react": "^2.0.8",
     "@radix-ui/react-dialog": "^1.0.5",
@@ -39,7 +41,7 @@
     "lodash": "^4.17.21",
     "lucide-react": "^0.454.0",
     "mdast-util-find-and-replace": "^3.0.1",
-    "next": "^14.2.3",
+    "next": "^15.0.2",
     "npm": "^10.8.0",
     "postcss": "^8.4.31",
     "posthog-js": "^1.176.0",
@@ -71,5 +73,8 @@
     "eslint": "^8.48.0",
     "eslint-config-next": "^14.1.0",
     "prettier": "2.8.8"
+  },
+  "overrides": {
+    "react-is": "^19.0.0-rc-69d4b800-20241021"
   }
 }
diff --git a/web/public/Wikipedia.svg b/web/public/Wikipedia.svg
index dc32f9848f4..ee4a3caa55f 100644
--- a/web/public/Wikipedia.svg
+++ b/web/public/Wikipedia.svg
@@ -1 +1,535 @@
-<?xml version="1.0" encoding="utf-8"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd" [<!ENTITY st0 "opacity:.22;fill:#231F20;"><!ENTITY st1 "opacity:.34;fill:url(#SVGID_25_);"><!ENTITY st2 "opacity:.19;fill:#231F20;"><!ENTITY st3 "opacity:.18;fill:url(#SVGID_42_);"><!ENTITY st4 "opacity:.73;fill:#FFF;"><!ENTITY st5 "opacity:.35;fill:url(#SVGID_6_);"><!ENTITY st6 "opacity:.53;fill:#FFF;"><!ENTITY st7 "opacity:.43;fill:url(#SVGID_23_);"><!ENTITY st8 "opacity:.25;fill:#231F20;"><!ENTITY st9 "opacity:.53;fill:url(#SVGID_9_);"><!ENTITY st10 "opacity:.88;fill:#231F20;"><!ENTITY st11 "opacity:.31;fill:url(#SVGID_24_);"><!ENTITY st12 "fill:#232323;"><!ENTITY st13 "opacity:.68;fill:url(#SVGID_22_);"><!ENTITY st14 "opacity:.28;fill:#231F20;"><!ENTITY st15 "opacity:.38;fill:url(#SVGID_43_);"><!ENTITY st16 "fill:url(#SVGID_1_);"><!ENTITY st17 "fill:url(#SVGID_2_);"><!ENTITY st18 "fill:url(#SVGID_4_);"><!ENTITY st19 "opacity:.45;fill:url(#SVGID_5_);"><!ENTITY st20 "opacity:.17;fill:#FFF;"><!ENTITY st21 "opacity:.3;fill:#505050;"><!ENTITY st22 "fill:#616161;"><!ENTITY st23 "fill:#FFF;"><!ENTITY st24 "opacity:.2;fill:url(#SVGID_41_);"><!ENTITY st25 "opacity:.69;fill:url(#SVGID_3_);"><!ENTITY st26 "opacity:.45;fill:url(#SVGID_39_);"><!ENTITY st27 "opacity:.96;fill:url(#SVGID_10_);"><!ENTITY st28 "opacity:.3;fill:#FFF;"><!ENTITY st29 "fill:#9FA2A3;"><!ENTITY st30 "opacity:.73;fill:url(#SVGID_8_);"><!ENTITY st31 "fill:#6B6B6B;"><!ENTITY st32 "opacity:.36;fill:#231F20;"><!ENTITY st33 "opacity:.5;fill:url(#SVGID_7_);"><!ENTITY st34 "opacity:.1;fill:#232323;"><!ENTITY st35 "fill:url(#SVGID_11_);"><!ENTITY st36 "fill:url(#SVGID_12_);"><!ENTITY st37 "fill:url(#SVGID_13_);"><!ENTITY st38 "fill:url(#SVGID_14_);"><!ENTITY st39 "fill:url(#SVGID_15_);"><!ENTITY st40 "fill:url(#SVGID_16_);"><!ENTITY st41 "fill:url(#SVGID_17_);"><!ENTITY st42 "fill:url(#SVGID_18_);"><!ENTITY st43 "fill:url(#SVGID_19_);"><!ENTITY st44 "fill:url(#SVGID_20_);"><!ENTITY st45 "fill:url(#SVGID_21_);"><!ENTITY st46 "fill:url(#SVGID_26_);"><!ENTITY st47 "fill:url(#SVGID_27_);"><!ENTITY st48 "fill:url(#SVGID_28_);"><!ENTITY st49 "fill:url(#SVGID_29_);"><!ENTITY st50 "fill:url(#SVGID_30_);"><!ENTITY st51 "fill:url(#SVGID_31_);"><!ENTITY st52 "fill:url(#SVGID_33_);"><!ENTITY st53 "fill:url(#SVGID_34_);"><!ENTITY st54 "fill:url(#SVGID_35_);"><!ENTITY st55 "fill:url(#SVGID_36_);"><!ENTITY st56 "fill:url(#SVGID_37_);"><!ENTITY st57 "fill:url(#SVGID_38_);"><!ENTITY st58 "fill:url(#SVGID_40_);"><!ENTITY st59 "opacity:.1;fill:#231F20;"><!ENTITY st60 "opacity:.5;fill:#141414;"><!ENTITY st61 "opacity:.38;fill:#231F20;"><!ENTITY st62 "opacity:.83;fill:url(#SVGID_32_);">]><svg version="1.1" xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="103px" height="94px" viewBox="0 0 103 94"><title>Wikipedia logo version 2</title><radialGradient id="SVGID_1_" cx="244.5713" cy="-427.1392" r="68.6868" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#FFF"/><stop offset=".4835" stop-color="#EAEAEB"/><stop offset=".9451" stop-color="#A9ABAE"/><stop offset="1" stop-color="#999B9E"/></radialGradient><path style="&st16;" d="M49.85,17.003c0.014-0.606-0.392-1.27-0.392-1.27l-0.025-0.058c0,0-0.487-0.949-1.301-1.228c-0.815-0.278-1.478,0.342-1.478,0.342s-0.114,0.131-0.428,0.494c-0.313,0.364-0.507,0.666-1.198,0.938c-0.692,0.271-1.379,0.204-1.743,0.033c-0.365-0.172-0.457-0.537-0.457-0.537s-0.229-0.722-0.313-1.049c-0.086-0.331-0.308-1.694-0.308-1.694s-0.492-2.747-0.535-3.304c0,0,1.475-0.126,3.686-0.775c2.3-0.673,3.043-1.206,3.043-1.206s-0.432-0.156-0.484-0.662c-0.051-0.507-0.089-1.19-0.089-1.19s-0.089-0.5,0.483-1.139c0.572-0.64,1.354-0.863,1.763-0.953c0.41-0.089,1.281-0.17,2.092-0.134c0.813,0.038,1.266,0.112,1.594,0.291c0.327,0.178,0.356,0.61,0.356,0.61l-0.009,1.146c0,0-0.035,0.402,0.262,0.529c0,0,0.505,0.305,2.197,0.133c0,0,0.719-0.126,1.845-0.46c1.125-0.335,4.129-1.229,4.554-1.341c0.426-0.111,0.953-0.291,1.645-0.469c0.693-0.179,2.041-0.626,2.309-0.73c0.27-0.104,1.811-0.618,2.928-0.81c1.114-0.195,2.226-0.186,2.813,0.149c0,0,0.357,0.304,0.521,0.662c0.162,0.358,0.476,0.863,0.92,1.088c0.457,0.227,0.754,0.371,1.877,0.273c0,0,0.021-0.096-0.396-0.37c-0.417-0.277-0.991-0.701-0.991-0.701s-0.334-0.245-0.408-0.447c-0.072-0.202-0.043-0.306-0.043-0.306l0.877-1.406c0,0,0-0.172,0.506-0.238c0.506-0.067,1.074-0.134,1.742-0.313c0.67-0.178,0.789-0.312,0.789-0.312l0.578,0.178c0,0,3.547,2.853,4.655,3.583l0.198-0.239c0,0,0.437,0.018,0.828,0.172c0.393,0.154,0.979,0.562,0.979,0.562s1.613,1.31,2.072,2.2l0.222,0.679l-0.102,0.161c0,0,0.919,1.307,2.096,2.602c0,0,1.229,1.664,1.689,2.09c0,0-0.108-0.399-0.203-0.849l0.339-0.226l0.2-0.144l0.617,0.259c3.575,4.811,6.435,10.424,8.144,16.328l-0.121,0.484l0.396,0.501c1.129,4.212,1.729,8.643,1.729,13.211c0,1.122-0.038,2.236-0.109,3.339l-0.304,0.511l0.226,0.555C99.95,72.645,78.057,93.131,51.38,93.131c-18.502,0-34.702-9.854-43.637-24.601l-0.102-0.365l-0.205-0.151c-3.387-5.742-5.682-12.205-6.595-19.104l0.212-0.524l-0.336-0.482c-0.213-1.892-0.322-3.812-0.322-5.758c0-2.985,0.255-5.909,0.748-8.755l0.25-0.562l-0.087-0.328C2.463,26.451,4.689,20.783,7.78,15.7l0.684-0.384l0.081,0.032c0,0,0.233-0.169,0.354-0.217l0.076-0.023c0,0,1.179-1.971,1.625-2.601c0,0,0.542-0.348,0.745-0.407c0,0,0.124-0.016,0.189,0.076c0,0,0.496-0.432,1.699-2.054c0.004-0.005,0.007-0.011,0.012-0.017c0,0-0.114-0.076-0.131-0.174c-0.018-0.097,0.108-0.591,0.173-0.717s0.108-0.156,0.108-0.156s1.722-2.032,3.151-3.238c0,0,0.26-0.202,0.678-0.25c0,0,1.472-0.613,3.264-2.184c0,0,0.051-0.289,0.478-0.858c0.428-0.57,1.456-1.163,2.222-1.337c0.764-0.174,0.896-0.038,0.896-0.038l0.064,0.065l0.515,0.766c0,0,0.565-0.316,1.413-0.604c0.847-0.289,0.979-0.262,0.979-0.262l0.823,1.336l-0.987,2c0,0-0.644,1.421-1.655,2.185c0,0-0.472,0.284-1.12,0.127c-0.648-0.157-1.072,0.333-1.072,0.333l-0.17,0.14c0,0,0.14-0.024,0.346-0.103c0,0,0.158,0.065,0.274,0.223c0.114,0.158,0.913,1.175,0.913,1.175s0.005,0.837-0.415,1.938c-0.419,1.1-1.467,2.891-1.467,2.891s-0.733,1.424-1.075,2.253c-0.342,0.829-0.515,1.765-0.488,2.262c0,0,0.187,0.062,0.707-0.202c0.655-0.332,1.083,0.027,1.083,0.027s0.719,0.53,1.041,0.881c0.262,0.289,0.802,1.765,0.209,3.224c0,0-0.402,1.008-1.377,1.724c0,0-0.216,0.332-1.529,0.174c-0.368-0.043-0.585-0.276-1.372-0.2c-0.785,0.077-1.231,0.815-1.231,0.815l0.013-0.024c-0.692,0.999-1.154,2.458-1.154,2.458l-0.057,0.165c0,0-0.241,0.509-0.292,1.752c-0.053,1.284,0.284,3.109,0.284,3.109s7.876-1.387,9.88-0.055l0.58,0.532c0,0,0.046,0.174-0.031,0.376c-0.08,0.204-0.375,0.673-0.987,1.113c-0.611,0.438-1.222,1.583-0.313,2.304c1.034,0.818,1.691,0.766,3.43,0.468c1.74-0.297,2.898-1.269,2.898-1.269s0.972-0.72,0.783-1.628c-0.188-0.908-1.017-1.189-1.017-1.189s-0.658-0.423-0.141-1.238c0,0,0.141-0.689,2.553-1.316c2.414-0.626,6.812-1.52,10.557-1.989c0,0-2.539-8.223-0.738-9.289c0,0,0.438-0.296,1.224-0.408l0.72-0.037c0.131-0.027,0.343,0.005,0.796,0.045c0.453,0.038,1.001,0.076,1.678-0.441c0.676-0.519,0.697-0.819,0.697-0.819"/><radialGradient id="SVGID_2_" cx="197.6416" cy="-371.8613" r="0" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#FFF"/><stop offset=".4835" stop-color="#EAEAEB"/><stop offset=".9451" stop-color="#A9ABAE"/><stop offset="1" stop-color="#999B9E"/></radialGradient><path style="&st17;" d="M-16.122-14.641"/><linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="456.2744" y1="510.1602" x2="502.7757" y2="582.9122" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#8A8A8A"/><stop offset=".5687" stop-color="#606060"/><stop offset=".5914" stop-color="#FFF"/><stop offset=".6116" stop-color="#585858"/><stop offset="1" stop-color="#303030"/></linearGradient><path style="&st25;" d="M82.447,79.307l0.451,0.168c-0.032,0.061-0.067,0.119-0.101,0.179c-0.864,1.573-0.936,1.927-1.36,2.253c-0.424,0.326-0.58,0.539-1.443,0.695c-0.865,0.156-1.771,1.175-1.771,1.175s-0.875,0.835-0.48,0.866c0.396,0.025,1.177-0.609,1.177-0.609s0.497-0.354,0.625-0.255c0.125,0.099-0.539,0.751-0.539,0.751s-1.161,1.176-2.479,1.982l-0.396,0.213c0,0,1.869-1.392,2.764-2.395c0,0,0.185-0.271-0.185,0.057c-0.369,0.325-1.332,0.821-1.741,0.821c-0.41,0,0.016-0.563,0.016-0.563s0.596-0.768,1.812-1.758c1.22-0.994,2.239-0.922,2.239-0.922s0.227,0.043,0.253-0.271c0.028-0.313,0.312-0.863,0.312-0.863s0.453-1.032,1.674-2.749c1.218-1.714,1.77-2.195,1.77-2.195s1.588-1.629,2.423-3.259c0,0,0.581-1.006-0.364-0.255c-0.951,0.753-2.211,1.7-3.44,2.014c-1.233,0.312-1.349-0.836-1.349-0.836s-0.271-1.884,1.049-3.344c1.188-1.316,2.492-1.273,3.684-1.415c1.188-0.144,2.21-1.571,2.21-1.571s0.82-0.922,1.289-3.797c0.218-1.337,0.067-2.244-0.144-2.818l0.021-0.647l-0.368-0.021c-0.078-0.106-0.135-0.153-0.135-0.153s-0.297-0.299-0.607-0.524c-0.313-0.227-0.692-0.649-1.063-1.457c0,0-1.019-2.11-0.792-5.156c0.227-3.047,2.762-2.762,2.762-2.762s1.475,0.143,1.76-0.298c0.283-0.438,0.553-0.993,0.649-2.223c0.101-1.233-0.396-2.408-0.396-2.408s-0.186-0.609-1.049-0.708c-0.863-0.1-1.051,0.169-1.051,0.169s-0.255,0.072-0.723,1.09c-0.471,1.021-0.75,1.488-1.602,1.702c-0.849,0.211-2.023,0.099-2.549-0.992c-0.515-1.072-1.757-3.693-2.351-5.772l0.084-0.735l-0.325-0.471c-0.063-0.396-0.117-0.846-0.13-1.236c-0.026-0.992-0.312-2.097-0.638-3.2c-0.326-1.106-1.459-2.972-1.672-3.399c-0.324-0.667-0.621-0.949-1.021-0.893c-0.396,0.056-0.339,0.056-0.513,0.056c-0.168,0-0.266,0.212-0.311,0.425c-0.042,0.212-0.375,1.315-1.104,1.812c-0.752,0.51-1.147,0.737-2.438,0.85c0,0-1.487,0.099-2.661-2.097C71,29.36,71.301,27.96,71.398,27.45c0.099-0.51,0.539-1.247,1.229-1.388c0.693-0.141,1.119-0.184,1.119-0.184s1.122-0.01,1.603-0.071c0.551-0.071,0.283-1.006,0.283-1.006s-0.361-2.371-2.348-4.318l-0.096-0.487l-0.756-0.381c-1.881-2.04-4.354-5.354-4.354-5.354s-1.105-1.048-0.17-2.493c0,0,0.438-0.963,1.742-0.792c0.419,0.081,0.457,0.123,0.818,0.199c0.481,0.099,1.008,0.225,1.478-0.398c0,0,0.438-0.792-0.271-1.812s-0.923-1.742-1.785-1.687c0,0-0.397-0.055-0.793,0.384C68.702,8.1,67.682,8.378,67.086,8.44c-0.679,0.071-2.252-0.528-3.156-2.082c-0.513-0.874-0.752-1.35-0.865-1.595l0.231-0.34l0.72,0.08c0.186,0.358,0.549,1.056,0.978,1.812c0.552,0.978,1.048,1.118,1.373,1.261c0.325,0.141,1.049-0.071,1.431-0.283c0.385-0.213,0.766-0.695,1.43-0.738c0.668-0.043,1.629,0.071,2.311,0.793c0.682,0.723,1.531,2.224,1.459,2.932c-0.068,0.708-0.111,1.403-1.035,1.699c-0.921,0.298-1.303,0.043-1.884-0.084c-0.581-0.128-0.864-0.072-1.104,0.211c-0.242,0.284-0.512,0.892-0.412,1.162c0.102,0.27,0.186,0.454,0.75,1.262c0.566,0.806,3.474,4.25,4.031,4.766l-0.152,0.698l0.968,0.176c0.625,0.724,1.358,1.668,1.687,2.263c0.493,0.907,0.752,2.337,0.779,3.002c0.025,0.666-0.299,0.963-0.299,0.963s-0.313,0.524-2.012,0.524c-1.517,0-1.614,0.297-1.614,0.297s-0.47,0.467-0.369,1.615c0.103,1.146,0.616,2.344,1.56,3.37c0.778,0.851,2.252-0.006,2.748-0.295c0.51-0.299,0.822-1.264,0.822-1.264s0.102-1.217,1.432-1.217c1.377,0,1.969,1.318,1.969,1.318s0.963,1.295,1.941,4.121c0.481,1.389,0.566,2.281,0.566,2.281l0.126,1.668l-0.513,0.892l0.732,0.312c0.135,0.541,0.348,1.24,0.686,2.044c0,0,1.008,2.381,1.443,3.286c0.44,0.906,0.896,0.766,1.264,0.808c0,0,0.85,0.113,1.16-0.282c0.313-0.398,0.61-1.092,0.61-1.092s0.663-1.812,2.138-1.657c1.475,0.156,1.84,1.092,2.096,2.168c0.256,1.074,0.313,3.229-0.479,4.545c-0.798,1.318-1.688,1.135-1.688,1.135s-1.813-0.17-2.225,0.213c-0.41,0.382-0.623,0.724-0.681,1.613c-0.058,0.894,0.026,2.336,0.751,4.08c0.631,1.523,1.146,1.361,1.432,1.728c0.284,0.368,1.188,1.204,1.104,3.272c-0.082,2.067-0.791,4.149-1.586,5.439c-0.793,1.288-1.997,2.053-1.997,2.053s-0.338,0.211-1.076,0.311c-0.735,0.102-1.784,0.086-2.534,0.513c-0.75,0.426-0.992,1.501-1.063,1.971c-0.069,0.468-0.112,1.529,0.921,1.413c1.034-0.109,2.577-1.4,2.577-1.4s1.486-1.104,1.685-0.752c0.199,0.354-0.636,1.784-0.636,1.784s-1.035,1.562-1.898,2.523c-0.709,0.791-1.641,1.868-2.431,3.102L82.447,79.307L82.447,79.307z"/><linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="425.2861" y1="502.9512" x2="445.7861" y2="598.6606" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#A8A9AB"/><stop offset="1" stop-color="#636668"/></linearGradient><path style="&st18;" d="M23.441,22.732c-0.007,0.008-0.013,0.018-0.021,0.025C23.428,22.75,23.434,22.74,23.441,22.732z"/><linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="421.0684" y1="504.3701" x2="441.068" y2="597.7433" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#A8A9AB"/><stop offset="1" stop-color="#636668"/></linearGradient><path style="&st19;" d="M38.188,89.707l0.163,0.01c-0.163-0.147-0.37-0.281-0.62-0.414c-0.699-0.371-3.731-2.375-4.669-3.009c-0.936-0.633-2.575-1.7-3.297-1.478c-0.554,0.172-0.475,0.394-0.804,0.556c-0.331,0.157-0.687,0.052-1.504-0.384c-0.818-0.434-1.424-0.725-3.02-2.239c-1.245-1.185,1.595-0.118,1.595-0.118s0.619,0.262,0.343-0.385c-0.277-0.646-1.676-2.333-2.994-3.691c-1.068-1.098-1.674-1.86-2.435-2.97l-0.566-0.661l0.007-0.166c-0.018-0.024-0.035-0.054-0.052-0.078c0,0-1.874-3.377-1.676-4.617c0,0,0.068-0.828,1.241-1.188c0.817-0.251,0.71,0.542,3.112,0.975c0,0,2.07,0.559,2.611-0.945c0.539-1.505-0.567-3.393-0.567-3.393s-1.449-2.656-3.244-2.758c-0.95-0.053-0.692,0.586-2.125,0.926c0,0-1.913,0.184-2.519-0.963c-0.734-1.389-1.04-2.969-1.015-4.022c0.022-1,0.054-1.079,0.119-1.371c0.045-0.206,0.192-0.582,0.254-1.128l-0.134-0.667l0.204-0.501c0.002-0.26-0.004-0.535-0.021-0.83c-0.091-1.66-0.213-4.221-0.437-5.71c-0.223-1.491-0.633-3.798-1.991-3.866c0,0-0.671-0.051-1.634,0.885c-0.884,0.856-1.684-0.708-1.728-1.663c-0.053-1.121,0.131-2.254,0.409-2.795c0.277-0.541,1.042-0.567,1.186-0.555c0.146,0.013,0.555,0.171,1.043,0.474c0.488,0.305,1.16,0.305,1.557-0.092c0.395-0.394,0.947-0.856,1.173-2.598c0.224-1.741,0.224-3.547,0.013-5.71l-0.225-0.484l1.339-0.289c-0.001-0.017-0.021-0.126-0.032-0.193c0-0.002,0-0.003,0-0.005c-0.002-0.017-0.005-0.032-0.007-0.049c-0.001-0.003-0.001-0.005-0.001-0.007c-0.003-0.019-0.007-0.038-0.009-0.057c0-0.001-0.001-0.001-0.001-0.003c-0.003-0.02-0.006-0.042-0.009-0.062c0-0.001,0-0.001,0-0.001c-0.004-0.023-0.007-0.045-0.011-0.068c0-0.004-0.001-0.006-0.001-0.008c-0.002-0.022-0.006-0.045-0.008-0.066c-0.001-0.006-0.001-0.01-0.003-0.017c-0.002-0.023-0.005-0.046-0.009-0.069c-0.001-0.004-0.001-0.007-0.002-0.014c-0.003-0.026-0.007-0.05-0.011-0.076c0-0.002,0-0.004,0-0.005c-0.004-0.024-0.008-0.05-0.011-0.076c-0.001-0.007-0.001-0.01-0.003-0.017c-0.002-0.025-0.006-0.052-0.009-0.08c-0.001-0.005-0.001-0.011-0.002-0.021c-0.005-0.027-0.007-0.053-0.011-0.081c-0.001-0.006-0.002-0.013-0.002-0.019c-0.002-0.029-0.006-0.058-0.01-0.087c0-0.004-0.001-0.008-0.003-0.014c-0.002-0.028-0.005-0.056-0.009-0.082c-0.001-0.006-0.001-0.011-0.002-0.016c-0.002-0.031-0.006-0.06-0.01-0.09c0-0.006-0.001-0.015-0.002-0.021c-0.004-0.03-0.006-0.061-0.011-0.09c0-0.007-0.001-0.015-0.002-0.022c-0.003-0.03-0.006-0.062-0.01-0.093c0-0.006-0.002-0.012-0.002-0.019c-0.003-0.032-0.005-0.063-0.009-0.094c0-0.002,0-0.005,0-0.009c-0.004-0.032-0.005-0.066-0.01-0.1c0-0.005,0-0.012-0.001-0.02c-0.002-0.033-0.005-0.065-0.007-0.098c-0.001-0.007-0.001-0.014-0.001-0.021c-0.004-0.033-0.006-0.067-0.008-0.099c0-0.005,0-0.012-0.001-0.02c-0.002-0.033-0.006-0.069-0.007-0.102c0-0.003,0-0.007-0.001-0.01c-0.002-0.033-0.004-0.066-0.006-0.1c-0.001-0.006-0.001-0.011-0.001-0.017c-0.001-0.032-0.003-0.068-0.005-0.1c0-0.008,0-0.014-0.001-0.021c-0.001-0.031-0.002-0.065-0.003-0.099c-0.001-0.006-0.001-0.013-0.001-0.021c-0.002-0.033-0.003-0.066-0.004-0.1c0-0.005,0-0.009,0-0.014c-0.001-0.032-0.001-0.066-0.002-0.099c0-0.003,0-0.005,0-0.009c0-0.034,0-0.067-0.001-0.101c0-0.005,0-0.013,0-0.017c0-0.033,0-0.067,0-0.098c0-0.005,0.001-0.012,0.001-0.019c0-0.032,0-0.065,0.001-0.095c0-0.005,0-0.009,0-0.015c0.001-0.033,0.001-0.065,0.003-0.099c0.052-1.244,0.292-1.752,0.292-1.752l0.057-0.164c0,0,0.46-1.459,1.154-2.459l-0.013,0.024c0,0,0.446-0.738,1.231-0.814c0.785-0.079,1.004,0.156,1.371,0.2c0.04,0.004,0.078,0.008,0.115,0.013c0.013,0.002,0.025,0.002,0.037,0.004c0.025,0.002,0.051,0.004,0.075,0.006c0.014,0.001,0.027,0.003,0.041,0.003c0.021,0.001,0.043,0.003,0.064,0.004c0.014,0.001,0.028,0.002,0.041,0.003c0.02,0.001,0.04,0.001,0.058,0.003c0.014,0,0.026,0,0.042,0c0.019,0.001,0.036,0.002,0.055,0.002c0.013,0.001,0.026,0.001,0.038,0.002c0.017,0,0.034,0,0.051,0c0.011,0,0.023,0,0.034,0c0.017,0,0.033,0,0.05,0c0.011,0,0.02-0.001,0.032-0.001c0.016-0.001,0.031-0.001,0.046-0.001c0.011-0.001,0.02-0.001,0.03-0.002c0.016,0,0.03-0.002,0.045-0.002c0.009,0,0.019,0,0.026-0.001c0.016-0.001,0.03-0.002,0.044-0.004c0.006,0,0.016-0.001,0.023-0.002c0.015-0.001,0.029-0.001,0.044-0.003c0.006-0.001,0.013-0.002,0.02-0.002c0.016-0.002,0.03-0.004,0.045-0.008c0.004,0,0.008,0,0.013-0.001c0.019-0.002,0.036-0.005,0.052-0.008l0,0c0.035-0.006,0.068-0.014,0.098-0.021c0,0,0,0,0.002-0.002c0.012-0.002,0.026-0.005,0.039-0.01c0.002,0,0.004,0,0.008-0.001c0.009-0.003,0.019-0.006,0.028-0.009c0.004,0,0.007-0.002,0.01-0.003c0.009-0.003,0.019-0.007,0.026-0.009c0.002-0.001,0.005-0.002,0.008-0.003c0.008-0.003,0.015-0.006,0.021-0.009c0.004-0.001,0.006-0.003,0.009-0.003c0.007-0.004,0.014-0.005,0.02-0.009c0.003-0.001,0.006-0.003,0.008-0.004c0.005-0.002,0.012-0.005,0.019-0.007c0.001-0.001,0.003-0.002,0.005-0.004c0.005-0.003,0.01-0.005,0.016-0.007c0.002-0.002,0.003-0.002,0.006-0.004c0.004-0.001,0.008-0.005,0.012-0.007c0.002-0.001,0.004-0.001,0.005-0.003c0.004-0.002,0.008-0.006,0.012-0.008c0.001,0,0.003-0.002,0.004-0.003c0.003-0.003,0.007-0.004,0.011-0.007c0.001-0.001,0.001-0.001,0.002-0.003c0.004-0.001,0.006-0.005,0.008-0.008h0.002c0.003-0.002,0.005-0.005,0.008-0.007l0.001-0.001c0.002-0.002,0.004-0.004,0.006-0.006s0.002-0.002,0.003-0.003c0.001,0,0.002-0.002,0.002-0.002c0.001-0.003,0.001-0.003,0.003-0.005c0.003-0.001,0.003-0.004,0.003-0.004c0.328-0.241,0.593-0.516,0.797-0.775c0.014-0.017,0.026-0.033,0.04-0.05c0.002-0.004,0.005-0.009,0.008-0.012c0.013-0.015,0.026-0.031,0.038-0.048c0.002-0.004,0.006-0.008,0.007-0.011c0.012-0.018,0.025-0.033,0.038-0.05c0.001,0,0.001,0,0.001-0.001c0.039-0.054,0.075-0.109,0.109-0.159c0-0.002,0.002-0.004,0.003-0.007c0.01-0.016,0.02-0.029,0.03-0.044c0.001-0.004,0.005-0.007,0.007-0.011c0.008-0.014,0.017-0.029,0.024-0.042c0.003-0.004,0.005-0.009,0.009-0.013c0.008-0.014,0.017-0.028,0.024-0.042l0.001-0.002c0.017-0.027,0.032-0.055,0.046-0.079c0.002-0.003,0.004-0.008,0.007-0.012c0.005-0.009,0.01-0.021,0.016-0.03c0.003-0.007,0.007-0.014,0.012-0.02c0.004-0.008,0.01-0.017,0.014-0.024c0.002-0.008,0.006-0.017,0.011-0.023c0.004-0.007,0.009-0.016,0.012-0.022c0.004-0.007,0.007-0.014,0.011-0.021c0.002-0.006,0.007-0.011,0.01-0.018c0.066-0.13,0.097-0.207,0.097-0.207c0.594-1.459,0.052-2.935-0.21-3.224c-0.32-0.354-1.041-0.883-1.041-0.883s-0.426-0.357-1.08-0.025c-0.521,0.264-0.711,0.201-0.711,0.201c-0.024-0.498,0.149-1.432,0.491-2.263c0.341-0.829,1.075-2.253,1.075-2.253s1.047-1.792,1.467-2.89c0.418-1.102,0.415-1.94,0.415-1.94s-0.795-1.019-0.91-1.177c-0.115-0.158-0.272-0.223-0.272-0.223c-0.205,0.078-0.345,0.103-0.345,0.103l0.169-0.14c0,0,0.424-0.492,1.073-0.334c0.648,0.158,1.119-0.126,1.119-0.126c1.011-0.764,1.654-2.187,1.654-2.187l0.988-1.997L27.059,1.12c0,0-0.131-0.028-0.979,0.259c0,0-0.773,1.905-1.725,3.087c0,0-0.374,0.552-0.664,0.416c-0.289-0.134-0.629-0.144-0.91-0.085c-0.281,0.06-1.156,0.349-1.948,1.413c-0.79,1.064-0.883,1.863-0.458,2.101c0.425,0.238,1.045-0.076,1.42-0.407c0.375-0.333,0.638-0.485,0.757-0.47c0.118,0.02,0.093,0.315,0.068,0.41c-0.026,0.094-0.154,1.364-1.625,3.913c-0.369,0.64-0.845,1.589-1.069,2.046l0.608,0.447l-0.999,0.503c-0.266,0.758-0.605,1.945-0.471,2.61c0.204,1.011,1.158,1.011,1.158,1.011s0.621,0.027,1.344-0.348c0.722-0.375,0.892,1.242,0.892,1.505c0,0.264-0.007,1.726-1.02,2.442c0,0-0.409,0.313-0.985,0.313c-0.579,0-0.954-0.169-0.954-0.169s-0.984-0.272-1.59,0.083c-0.604,0.358-1.326,1.098-1.897,2.17c-0.569,1.072-1.045,2.986-1.019,4.397c0.02,1.111,0.18,2.162,0.291,2.879l0.667,0.435l-0.543,0.623c0.079,1.136,0.168,3.363,0.155,4.109c-0.018,0.911-0.154,2.84-1.115,3.292c0,0-0.324,0.171-0.868-0.238s-1.132-0.426-1.37-0.435c-0.238-0.007-1.285,0.162-1.692,1.311c-0.408,1.145-0.375,2.688-0.29,3.597c0.086,0.91,0.876,2.458,1.872,2.458c0,0,0.484,0.035,1.055-0.553c0.568-0.586,0.902-0.637,1.156-0.629c0.255,0.009,0.749,0.272,1.072,2.851c0.307,2.442,0.437,4.442,0.414,6.668l0.605,0.392l-0.758,0.768c-0.042,0.199-0.089,0.417-0.142,0.626c-0.169,0.682-0.187,1.364-0.024,2.569c0.161,1.21,0.811,3.72,1.754,4.375c1.252,0.871,2.071,0.856,2.916,0.791c0.842-0.067,1.424-0.712,1.424-0.712s0.331-0.342,0.685-0.237c0.356,0.104,1.346,0.66,2.058,2.084c0.713,1.425,0.871,2.992-0.316,3.272c-1.187,0.272-3.231-0.846-3.231-0.846s-1.161-0.647-2.109,0.064c-0.951,0.713-0.977,1.807-0.502,3.15c0.261,0.738,0.782,1.938,1.513,3.188l0.721,0.302l-0.193,0.551c0.492,0.748,1.055,1.479,1.678,2.105c0,0,2.466,2.729,3.838,4.457c0,0,0.08,0.157-0.158,0.016c-0.238-0.146-1.266-0.621-1.82-0.566c-0.555,0.054-0.45,0.395-0.45,0.395s0.238,1.254,4.01,3.365c0,0,1.359,0.766,2.216,0.766c0,0,0.277,0.039,0.619-0.346c0.346-0.381,0.45-0.341,0.688-0.262c0.237,0.076,0.553,0.249,1.741,1.105c1.188,0.857,3.496,2.176,4.325,2.731c0.83,0.555,0.793,0.657,0.621,1.054c-0.171,0.396,0.593,0.619,0.593,0.619s1.899,0.855,2.928,0.846c1.029-0.016,1.464-0.119,2.097,0.012c0.632,0.133,1.28,0.291,1.345,0.346c0.066,0.053-0.315,0.038-0.315,0.038s-2.362-0.276-2.494-0.21c-0.13,0.066,0.014,0.184,0.99,0.436v0.006c1.245,0.217,2.507,0.387,3.782,0.51c-0.489-0.061-2.52-0.322-3.823-0.713c0,0-0.131-0.078,0.173-0.014c0.303,0.065,2.018,0.225,2.466,0.157c0.448-0.065-0.092-0.274-0.092-0.274s-0.897-0.425-2.889-0.582c0,0-0.803-0.055-1.503,0.014c-0.699,0.066-1.41-0.264-1.41-0.264s-1.028-0.317-0.78-0.646c0.126-0.165,0.137-0.336,0.065-0.502L38.188,89.707L38.188,89.707z"/><linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="444.7598" y1="550.8145" x2="473.8418" y2="550.8145" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#231F20"/><stop offset="1" stop-color="#474747"/></linearGradient><path style="&st5;" d="M62.342,86.16l-0.438,0.646c0.096,0.655-0.104,0.875-0.254,1.119c-0.156,0.26-0.59,0.592-0.375,1.185c0.218,0.59,1.687,0.401,2.535,0.144c0.852-0.26,1.569-0.49,2.134-0.693c0.562-0.198,0.719,0.086,0.719,0.086s0.246,0.175-0.574,0.606c-0.821,0.433-2.336,0.634-3.055,0.72c-0.724,0.086-0.506-0.043-1.586,0.33c-1.082,0.377-0.07,0.707-0.07,0.707c2.435,0.635,4.188,0.115,4.188,0.115l0.332-0.097c0.27-0.077,0.535-0.161,0.803-0.244c-2.021,0.456-3.326,0.149-3.739,0.095c-0.431-0.058-0.778-0.145-0.88-0.361c-0.102-0.215,0.479-0.2,0.479-0.2s1.683-0.188,3.24-0.69c1.557-0.506,1.932-0.98,1.871-1.341c-0.057-0.358-0.848-0.332-1.785-0.028c-0.937,0.305-2.334,0.75-2.662,0.821c-0.334,0.07-0.691,0.06-0.812-0.146c-0.114-0.203-0.216-0.53,0.146-0.905c0.36-0.376,0.402-0.694,0.402-0.694c0.055-0.254,0.057-0.523,0.021-0.8L62.342,86.16l0.545-0.118c-0.298-1.124-1.052-2.218-1.279-2.577c-0.314-0.507-0.836-0.793-2.393-0.535c-1.556,0.26-3.386,1.035-3.386,1.035s-1.888,0.997-3.298,0.812c-1.413-0.188-1.703-1.614-1.688-2.063c0.015-0.447,0.304-0.835,1.039-1.123c0.735-0.289,2.465-0.678,4.985-0.808s3.458-1.771,3.458-1.771c0.33-0.478,0.922-1.543-0.489-2.336c-1.41-0.791-1.441-0.593-2.119-1.107c-0.678-0.52-1.412-1.153-1.701-2.033c-0.025-0.084-0.066-0.197-0.104-0.292l-0.849-0.558l0.41-0.371c-0.34-0.67-0.693-1.271-0.693-1.271s-1.281-1.902-0.246-3.703c1.038-1.803,2.736-2.348,2.736-2.348s1.5-0.332,2.996,0.016c1.418,0.324,2.133-0.219,2.133-0.219s0.865-0.374,1.123-2.102c0.264-1.729-0.014-4.943-2.316-5.578c-2.307-0.633-3.527,0.563-3.527,0.563s-1.24,1.369-1.644,1.57c-0.404,0.201-1.022,0.563-1.022,0.563s-0.867,0.519-1.453,0.648c-0.393,0.086-1.791-1.771-1.933-3.201c-0.133-1.316-0.401-2.388-0.306-5.096l-0.485-0.63l0.537-0.533c0.101-2.999-0.417-5.116-1.188-6.461c-0.807-1.412-2.119-2.161-2.766-2.478c-0.648-0.318-2.147-0.462-3.17-0.086c-1.023,0.374-1.239,0.678-1.613,1.326c-0.376,0.648-0.836,0.605-0.836,0.605s-1.427,0.043-1.888-2.133c-0.646-3.049,0.359-4.035,0.359-4.035s0.318-0.476,1.369-0.619c1.053-0.144,1.73,0.115,2.537,0.315c0.806,0.202,1.41,0.116,2.419-0.374c1.008-0.491,1.442-1.297,1.238-2.739c-0.195-1.393-0.255-1.742-1.483-5.964l-0.875-0.46l0.583-0.579C44.87,23.032,44.8,20.35,44.8,20.35c-0.106-0.977,0.049-1.292,0.598-1.54c0.576-0.259,1.116-0.175,1.934-0.123c0.818,0.053,1.425-0.079,1.847-0.316c0.422-0.237,1.581-0.87,1.504-2.162c-0.08-1.292-1.109-2.824-1.953-3.351c-0.843-0.528-1.953-0.316-2.558,0.131c-0.606,0.45-0.845,1.029-1.214,1.214c-0.369,0.183-0.895,0.605-1.45,0.474c-0.554-0.132-0.581-1.371-0.818-2.77c-0.153-0.907-0.271-1.611-0.338-1.989l-0.678-0.254c0.044,0.557,0.535,3.304,0.535,3.304s0.223,1.364,0.308,1.692c0.086,0.329,0.314,1.05,0.314,1.05s0.092,0.364,0.456,0.535c0.365,0.172,1.052,0.24,1.743-0.032c0.692-0.271,0.886-0.572,1.199-0.938c0.315-0.364,0.429-0.493,0.429-0.493s0.663-0.622,1.478-0.343c0.813,0.278,1.299,1.228,1.299,1.228l0.028,0.06c0,0,0.403,0.661,0.392,1.269v-0.057c0,0-0.022,0.301-0.697,0.818c-0.676,0.519-1.226,0.479-1.678,0.442c-0.454-0.04-0.666-0.072-0.797-0.045l-0.719,0.038C45.178,18.303,44.74,18.6,44.74,18.6c-1.8,1.064,0.736,9.288,0.736,9.288l0,0L45.2,28.501c0,0,0.514,2.052,0.904,3.378c0.388,1.326,0.562,2.637,0.62,2.91c0.058,0.274,0.044,0.762-0.317,1.051c-0.359,0.289-1.239,0.534-1.425,0.562c-0.187,0.029-0.535-0.042-0.996-0.201c-0.462-0.158-0.922-0.187-0.922-0.187s-1.11-0.188-2.291,0.173c-1.182,0.359-2.076,1.569-2.148,3.083c-0.071,1.513-0.057,2.278,0.535,3.617c0.59,1.34,1.657,2.104,2.463,2.118c0.808,0.014,1.469-0.403,1.931-1.051c0.459-0.65,0.59-0.751,0.59-0.751c0.548-0.302,1.944-0.433,2.651-0.172c0.708,0.258,2.007,1.073,2.723,2.679c0.579,1.298,0.76,2.75,0.729,5.363l0.584,0.448l-0.61,0.703c-0.007,0.246-0.016,0.498-0.026,0.761c-0.143,3.876,0.72,5.864,0.72,5.864c0.491,1.065,1.023,1.83,1.745,2.318c0.719,0.489,1.699,0.156,2.018,0c0.315-0.158,1.194-0.563,1.943-1.037c0.749-0.477,0.725-0.679,1.629-1.515c0.907-0.833,2.076-0.604,2.076-0.604s1.107,0.028,1.74,1.313c0.637,1.283,0.509,3.109,0.347,3.773c-0.158,0.662-0.444,1.097-1.063,0.979c-0.621-0.114-1.645-0.217-2.019-0.231c-0.375-0.014-1.433-0.049-2.394,0.203c-0.938,0.244-2.205,0.92-3.414,2.883c-0.938,1.52-0.478,3.013-0.262,3.603c0.17,0.462,0.635,1.104,1.043,1.896l0.756,0.252l-0.35,0.656c0.398,0.963,0.701,1.493,1.305,2.151c0.648,0.705,1.672,1.251,2.881,1.714c1.213,0.462,0.662,1.282,0.662,1.282c-0.69,1.497-2.75,1.557-3.354,1.628c-0.604,0.07-1.872,0.188-3.058,0.447c-1.182,0.261-2.291,0.418-2.954,1.182c-0.661,0.764-0.402,1.557-0.013,2.393c0.388,0.834,1.427,1.28,2.853,1.226c1.426-0.058,2.35-0.476,3.214-0.851s2.362-0.809,2.81-0.937c0.445-0.129,1.051-0.39,1.498,0.26c0.482,0.701,0.994,1.697,1.229,2.45L62.342,86.16L62.342,86.16z"/><path style="&st34;" d="M101.566,51.162c0,0,0.347-3.236,0.457-4.392c0.018-0.173,0.031-0.343,0.045-0.513l-0.098-0.241c-0.459,5.815-0.938,7.727-0.938,7.727s0.013-0.037,0.031-0.101c0.189-0.817,0.357-1.646,0.51-2.48C101.568,51.162,101.566,51.162,101.566,51.162L101.566,51.162z"/><path style="&st34;" d="M91.268,12.579l0.328,0.413l0.279,0.5c1.021,1.648,2.846,4.295,2.846,4.295s0.739,0.958,0.905,0.968c0.164,0.01-0.084-0.741-0.084-0.741s-0.271-0.979,0.517,0.298c0.73,1.19,1.207,2.359,1.317,2.72c0.114,0.361,0.042,0.411,0.042,0.411s-0.146,0.072-0.146,0.33c0,0.505,0.496,1.659,0.496,1.659s1.545,3.926,2.07,5.563c0.526,1.641,1.133,4.368,1.133,4.368s0.187,0.804,0.207,1.174c0.021,0.371-0.063,0.381-0.229,0.186c-0.164-0.196-0.329-0.072-0.329-0.072s-0.175,0.072-0.175,0.773c0,0.7,0.085,2.346,0.217,3.615c0.152,1.505,0.616,2.133,0.616,2.133s0.413,0.618,0.526-0.577c0.096-1.028,0.185-0.228,0.185-0.228c0.196,1.557,0.062,4.544,0.03,4.955c-0.019,0.218-0.032,0.433-0.049,0.64l0.133-0.221c0.201-2.971,0.06-5.359,0.06-5.359s-0.144-1.323-0.3-1.311c-0.155,0.01-0.211,0.701-0.211,0.701s-0.065,0.467-0.156,0.456c-0.088-0.011-0.369,0.022-0.566-1.412c-0.199-1.436-0.156-2.949-0.156-2.949s-0.043-0.155,0.048-0.189c0.09-0.034,0.188,0.1,0.188,0.1s0.133,0.189,0.287,0.033c0.156-0.154,0.19-0.622-0.301-3.08c-0.288-1.454-0.711-2.896-1.006-3.832l-0.294-0.333l-0.058-0.718c0,0-0.311-0.913-1.033-2.737c-0.723-1.824-0.846-2.458-0.846-2.458s-0.045-0.2,0.066-0.234c0.111-0.032,0.091-0.178,0.091-0.178s-0.013-0.245-0.278-0.99c-0.268-0.746-0.426-1.281-1.356-2.86c-0.869-1.468-1.124-1.558-1.124-1.558s-0.426-0.234-0.112,0.591c0.313,0.823-0.075,0.232-0.075,0.232c-0.925-1.177-2.838-4.292-2.838-4.292l-0.537-0.373l-0.508-1.261l-0.015,0.01"/><path style="&st34;" d="M1.547,32.696l-0.183,0.37c-0.472,2.495-0.625,5.135-0.62,5.31c0.01,0.208-0.036,0.211-0.075,0.178c-0.042-0.035,0.03-0.16-0.048-0.16c-0.079,0-0.108,0.211-0.108,0.211L0.512,38.6c-0.021,0.288-0.038,0.574-0.053,0.861l0.016-0.003c0.068-0.098,0.097-0.028,0.097-0.028s-0.029,0.126,0.097,0.056c0.126-0.07,0.21-1.379,0.21-1.379s0.014-2.813,0.836-6.14c0.819-3.327,1.281-4.259,1.281-4.259s0.154-0.418,0.138-0.083C3.12,27.958,3.33,27.986,3.33,27.986c0.375-0.054,0.821-1.125,1.267-2.493c0.445-1.363,0.668-2.589,0.668-2.7c0-0.11-0.055-0.194-0.152-0.138c-0.098,0.056-0.125,0.014-0.125,0.014c-0.014-0.208,0.361-1.127,0.361-1.127c1.253-3.202,3.104-5.694,3.104-5.694l0.09-0.504c-0.164,0.254-0.27,0.419-0.421,0.661c-0.056,0.089-0.042,0.297-0.001,0.32c-0.201,0.191-0.365,0.35-0.476,0.456c-2.707,4.473-3.059,6.556-3.059,6.556c-0.017,0.214,0.004,0.311,0.111,0.306c0.065-0.003,0.251-0.349,0.116,0.354c-0.09,0.468-0.524,1.708-0.693,2.212c-0.171,0.505-0.358,0.85-0.495,0.826C3.49,27.01,3.49,26.801,3.49,26.801s-0.042-0.546-0.398,0.245c-0.356,0.791-0.713,1.859-1.425,4.65c-0.031,0.125-0.063,0.251-0.092,0.38L1.547,32.696L1.547,32.696z"/><path style="&st34;" d="M4.841,62.626c-0.15-0.401-0.264-0.722-0.179-0.581c0.085,0.143,0.198,0.432,0.462,0.725c0.263,0.291,0.442-0.226-0.622-3.104s-1.354-3.04-1.402-3.095c-0.046-0.058-0.215-0.237-0.167,0.167c0.045,0.404,0.018,0.656-0.51-1.146c-0.528-1.806-0.996-4.844-1.025-5.089c-0.027-0.243-0.169-1.778-0.396-3.594c-0.226-1.814-0.246-3.743-0.207-4.28c0.039-0.537-0.066-0.828-0.123-0.837c-0.056-0.008-0.094,0.047-0.131,0.284c-0.038,0.234-0.01,0.395-0.067,0.385c-0.057-0.009-0.076-0.471-0.076-0.471H0.391c0,0.05,0,0.1,0,0.151c0,0.174,0.001,0.345,0.002,0.519l0.039,0.402c0.033,0.597,0.129,0.354,0.135,0.246c0.006-0.109,0.03-0.329,0.03-0.329s0.103-0.884,0.084,0.02c-0.019,0.904,0.236,4.563,0.236,4.563c0.019,0.236,0.041,0.479,0.068,0.729l0.063,0.092l-0.042,0.104c0.265,2.425,0.795,5.086,0.795,5.086c0.507,2.417,1.11,3.846,1.308,4.25c0.198,0.405,0.236,0.085,0.17-0.271c-0.066-0.357,0.546,0.688,0.873,1.674c0.332,0.99,0.556,1.815,0.556,1.815s0.254,0.781,0.142,0.828c-0.113,0.046-0.292-0.293-0.292-0.293s-0.473-0.835-0.274-0.228c0.398,1.231,1.6,3.822,1.6,3.822l1.396,2.471C6.282,65.836,4.982,63.004,4.841,62.626L4.841,62.626z"/><path style="&st34;" d="M7.281,67.639c0.069,0.125,0.136,0.246,0.202,0.359L7.281,67.639z"/><path style="&st34;" d="M20.722,2.874C20.51,3.216,20.48,3.388,20.48,3.388s0.112-0.118,0.183-0.237C20.733,3.033,20.722,2.874,20.722,2.874z"/><path style="&st34;" d="M17.216,5.572c-0.417,0.048-0.677,0.25-0.677,0.25S16.889,5.761,17.216,5.572z"/><path style="&st34;" d="M84.005,4.909c0,0,0.247-0.03,0.347,0.02c0.101,0.049,0.066-0.158,0.066-0.158s-0.287-0.406-0.322-0.556c-0.32-0.089-0.611-0.1-0.611-0.1l-0.028,0.034c-0.01,0.075-0.036,0.188-0.012,0.297C83.441,4.448,83.917,4.811,84.005,4.909L84.005,4.909z"/><path style="&st34;" d="M87.487,7.88l0.095-0.152l-0.223-0.679c-0.004-0.012-0.012-0.022-0.021-0.036c-0.007,0.066-0.049,0.125-0.172,0.115c0,0-0.099-0.03-0.011,0.198C87.219,7.469,87.355,7.699,87.487,7.88L87.487,7.88z"/><path style="&st34;" d="M101.566,51.162c0,0,0.347-3.236,0.457-4.392c0.018-0.173,0.031-0.343,0.045-0.513l-0.098-0.241c-0.459,5.815-0.938,7.727-0.938,7.727s0.013-0.037,0.031-0.101c0.189-0.817,0.357-1.646,0.51-2.48C101.568,51.162,101.566,51.162,101.566,51.162L101.566,51.162z"/><linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="266.4922" y1="-395.2783" x2="295.9644" y2="-485.0349" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset=".0094" stop-color="#FCFCFC"/><stop offset=".0655" stop-color="#EEEEEE"/><stop offset=".1342" stop-color="#E5E5E5"/><stop offset=".2515" stop-color="#E3E3E3"/><stop offset=".3357" stop-color="#8A8A8A"/><stop offset=".4422" stop-color="#B8B8B8"/><stop offset="1" stop-color="#3B3B3B"/></linearGradient><path style="&st33;" d="M79.003,84.528c0,0,0.667-0.653,0.539-0.752c-0.128-0.101-0.623,0.256-0.623,0.256s-0.073,0.062-0.185,0.142l0.393-0.252c0,0-0.038,0.238-0.355,0.555c0,0-0.094,0.094-0.258,0.248c-0.957,0.938-2.386,1.998-2.386,1.998l0.396-0.211C77.844,85.703,79.003,84.528,79.003,84.528z"/><linearGradient id="SVGID_8_" gradientUnits="userSpaceOnUse" x1="460.4629" y1="512.5557" x2="509.5884" y2="573.3062" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EFF0F0"/><stop offset=".5914" stop-color="#F0F1F2"/><stop offset=".599" stop-color="#787878"/><stop offset=".6456" stop-color="#EEEFF0"/><stop offset="1" stop-color="#D8D9DB"/></linearGradient><path style="&st30;" d="M82.984,39.705l0.51-0.891l-0.127-1.667c0,0-0.085-0.893-0.566-2.28c-0.979-2.828-1.941-4.122-1.941-4.122s-0.592-1.318-1.969-1.318c-1.33,0-1.431,1.219-1.431,1.219s-0.312,0.963-0.821,1.261c-0.5,0.29-1.972,1.146-2.748,0.296c-0.941-1.026-1.461-2.225-1.56-3.372c-0.101-1.146,0.367-1.615,0.367-1.615s0.1-0.296,1.614-0.296c1.703,0,2.014-0.525,2.014-0.525s0.324-0.296,0.297-0.963s-0.284-2.097-0.779-3.001c-0.324-0.596-1.058-1.537-1.684-2.261l-0.967-0.178l0.15-0.699c-0.56-0.514-3.465-3.959-4.031-4.766c-0.564-0.808-0.65-0.993-0.75-1.262c-0.099-0.269,0.17-0.877,0.412-1.161c0.238-0.283,0.521-0.341,1.104-0.212c0.58,0.127,0.961,0.381,1.886,0.084c0.919-0.297,0.962-0.992,1.033-1.699c0.071-0.708-0.78-2.209-1.458-2.932c-0.684-0.721-1.645-0.836-2.311-0.792c-0.664,0.042-1.047,0.523-1.433,0.737c-0.382,0.213-1.103,0.425-1.429,0.284c-0.326-0.142-0.823-0.284-1.375-1.261c-0.43-0.76-0.794-1.459-0.979-1.817L63.299,4.42v0.012c0,0,0.633,1.654,1.633,2.811c0.998,1.157,2.266,0.919,2.266,0.919s0.82-0.089,1.533-0.772c0.711-0.683,1.761-0.148,2.024,0.04c0.269,0.189,0.853,0.911,1.478,2.127c0.621,1.216-0.355,2.058-0.355,2.058s-0.555,0.535-1.691,0.088c-1.14-0.443-1.813,0.259-1.986,0.614c-0.182,0.357-0.508,0.99,0.076,1.73c0.584,0.742,3.178,4.273,4.916,5.945c1.74,1.672,2.314,3.047,2.682,4.342c0.365,1.297,0.079,1.899-0.521,2.018c-0.604,0.118-1.148,0.021-2.086,0.187c-0.94,0.17-1.349,0.367-1.543,1.653c-0.199,1.286,0.562,3.373,1.67,4.361c1.106,0.989,2.334,0.386,2.76,0.228c0.427-0.159,1.352-0.653,1.681-2.027c0.188-0.783,0.851-0.721,0.851-0.721s0.563-0.071,0.854,0.117c0.287,0.19,0.633,0.525,1.402,1.87c0.772,1.346,1.453,3.146,1.724,4.738C82.924,38.35,82.729,38.576,82.984,39.705c0.256,1.128,1.078,3.245,1.466,4.074c0.383,0.832,0.78,1.662,0.989,2.107c0.205,0.445,0.531,0.782,1.443,0.802c0.908,0.02,1.273-0.228,1.541-0.662c0.27-0.435,0.612-1.088,0.713-1.316c0.1-0.228,0.467-0.911,1.146-1.02c0.685-0.108,1.762,0.01,2.106,1.198c0.313,1.071,0.76,2.622-0.158,4.5c-0.65,1.334-1.129,0.859-2.451,0.948c0,0-1.165-0.01-1.781,0.921c-0.611,0.93-0.416,2.61-0.286,3.877s0.988,3.113,1.621,3.563c0.636,0.443,0.86,0.849,1.08,1.256c0.216,0.404,0.534,1.205,0.216,3.313c-0.313,2.106-0.979,3.74-1.867,4.521c-0.024,0.021-0.05,0.043-0.07,0.063c-0.067,0.065-0.141,0.135-0.219,0.201c-0.537,0.521-0.371,0.543-0.889,0.793c-0.594,0.289-0.988,0.207-1.958,0.365c-0.97,0.16-1.583,0.327-2.088,0.821c-0.503,0.495-1.243,1.409-0.979,3.187c0.148,0.986,1.318,0.584,2.229,0.111c-0.274,0.125-0.553,0.221-0.798,0.246c-1.033,0.113-0.991-0.949-0.921-1.415c0.069-0.47,0.313-1.544,1.063-1.97s1.799-0.41,2.533-0.512c0.738-0.101,1.076-0.313,1.076-0.313s1.205-0.766,1.997-2.055c0.793-1.289,1.502-3.371,1.587-5.438c0.084-2.068-0.821-2.902-1.104-3.271c-0.283-0.366-0.799-0.203-1.431-1.729c-0.724-1.74-0.81-3.188-0.751-4.079c0.057-0.892,0.27-1.231,0.682-1.612c0.41-0.383,2.223-0.213,2.223-0.213s0.893,0.185,1.686-1.134c0.793-1.317,0.738-3.471,0.481-4.546c-0.253-1.076-0.623-2.013-2.097-2.168c-1.471-0.152-2.138,1.66-2.138,1.66s-0.297,0.693-0.608,1.092c-0.312,0.395-1.16,0.285-1.16,0.285c-0.37-0.044-0.821,0.099-1.264-0.81c-0.438-0.906-1.442-3.286-1.442-3.286c-0.339-0.809-0.556-1.512-0.688-2.055L82.984,39.705L82.984,39.705z"/><linearGradient id="SVGID_9_" gradientUnits="userSpaceOnUse" x1="272.8721" y1="-392.8257" x2="302.4699" y2="-482.9646" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st9;" d="M82.487,79.313l0.801-0.479c0.788-1.231,1.72-2.31,2.431-3.103c0.863-0.961,1.896-2.522,1.896-2.522s0.835-1.431,0.638-1.784c-0.13-0.23-0.704,0.02-1.687,0.752c-3.674,2.746-3.971,1.246-3.971,1.246c0.422,2.438,4.729-1.633,5.108-1.375c-0.063,0.563-0.457,1.172-1.25,2.25c0,0-0.388,0.555-0.78,0.953c-0.369,0.432-0.578,0.641-0.578,0.641s-0.088,0.09-0.125,0.125c-0.471,0.541-1.024,1.242-1.595,2.018c-0.019,0.021-0.104,0.113-0.125,0.143c-0.295,0.416-0.491,0.73-0.702,1.062c-0.014,0.022-0.064,0.011-0.076,0.034c0-0.002-0.013,0.014-0.025,0.037c-0.245,0.39-0.373,0.713-0.508,0.959c-0.012,0.029-0.021,0.065-0.03,0.095c0,0-0.319,0.665-0.457,1.067c-0.14,0.405-0.12,0.547-0.623,0.625c-0.504,0.078-0.276-0.053-1.021,0.196c0,0-0.403,0.199-0.938,0.571c-0.027,0.021-0.057,0.042-0.082,0.063c-0.736,0.604-1.247,1.119-1.534,1.436c-0.051,0.063-0.099,0.13-0.146,0.195c0,0-0.157,0.168,0.051,0.188c0.206,0.021,0.633-0.01,1.008-0.169l0.088-0.057c-0.186,0.103-0.373,0.174-0.513,0.162c-0.396-0.026,0.479-0.864,0.479-0.864s0.906-1.019,1.771-1.175c0.862-0.156,1.021-0.371,1.444-0.693c0.426-0.327,0.494-0.682,1.359-2.254c0.03-0.059,0.064-0.115,0.098-0.176L82.487,79.313L82.487,79.313z"/><linearGradient id="SVGID_10_" gradientUnits="userSpaceOnUse" x1="444.6943" y1="510.9561" x2="469.7246" y2="592.0699" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#FFF"/><stop offset=".7473" stop-color="#F9F9F9"/><stop offset="1" stop-color="#D5D7D8"/></linearGradient><path style="&st27;" d="M55.064,72.686l0.408-0.377c-0.34-0.668-0.693-1.269-0.693-1.269s-1.282-1.901-0.245-3.703c1.036-1.803,2.737-2.348,2.737-2.348s1.5-0.332,2.996,0.017c1.418,0.323,2.133-0.22,2.133-0.22s0.865-0.376,1.123-2.104c0.261-1.729-0.014-4.94-2.317-5.576c-2.306-0.633-3.528,0.563-3.528,0.563s-1.242,1.369-1.644,1.57c-0.404,0.2-1.024,0.562-1.024,0.562s-0.865,0.52-1.453,0.648c-0.39,0.087-1.788-1.771-1.931-3.2c-0.133-1.313-0.4-2.385-0.305-5.084c0-0.005,0-0.01,0-0.017l-0.486-0.629l0.537-0.526c0.102-3-0.418-5.119-1.188-6.463c-0.805-1.414-2.118-2.163-2.766-2.479c-0.647-0.317-2.146-0.461-3.169-0.086c-1.022,0.375-1.237,0.677-1.613,1.325c-0.376,0.65-0.835,0.606-0.835,0.606s-1.427,0.044-1.89-2.132c-0.644-3.049,0.36-4.036,0.36-4.036s0.318-0.475,1.369-0.619c1.053-0.144,1.73,0.115,2.536,0.317c0.807,0.2,1.41,0.114,2.42-0.374c1.009-0.49,1.442-1.298,1.24-2.738c-0.196-1.397-0.249-1.727-1.484-5.966l-0.874-0.458l0.582-0.579c-1.182-4.271-1.257-6.961-1.257-6.961c-0.105-0.975,0.049-1.29,0.598-1.537c0.576-0.261,1.117-0.177,1.934-0.125c0.819,0.052,1.425-0.079,1.847-0.316c0.423-0.236,1.583-0.87,1.503-2.163c-0.078-1.292-1.108-2.823-1.951-3.35c-0.844-0.528-1.952-0.317-2.56,0.132c-0.606,0.447-0.843,1.028-1.213,1.212c-0.368,0.185-0.896,0.607-1.45,0.474c-0.554-0.132-0.581-1.372-0.818-2.77c-0.155-0.915-0.276-1.614-0.342-1.989l-0.674-0.254c0.043,0.557,0.535,3.304,0.535,3.304l0.294,1.624c0,0,0,0.007,0,0.02c0.006,0.018,0.009,0.036,0.013,0.05c0.019,0.079,0.049,0.18,0.082,0.289c0.114,0.215,0.37,0.456,0.942,0.502c1.076,0.089,1.772-0.468,2.025-0.709c0.254-0.239,0.86-0.911,0.86-0.911s0.329-0.632,1.253-0.494c0.922,0.14,1.238,0.773,1.403,1.013c0.167,0.242,1.57,1.961,0.672,2.861c-0.039,0.039-0.093,0.089-0.152,0.146c-0.104,0.111-0.245,0.246-0.446,0.399c-0.675,0.517-1.226,0.48-1.678,0.442c-0.453-0.039-0.665-0.07-0.795-0.043l-0.72,0.038c-0.787,0.11-1.224,0.408-1.224,0.408c-1.8,1.065,0.735,9.287,0.735,9.287s0.671,2.029,0.973,2.979c0.301,0.949,0.496,1.625,0.791,3.264c0.295,1.639-0.231,2.092-0.525,2.251c-0.294,0.158-0.984,0.568-1.77,0.604c-0.783,0.034-1.027-0.368-2.371-0.432c-1.345-0.065-2.246,0.345-2.661,0.906c-0.417,0.561-0.913,1.862-0.675,3.4c0.237,1.541,0.504,2.064,1.092,2.748c0.59,0.681,1.165,0.97,1.978,0.696c0.792-0.266,0.764-0.756,1.173-1.164c0.517-0.519,0.855-0.792,2.063-0.821c1.208-0.029,1.979,0.288,2.594,0.784c0.619,0.496,1.934,1.646,2.294,3.356c0.359,1.713,0.424,2.268,0.424,3.676s-0.101,2.978-0.064,4.381c0.036,1.4,0.187,2.841,0.577,3.795c0.386,0.955,0.926,1.755,1.4,2.18c0.475,0.426,0.896,0.438,1.373,0.252c0.475-0.188,1.511-0.771,2.373-1.324c0.861-0.555,0.797-0.99,1.576-1.502c0.875-0.576,1.799-0.605,2.457-0.486c0.661,0.112,1.676,0.631,2.092,1.889C63.059,60.58,63,61.998,63,61.998s0.035,1.186-0.445,1.876c-0.48,0.688-1.272,0.616-1.625,0.545c-0.354-0.071-1.094-0.136-1.094-0.136s-1.451-0.165-2.563,0.094c-1.105,0.258-2.077,1.085-2.73,1.896c-0.652,0.813-0.941,1.57-0.984,2.158c-0.043,0.59,0.027,1.595,0.642,2.572c0.612,0.979,0.712,1.432,1.409,2.827c0.695,1.396,2.15,2.17,2.201,2.206c0.05,0.037,1.388,0.523,1.89,0.949c0.505,0.425,0.555,0.826,0.411,1.208c-0.145,0.381-0.438,1.094-1.604,1.604c-1.166,0.512-2.591,0.523-3.496,0.617c-0.906,0.094-2.651,0.332-3.697,0.834c-1.043,0.503-0.97,1.454-0.97,1.454s-0.028,1.556,1.337,1.983c1.365,0.434,2.64,0,3.201-0.237c0.562-0.238,1.487-0.583,1.487-0.583s1.791-0.631,2.752-0.848c0.965-0.217,1.533-0.323,2.188,0.832c0.652,1.158,1.014,1.886,1.078,2.625c0.064,0.74-0.209,1.148-0.461,1.432c-0.25,0.279-0.217,0.46-0.217,0.46c-0.105,0.873,1.182,0.763,1.182,0.763s0.041-0.004,0.11-0.018c-0.26,0.021-0.499-0.021-0.59-0.178c-0.116-0.202-0.217-0.531,0.146-0.906c0.359-0.374,0.402-0.693,0.402-0.693c0.305-1.439-1.038-3.371-1.354-3.875c-0.315-0.503-0.836-0.791-2.394-0.531c-1.556,0.26-3.386,1.037-3.386,1.037s-1.891,0.995-3.299,0.809c-1.413-0.188-1.701-1.614-1.687-2.063c0.016-0.444,0.304-0.836,1.038-1.122c0.733-0.289,2.464-0.679,4.984-0.809c2.522-0.128,3.458-1.771,3.458-1.771c0.331-0.478,0.923-1.543-0.489-2.338c-1.412-0.789-1.44-0.589-2.116-1.104c-0.68-0.521-1.412-1.153-1.701-2.034c-0.026-0.084-0.07-0.198-0.108-0.291L55.064,72.686L55.064,72.686z"/><linearGradient id="SVGID_11_" gradientUnits="userSpaceOnUse" x1="390.042" y1="485.6797" x2="390.042" y2="485.6797" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#FFF"/><stop offset=".7473" stop-color="#F9F9F9"/><stop offset="1" stop-color="#D5D7D8"/></linearGradient><path style="&st35;" d="M-16.122-14.641"/><linearGradient id="SVGID_12_" gradientUnits="userSpaceOnUse" x1="390.042" y1="485.6797" x2="390.042" y2="485.6797" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#FFF"/><stop offset=".7473" stop-color="#F9F9F9"/><stop offset="1" stop-color="#D5D7D8"/></linearGradient><path style="&st36;" d="M-16.122-14.641"/><path style="&st34;" d="M92.002,13.363c1.021,1.649,2.844,4.295,2.844,4.295s0.256,0.332,0.5,0.605l0.01-0.011c0.011-0.375-0.051-0.571-0.06-0.621l-0.091-0.274c-0.021-0.367,0.438,0.095,0.611,0.288c-0.498-0.754-0.659-0.811-0.659-0.811s-0.423-0.234-0.111,0.59c0.312,0.824-0.075,0.233-0.075,0.233c-0.924-1.177-2.838-4.293-2.838-4.293l-0.553-0.383L92.002,13.363L92.002,13.363z"/><path style="&st34;" d="M101.539,51.352c0.014-0.063,0.023-0.125,0.034-0.188c-0.004,0-0.009,0-0.009,0s0.005-0.03,0.013-0.089C101.563,51.17,101.551,51.262,101.539,51.352L101.539,51.352z"/><path style="&st34;" d="M100.822,35.679c0.088-0.033,0.188,0.1,0.188,0.1s0.135,0.189,0.289,0.033c0.107-0.104,0.155-0.353,0.025-1.257c-0.004,0.229-0.053,0.409-0.137,0.59c-0.139,0.296-0.314,0.02-0.441-0.081c-0.129-0.098-0.168,0.07-0.168,0.07l-0.004,0.162c0,0.7,0.087,2.346,0.217,3.617c0.063,0.605,0.173,1.071,0.287,1.408l0.041,0.076c0.089,0.148,0.188,0.343,0.307,0.255c0.116-0.089,0.274-0.582,0.274-0.582l0.128-0.591c0.191,0.113,0.291,0.529,0.341,0.962c-0.002-0.037-0.004-0.056-0.004-0.056s-0.144-1.324-0.3-1.313c-0.155,0.01-0.21,0.701-0.21,0.701s-0.066,0.468-0.157,0.456c-0.088-0.011-0.365,0.022-0.564-1.412c-0.201-1.436-0.158-2.949-0.158-2.949S100.732,35.713,100.822,35.679L100.822,35.679z"/><path style="&st34;" d="M99.696,28.566l0.298,0.341c0.479,1.494,0.996,3.843,1.092,4.264c-0.027-0.139-0.056-0.286-0.088-0.441c-0.288-1.454-0.711-2.896-1.006-3.832L99.696,28.566l-0.05-0.702c-0.004-0.01-0.006-0.016-0.006-0.016s-0.312-0.913-1.033-2.737c-0.725-1.824-0.848-2.458-0.848-2.458s-0.043-0.2,0.066-0.234c0.109-0.032,0.09-0.178,0.09-0.178s-0.013-0.245-0.277-0.99c-0.182-0.503-0.312-0.911-0.662-1.607c0.281,0.585,0.463,1.052,0.524,1.259l0.028,0.068c0,0,0.099,0.148,0.066,0.552c-0.027,0.403-0.146,0.452-0.146,0.452l0.022,0.14c0.141,0.538,0.418,1.187,0.418,1.187s1.065,2.709,1.748,4.54L99.696,28.566L99.696,28.566z"/><linearGradient id="SVGID_13_" gradientUnits="userSpaceOnUse" x1="274.5342" y1="-396.1577" x2="255.2091" y2="-490.1944" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st37;" d="M42.958,91.536c0.938,0.177,1.723,0.358,1.889,0.396C44.514,91.822,43.895,91.653,42.958,91.536z"/><linearGradient id="SVGID_14_" gradientUnits="userSpaceOnUse" x1="422.5586" y1="518.7568" x2="427.2878" y2="578.1768" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset=".2198" stop-color="#989A9C"/><stop offset=".2527" stop-color="#FFF"/></linearGradient><path style="&st38;" d="M20.381,74.92l0.007-0.164l-0.052-0.08c0,0-1.874-3.375-1.676-4.615c0,0,0.069-0.827,1.241-1.187c0.817-0.25,0.71,0.538,3.112,0.976c0,0,2.07,0.557,2.611-0.946c0.539-1.507-0.566-3.394-0.566-3.394s-1.45-2.656-3.244-2.756c-0.95-0.055-0.692,0.583-2.125,0.924c0,0-1.913,0.185-2.519-0.963c-0.733-1.389-1.015-2.968-1.015-4.021c0-1.058,0.045-1.001,0.126-1.405c0.045-0.219,0.186-0.548,0.248-1.09l-0.134-0.675l0.204-0.499c0.002-0.26-0.004-0.535-0.021-0.83c-0.092-1.661-0.211-4.221-0.436-5.711c-0.223-1.491-0.633-3.799-1.991-3.865c0,0-0.671-0.052-1.636,0.885c-0.882,0.856-1.682-0.708-1.726-1.663c-0.052-1.121,0.131-2.255,0.409-2.795c0.277-0.541,1.042-0.566,1.186-0.554c0.146,0.012,0.555,0.17,1.042,0.474c0.489,0.304,1.161,0.304,1.558-0.092c0.395-0.395,0.948-0.856,1.173-2.598c0.225-1.741,0.225-3.547,0.013-5.71l-0.224-0.485l1.339-0.288c-0.001-0.016-0.021-0.125-0.032-0.195c0,0,0-0.001-0.001-0.002c-0.001-0.017-0.004-0.033-0.007-0.052c0-0.002,0-0.004,0-0.005c-0.003-0.019-0.007-0.038-0.009-0.057c-0.001-0.001-0.001-0.001-0.001-0.003c-0.003-0.022-0.006-0.042-0.009-0.062c0-0.001,0-0.001,0-0.001c-0.004-0.022-0.006-0.045-0.011-0.067c0-0.003-0.001-0.006-0.001-0.007c-0.002-0.022-0.006-0.045-0.008-0.068c-0.001-0.005-0.001-0.01-0.003-0.015c-0.002-0.023-0.005-0.047-0.009-0.069c-0.001-0.004-0.002-0.01-0.002-0.014c-0.003-0.026-0.007-0.05-0.012-0.076c0-0.002,0-0.005,0-0.005c-0.004-0.025-0.008-0.05-0.012-0.076c0-0.007-0.001-0.012-0.002-0.018c-0.002-0.024-0.006-0.052-0.009-0.079c-0.001-0.005-0.001-0.011-0.003-0.021c-0.004-0.027-0.006-0.053-0.01-0.081c-0.001-0.007-0.002-0.013-0.003-0.02c-0.003-0.028-0.005-0.057-0.009-0.087c-0.001-0.003-0.001-0.008-0.003-0.013c-0.003-0.028-0.005-0.056-0.009-0.082c-0.001-0.006-0.001-0.011-0.002-0.017c-0.003-0.031-0.006-0.06-0.01-0.091c-0.001-0.007-0.001-0.014-0.002-0.02c-0.004-0.03-0.006-0.062-0.011-0.09c0-0.007-0.001-0.015-0.002-0.022c-0.003-0.031-0.006-0.063-0.01-0.094c-0.001-0.006-0.002-0.011-0.002-0.018c-0.003-0.032-0.005-0.063-0.009-0.094c0-0.003,0-0.005-0.001-0.009c-0.003-0.033-0.004-0.068-0.009-0.1c0-0.005,0-0.013-0.001-0.02c-0.002-0.035-0.005-0.065-0.007-0.099c-0.001-0.008-0.001-0.015-0.002-0.021c-0.003-0.032-0.005-0.066-0.007-0.099c0-0.005-0.001-0.011-0.001-0.02c-0.002-0.033-0.006-0.068-0.007-0.101c0-0.003-0.001-0.008-0.001-0.009c-0.002-0.033-0.004-0.066-0.007-0.1c0-0.006,0-0.012,0-0.017c-0.001-0.032-0.003-0.068-0.005-0.1c0-0.008,0-0.014-0.001-0.022c-0.001-0.033-0.004-0.067-0.005-0.098c0-0.006,0-0.013,0-0.021c-0.002-0.033-0.003-0.066-0.004-0.101c0-0.005,0-0.008,0-0.013c-0.001-0.032-0.002-0.066-0.002-0.099c0-0.003,0-0.005,0-0.009c0-0.034-0.001-0.067-0.001-0.102c0-0.005,0-0.012,0-0.016c0-0.033,0-0.067,0-0.098c0-0.005,0-0.012,0-0.019c0.001-0.032,0.001-0.065,0.001-0.096c0-0.004,0.001-0.009,0.001-0.014c0-0.033,0.001-0.066,0.003-0.1c0.052-1.243,0.291-1.751,0.291-1.751l0.058-0.166c0,0,0.46-1.458,1.152-2.458l-0.011,0.024c0,0,0.446-0.738,1.231-0.816c0.785-0.077,1.003,0.158,1.371,0.202c0.04,0.004,0.078,0.008,0.115,0.013c0.013,0.001,0.025,0.002,0.037,0.004c0.025,0.002,0.051,0.003,0.074,0.006c0.014,0.001,0.028,0.002,0.042,0.003c0.021,0.001,0.043,0.002,0.064,0.004c0.014,0.001,0.028,0.001,0.041,0.002c0.02,0.001,0.04,0.002,0.058,0.002c0.013,0.002,0.026,0.002,0.04,0.002c0.021,0.001,0.037,0.002,0.055,0.002c0.014,0,0.026,0.001,0.039,0.001c0.016,0,0.034,0.001,0.051,0.001c0.011,0,0.023,0,0.034,0c0.017,0,0.032,0,0.05-0.001c0.01,0,0.02,0,0.032-0.001c0.016,0,0.031,0,0.046-0.001c0.011,0,0.02-0.001,0.03-0.001c0.015,0,0.03-0.002,0.045-0.002c0.009,0,0.017-0.001,0.026-0.001c0.015-0.001,0.03-0.003,0.044-0.004c0.006-0.001,0.016-0.002,0.022-0.002c0.016-0.001,0.03-0.002,0.044-0.004c0.007-0.001,0.014-0.001,0.019-0.002c0.016-0.002,0.03-0.004,0.045-0.007c0.006,0,0.009,0,0.014-0.001c0.019-0.002,0.036-0.006,0.052-0.008l0,0c0.035-0.008,0.068-0.014,0.098-0.021c0,0,0-0.002,0.002-0.002c0.012-0.002,0.025-0.005,0.039-0.01c0.002,0,0.004-0.001,0.007-0.001c0.01-0.003,0.02-0.006,0.029-0.009c0.003-0.001,0.007-0.002,0.01-0.004c0.009-0.002,0.018-0.006,0.026-0.008c0.002-0.002,0.005-0.003,0.008-0.003c0.008-0.003,0.015-0.006,0.021-0.009c0.003-0.001,0.006-0.003,0.009-0.004c0.006-0.003,0.014-0.004,0.02-0.009c0.003-0.001,0.006-0.002,0.008-0.003c0.005-0.002,0.012-0.005,0.019-0.007c0.001-0.002,0.003-0.003,0.005-0.004c0.005-0.004,0.01-0.005,0.015-0.008c0.003-0.001,0.004-0.003,0.008-0.003c0.004-0.002,0.008-0.005,0.012-0.008c0,0,0.004,0,0.005-0.002c0.004-0.003,0.008-0.006,0.012-0.008c0.001-0.002,0.002-0.002,0.004-0.003c0.003-0.003,0.007-0.004,0.01-0.008c0.002,0,0.002,0,0.002-0.002c0.003-0.001,0.007-0.005,0.011-0.008c0,0,0.001,0,0.001-0.001c0.004-0.002,0.005-0.004,0.009-0.007h0.001c0.002-0.002,0.004-0.004,0.006-0.007c0.001-0.001,0.002-0.001,0.003-0.002s0.002-0.002,0.002-0.002c0.001-0.003,0.001-0.003,0.003-0.005c0.002-0.002,0.004-0.004,0.004-0.004c0.328-0.241,0.591-0.516,0.797-0.775c0.014-0.017,0.026-0.034,0.04-0.05c0.002-0.004,0.005-0.009,0.008-0.012c0.013-0.016,0.026-0.032,0.038-0.05c0.002-0.003,0.006-0.006,0.007-0.01c0.012-0.018,0.025-0.032,0.038-0.05c0,0,0,0,0.001,0c0.039-0.055,0.075-0.109,0.109-0.159c0-0.003,0.002-0.006,0.003-0.008c0.01-0.015,0.021-0.028,0.03-0.044c0.001-0.003,0.004-0.007,0.007-0.01c0.008-0.016,0.017-0.029,0.024-0.042c0.002-0.004,0.005-0.009,0.009-0.013c0.008-0.014,0.017-0.028,0.023-0.042c0.001-0.001,0.001-0.002,0.002-0.002c0.017-0.028,0.032-0.055,0.046-0.079c0.002-0.003,0.004-0.008,0.006-0.013c0.006-0.01,0.01-0.021,0.017-0.029c0.003-0.007,0.007-0.014,0.012-0.02c0.004-0.008,0.009-0.017,0.014-0.024c0.002-0.008,0.006-0.017,0.01-0.023c0.004-0.007,0.009-0.016,0.012-0.023c0.004-0.006,0.006-0.014,0.011-0.021c0.002-0.006,0.007-0.013,0.01-0.021c0.066-0.128,0.097-0.205,0.097-0.205c0.593-1.459,0.052-2.936-0.21-3.225c-0.32-0.353-1.041-0.882-1.041-0.882s-0.288-0.241-0.751-0.144c0.349-0.049,0.791,0.091,0.966,0.558c0.277,0.734,0.376,1.335,0.212,2.33c0,0-0.26,1.387-1.384,2.233c-1.125,0.848-1.923,0.096-2.885,0.13c-0.962,0.032-1.516,0.701-1.809,1.157c-0.293,0.457-1.417,2.2-1.319,5.067c0.097,2.868,0.291,4.301,0.325,5.558c0.033,1.205,0.178,3.976-0.635,5.278c-0.815,1.303-1.628,0.65-2.2,0.309c-0.571-0.341-1.223-0.245-1.744,0.131c-0.521,0.375-0.833,1.124-0.848,3.324c-0.016,2.364,1.532,2.606,1.532,2.606s0.293,0.146,0.945-0.537c0,0,0.651-0.685,1.253-0.603c0.604,0.082,0.995,0.716,1.255,1.808c0.261,1.092,0.796,5.621,0.717,8.668c-0.034,1.271-0.62,1.286-0.36,3.617c0,0,0.409,3.13,1.401,4.089c0.995,0.962,2.378,0.781,2.706,0.75c0.324-0.032,0.7-0.26,0.7-0.26s0.309-0.197,0.537-0.374c0.23-0.182,0.522-0.428,1.011-0.277c0.489,0.146,1.645,0.896,2.557,2.571c0.915,1.678,0.496,3.317-0.26,3.521c-0.668,0.182-0.848,0.229-1.971-0.05c-1.124-0.274-1.451-0.567-1.957-0.766c-0.504-0.196-1.043-0.263-1.547,0.114c-0.505,0.373-1.345,1.057-0.343,3.32c0.961,2.174,1.692,3.797,3.518,5.623c-0.522-0.607-0.956-1.188-1.427-1.871L20.381,74.92L20.381,74.92z"/><linearGradient id="SVGID_15_" gradientUnits="userSpaceOnUse" x1="237.3721" y1="-388.3604" x2="218.8474" y2="-478.5023" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st39;" d="M22.494,7.387l-0.05,0.025C22.45,7.41,22.469,7.401,22.494,7.387z"/><linearGradient id="SVGID_16_" gradientUnits="userSpaceOnUse" x1="259.9063" y1="-479.3379" x2="259.8987" y2="-479.3752" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st40;" d="M46.028,92.85c0.13,0.014,0.26,0.026,0.391,0.041c-0.114-0.016-0.31-0.039-0.561-0.074C45.916,92.828,45.972,92.838,46.028,92.85L46.028,92.85z"/><path style="&st29;" d="M24.089,5.489c-0.649,0.36-0.7-0.016-1.141,0.017c-0.439,0.033-1.026,0.212-1.596,1.043c-0.571,0.831-0.586,1.89,0.326,1.417c0,0,0.436-0.428,0.815-0.579c0.081-0.043,0.24-0.126,0.406-0.174l0.144-0.117c0,0,0.424-0.491,1.073-0.333c0.648,0.156,1.119-0.129,1.119-0.129c1.01-0.761,1.655-2.184,1.655-2.184l0.987-1.998l-0.826-1.336c0,0-0.698,1.489-1.383,2.53C24.986,4.692,25.033,4.972,24.089,5.489L24.089,5.489z"/><path style="&st29;" d="M19.871,16.292c0,0-0.424,1.89,1.156,1.597c0,0,0.006-0.002,0.022-0.007c-0.062,0.003-0.089-0.006-0.089-0.006c-0.025-0.497,0.149-1.432,0.49-2.261c0.341-0.83,1.075-2.254,1.075-2.254s1.047-1.791,1.467-2.89c0.42-1.102,0.416-1.939,0.416-1.939s-0.8-1.019-0.915-1.176c-0.115-0.157-0.272-0.223-0.272-0.223c-0.054,0.019-0.103,0.036-0.146,0.051c0.115-0.007,0.221,0.021,0.283,0.114c0.213,0.31-0.39,2.036-0.39,2.036s-0.522,1.238-1.548,3.03C20.393,14.157,19.871,16.292,19.871,16.292L19.871,16.292z"/><linearGradient id="SVGID_17_" gradientUnits="userSpaceOnUse" x1="268.9033" y1="-394.6382" x2="249.4966" y2="-489.0725" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st41;" d="M38.185,89.697l0.166,0.02c-0.134-0.119-0.305-0.236-0.497-0.347c0.184,0.113,0.489,0.358,0.312,0.665c-0.245,0.424-0.048,0.457,0.489,0.863c0,0,0.022,0.013,0.061,0.033c-0.156-0.107-0.253-0.234-0.156-0.362c0.125-0.166,0.136-0.334,0.065-0.499L38.185,89.697L38.185,89.697z"/><linearGradient id="SVGID_18_" gradientUnits="userSpaceOnUse" x1="254.4561" y1="-391.5991" x2="235.0337" y2="-486.1104" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st42;" d="M24.437,82.749c-1.245-1.185,1.595-0.118,1.595-0.118s0.619,0.262,0.343-0.385c-0.246-0.57-1.373-1.963-2.543-3.219l0.144,0.156c0,0,2.346,2.721,2.183,2.966c-0.164,0.245-1.108-0.325-1.108-0.325s-1.401-0.539-1.206,0.13c0.143,0.491,1.059,1.271,1.536,1.649C25.109,83.372,24.798,83.09,24.437,82.749z"/><linearGradient id="SVGID_19_" gradientUnits="userSpaceOnUse" x1="262.6514" y1="-392.9692" x2="243.1559" y2="-487.8355" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st43;" d="M26.977,84.73c0.424,0.256,0.903,0.513,1.365,0.676c0,0,0.831,0.293,1.337-0.114c0.503-0.403,0.503-0.601,1.238-0.243c0,0,1.388,0.696,2.382,1.444c0.98,0.735,3.557,2.336,4.396,2.791c-0.764-0.417-3.712-2.365-4.633-2.99c-0.936-0.633-2.574-1.698-3.297-1.476c-0.554,0.172-0.474,0.396-0.804,0.555c-0.331,0.158-0.688,0.055-1.504-0.383C27.291,84.9,27.134,84.818,26.977,84.73L26.977,84.73z"/><linearGradient id="SVGID_20_" gradientUnits="userSpaceOnUse" x1="271.5479" y1="-390.9575" x2="251.1904" y2="-490.0176" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st44;" d="M39.903,91.448c0.37,0.104,0.672,0.06,0.787,0.035c-0.678,0.04-1.35-0.269-1.35-0.269s-0.067-0.021-0.165-0.061C39.413,91.268,39.689,91.385,39.903,91.448z"/><linearGradient id="SVGID_21_" gradientUnits="userSpaceOnUse" x1="274.6582" y1="-395.8442" x2="255.2559" y2="-490.2569" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset="1" stop-color="#E4E5E6"/></linearGradient><path style="&st45;" d="M44.293,92.169c0,0-1.344-0.163-1.939-0.163c0,0-0.037,0.064,0.226,0.158c-0.021-0.021-0.031-0.049,0.189,0c0.304,0.064,2.018,0.225,2.465,0.158c0.448-0.068-0.091-0.278-0.091-0.278s-0.088-0.041-0.267-0.102C44.553,92.201,44.293,92.169,44.293,92.169L44.293,92.169z"/><path style="&st34;" d="M4.662,62.045c0.085,0.143,0.198,0.432,0.462,0.725c0.146,0.16,0.266,0.072,0.162-0.525c-0.253-0.182-0.407-0.318-0.464-0.371c-0.113-0.013-0.263-0.297-0.263-0.297s-0.471-0.835-0.274-0.227c0.398,1.23,1.6,3.821,1.6,3.821l1.396,2.47c-0.999-1.803-2.299-4.633-2.44-5.013C4.691,62.223,4.577,61.904,4.662,62.045L4.662,62.045z"/><path style="&st34;" d="M1.547,32.696l0.032-0.643c0.027-0.122,0.057-0.241,0.087-0.36c0.712-2.789,1.069-3.858,1.425-4.649c0.356-0.792,0.398-0.245,0.398-0.245s0,0.209,0.136,0.234c0.135,0.023,0.324-0.32,0.494-0.826c0.17-0.504,0.604-1.745,0.693-2.212c0.135-0.704-0.051-0.358-0.115-0.354c-0.108,0.005-0.126-0.091-0.113-0.306c0,0,0.382-2.122,3.064-6.563c0.18-0.17,0.321-0.307,0.47-0.449c-0.055-0.052-0.033-0.265,0.001-0.32c0.136-0.214,0.275-0.435,0.422-0.661l-0.09,0.504c0,0-1.85,2.492-3.104,5.694c0,0-0.342,0.835-0.359,1.094c-0.025,0.154-0.104,0.739,0.152,0.582l0.065-0.048c-0.093,0.484-0.295,1.37-0.607,2.325c-0.288,0.878-0.573,1.633-0.841,2.078l-0.002-0.004c-0.08,0.067-0.098-0.016-0.146-0.21c-0.048-0.198-0.113-0.198-0.113-0.198c-0.179,0-0.324,0.202-0.41,0.359c-0.04,0.056-0.089,0.19-0.089,0.19s-0.461,0.934-1.281,4.26c-0.822,3.328-0.836,6.14-0.836,6.14s-0.084,1.309-0.21,1.379c-0.126,0.07-0.097-0.056-0.097-0.056s-0.045-0.052-0.114,0.045c0,0,0.02-0.76,0.044-0.875c0,0,0.041-0.206,0.119-0.206s0.006,0.125,0.048,0.16c0.039,0.032,0.084,0.03,0.075-0.178c-0.005-0.176,0.147-2.816,0.621-5.312L1.547,32.696L1.547,32.696z"/><path style="&st34;" d="M0.392,41.994c0-0.002,0-0.004,0-0.005c0,0,0.018,0.217,0.042,0.356l-0.003,0.01c0.078,0.357,0.187,0.357,0.187,0.357c0.008-0.096,0.087-0.273,0.183-0.458c0.007,0.106,0.007,0.231-0.004,0.375c-0.037,0.537-0.019,2.466,0.209,4.28c0.225,1.815,0.367,3.349,0.394,3.594c0.029,0.245,0.498,3.283,1.025,5.089c0.285,0.967,0.422,1.344,0.483,1.424l0.008,0.049c0,0,0.097,0.184,0.348,0.32c0,0,0.111-0.097,0.112-0.412c0.018,0.031,0.037,0.065,0.057,0.105c-0.083,0.262-0.105,0.426-0.105,0.426l0,0c-0.042-0.043-0.06-0.031-0.046,0.045c0.067,0.357,0.027,0.68-0.169,0.272c-0.198-0.403-0.8-1.832-1.307-4.251c0,0-0.531-2.659-0.795-5.084l0.042-0.105L0.989,48.29c-0.027-0.248-0.048-0.491-0.067-0.729c0,0-0.255-3.657-0.237-4.562c0.019-0.904-0.085-0.02-0.085-0.02s-0.021,0.219-0.028,0.329c-0.008,0.109-0.103,0.352-0.136-0.246C0.459,43.322,0.392,42.261,0.392,41.994L0.392,41.994z"/><path style="&st34;" d="M24.672,1.974l-0.53-0.753l-0.066-0.065c0,0-0.131-0.136-0.896,0.038l-0.11,0.022c0,0,0.38-0.094,0.168,0.191c-0.212,0.285-0.56,0.496-1.172,1.15c-0.612,0.655-0.411,0.803-0.01,0.668c0.401-0.138,1.188-0.499,2.606-1.243L24.672,1.974z"/><path style="&st34;" d="M23.106,1.21c-0.022,0.003-0.036,0.006-0.036,0.006L23.106,1.21z"/><path style="&st34;" d="M23.637,1.811c0.31-0.272,0.429-0.484,0.454-0.641l-0.015-0.014c0,0-0.131-0.136-0.896,0.038l-0.075,0.015c0.09-0.015,0.303-0.032,0.133,0.198c-0.212,0.285-0.56,0.496-1.172,1.151c-0.612,0.656-0.411,0.803-0.01,0.667c0.106-0.038,0.239-0.088,0.4-0.157C22.599,2.755,23.046,2.325,23.637,1.811z"/><path style="&st34;" d="M83.482,4.115l-0.2,0.235c0,0,0.136,0.081,0.208,0.141c0.008,0.005,0.014,0.01,0.021,0.012c-0.043-0.034-0.07-0.056-0.07-0.056c-0.023-0.109,0.004-0.223,0.014-0.297L83.482,4.115c0,0,0.055,0.002,0.143,0.011L83.482,4.115L83.482,4.115z"/><path style="&st6;" d="M62.559,90.319c0,0,1.686-0.187,3.239-0.691c1.558-0.504,1.935-0.981,1.874-1.341c-0.037-0.238-0.396-0.305-0.906-0.238c0.271-0.021,0.514,0.032,0.354,0.332c-0.313,0.582-0.861,0.782-0.861,0.782s-0.267,0.19-0.89,0.371c-0.806,0.245-1.794,0.375-2.335,0.438c-0.691,0.082-0.521-0.033-1.465,0.291c-0.023,0.016-0.047,0.025-0.065,0.043c-0.289,0.237,1.071,0.514,1.071,0.514s1.302,0.361,3.257,0.23l0.067-0.021c0.267-0.078,0.533-0.161,0.8-0.245c-2.021,0.457-3.324,0.149-3.737,0.095c-0.434-0.058-0.777-0.144-0.88-0.359C61.98,90.305,62.559,90.319,62.559,90.319L62.559,90.319z"/><path style="&st34;" d="M63.301,4.417l0.717,0.076c1.429-0.405,2.651-0.776,2.651-0.776s1.265-0.41,1.901-0.636c0.635-0.227,1.09-0.313,1.656-0.41c0.563-0.096,1.309-0.14,1.709-0.131c0.398,0.007,0.528,0.122,0.528,0.122s0.166,0.131,0.245,0.27c0.077,0.138,0.74,1.891,2.975,2.005c2.231,0.112,2.261-1.096,2.063-1.464c-0.226-0.427-0.896-0.863-0.896-0.863s-0.898-0.575-1.09-0.846c-0.192-0.271,0.033-0.358,0.104-0.376c0.066-0.018,2.433-0.497,2.729-0.608l0.021-0.02l-0.563-0.174c0,0-0.119,0.134-0.789,0.312c-0.67,0.179-1.233,0.246-1.742,0.313c-0.504,0.067-0.504,0.239-0.504,0.239l-0.879,1.406c0,0-0.029,0.104,0.043,0.305c0.073,0.202,0.41,0.448,0.41,0.448s0.573,0.424,0.99,0.699c0.418,0.275,0.395,0.373,0.395,0.373c-1.123,0.095-1.418-0.048-1.875-0.276c-0.445-0.223-0.76-0.729-0.922-1.086c-0.162-0.357-0.521-0.663-0.521-0.663c-0.589-0.336-1.696-0.343-2.813-0.15c-1.115,0.194-2.656,0.707-2.923,0.812c-0.271,0.104-1.616,0.551-2.309,0.729c-0.694,0.179-1.224,0.357-1.646,0.47c-0.426,0.11-3.431,1.005-4.557,1.339c-1.124,0.333-1.848,0.461-1.848,0.461c-1.688,0.171-2.193-0.134-2.193-0.134c-0.297-0.125-0.262-0.527-0.262-0.527l0.009-1.147c0,0-0.029-0.433-0.356-0.611c-0.328-0.179-0.779-0.252-1.593-0.29c-0.811-0.038-1.683,0.044-2.093,0.134c-0.408,0.09-1.19,0.313-1.764,0.952c-0.573,0.641-0.481,1.141-0.481,1.141s0.035,0.683,0.088,1.19c0.052,0.504,0.484,0.662,0.484,0.662s-0.744,0.532-3.045,1.206c-2.209,0.648-3.682,0.774-3.682,0.774l0.678,0.254c0,0,5.468-1.015,7.148-2.616c0,0,0.624-0.293,0.02-0.88c-0.606-0.585-0.897-0.761-0.897-0.761s-0.41-0.224,0.019-0.771c0.43-0.547,0.923-0.795,1.352-0.933c0.429-0.138,1.753-0.25,2.925-0.093c0,0,0.491,0.041,0.459,0.408c-0.034,0.366-0.088,0.872-0.077,1.028c0.008,0.158,0.023,0.515,0.398,0.845c0.378,0.332,1.099,0.453,1.099,0.453s1.257,0.228,2.843-0.217c1.584-0.445,3.642-1.14,5.431-1.629L63.301,4.417L63.301,4.417z"/><path style="&st12;" d="M-16.122-14.641"/><path style="&st22;" d="M48.462,6.628c0.31,0.207,0.476,0.221,0.5,0.421c0.055,0.339-0.56,0.64-0.56,0.64l-0.006-0.011c0,0-0.431-0.157-0.484-0.664c-0.052-0.505-0.088-1.19-0.088-1.19s0.001,0.2,0.046,0.26C48.004,6.256,48.087,6.378,48.462,6.628z"/><path style="&st20;" d="M82.447,79.307l0.451,0.17c1.104-0.617,1.496-0.809,1.759-0.963c1.183-0.703,2.592-1.563,2.963-1.855c0,0,0.761-0.518,0.116,0.195s-0.969,1.007-0.969,1.007s-0.625,0.626-0.471,0.782c0,0,0.166,0.246,1.193-0.687c1.023-0.929,2.15-2.258,2.275-2.44c0.127-0.188,0.146-0.293,0.146-0.293s0.107-0.215,0.273-0.393c0.145-0.15,1.377-1.496,1.994-2.121c0,0,0.002,0.001,0.006,0.003c0.273-0.362,0.541-0.729,0.806-1.102c-0.358,0.379-1.724,1.829-2.483,2.684c0,0-0.713,0.763-0.938,1.056s-0.225,0.47-0.225,0.47s-0.117,0.196-0.392,0.519s-1.24,1.186-1.24,1.186s-0.577,0.47-0.754,0.478c-0.179,0.011,0.431-0.538,0.431-0.538s0.588-0.625,0.967-1.123c0.382-0.498,0.137-0.47,0.137-0.47s-0.186-0.049-0.986,0.459c-0.8,0.508-1.367,0.858-1.367,0.858s-1.722,0.986-2.814,1.623c-1.096,0.636-3.6,1.908-5.021,2.492c-1.43,0.588-2.162,0.715-2.035,0.527c0.127-0.186,0.461-0.459,0.461-0.459s0.399-0.4,0.399-0.803c0,0,0.128-0.586-1.604-0.223c-1.729,0.36-3.293,1.213-3.293,1.213s-2.571,1.182-1.965,1.887c0,0,0.117,0.186,0.635,0.352c0.52,0.166-0.92,0.606-0.92,0.606c-1.365,0.448-2.413,0.651-3.74,0.926c-1.963,0.403-3.564,0.761-4.165,0.894c-0.165,0.035-0.253,0.059-0.253,0.059s-1.212,0.292-3.229,1.072c-2.015,0.783-5.972,1.43-5.972,1.43s-2.542,0.293-2.777,0.627c-0.234,0.331,0.177,0.499,0.177,0.499s0.362,0.224,1.671,0.283c0,0,0.451,0,0.471,0.036c0.018,0.039,0.046,0.068-0.235,0.156c-0.286,0.088-0.854,0.314-2.778,0.558c-1.936,0.245-1.896-0.067-1.896-0.067s-0.01-0.076,0.078-0.216c0.087-0.134,0.009-0.369-0.293-0.535c0,0-0.419-0.272-1.829-0.262c-1.408,0.009-4.212,0.017-6.833-0.14c-2.374-0.143-5.59-0.551-6.099-0.664c0,0-0.117-0.029-0.206-0.117c-0.088-0.09-0.646-0.422-1.164-0.733c-0.517-0.313-2.073-0.907-2.073-0.907s-2.011-0.783-1.945-0.521c0.015,0.063,0.13,0.153,0.268,0.246c0.351,0.188,0.704,0.375,1.06,0.56l0.002-0.002c0,0-0.743-0.402-0.538-0.402s0.438,0.109,0.438,0.109s1.213,0.332,1.966,0.686c0.753,0.353,1.407,0.83,1.407,0.83s0.929,0.549,2.319,0.732c1.346,0.182,3.174,0.389,3.777,0.448l0.594-0.272l0.433,0.354c1.106,0.068,2.575,0.146,2.575,0.146s2.976,0.111,4.605-0.019c0.733-0.063,0.507,0.317,0.507,0.317s-0.214,0.354,0.206,0.529c0,0,0.771,0.439,3.343,0.157c2.573-0.286,3.138-0.862,3.138-0.862s0.299-0.275-0.351-0.398c-0.513-0.1-0.513-0.051-1.175-0.117c-0.665-0.067-0.998-0.205-0.557-0.323c0.441-0.114,1.174-0.175,1.174-0.175s2.249-0.313,4.066-0.783c0,0,1.938-0.458,3.861-1.134c0.756-0.265,1.395-0.459,1.887-0.599l0.438-0.644l0.644,0.372c0.065-0.014,0.103-0.021,0.103-0.021s2.306-0.539,3.274-0.703c0.966-0.168,3.154-0.637,4.087-1.086c0.928-0.448,1.396-0.805,1.505-1.075c0.107-0.272-0.393-0.431-0.393-0.431s-0.588-0.138-0.508-0.34c0.075-0.205,0.293-0.382,1.213-0.793c0.918-0.41,2.07-0.859,3.227-1.144c1.154-0.282,0.732,0.194,0.732,0.194s-0.692,0.705-0.783,0.979c-0.086,0.273,0.029,0.285,0.119,0.333c0.088,0.05,0.646,0.028,1.022-0.067c0.383-0.099,3.464-1.271,5.341-2.347c0.049-0.026,0.094-0.054,0.139-0.08L82.447,79.307z"/><linearGradient id="SVGID_22_" gradientUnits="userSpaceOnUse" x1="221.1826" y1="-454.5649" x2="221.373" y2="-454.5649" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#808080"/><stop offset=".0868" stop-color="#7A7A7A"/><stop offset=".36" stop-color="#6B6B6B"/><stop offset=".5192" stop-color="#686868"/><stop offset=".6377" stop-color="#5F5F5F"/><stop offset=".7431" stop-color="#4E4E4E"/><stop offset=".8408" stop-color="#383838"/><stop offset=".9324" stop-color="#1B1B1B"/><stop offset="1" stop-color="#000"/></linearGradient><path style="&st13;" d="M7.432,68.01l0.178,0.131c-0.105-0.099-0.167-0.155-0.167-0.155s-0.008,0.003-0.024,0.003C7.423,67.993,7.429,68.002,7.432,68.01L7.432,68.01z"/><linearGradient id="SVGID_23_" gradientUnits="userSpaceOnUse" x1="221.4043" y1="-449.8027" x2="316.0254" y2="-449.8027" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#808080"/><stop offset=".0868" stop-color="#7A7A7A"/><stop offset=".3817" stop-color="#828282"/><stop offset=".5185" stop-color="#808080"/><stop offset=".5677" stop-color="#797979"/><stop offset=".6027" stop-color="#6E6E6E"/><stop offset=".608" stop-color="#6B6B6B"/><stop offset="1" stop-color="#4E4E4E"/></linearGradient><path style="&st7;" d="M90.412,59.954l-0.371-0.021c-1.807,1.463-2.342,1.938-3.781,2.955c0,0-1.644,1.228-3.44,2.196c-1.804,0.97-3.919,0.853-3.919,0.853s-0.573-0.062-0.295-0.477c0.275-0.415,0.634-1.425,0.771-1.938c0.141-0.516,0.415-1.82-0.633-2.235s-3.018,0.196-3.018,0.196s-4.525,1.205-6.371,3.23c-1.793,1.969-0.286,2.846-0.286,2.846s0.702,0.49,1.386,0.73c0.645,0.229,0.516,0.436,0.354,0.555c0,0-2.021,1.287-4.408,1.974c0,0-3.268,1.001-6.441,1.206c-3.154,0.203-4.495,0.283-4.495,0.283l-0.399,0.379l-0.758-0.252c-1.283,0.154-3.455,0.463-5.596,0.979c-2.622,0.64-5.123,1.147-6.903,2.296c0,0-1.333,0.887-0.566,1.746c0.765,0.862,1.565,1.23,1.565,1.23s0.472,0.207,0.464,0.336c-0.009,0.164-0.015,0.309-0.367,0.512c0,0-1.394,0.814-4.108,0.859c-2.714,0.045-3.911,0.021-4.707-0.695c-0.658-0.591,0.165-1.844,0.165-1.844s0.33-0.612,0-1.453c-0.33-0.84-2.218-0.854-2.218-0.854l-2.615-0.134c-3.095-0.081-7.182-0.427-9.001-0.653c0,0-0.012-0.002-0.033-0.006l-0.006,0.166l-0.721-0.303c-1.139-0.221-3.243-0.668-4.075-1.084c-0.759-0.38-1.167-0.313-1.066-1.102c0.101-0.769-0.753-1.836-0.753-1.836s-1.188-1.287-2.257-2.086c-1.069-0.804-1.523-0.564-1.523-0.564s-0.484,0.258-0.049,1.296c0.436,1.04,0.86,1.403,0.86,1.403s0.208,0.22,0.089,0.279c-0.118,0.06-0.484-0.219-0.789-0.478c-0.253-0.21-1.885-1.742-2.456-2.276l0.105,0.356c0.019,0.028,0.035,0.062,0.052,0.086c0.184,0.291,0.855,1.269,2.155,2.28c1.549,1.213,1.559,0.729,1.559,0.729s0.061-0.399-0.297-0.84c-0.359-0.44-0.934-1.373-0.791-1.715c0.144-0.339,0.309-0.31,0.309-0.31s0.133-0.051,0.596,0.299c0.462,0.351,1.498,1.076,2.011,1.703c0.513,0.623,0.48,1.181,0.48,1.181s-0.102,0.563,0.453,1.17c0.553,0.604,1.733,1.714,5.859,2.351c0.025,0.004,0.034,0.006,0.059,0.01l0.193-0.551l0.573,0.663c3.598,0.521,5.555,0.563,5.555,0.563s4.709,0.162,5.982,0.162c1.272,0,1.035,0.666,1.035,0.666s-0.072,0.359-0.225,0.646c-0.155,0.287-0.524,1.365-0.144,1.939c0,0,0.585,1.427,4.381,1.527c0,0,3.324,0.268,5.643-0.688c2.319-0.954,0.226-2.275,0.226-2.275s-0.794-0.481-1.13-0.739c-0.308-0.234-0.184-0.481-0.121-0.646c0.06-0.162,0.297-0.359,0.563-0.492c0.266-0.134,1.239-0.654,5.365-1.722c4.124-1.069,6.587-1.183,6.587-1.183s0.02-0.002,0.055-0.004l0.338-0.656l0.854,0.556c0.732-0.06,1.681-0.129,2.526-0.171c1.691-0.082,4.341-0.471,5.879-0.807c1.54-0.343,3.869-1.062,5.592-1.951c1.725-0.895,1.809-1.519,1.809-1.519s0.328-0.475-0.392-0.995c-0.719-0.523-1.036-0.382-1.673-1.027c-0.637-0.646,0.557-1.62,0.557-1.62s0.612-0.861,4.021-2.175c3.403-1.313,3.979-0.873,4.153-0.729s0.195,0.615,0.123,0.935c-0.069,0.317-0.494,1.455-0.721,2.053c-0.227,0.594-0.316,1.406,0.605,1.601c0.923,0.194,2.215-0.008,3.428-0.442c2.893-1.033,3.756-2.295,8.534-5.764c0.012-0.008,0.021-0.017,0.03-0.021L90.412,59.954l0.689,0.108c1.978-1.573,3.869-3.571,3.869-3.571s1.258-1.261,1.889-2.356c0.595-1.026,0.027,0.89,0.027,0.89s-0.32,1.516,0.19,2.077c0.405,0.445,1.563-0.795,1.563-0.795s0.688-0.789,0.965-2.061c0.408-1.875,0.185-2.248,0.185-2.248s-0.246-0.389-0.093-0.852c0.154-0.459,1.158-3.047,1.98-4.01l0.502-0.563c0-0.008,0.002-0.02,0.002-0.027l-0.224-0.557l0.304-0.512c0,0-0.279,0.322-1.404,2.177c-1.266,2.087-1.467,3.771-1.467,3.771s-0.119,0.653-0.054,1.034c0.063,0.355,0.188,0.519,0.192,0.622c0.009,0.104-0.073,0.959-0.508,1.773c-0.438,0.814-0.815,1.031-0.815,1.031s-0.756,0.545-0.86,0.157c-0.104-0.39-0.074-0.72-0.035-0.966c0.035-0.248,0.289-1.579,0.252-2.072c-0.035-0.494-0.479-0.098-0.479-0.098s-0.104,0.119-0.298,0.366s-1.288,1.637-1.705,2.125c-0.988,1.157-1.886,1.989-4.292,3.93c-0.007,0.003-0.015,0.011-0.019,0.015L90.412,59.954L90.412,59.954z"/><linearGradient id="SVGID_24_" gradientUnits="userSpaceOnUse" x1="214.5928" y1="-431.356" x2="314.4043" y2="-431.356" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#555555"/><stop offset="1" stop-color="#231F20"/></linearGradient><path style="&st11;" d="M1.193,48.543l0.104,0.975c0.235,0.285,0.577,0.677,1.071,1.187c1.084,1.121,1.427,0.416,1.427,0.416s0.253-0.705-0.054-1.985C3.434,47.85,3.488,47.09,4.066,47.11c0.578,0.019,2.258,0.361,2.782,1.752c0,0,0.217,0.649,0.018,1.479c-0.2,0.834-0.162,1.57-0.018,2.295c0.145,0.725,0.759,1.407,1.464,1.971c0.706,0.562,2.746,1.535,4.734,1.66c1.987,0.127,2.601,0.021,2.601,0.021l0.746-0.767l0.134,0.668c0.812-0.09,2.116-0.229,3.62-0.393c2.528-0.271,4.227-0.544,5.798-0.308c1.573,0.235,2.079,1.462,2.079,1.462s0.108,0.314-0.253,0.544c-0.349,0.224-0.749,0.547-0.948,0.89c-0.199,0.346-0.411,1.068,0.16,2.035c0.572,0.963,2.142,1.592,3.478,1.432c1.335-0.155,3.335-0.67,4.52-0.979c1.287-0.337,2.424-0.971,2.572-1.98c0.147-1.008-1.534-2.295-1.534-2.295s-0.812-0.609-0.91-0.75c-0.1-0.139,0.099-0.197,0.099-0.197s0.949-0.229,1.357-0.414c0.404-0.189,1.522-0.646,3.353-1.219s5.608-1.248,5.608-1.248s2.084-0.332,4.685-0.543l0.622-0.702l0.485,0.625c0.409-0.024,0.825-0.046,1.243-0.063c3.572-0.129,5.344-0.554,7.242-0.979c1.897-0.427,4.568-1.978,4.965-2.276c0.396-0.295,1.229-0.66,1.396-1.957c0.168-1.295-1.364-2.157-1.364-2.157s-1.218-0.644-1.475-0.93c-0.258-0.287-0.02-0.562-0.02-0.562s0.689-1.485,2.896-2.354c2.205-0.872,3.689-1.107,4.618-1.208c0.932-0.099,1.245,0.237,1.374,0.396c0.128,0.157,0.128,0.485,0.021,0.821c-0.102,0.308-0.444,1.038-0.645,1.395c-0.197,0.356-0.523,1.216-0.316,1.622c0.208,0.405,0.843,0.593,1.662,0.445c0.821-0.149,2.988-0.761,4.888-1.553c1.9-0.792,5.073-2.345,5.073-2.345s0.009-0.004,0.022-0.012l0.086-0.729l0.729,0.295c1.02-0.562,2.764-1.58,4.01-2.631c1.871-1.573,3.699-3.225,4.166-3.639c0.465-0.417,0.892-0.752,1.307-0.732c0.414,0.021,0.732,0.317,0.988,1.434c0.258,1.118,0.308,2.038,0.426,2.582c0.117,0.543,0.285,1.175,0.931,1.304c0.646,0.129,1.513-0.434,1.838-0.713c0.33-0.276,0.92-1.176,0.882-2.382c0,0,0.068-1.604-0.761-3.127c0,0-0.351-0.614-0.479-0.782c-0.088-0.118-0.155-0.238-0.01-0.525c0.148-0.286,0.467-0.821,1.062-1.156c0.448-0.256,0.88-0.316,1.128-0.396c0,0,0.275-0.067,0.626-0.261l-0.126-0.412l0.289,0.316c0.404-0.239,0.652,0.045,0.652,0.045l-0.392-0.501l0.119-0.484c0,0-0.304-0.163-0.685-0.088c-0.383,0.078-0.42,0.362-1.014,0.458c-0.593,0.096-1.275,0.306-1.945,1.319c-0.67,1.011,0,2.271,0,2.271s0.359,0.592,0.533,0.896c0.172,0.306,1.066,2.215,0.037,3.608c0,0-0.552,0.643-1.525,0.86c-0.86,0.19-0.642-0.816-0.729-1.355c0,0-0.129-2.281-1.237-3.588c-0.976-1.146-2.746,0.888-3.629,1.566c-0.822,0.629-3.228,3.112-6.684,4.925l-0.51,0.892l-0.324-0.472c-1.658,0.827-5.418,2.656-7.87,3.514c0,0-1.875,0.762-2.64,0.782c0,0-0.17,0.006-0.034-0.179c0.133-0.185,0.276-0.322,0.507-0.737c0.23-0.418,0.646-1.357,0.646-2.327c0-0.969-1.119-1.917-2.68-1.748c-1.561,0.167-3.052,0.6-4.849,1.292c-1.796,0.692-3.343,2.159-3.55,3.375c-0.209,1.216,1.105,1.92,1.105,1.92s1.484,0.751,1.674,1.157c0.188,0.406,0.049,0.783,0.049,0.783s-0.129,0.406-0.783,0.782c-0.649,0.377-2.42,1.287-2.42,1.287s-2.207,1.217-5.562,1.512c0,0-3.058,0.26-4.817,0.348c-0.022,0.002-0.046,0.002-0.069,0.003l-0.541,0.53l-0.587-0.449c-1.64,0.136-3.54,0.359-3.54,0.359s-4.29,0.609-8.219,1.822c-3.336,1.027-3.971,1.594-3.971,1.594s-0.711,0.596-0.118,1.453c0,0,0.341,0.539,1.03,1.08c0.685,0.541,1.237,1.057,1.159,1.317c-0.08,0.265-0.29,0.382-0.29,0.382s-0.961,0.856-4.894,1.518c0,0-2.531,0.488-3.31-0.275c-0.778-0.766-0.435-1.279-0.435-1.279s0.052-0.173,0.62-0.527c0,0,0.951-0.596,0.647-1.727c-0.303-1.135-2.129-2.545-4.101-2.533c0,0-2.303,0.006-8.196,0.762c0,0-0.008,0-0.027,0.005l-0.204,0.501l-0.605-0.393c-0.324,0.039-0.713,0.076-1.114,0.107c-1.012,0.07-2.477-0.057-4.3-0.596c-1.824-0.543-2.963-1.535-3.126-2.133c-0.114-0.412,0.035-0.996,0.035-0.996S7.62,50.2,7.405,49.115c-0.22-1.083-0.582-1.934-1.631-2.567c-1.048-0.632-1.643-0.522-1.643-0.522s-0.939-0.144-0.904,1.446c0.037,1.59,0.235,2.256,0.235,2.256s0.181,0.562-0.108,0.635c-0.288,0.072-0.74-0.361-0.74-0.361s-1.021-0.924-1.619-1.688l0.048,0.066l-0.214,0.525c0,0,0.067,0.115,0.266,0.375L1.193,48.543L1.193,48.543z"/><linearGradient id="SVGID_25_" gradientUnits="userSpaceOnUse" x1="214.8965" y1="-410.8818" x2="306.501" y2="-410.8818" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#A0A0A0"/><stop offset=".0769" stop-color="#656767"/><stop offset="1" stop-color="#717375"/></linearGradient><path style="&st1;" d="M91.075,11.742l0.183,0.846c0,0-0.563,0.313-1.678,0.613c-1.113,0.3-2.188,0.801-2.188,0.801s-0.89,0.289-0.551,1.013c0.338,0.726,0.838,1.076,0.838,1.076s0.711,0.69,0.736,1.213c0.021,0.526-0.199,0.765-0.764,1.076c-0.563,0.313-1.075,0.375-1.075,0.375s-1.338,0.24-2.001-0.387c-0.663-0.626-0.787-1.663-0.787-1.663s-0.05-0.876-1.148-1.251c-1.102-0.375-2.453,0.425-2.453,0.425s-1.063,0.563-2.2,1.3c-1.14,0.738-3.291,1.64-4.642,2.114l-0.152,0.699l-0.758-0.382c-1.422,0.489-3.271,1.109-3.271,1.109S66.652,21.645,65,21.871c-1.648,0.224-2.016-0.014-2.238-0.238c-0.228-0.224,0.039-1.012,0.039-1.012s0.674-1.376,0.348-2.09c-0.324-0.714-2.451-0.9-2.486-0.9c-0.04,0-2.318-0.265-4.451,0.932c-1.895,1.062-2.143,1.642-2.143,1.642s-0.604,0.938,0.113,1.867c0.807,1.051,1.879,1.146,1.879,1.146s1.021,0.079,1.258,0.38c0.24,0.299,0.076,0.626,0.076,0.626s-0.336,0.925-2.228,1.312c0,0-3.181,0.933-9.113,1.776l-0.582,0.579c-3.743,0.47-8.143,1.363-10.555,1.989c-2.412,0.627-2.553,1.317-2.553,1.317c-0.519,0.813,0.141,1.236,0.141,1.236s0.829,0.283,1.017,1.19c0.19,0.91-0.783,1.629-0.783,1.629s-1.159,0.972-2.898,1.269c-1.739,0.297-2.396,0.35-3.429-0.47c-0.91-0.721-0.298-1.863,0.312-2.301c0.612-0.438,0.909-0.91,0.988-1.112c0.079-0.204,0.032-0.377,0.032-0.377l-0.58-0.534c-2.005-1.33-9.883,0.063-9.883,0.063s0,0.002,0,0l-1.341,0.289l-0.673-0.435c0,0-2.291,0.604-4.876,0.589c-2.712-0.014-1.27-2.128-1.27-2.128s0.638-1.118,0.75-1.764c0,0,0.224-1.745-1.42-1.631c-1.645,0.112-1.968,1.546-1.968,1.546s-0.112,0.801-0.089,1.392c0.021,0.594-0.002,1.847-0.742,2.56c-0.737,0.713-1.529,0.495-1.529,0.495s-1.331-0.582-1.595-0.718c-0.274-0.142-0.517-0.151-0.751-0.066c-0.02,0.007-0.039,0.018-0.057,0.029l-0.175,0.75l0.025-0.623c-0.156,0.176-0.271,0.42-0.271,0.42l0.088,0.327l-0.248,0.565c-0.002,0.012-0.005,0.023-0.006,0.035l0.008,0.003c0,0,0.087-0.196,0.222-0.357l0.182-0.369L1.493,32.94c0.055-0.044,0.112-0.07,0.172-0.074c0.281-0.017,0.629,0.158,0.629,0.158s1.135,0.611,1.642,0.716s0.875-0.065,1.205-0.292c0.527-0.365,1.143-1.121,1.4-1.839c0.229-0.646,0.279-2.394,0.279-2.394l0.004,0.014c0,0,0-0.421-0.011-0.518c-0.012-0.098-0.075-0.553,0.204-0.783c0.278-0.234,0.459-0.347,1.144-0.364c0.68-0.017,0.886,0.38,0.886,0.38S9.28,28.269,9.28,28.48c0,0.21-0.068,0.34-0.068,0.34s-0.371,0.626-0.5,0.934c-0.13,0.307-0.636,1.323-0.489,2.177c0.148,0.852,1.479,1.251,1.479,1.251s1.062,0.25,2.575,0.027l3.12-0.503l0.543-0.624l0.218,0.474c0.805-0.147,2.14-0.369,3.583-0.511c2.326-0.228,4.787-0.262,5.821-0.132c0,0,0.484,0.061,0.405,0.234c-0.062,0.136-0.421,0.415-0.421,0.415s-0.135,0.081-0.654,0.667s-0.671,1.155-0.693,1.661c-0.024,0.505,0.516,1.995,2.415,2.394c1.901,0.397,4.077-0.341,4.077-0.341s1.427-0.314,2.786-1.674c1.41-1.411,0.012-3.108,0.012-3.108s-0.22-0.304-0.647-0.442c0,0-0.187-0.05-0.079-0.17c0.108-0.12,0.37-0.647,1.37-0.905c1-0.257,3.977-1.198,11.065-2.135l0.274-0.617l0.874,0.459c2.708-0.373,4.551-0.694,7.827-1.454c0,0,3.04-0.539,3.675-1.651c0.368-0.65,1.032-1.539-0.045-2.434c0,0-0.125-0.161-0.791-0.312c-0.666-0.151-1.179-0.252-1.738-0.653c-0.563-0.403-0.551-0.869-0.551-0.869s-0.047-0.445,0.862-0.958c0,0,0.851-0.623,2.54-1.158c1.691-0.533,3.648-0.296,3.648-0.296s0.82,0.089,0.685,0.643c-0.14,0.555-0.604,1.482-0.622,1.959c-0.021,0.474,0.012,1.027,0.938,1.414c0.931,0.386,2.267,0.266,2.267,0.266s1.386-0.069,3.647-0.801c2.265-0.731,4.594-1.573,4.594-1.573l-0.094-0.483l0.971,0.173c0.591-0.22,1.406-0.539,2.285-0.936c1.801-0.81,2.656-1.488,3.48-1.958c0,0,1.815-1.118,2.887-1.088c0,0,0.25-0.009,0.272,0.32c0.022,0.329,0.104,0.814,0.218,1.096c0.111,0.281,0.734,1.822,2.729,2.048c1.993,0.228,2.846-1.118,2.846-1.118s0.683-1.049-0.493-2.296c0,0-0.961-1.028-0.99-1.107c0,0-0.104-0.155,0.168-0.233c0.269-0.078,3.152-0.884,4.268-1.398c0,0,0.012-0.005,0.036-0.015l-0.271-0.485l0.535,0.365c0.202-0.101,0.449-0.246,0.544-0.395c0.171-0.271-0.054-0.797-0.054-0.797l0.006-0.007c-0.133-0.184-0.266-0.366-0.4-0.546l-0.606-0.256l-0.06,0.033L91.075,11.742"/><path style="&st34;" d="M8.12,16.324l-0.291,0.435c0.134-0.023,0.244-0.056,0.244-0.056s0.404-0.066,1.112,0.12c0.917,0.244,2.067-0.496,2.067-0.496s0.872-0.523,1.274-1.381l0.361-1.005c0,0,0.291-0.972,1.105-1.281c0,0,0.317-0.171,0.831-0.177c0.513-0.005,0.392,0.354,0.392,0.354s-0.056,0.233-0.586,0.912c-0.529,0.677-0.671,1.196-0.671,1.196s-0.318,0.722,0.193,0.957c0,0,0.24,0.141,0.939-0.091c0.7-0.233,3.02-0.843,4.438-1.06l0.993-0.506c-0.313-0.23-0.602-0.444-0.602-0.444c-2.388,0.442-4.168,0.995-4.775,1.226c-0.609,0.23-0.62,0.082-0.62,0.082c-0.211-0.248,0.435-1.022,0.435-1.022s0.655-0.894,0.913-1.715c0.259-0.82-0.892-0.707-0.892-0.707c-0.758,0.121-1.303,0.48-1.93,1.236c-0.486,0.586-0.782,1.389-0.893,1.814c-0.071,0.267-0.193,0.515-0.193,0.515s-0.091,0.165-0.45,0.322c-0.416,0.182-1.228,0.396-1.228,0.396s-0.54,0.045-0.931-0.15c-0.24-0.118-0.901,0.05-0.901,0.05l0.091-0.504l-0.082-0.032l-0.683,0.383c-0.115,0.189-0.228,0.379-0.341,0.569c-0.063,0.146-0.179,0.475,0.044,0.51c0.05,0.008,0.113,0.008,0.164,0.008L8.12,16.324L8.12,16.324z"/><path style="&st34;" d="M13.295,9.19c0,0,0.342-0.271,0.342-0.026c0,0.248-0.333,0.69-0.333,0.69s-0.107,0.134-0.172,0.134C13.068,9.987,13.15,9.516,13.295,9.19L13.295,9.19z"/><path style="&st12;" d="M-16.122-14.641"/><path style="&st34;" d="M10.595,12.501c0,0-0.388,0.46,0,0.52l0.838-0.828c-0.037-0.037-0.054-0.055-0.157-0.112C11.277,12.081,10.752,12.343,10.595,12.501L10.595,12.501z"/><path style="&st28;" d="M77.807,85.745c0,0-1.75,0.806-3.396,1.603c-1.647,0.803-2.25,1.14-2.25,1.14s-0.088,0.049-0.031,0.082c0.056,0.028-0.008,0.063-0.008,0.063c-0.847,0.409-2.703,1.07-2.703,1.07s-0.167,0.057-0.157,0.02c0.006-0.041,0.199-0.105,0.199-0.105s0.381-0.146,0.283-0.217c-0.098-0.072-1.21,0.305-1.21,0.305s-1.949,0.621-2.894,0.887c-0.941,0.269-2.75,0.735-2.75,0.735c-1.201,0.298-3.75,0.798-4.315,0.901c-0.562,0.104-0.417,0.064-0.417,0.064s0.492-0.189-0.411-0.226c-1.146-0.05-2.362,0.112-3.612,0.288c-1.25,0.18-1.178,0.324-1.178,0.324s-0.04,0.154,0.708,0.14c0.752-0.019,0.534,0.046,0.534,0.046c-1.396,0.202-4.2,0.238-4.2,0.238l0.005,0.01c0.456,0.014,0.916,0.018,1.376,0.018c0.608,0,1.218-0.01,1.822-0.031c0.241-0.021,1.483-0.15,1.925-0.244c0.486-0.105-0.093-0.105-0.093-0.105l-0.74-0.023c0,0-0.776-0.026-0.052-0.158c0.729-0.133,1.834-0.192,2.388-0.252c0.83-0.094,0.541,0.105,0.541,0.105l-0.04,0.023c0,0,0.014,0.094,0.528,0.066c0.515-0.024,4.721-0.804,7.069-1.487c2.347-0.688,4.102-1.255,4.102-1.255s0.157-0.055,0.066,0.025c-0.094,0.078-0.463,0.2-0.463,0.2s-0.595,0.3,0.844-0.108c0.066-0.02,0.134-0.039,0.197-0.06c1.234-0.469,2.446-0.983,3.635-1.543c0.029-0.028,0.131-0.146-0.021-0.101c-0.172,0.056,0-0.065,0.187-0.131c0.184-0.066,2.267-1.162,3.363-1.608l0.002,0.003c0.394-0.227,0.781-0.455,1.17-0.691L77.807,85.745L77.807,85.745z"/><linearGradient id="SVGID_26_" gradientUnits="userSpaceOnUse" x1="217.6563" y1="-436.751" x2="217.6563" y2="-436.751" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset=".078" stop-color="#F4F4F4"/><stop offset=".3807" stop-color="#CECECE"/><stop offset=".5396" stop-color="#BFBFBF"/><stop offset=".8357" stop-color="#7C7C7C"/><stop offset=".8996" stop-color="#A8A8A8"/><stop offset=".9093" stop-color="#9A9A9A"/><stop offset=".9327" stop-color="#7D7D7D"/><stop offset=".9558" stop-color="#686868"/><stop offset=".9785" stop-color="#5B5B5B"/><stop offset="1" stop-color="#575757"/></linearGradient><path style="&st46;" d="M3.893,50.249L3.893,50.249L3.893,50.249z"/><linearGradient id="SVGID_27_" gradientUnits="userSpaceOnUse" x1="214.3262" y1="-436.5107" x2="322.8717" y2="-424.4851" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset=".078" stop-color="#F4F4F4"/><stop offset=".3807" stop-color="#CECECE"/><stop offset=".5396" stop-color="#BFBFBF"/><stop offset=".8357" stop-color="#7C7C7C"/><stop offset=".8996" stop-color="#A8A8A8"/><stop offset=".9093" stop-color="#9A9A9A"/><stop offset=".9327" stop-color="#7D7D7D"/><stop offset=".9558" stop-color="#686868"/><stop offset=".9785" stop-color="#5B5B5B"/><stop offset="1" stop-color="#575757"/></linearGradient><path style="&st47;" d="M1.193,48.543l0.107,0.98c0.236,0.285,0.579,0.675,1.067,1.181c1.084,1.121,1.428,0.416,1.428,0.416s0.072-0.201,0.092-0.586c-0.042,0.216-0.209,0.403-0.788-0.056c-0.298-0.237-0.688-0.627-1.076-1.049c-0.326-0.326-0.723-0.742-1.021-1.117l0.048,0.065l-0.213,0.526c0,0,0.069,0.115,0.268,0.376L1.193,48.543L1.193,48.543z"/><linearGradient id="SVGID_28_" gradientUnits="userSpaceOnUse" x1="213.8887" y1="-436.4771" x2="336.8819" y2="-422.851" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#FFF"/><stop offset=".078" stop-color="#F4F4F4"/><stop offset=".3807" stop-color="#CECECE"/><stop offset=".5396" stop-color="#BFBFBF"/><stop offset=".8357" stop-color="#7C7C7C"/><stop offset=".8996" stop-color="#A8A8A8"/><stop offset=".9093" stop-color="#9A9A9A"/><stop offset=".9327" stop-color="#7D7D7D"/><stop offset=".9558" stop-color="#686868"/><stop offset=".9785" stop-color="#5B5B5B"/><stop offset="1" stop-color="#575757"/></linearGradient><path style="&st48;" d="M3.741,49.133c-0.006-0.027-0.013-0.054-0.02-0.078c0.012,0.088,0.028,0.179,0.043,0.272c0,0,0.094,0.394,0.12,0.753C3.87,49.813,3.83,49.498,3.741,49.133z"/><linearGradient id="SVGID_29_" gradientUnits="userSpaceOnUse" x1="411.2539" y1="557.002" x2="507.2363" y2="537.6277" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset="1" stop-color="#BFC0C2"/></linearGradient><path style="&st49;" d="M99.696,28.566l0.29,0.316c0.406-0.238,0.654,0.045,0.654,0.045l-0.386-0.494c-0.383,0.082-1.093,0.256-2.233,0.61c0,0-1.353,0.594-1.59,1.532c-0.016,0.332,0.049,0.641,0.116,0.876c0.753,1.429,0.913,1.666,0.913,1.666c0.499,1.091,0.422,1.993,0.25,2.824c-0.188,0.921-1.188,1.668-1.794,1.842c-0.608,0.171-0.953,0-1.14-0.483c-0.188-0.485-0.157-0.845-0.438-2.34c-0.28-1.499-0.653-2.2-1.216-2.372c-0.563-0.173-1.313,0.468-1.749,0.811c-0.438,0.344-2.417,2.152-2.417,2.152s-2.324,2.091-5.743,4.026c-3.417,1.935-7.69,3.418-7.69,3.418s-2.842,1.092-3.525,0.998c-0.688-0.093-0.516-0.718-0.297-1.076c0.219-0.36,0.516-0.937,0.766-1.545c0.313-0.756,0.201-1.654-0.359-1.967c-0.562-0.311-1.248-0.468-3.523,0.094s-3.197,1.076-3.838,1.451c-0.643,0.376-1.576,1.233-1.842,1.716c-0.266,0.484-0.563,0.983,0.126,1.576c0.688,0.594,0.763,0.485,1.56,1.046c0.797,0.562,0.922,0.64,0.969,1.466c0.045,0.827-0.766,1.341-0.766,1.341s-3.123,2.082-6.602,2.777c-2.34,0.469-6.833,0.639-6.833,0.639s-1.327-0.045-5.384,0.547c-4.056,0.594-6.787,1.514-7.738,1.856c-0.952,0.343-2.34,0.81-3.136,1.17c-0.795,0.358-0.375,0.89-0.203,1.015c0.172,0.123,1.185,1.074,1.498,1.354c0.312,0.282,1.466,1.279,0.108,2.108c-1.356,0.826-3.603,1.264-3.603,1.264s-3.121,0.654-4.119,0.234c-1.001-0.422-1.2-0.778-1.452-1.358c-0.313-0.718-0.016-1.263,0.794-1.78c0.77-0.486,0.469-1.199,0.469-1.199c-0.983-1.9-3.058-2.058-4.774-1.936c-2.72,0.198-5.358,0.471-7.425,0.734c-3.059,0.39-4.541-0.063-5.992-0.516c-1.452-0.453-2.997-1.529-3.154-2.498c-0.027-0.097-0.039-0.199-0.042-0.307c-0.078-0.758,0.074-1.146,0.184-1.784c0.108-0.625-0.043-1.532-0.084-1.758c-0.008-0.026-0.016-0.058-0.023-0.085c-0.507-1.304-1.725-1.903-2.668-2.058c-0.953-0.157-0.983,0.857-0.983,0.857s-0.002,0.012-0.006,0.04c0.078-0.208,0.222-0.315,0.444-0.31c0.577,0.02,2.259,0.361,2.781,1.754c0,0,0.218,0.648,0.019,1.479c-0.199,0.832-0.162,1.571-0.019,2.295c0.145,0.725,0.759,1.408,1.465,1.969c0.704,0.562,2.745,1.535,4.734,1.66c1.814,0.117,2.483,0.037,2.587,0.023l0.759-0.768l0.135,0.666c0.81-0.088,2.115-0.229,3.619-0.394c2.529-0.271,4.227-0.541,5.8-0.306c1.572,0.232,2.078,1.463,2.078,1.463s0.107,0.313-0.253,0.542c-0.349,0.226-0.75,0.548-0.947,0.891c-0.2,0.345-0.411,1.066,0.159,2.033c0.572,0.965,2.142,1.595,3.478,1.435c1.336-0.158,3.336-0.672,4.521-0.98c1.286-0.336,2.424-0.969,2.572-1.979c0.148-1.009-1.534-2.297-1.534-2.297s-0.81-0.611-0.909-0.75c-0.1-0.14,0.099-0.197,0.099-0.197s0.95-0.229,1.356-0.416c0.403-0.19,1.523-0.644,3.353-1.217c1.831-0.572,5.609-1.248,5.609-1.248s2.09-0.332,4.694-0.543l0.612-0.705l0.493,0.627c0.406-0.023,0.819-0.045,1.235-0.061c3.572-0.129,5.343-0.555,7.24-0.979c1.897-0.426,4.569-1.979,4.965-2.276c0.396-0.296,1.229-0.662,1.395-1.958c0.17-1.294-1.363-2.157-1.363-2.157s-1.218-0.643-1.476-0.929c-0.256-0.288-0.019-0.562-0.019-0.562s0.689-1.485,2.896-2.355c2.207-0.872,3.69-1.106,4.619-1.207c0.931-0.099,1.247,0.237,1.375,0.395c0.128,0.158,0.128,0.485,0.021,0.821c-0.101,0.308-0.443,1.039-0.644,1.396c-0.199,0.356-0.522,1.216-0.317,1.622c0.211,0.405,0.842,0.592,1.662,0.444c0.822-0.147,2.987-0.761,4.889-1.553c1.897-0.793,5.074-2.344,5.074-2.344s0.01-0.005,0.021-0.013l0.086-0.729l0.729,0.296c1.021-0.562,2.764-1.582,4.01-2.63c1.871-1.574,3.699-3.225,4.166-3.641c0.465-0.415,0.89-0.751,1.305-0.732c0.416,0.021,0.734,0.318,0.99,1.434c0.258,1.119,0.306,2.038,0.426,2.583c0.117,0.542,0.285,1.176,0.929,1.305c0.644,0.128,1.513-0.436,1.841-0.713c0.326-0.277,0.918-1.176,0.879-2.383c0,0,0.068-1.605-0.762-3.127c0,0-0.348-0.614-0.477-0.782c-0.088-0.119-0.155-0.238-0.008-0.523c0.146-0.288,0.465-0.821,1.059-1.157c0.449-0.256,0.881-0.316,1.128-0.396c0,0,0.28-0.069,0.636-0.266L99.696,28.566L99.696,28.566z"/><path style="&st23;" d="M1.547,32.696l-0.05,0.239c0.053-0.041,0.111-0.068,0.17-0.072c0.281-0.018,0.629,0.158,0.629,0.158s1.135,0.61,1.642,0.716c0.507,0.105,0.875-0.065,1.205-0.292c0.25-0.174,0.521-0.434,0.763-0.737c-0.29,0.29-0.725,0.548-1.365,0.595c0,0-0.571-0.049-1.419-0.587c-0.846-0.537-1.327-0.231-1.729,0.105l0,0l-0.249,0.564c-0.001,0.013-0.003,0.023-0.006,0.036l0.009,0.001c0,0,0.087-0.192,0.217-0.352L1.547,32.696L1.547,32.696z"/><path style="&st4;" d="M6.387,31.961c0.059-0.12,0.112-0.238,0.155-0.354c0.05-0.14,0.09-0.334,0.125-0.55c-0.096,0.44-0.223,0.73-0.223,0.73S6.428,31.853,6.387,31.961L6.387,31.961z"/><path style="&st4;" d="M6.807,28.654c-0.038,0.439-0.038,0.987-0.038,1.493l0,0c0.04-0.509,0.052-0.935,0.052-0.935l0.004,0.014c0,0,0-0.421-0.011-0.518C6.811,28.695,6.809,28.677,6.807,28.654z"/><linearGradient id="SVGID_30_" gradientUnits="userSpaceOnUse" x1="413.5137" y1="516.4121" x2="417.999" y2="516.4121" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#E2E3E4"/><stop offset=".5055" stop-color="#FFF"/></linearGradient><path style="&st50;" d="M8.12,16.324l-0.291,0.435c0.087-0.014,0.089-0.007,0.244-0.056c0,0,0.404-0.066,1.112,0.12c0.917,0.244,2.067-0.496,2.067-0.496s0.272-0.164,0.583-0.458c-0.4,0.24-1.385,0.762-2.132,0.585c-0.961-0.229-0.994-0.017-1.336-0.049c-0.292-0.028-0.292-0.11-0.275-0.314c0.002-0.032,0.023-0.054,0.03-0.092c0.05-0.261,0.339-0.689,0.339-0.689l-0.684,0.383c-0.115,0.189-0.227,0.378-0.34,0.569c-0.063,0.146-0.179,0.475,0.044,0.509c0.055,0.009,0.11,0.011,0.163,0.009L8.12,16.324L8.12,16.324z"/><path style="&st23;" d="M17.019,15.036c-1.027,0.278-1.972,0.734-2.494,0.604c-0.438-0.109-0.084-0.76,0.042-0.965c0.069-0.146,0.167-0.292,0.249-0.406c0.31-0.506,0.833-1.351,0.833-1.351s0,0,0-0.002c0.017-0.032,0.033-0.067,0.049-0.102c0.067-0.18,0.12-0.537-0.472-0.597c-0.799-0.082-1.613,0.619-1.613,0.619s-0.547,0.301-0.838,1.417l0.112-0.313c0,0,0.291-0.973,1.105-1.281c0,0,0.317-0.171,0.831-0.177c0.513-0.005,0.392,0.354,0.392,0.354s-0.056,0.233-0.586,0.912c-0.529,0.677-0.671,1.195-0.671,1.195s-0.319,0.723,0.193,0.957c0,0,0.238,0.141,0.939-0.091c0.7-0.233,3.02-0.843,4.438-1.06l0.993-0.506C19.849,14.299,18.012,14.769,17.019,15.036L17.019,15.036z"/><path style="&st23;" d="M-16.122-14.641"/><path style="&st23;" d="M-16.122-14.641"/><path style="&st59;" d="M90.578,74.217c0.006-0.004,0.009-0.008,0.014-0.013C90.587,74.21,90.582,74.213,90.578,74.217z"/><path style="&st59;" d="M90.821,73.951c0.001-0.002,0.002-0.004,0.003-0.004C90.823,73.947,90.822,73.949,90.821,73.951z"/><path style="&st59;" d="M90.738,74.042c0.003-0.004,0.008-0.009,0.011-0.013C90.746,74.033,90.741,74.038,90.738,74.042z"/><path style="&st8;" d="M87.9,76.68l-0.164,0.174c-0.646,0.716-0.969,1.007-0.969,1.007l-0.242,0.269l0,0c-0.002,0.006-0.006,0.012-0.012,0.014c0.008,0.002,0.018,0.004,0.041,0.004c0,0,0.229-0.002,0.588-0.219c-0.072,0.041-0.139,0.068-0.181,0.07c-0.177,0.012,0.431-0.538,0.431-0.538s0.588-0.626,0.967-1.125c0.382-0.497,0.138-0.468,0.138-0.468s-0.087-0.021-0.397,0.125C88.125,76.018,88.268,76.223,87.9,76.68L87.9,76.68z"/><path style="&st59;" d="M90.657,74.133c0.003-0.006,0.007-0.013,0.013-0.017C90.664,74.121,90.662,74.128,90.657,74.133z"/><path style="&st0;" d="M72.945,81.21c0,0,2.215-0.931,3.402-0.79c0,0,0.611,0.059-0.062,0.711c-0.672,0.652-0.892,1.168,0.396,0.752c0,0,0.094-0.033,0.244-0.086c-0.504,0.143-0.734,0.143-0.658,0.033c0.127-0.188,0.461-0.461,0.461-0.461s0.399-0.4,0.399-0.801c0,0,0.128-0.588-1.604-0.225c-1.01,0.209-1.969,0.59-2.588,0.867C72.941,81.211,72.943,81.211,72.945,81.21L72.945,81.21z"/><path style="&st59;" d="M72.234,81.555l-0.004,0.002C72.232,81.557,72.234,81.555,72.234,81.555z"/><path style="&st59;" d="M88.057,76.014c-0.014,0.008-0.026,0.015-0.043,0.021C88.028,76.027,88.043,76.021,88.057,76.014z"/><path style="&st14;" d="M78.305,81.299c1.448-0.521,3.93-1.854,5.023-2.492C82.232,79.442,79.73,80.717,78.305,81.299z"/><path style="&st59;" d="M87.512,76.325c0.026-0.017,0.055-0.032,0.08-0.049C87.566,76.293,87.538,76.309,87.512,76.325z"/><path style="&st59;" d="M91.618,73.084c0.008-0.011,0.019-0.023,0.026-0.029C91.637,73.061,91.626,73.073,91.618,73.084z"/><path style="&st59;" d="M91.434,73.281c0.01-0.012,0.019-0.02,0.023-0.027C91.451,73.264,91.441,73.27,91.434,73.281z"/><path style="&st59;" d="M91.521,73.188c0.01-0.012,0.02-0.022,0.029-0.032C91.539,73.165,91.53,73.178,91.521,73.188z"/><path style="&st59;" d="M90.9,73.863c0.003-0.006,0.007-0.01,0.012-0.016C90.907,73.854,90.903,73.857,90.9,73.863z"/><path style="&st59;" d="M71.246,82.088c-0.014,0.008-0.023,0.018-0.036,0.022C71.221,82.105,71.232,82.096,71.246,82.088z"/><path style="&st59;" d="M91.346,73.375c0.008-0.008,0.014-0.015,0.021-0.021C91.359,73.36,91.354,73.367,91.346,73.375z"/><path style="&st59;" d="M91.077,73.669c0.006-0.006,0.01-0.011,0.013-0.015C91.087,73.659,91.083,73.663,91.077,73.669z"/><path style="&st59;" d="M91.251,73.479c0.007-0.006,0.015-0.012,0.021-0.021C91.266,73.463,91.255,73.473,91.251,73.479z"/><path style="&st59;" d="M90.987,73.768c0.004-0.006,0.01-0.012,0.016-0.021C90.997,73.756,90.991,73.762,90.987,73.768z"/><path style="&st59;" d="M91.161,73.578c0.006-0.006,0.011-0.014,0.017-0.02C91.172,73.564,91.165,73.572,91.161,73.578z"/><path style="&st59;" d="M71.816,81.764c-0.008,0.006-0.02,0.012-0.026,0.017C71.799,81.774,71.809,81.77,71.816,81.764z"/><path style="&st59;" d="M72.044,81.646c-0.006,0.004-0.012,0.006-0.019,0.009C72.032,81.652,72.038,81.649,72.044,81.646z"/><path style="&st59;" d="M71.305,82.055c-0.011,0.008-0.021,0.014-0.031,0.021C71.283,82.067,71.294,82.063,71.305,82.055z"/><path style="&st59;" d="M71.969,81.684c-0.008,0.004-0.015,0.009-0.021,0.014C71.953,81.692,71.961,81.688,71.969,81.684z"/><path style="&st59;" d="M72.109,81.613c-0.004,0.005-0.01,0.006-0.016,0.01C72.102,81.619,72.105,81.617,72.109,81.613z"/><path style="&st59;" d="M72.008,81.665c-0.007,0.004-0.013,0.007-0.021,0.011C71.994,81.672,72.001,81.669,72.008,81.665z"/><path style="&st59;" d="M72.206,81.566c-0.003,0.002-0.005,0.003-0.007,0.004C72.201,81.569,72.203,81.568,72.206,81.566z"/><path style="&st59;" d="M72.148,81.594c-0.008,0.004-0.016,0.01-0.024,0.015C72.135,81.604,72.141,81.598,72.148,81.594z"/><path style="&st59;" d="M71.928,81.707c-0.01,0.003-0.02,0.01-0.027,0.016C71.908,81.715,71.918,81.711,71.928,81.707z"/><path style="&st59;" d="M72.193,81.575c-0.006,0.001-0.01,0.003-0.014,0.004C72.184,81.578,72.189,81.575,72.193,81.575z"/><path style="&st59;" d="M72.172,81.585c-0.004,0.003-0.008,0.005-0.015,0.007C72.164,81.59,72.168,81.585,72.172,81.585z"/><path style="&st59;" d="M72.079,81.631c-0.008,0.002-0.013,0.004-0.019,0.006C72.066,81.635,72.071,81.633,72.079,81.631z"/><path style="&st59;" d="M71.866,81.739c-0.011,0.002-0.021,0.008-0.027,0.013C71.848,81.747,71.857,81.741,71.866,81.739z"/><path style="&st59;" d="M71.555,81.906c-0.018,0.012-0.033,0.02-0.052,0.029C71.521,81.924,71.539,81.918,71.555,81.906z"/><path style="&st59;" d="M72.219,81.561c0,0.002-0.002,0.002-0.002,0.004C72.217,81.563,72.219,81.563,72.219,81.561z"/><path style="&st59;" d="M71.421,81.986c-0.011,0.006-0.022,0.01-0.03,0.02C71.398,81.996,71.41,81.992,71.421,81.986z"/><path style="&st59;" d="M71.363,82.02c-0.012,0.005-0.021,0.013-0.032,0.021C71.342,82.032,71.354,82.024,71.363,82.02z"/><path style="&st59;" d="M71.482,81.948c-0.014,0.006-0.023,0.015-0.036,0.021C71.458,81.963,71.471,81.956,71.482,81.948z"/><path style="&st59;" d="M71.768,81.79c-0.01,0.004-0.018,0.009-0.024,0.017C71.751,81.799,71.758,81.794,71.768,81.79z"/><path style="&st59;" d="M91.708,72.984c0.013-0.014,0.024-0.028,0.04-0.042C91.734,72.956,91.723,72.971,91.708,72.984z"/><path style="&st59;" d="M71.717,81.818c-0.01,0.004-0.019,0.009-0.027,0.015C71.698,81.827,71.707,81.822,71.717,81.818z"/><path style="&st59;" d="M71.665,81.846c-0.011,0.006-0.021,0.014-0.028,0.018C71.645,81.858,71.654,81.852,71.665,81.846z"/><path style="&st59;" d="M71.609,81.875c-0.01,0.008-0.021,0.014-0.03,0.02C71.589,81.889,71.602,81.881,71.609,81.875z"/><path style="&st59;" d="M92.443,72.197c-0.021,0.021-0.037,0.039-0.055,0.061C92.406,72.238,92.423,72.217,92.443,72.197z"/><path style="&st59;" d="M92.359,72.287c-0.021,0.021-0.041,0.044-0.061,0.066C92.318,72.331,92.339,72.309,92.359,72.287z"/><path style="&st59;" d="M92.501,72.135c-0.011,0.015-0.021,0.023-0.028,0.033C92.48,72.157,92.49,72.148,92.501,72.135z"/><path style="&st59;" d="M91.786,72.899c0.022-0.021,0.042-0.042,0.062-0.063C91.828,72.857,91.807,72.878,91.786,72.899z"/><path style="&st59;" d="M92.273,72.379c-0.014,0.016-0.025,0.029-0.041,0.045C92.245,72.409,92.26,72.395,92.273,72.379z"/><path style="&st59;" d="M92.579,72.053c-0.013,0.012-0.021,0.021-0.03,0.032C92.559,72.074,92.569,72.063,92.579,72.053z"/><path style="&st59;" d="M92.762,71.858c-0.002,0-0.002,0-0.002,0.001C92.76,71.858,92.76,71.858,92.762,71.858z"/><path style="&st59;" d="M92.703,71.919c-0.004,0.005-0.01,0.013-0.014,0.017C92.693,71.928,92.699,71.924,92.703,71.919z"/><path style="&st59;" d="M87.938,76.074c-0.017,0.008-0.027,0.016-0.043,0.023C87.907,76.09,87.922,76.082,87.938,76.074z"/><path style="&st59;" d="M92.648,71.98c-0.012,0.011-0.021,0.021-0.029,0.029C92.629,72,92.637,71.991,92.648,71.98z"/><path style="&st59;" d="M87.666,76.229c-0.018,0.013-0.031,0.021-0.051,0.031C87.633,76.25,87.648,76.241,87.666,76.229z"/><path style="&st59;" d="M48.204,91.026c0,0-0.001,0-0.003,0.002C48.203,91.026,48.203,91.026,48.204,91.026z"/><path style="&st59;" d="M48.073,91.03c-0.007,0-0.013,0-0.021,0C48.06,91.03,48.066,91.03,48.073,91.03z"/><path style="&st59;" d="M48.136,91.028c-0.007,0.002-0.013,0.002-0.02,0.002C48.123,91.03,48.129,91.03,48.136,91.028z"/><path style="&st59;" d="M48.005,91.032c-0.003,0-0.007,0-0.012,0C47.998,91.032,48.002,91.032,48.005,91.032z"/><path style="&st59;" d="M47.94,91.035c-0.004,0-0.009,0-0.015,0C47.93,91.035,47.935,91.035,47.94,91.035z"/><path style="&st59;" d="M87.869,76.111c-0.014,0.006-0.021,0.015-0.039,0.021C87.848,76.126,87.857,76.117,87.869,76.111z"/><path style="&st59;" d="M87.807,76.146c-0.014,0.009-0.025,0.018-0.041,0.021C87.779,76.164,87.793,76.155,87.807,76.146z"/><path style="&st59;" d="M87.738,76.186c-0.016,0.011-0.028,0.018-0.044,0.024C87.71,76.203,87.723,76.196,87.738,76.186z"/><path style="&st59;" d="M87.951,76.067c0.02-0.011,0.035-0.021,0.054-0.027C87.986,76.047,87.971,76.057,87.951,76.067z"/><path style="&st59;" d="M87.719,77.52c0,0-0.082,0.064-0.193,0.147C87.592,77.621,87.664,77.566,87.719,77.52z"/><path style="&st59;" d="M92.205,72.452c-0.021,0.022-0.042,0.046-0.064,0.067C92.163,72.498,92.186,72.475,92.205,72.452z"/><path style="&st59;" d="M28.05,87.006c-0.019-0.007-0.037-0.016-0.056-0.021C28.013,86.991,28.031,86.999,28.05,87.006z"/><path style="&st59;" d="M71.051,82.217c-0.013,0.008-0.021,0.017-0.037,0.021C71.027,82.23,71.038,82.223,71.051,82.217z"/><path style="&st59;" d="M45.854,89.871c-0.024,0-0.049-0.004-0.074-0.004C45.806,89.867,45.831,89.871,45.854,89.871z"/><path style="&st59;" d="M27.678,86.883c0.02,0.004,0.039,0.01,0.058,0.019C27.717,86.893,27.696,86.887,27.678,86.883z"/><path style="&st59;" d="M27.555,86.844c0.038,0.012,0.078,0.022,0.119,0.037C27.633,86.869,27.593,86.854,27.555,86.844z"/><path style="&st59;" d="M27.862,86.941c-0.041-0.014-0.082-0.026-0.123-0.04C27.78,86.913,27.82,86.928,27.862,86.941z"/><path style="&st59;" d="M91.88,72.799c0.015-0.016,0.026-0.027,0.04-0.043C91.907,72.771,91.895,72.784,91.88,72.799z"/><path style="&st59;" d="M28.058,87.006c0.04,0.016,0.079,0.026,0.116,0.041C28.137,87.033,28.098,87.021,28.058,87.006z"/><path style="&st59;" d="M91.977,72.695c0.014-0.016,0.026-0.027,0.041-0.044C92.002,72.668,91.989,72.68,91.977,72.695z"/><path style="&st59;" d="M92.057,72.612c0.021-0.022,0.039-0.041,0.061-0.062C92.096,72.571,92.076,72.59,92.057,72.612z"/><path style="&st59;" d="M27.991,86.985c-0.021-0.009-0.041-0.017-0.062-0.022C27.95,86.969,27.97,86.977,27.991,86.985z"/><path style="&st59;" d="M71.133,82.162c0.021-0.013,0.037-0.021,0.055-0.036C71.17,82.139,71.152,82.149,71.133,82.162z"/><path style="&st59;" d="M46.361,89.926c-0.016-0.002-0.035-0.004-0.053-0.008C46.325,89.922,46.345,89.924,46.361,89.926z"/><path style="&st59;" d="M71.122,82.17c-0.019,0.012-0.034,0.021-0.049,0.032C71.088,82.191,71.105,82.18,71.122,82.17z"/><path style="&st59;" d="M46.439,89.939c-0.017-0.002-0.032-0.006-0.049-0.008C46.407,89.934,46.422,89.938,46.439,89.939z"/><path style="&st59;" d="M46.513,89.953c-0.016-0.005-0.031-0.007-0.047-0.011C46.481,89.946,46.497,89.948,46.513,89.953z"/><path style="&st59;" d="M46.096,89.891c-0.031-0.002-0.065-0.006-0.1-0.01C46.031,89.885,46.065,89.889,46.096,89.891z"/><path style="&st59;" d="M46.278,89.913c-0.02-0.002-0.041-0.004-0.061-0.006C46.237,89.909,46.259,89.911,46.278,89.913z"/><path style="&st59;" d="M46.19,89.904c-0.028-0.005-0.054-0.008-0.084-0.012C46.136,89.896,46.163,89.899,46.19,89.904z"/><path style="&st59;" d="M45.971,89.881c-0.025-0.005-0.051-0.006-0.079-0.009C45.92,89.875,45.945,89.877,45.971,89.881z"/><path style="&st59;" d="M27.926,86.961c-0.02-0.004-0.039-0.011-0.058-0.019C27.888,86.953,27.908,86.957,27.926,86.961z"/><path style="&st59;" d="M47.395,90.975c0.001,0.004,0.003,0.004,0.004,0.004C47.398,90.979,47.396,90.979,47.395,90.975z"/><path style="&st59;" d="M47.375,90.969c0.001,0,0.002,0,0.003,0C47.378,90.969,47.376,90.969,47.375,90.969z"/><path style="&st59;" d="M47.418,90.982L47.418,90.982L47.418,90.982z"/><path style="&st59;" d="M47.465,90.996c0.003,0,0.004,0.001,0.006,0.001C47.47,90.997,47.468,90.996,47.465,90.996z"/><path style="&st59;" d="M47.442,90.988c0.001,0,0.002,0.002,0.002,0.002L47.442,90.988z"/><path style="&st59;" d="M47.342,90.955L47.342,90.955L47.342,90.955z"/><path style="&st59;" d="M46.589,89.967c-0.02-0.002-0.036-0.009-0.054-0.013C46.553,89.959,46.57,89.965,46.589,89.967z"/><path style="&st59;" d="M46.764,90.012c-0.011-0.004-0.021-0.008-0.033-0.012C46.742,90.006,46.753,90.008,46.764,90.012z"/><path style="&st59;" d="M49.049,90.953c-0.01,0.001-0.021,0.002-0.032,0.004C49.028,90.955,49.038,90.954,49.049,90.953z"/><path style="&st59;" d="M46.659,89.984c-0.021-0.006-0.039-0.01-0.061-0.014C46.619,89.975,46.639,89.979,46.659,89.984z"/><path style="&st59;" d="M47.767,91.028c0.003,0,0.003,0,0.006,0C47.773,91.028,47.77,91.028,47.767,91.028z"/><path style="&st59;" d="M47.821,91.03c0.003,0,0.007,0,0.013,0C47.828,91.03,47.824,91.03,47.821,91.03z"/><path style="&st59;" d="M45.214,89.855c0.047,0,0.092,0,0.139,0C45.308,89.855,45.261,89.855,45.214,89.855z"/><path style="&st59;" d="M47.716,91.026c0.005,0,0.009,0,0.013,0.002C47.727,91.026,47.722,91.026,47.716,91.026z"/><path style="&st32;" d="M46.772,90.014c0,0,0.158,0.021,0.059,0.338c-0.1,0.313,0.06,0.451,0.475,0.594l0.023,0.006c0,0,0.001,0,0.002,0c-0.079-0.04-0.074-0.08-0.074-0.08s-0.01-0.078,0.078-0.216c0.087-0.136,0.009-0.369-0.293-0.536C47.044,90.118,46.967,90.07,46.772,90.014L46.772,90.014L46.772,90.014z"/><path style="&st59;" d="M47.869,91.032c0.006,0,0.012,0,0.018,0C47.881,91.032,47.875,91.032,47.869,91.032z"/><path style="&st59;" d="M47.495,91.002c0.001,0,0.004,0,0.004,0C47.497,91.002,47.496,91.002,47.495,91.002z"/><path style="&st59;" d="M47.677,91.024c0.004,0,0.007,0,0.01,0C47.684,91.024,47.681,91.024,47.677,91.024z"/><path style="&st59;" d="M47.529,91.006c0.001,0,0.002,0,0.002,0S47.53,91.006,47.529,91.006z"/><path style="&st59;" d="M47.559,91.01c0.002,0.002,0.005,0.002,0.008,0.002C47.566,91.012,47.561,91.01,47.559,91.01z"/><path style="&st59;" d="M47.594,91.016c0.002,0,0.004,0,0.008,0C47.599,91.016,47.596,91.016,47.594,91.016z"/><path style="&st59;" d="M46.711,89.997c-0.011-0.002-0.026-0.007-0.043-0.011C46.685,89.99,46.701,89.995,46.711,89.997z"/><path style="&st59;" d="M28.748,87.256c-0.008-0.003-0.018-0.007-0.025-0.01C28.731,87.25,28.739,87.254,28.748,87.256z"/><path style="&st59;" d="M28.776,87.27c-0.007-0.002-0.016-0.008-0.024-0.012C28.761,87.262,28.769,87.268,28.776,87.27z"/><path style="&st59;" d="M28.681,87.23c-0.008-0.004-0.021-0.006-0.03-0.01C28.66,87.225,28.672,87.227,28.681,87.23z"/><path style="&st59;" d="M45.66,89.862c0.026,0,0.052,0.005,0.074,0.005C45.711,89.865,45.686,89.865,45.66,89.862z"/><path style="&st59;" d="M28.797,87.275c-0.005-0.002-0.013-0.006-0.021-0.008C28.784,87.271,28.792,87.273,28.797,87.275z"/><path style="&st59;" d="M27.38,86.794c0.038,0.011,0.074,0.022,0.112,0.031C27.453,86.816,27.417,86.803,27.38,86.794z"/><path style="&st59;" d="M27.492,86.825c0.021,0.004,0.039,0.011,0.058,0.019C27.53,86.836,27.512,86.829,27.492,86.825z"/><path style="&st59;" d="M28.814,87.284c-0.003-0.002-0.006-0.005-0.012-0.007C28.808,87.279,28.811,87.282,28.814,87.284z"/><path style="&st59;" d="M28.64,87.219c-0.011-0.006-0.021-0.01-0.03-0.014C28.621,87.21,28.629,87.213,28.64,87.219z"/><path style="&st61;" d="M27.687,87.239c-0.014-0.013-0.023-0.022-0.03-0.032c-0.263-0.197-0.343-0.418-0.343-0.418l0.009-0.011c0.017,0.007,0.034,0.011,0.052,0.014c-0.291-0.075-0.5-0.104-0.479-0.018c0.015,0.061,0.131,0.153,0.269,0.245c0.174,0.094,0.346,0.187,0.522,0.279V87.239L27.687,87.239z"/><path style="&st59;" d="M28.716,87.245c-0.009-0.003-0.018-0.006-0.027-0.01C28.699,87.239,28.708,87.242,28.716,87.245z"/><path style="&st59;" d="M28.602,87.202c-0.013-0.004-0.027-0.01-0.042-0.015C28.574,87.192,28.59,87.198,28.602,87.202z"/><path style="&st59;" d="M28.841,87.293c0,0-0.006-0.004-0.022-0.009C28.835,87.29,28.841,87.293,28.841,87.293z"/><path style="&st59;" d="M28.232,87.068c-0.018-0.006-0.031-0.014-0.047-0.018C28.201,87.055,28.215,87.063,28.232,87.068z"/><path style="&st59;" d="M45.534,89.858c0.028,0,0.054,0.002,0.081,0.002C45.587,89.858,45.562,89.858,45.534,89.858z"/><path style="&st59;" d="M45.397,89.855c0.029,0,0.06,0,0.089,0C45.457,89.855,45.427,89.855,45.397,89.855z"/><path style="&st59;" d="M28.348,87.11c-0.034-0.013-0.07-0.025-0.106-0.039C28.278,87.085,28.314,87.098,28.348,87.11z"/><path style="&st59;" d="M28.506,87.168c-0.013-0.005-0.025-0.01-0.037-0.015C28.481,87.158,28.494,87.162,28.506,87.168z"/><path style="&st59;" d="M28.401,87.129c-0.013-0.006-0.028-0.011-0.043-0.015C28.373,87.118,28.388,87.123,28.401,87.129z"/><path style="&st59;" d="M28.557,87.186c-0.013-0.004-0.025-0.012-0.04-0.017C28.531,87.176,28.544,87.182,28.557,87.186z"/><path style="&st59;" d="M28.458,87.15c-0.016-0.008-0.031-0.014-0.047-0.018C28.427,87.137,28.442,87.143,28.458,87.15z"/><path style="&st59;" d="M70.02,84.389c0.004,0,0.006,0,0.006,0C70.023,84.389,70.023,84.389,70.02,84.389z"/><path style="&st59;" d="M70.789,84.087c-0.001,0.003-0.003,0.005-0.004,0.005C70.786,84.09,70.788,84.088,70.789,84.087z"/><path style="&st59;" d="M70.762,84.102c-0.002,0.002-0.004,0.002-0.006,0.004C70.758,84.104,70.76,84.104,70.762,84.102z"/><path style="&st59;" d="M70.732,84.116c-0.002,0.002-0.004,0.003-0.006,0.004C70.729,84.119,70.73,84.118,70.732,84.116z"/><path style="&st59;" d="M70.697,84.133c-0.001,0-0.002,0-0.002,0.002C70.695,84.133,70.696,84.133,70.697,84.133z"/><path style="&st59;" d="M70.417,84.249c0.006-0.001,0.009-0.002,0.011-0.002C70.426,84.247,70.423,84.248,70.417,84.249z"/><path style="&st59;" d="M70.389,84.262h0.002H70.389z"/><path style="&st59;" d="M70.662,84.146L70.662,84.146L70.662,84.146z"/><path style="&st59;" d="M70.316,84.289c0.002-0.002,0.007-0.003,0.009-0.003C70.321,84.286,70.318,84.287,70.316,84.289z"/><path style="&st59;" d="M70.285,84.299c0.003,0,0.005-0.002,0.009-0.004C70.29,84.297,70.288,84.299,70.285,84.299z"/><path style="&st59;" d="M70.354,84.275c0.002-0.002,0.002-0.002,0.002-0.002S70.354,84.273,70.354,84.275z"/><path style="&st59;" d="M48.946,90.965c-0.011,0-0.022,0.004-0.033,0.004C48.924,90.969,48.935,90.965,48.946,90.965z"/><path style="&st59;" d="M70.256,84.311c0.002,0,0.004,0,0.006-0.002C70.26,84.311,70.258,84.311,70.256,84.311z"/><path style="&st59;" d="M70.78,82.41c0.013-0.009,0.026-0.021,0.042-0.032C70.809,82.389,70.793,82.401,70.78,82.41z"/><path style="&st59;" d="M70.956,82.279c0.013-0.008,0.022-0.016,0.033-0.023C70.98,82.264,70.969,82.271,70.956,82.279z"/><path style="&st59;" d="M70.898,82.321c0.012-0.009,0.022-0.017,0.033-0.022C70.923,82.305,70.91,82.313,70.898,82.321z"/><path style="&st2;" d="M70.903,83.794c0.2,0.064,0.106,0.171-0.087,0.278l0.089-0.033c0,0,0.793-0.436,0.458-0.633c-0.338-0.198-1.129-0.275-0.613-0.969l0.02-0.02c-0.442,0.344-0.756,0.727-0.498,1.021C70.27,83.443,70.387,83.629,70.903,83.794z"/><path style="&st59;" d="M70.566,84.191c-0.002,0-0.006,0.002-0.008,0.003C70.561,84.193,70.564,84.191,70.566,84.191z"/><path style="&st59;" d="M70.631,84.163c-0.002,0.001-0.004,0.002-0.006,0.003C70.627,84.165,70.629,84.164,70.631,84.163z"/><path style="&st59;" d="M70.598,84.176c0,0.002-0.004,0.004-0.006,0.004C70.594,84.18,70.598,84.178,70.598,84.176z"/><path style="&st59;" d="M70.493,84.223c-0.003,0-0.003,0-0.007,0.002C70.49,84.223,70.49,84.223,70.493,84.223z"/><path style="&st59;" d="M70.459,84.233c-0.002,0-0.004,0.002-0.008,0.004C70.455,84.235,70.457,84.233,70.459,84.233z"/><path style="&st59;" d="M70.842,82.363c0.012-0.01,0.024-0.018,0.034-0.025C70.866,82.346,70.854,82.354,70.842,82.363z"/><path style="&st59;" d="M48.293,91.024c-0.007,0-0.016,0-0.023,0C48.277,91.024,48.286,91.024,48.293,91.024z"/><path style="&st59;" d="M48.444,91.014c-0.004,0-0.011,0-0.017,0C48.434,91.014,48.44,91.014,48.444,91.014z"/><path style="&st59;" d="M48.369,91.02c-0.009,0-0.017,0-0.027,0C48.352,91.02,48.359,91.02,48.369,91.02z"/><path style="&st8;" d="M50.023,89.904c0,0,0.362,0.225,1.673,0.285c0,0,0.45,0,0.468,0.035c0.016,0.028,0.036,0.056-0.068,0.102l0,0c0.06-0.021,0.793-0.254,0.476-0.391c-0.04-0.019-0.063-0.024-0.074-0.028c-0.006,0-0.013,0-0.019-0.003l0,0c-0.008-0.002-0.016-0.002-0.021-0.004c-0.007,0-0.009-0.001-0.013-0.001c-0.006-0.003-0.014-0.003-0.021-0.005c-0.004,0-0.009,0-0.015-0.002c-0.005,0-0.011-0.002-0.017-0.002c-0.004,0-0.011-0.002-0.017-0.004c-0.005,0-0.012,0-0.015,0c-0.008-0.002-0.014-0.002-0.018-0.004c-0.004,0-0.012,0-0.016-0.002c-0.005,0-0.012,0-0.018,0c-0.002-0.004-0.008-0.004-0.01-0.004c-0.011-0.002-0.015-0.003-0.021-0.003c-0.004,0-0.006-0.001-0.008-0.001c-0.01-0.001-0.021-0.001-0.028-0.002c0,0-0.002,0-0.004-0.002c-0.007,0-0.015,0-0.021,0c-0.005-0.002-0.007-0.002-0.012-0.002s-0.011,0-0.02,0c-0.004,0-0.006,0-0.008,0c-0.008-0.002-0.013-0.002-0.019-0.002c-0.003,0-0.007,0-0.009,0c-0.008,0-0.014-0.002-0.018-0.002c-0.005,0-0.009-0.003-0.012-0.003c-0.006,0-0.014,0-0.021,0c-0.004,0-0.004,0-0.006,0c-0.039-0.004-0.08-0.004-0.128-0.009c-0.002,0-0.004,0-0.004,0c-0.009,0-0.015-0.002-0.022-0.002c-0.002,0-0.007,0-0.011,0c-0.007,0-0.013,0-0.021,0c-0.004-0.002-0.01-0.002-0.014-0.002c-0.006,0-0.013-0.002-0.02-0.002c-0.005,0-0.011,0-0.016-0.002c-0.005,0-0.011,0-0.019,0c-0.01,0-0.02-0.002-0.028-0.004c-0.006,0-0.012,0-0.019,0c-0.008,0-0.018-0.002-0.028-0.002c-0.005,0-0.011-0.002-0.016-0.002c-0.008,0-0.019,0-0.024-0.001c-0.006-0.001-0.013-0.001-0.021-0.001c-0.007-0.002-0.018-0.002-0.026-0.004c-0.005,0-0.013-0.001-0.02-0.001c-0.008-0.001-0.018-0.003-0.026-0.003c-0.007-0.001-0.016-0.001-0.023-0.003c-0.01,0-0.019-0.002-0.027-0.002c-0.007-0.002-0.014-0.002-0.021-0.002c-0.012,0-0.024-0.003-0.035-0.005c-0.007,0-0.014,0-0.02,0c-0.02-0.004-0.037-0.006-0.057-0.006c-0.142-0.019-0.271-0.033-0.378-0.055l-0.187-0.025c0,0-0.47-0.014-0.653-0.316c-0.118-0.197,0.457-0.318,0.457-0.318s0.956-0.193,1.917-0.321c0,0-2.542,0.294-2.777,0.626C49.613,89.737,50.023,89.904,50.023,89.904L50.023,89.904z"/><path style="&st59;" d="M69.986,84.401L69.986,84.401L69.986,84.401z"/><path style="&st59;" d="M69.989,84.399c0.001,0,0.001,0,0.004,0C69.991,84.399,69.99,84.399,69.989,84.399z"/><path style="&st59;" d="M48.839,90.979c-0.007,0-0.015,0.002-0.021,0.002C48.824,90.98,48.831,90.979,48.839,90.979z"/><path style="&st59;" d="M48.727,90.988c-0.009,0-0.017,0.002-0.026,0.002C48.71,90.99,48.718,90.988,48.727,90.988z"/><path style="&st59;" d="M48.637,90.998c-0.009,0-0.019,0.001-0.029,0.003C48.619,90.999,48.628,90.998,48.637,90.998z"/><path style="&st59;" d="M48.55,91.004c-0.009,0-0.018,0.002-0.025,0.004C48.532,91.006,48.54,91.004,48.55,91.004z"/><path style="&st59;" d="M70.139,84.35c0.004,0,0.006-0.002,0.01-0.004C70.145,84.348,70.143,84.35,70.139,84.35z"/><path style="&st59;" d="M70.116,84.359c0.002-0.002,0.004-0.002,0.007-0.002C70.12,84.357,70.118,84.357,70.116,84.359z"/><path style="&st59;" d="M70.193,84.332c0.002,0,0.002-0.002,0.004-0.002C70.195,84.33,70.195,84.332,70.193,84.332z"/><path style="&st59;" d="M70.068,84.374c0.003,0,0.004,0,0.007-0.002C70.072,84.374,70.07,84.374,70.068,84.374z"/><path style="&st59;" d="M70.163,84.342c0.005,0,0.007-0.002,0.011-0.005C70.17,84.34,70.167,84.342,70.163,84.342z"/><path style="&st59;" d="M69.996,84.395c0.002,0,0.002,0,0.004,0C69.998,84.395,69.998,84.395,69.996,84.395z"/><path style="&st59;" d="M70.004,84.395c0.002,0,0.006-0.002,0.008-0.002C70.006,84.395,70.006,84.395,70.004,84.395z"/><path style="&st59;" d="M72.23,81.559c-0.002,0-0.004,0-0.004,0S72.229,81.559,72.23,81.559z"/><path style="&st59;" d="M70.053,84.379c0.002,0,0.002,0,0.004,0C70.053,84.379,70.053,84.379,70.053,84.379z"/><path style="&st59;" d="M70.036,84.385c0.001,0,0.003,0,0.004-0.002C70.039,84.385,70.037,84.385,70.036,84.385z"/><linearGradient id="SVGID_31_" gradientUnits="userSpaceOnUse" x1="414.2451" y1="568.2656" x2="509.0055" y2="545.7273" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset=".9505" stop-color="#B2B4B6"/></linearGradient><path style="&st51;" d="M7.61,68.141c-0.065-0.062-0.112-0.105-0.139-0.131L7.45,68.021L7.61,68.141L7.61,68.141z"/><linearGradient id="SVGID_32_" gradientUnits="userSpaceOnUse" x1="416.6992" y1="578.5645" x2="511.8228" y2="555.9398" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset=".9505" stop-color="#B2B4B6"/></linearGradient><path style="&st62;" d="M90.412,59.954l0.689,0.108c1.976-1.573,3.869-3.571,3.869-3.571s1.258-1.261,1.889-2.356c0.22-0.381,0.281-0.356,0.271-0.177c0.023-0.09,0.103-0.456-0.038-0.714c-0.094-0.176-0.381,0.288-0.83,0.861c0,0-2.013,2.926-5.798,5.816c-3.786,2.891-4.776,3.771-8.083,5.655c0,0-2.309,1.021-3.914,0.669c-1.246-0.271,0-1.547,0.271-2.699c0.271-1.146,0.063-1.58-0.225-1.807c-0.287-0.225-0.91-0.385-2.142-0.109c0,0-4.709,1.264-6.819,3.307c-1.918,1.854,0.478,2.619,1.021,2.875c0,0,0.78,0.338,0.719,0.672c-0.063,0.336-0.496,0.623-0.733,0.783c-0.239,0.16-3.338,1.977-8.324,2.764c-4.039,0.641-3.26,0.255-7.143,0.654c-3.881,0.399-4.952,0.72-8.068,1.453c-3.116,0.734-4.945,1.537-5.352,2.349c-0.336,0.671,0.479,1.103,0.991,1.407c0.511,0.304,1.423,0.781,1.119,1.293c-0.305,0.512-1.631,1.277-4.874,1.391c-3.243,0.114-4.569-0.336-5.16-1.04c-0.548-0.649-0.08-1.323,0.096-1.946c0,0,0.382-0.814,0.16-1.215c-0.224-0.398-0.737-0.494-1.278-0.559c-0.544-0.064-3.245-0.158-5.337-0.271c-2.372-0.127-5.208-0.211-8.611-0.928c0,0-2.237-0.441-3.69-1.262c-0.096-0.055-0.18-0.107-0.25-0.156c-0.11-0.059-0.194-0.122-0.25-0.209c-0.41-0.432-0.047-0.748-0.186-1.168c-0.121-0.359-0.352-0.878-0.896-1.501c-0.176-0.183-0.428-0.437-0.72-0.713c-0.08-0.069-0.165-0.144-0.254-0.214c-1.276-1.037-1.422-1.149-1.964-1.166c-0.542-0.019-0.235,0.895-0.129,1.246c0.041,0.136,0.139,0.328,0.242,0.508c-0.2-0.364-0.336-0.729-0.257-0.915c0.144-0.337,0.309-0.308,0.309-0.308s0.133-0.053,0.595,0.297c0.463,0.35,1.499,1.078,2.012,1.705c0.512,0.625,0.481,1.18,0.481,1.18s-0.103,0.563,0.451,1.17c0.555,0.604,1.733,1.714,5.859,2.349c0.021,0.005,0.041,0.005,0.06,0.009l0.193-0.549l0.568,0.663c0.006,0,0.01,0.001,0.016,0.002c3.592,0.519,5.544,0.563,5.544,0.563s4.709,0.164,5.982,0.164c1.271,0,1.035,0.664,1.035,0.664s-0.072,0.361-0.225,0.647c-0.153,0.288-0.524,1.365-0.144,1.94c0,0,0.585,1.426,4.382,1.527c0,0,3.324,0.267,5.643-0.688c2.317-0.954,0.224-2.277,0.224-2.277s-0.794-0.483-1.129-0.737c-0.308-0.233-0.184-0.48-0.122-0.646c0.061-0.163,0.297-0.355,0.564-0.492c0.265-0.134,1.241-0.652,5.365-1.722c4.124-1.067,6.587-1.183,6.587-1.183s0.021-0.004,0.062-0.006l0.334-0.656l0.845,0.559c0.732-0.061,1.686-0.129,2.537-0.17c1.691-0.083,4.341-0.475,5.879-0.811c1.539-0.342,3.869-1.059,5.591-1.951c1.724-0.891,1.808-1.519,1.808-1.519s0.328-0.472-0.391-0.995c-0.719-0.521-1.037-0.38-1.672-1.024c-0.638-0.646,0.553-1.619,0.553-1.619s0.615-0.865,4.021-2.177c3.408-1.313,3.98-0.873,4.156-0.728c0.175,0.142,0.195,0.613,0.123,0.933c-0.072,0.316-0.494,1.455-0.721,2.055c-0.227,0.592-0.316,1.402,0.604,1.6c0.924,0.193,2.215-0.009,3.427-0.443c2.893-1.033,3.757-2.295,8.535-5.764c0.011-0.01,0.021-0.016,0.029-0.023L90.412,59.954L90.412,59.954z"/><linearGradient id="SVGID_33_" gradientUnits="userSpaceOnUse" x1="415.4736" y1="573.4199" x2="510.5869" y2="550.7977" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset=".9505" stop-color="#B2B4B6"/></linearGradient><path style="&st52;" d="M100.895,47.596c-0.635,1.186-1.164,2.608-1.443,3.5c-0.045,0.213-0.061,0.33-0.061,0.33s-0.119,0.654-0.054,1.036c0.028,0.161,0.069,0.279,0.106,0.375c0.021,0.052,0.039,0.095,0.055,0.134c0.02,0.045,0.031,0.082,0.033,0.111c0.007,0.082-0.044,0.614-0.27,1.23l0,0c0,0,0,0,0,0.002c-0.063,0.176-0.143,0.359-0.24,0.539c-0.024,0.05-0.053,0.095-0.074,0.139c-0.458,0.814-1.098,1.457-1.604,1.532c-0.324,0.049-0.484-0.117-0.557-0.386c0.014,0.369,0.086,0.738,0.289,0.963c0.406,0.441,1.563-0.795,1.563-0.795s0.688-0.789,0.965-2.062c0.406-1.875,0.187-2.248,0.187-2.248s-0.247-0.389-0.093-0.853c0.152-0.461,1.156-3.047,1.979-4.01l0.502-0.562c0-0.009,0.002-0.02,0.002-0.029l-0.211-0.521c-0.129,0.13-0.259,0.284-0.385,0.454C101.405,46.763,101.178,47.129,100.895,47.596L100.895,47.596z"/><linearGradient id="SVGID_34_" gradientUnits="userSpaceOnUse" x1="414.7754" y1="570.4785" x2="509.8697" y2="547.861" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset=".9505" stop-color="#B2B4B6"/></linearGradient><path style="&st53;" d="M10.564,70.807L10.564,70.807c-0.146-0.092-0.315-0.229-0.469-0.356c-0.133-0.112-0.641-0.585-1.18-1.086c-0.212-0.194-0.411-0.382-0.589-0.55c-0.277-0.262-0.524-0.493-0.688-0.646l0.107,0.358c0.017,0.028,0.034,0.06,0.052,0.089c0.183,0.29,0.854,1.264,2.153,2.277c1.549,1.213,1.559,0.729,1.559,0.729s0.062-0.4-0.296-0.84c-0.063-0.076-0.131-0.167-0.202-0.27v0.002C11.011,70.516,12.023,71.998,10.564,70.807z"/><linearGradient id="SVGID_35_" gradientUnits="userSpaceOnUse" x1="414.915" y1="571.0664" x2="510.04" y2="548.4415" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset=".9505" stop-color="#B2B4B6"/></linearGradient><path style="&st54;" d="M10.678,69.98c0.103,0.186,0.219,0.371,0.333,0.533C11,70.501,10.833,70.253,10.678,69.98z"/><linearGradient id="SVGID_36_" gradientUnits="userSpaceOnUse" x1="416.1035" y1="576.0654" x2="511.2286" y2="553.4405" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#EDEDEE"/><stop offset=".4176" stop-color="#FFF"/><stop offset=".6264" stop-color="#F8F9F9"/><stop offset=".9505" stop-color="#B2B4B6"/></linearGradient><path style="&st55;" d="M96.887,55.023c0,0,0.227-0.76,0.243-1.066c-0.003,0.014-0.005,0.021-0.005,0.021s-0.513,1.443-0.333,2.16C96.771,55.579,96.887,55.023,96.887,55.023z"/><path style="&st34;" d="M63.301,4.417l0.728,0.072c1.426-0.402,2.643-0.772,2.643-0.772s1.265-0.41,1.901-0.637c0.635-0.226,1.09-0.313,1.654-0.409c0.565-0.096,1.311-0.14,1.709-0.131c0.4,0.007,0.531,0.122,0.531,0.122s0.166,0.131,0.244,0.27c0.077,0.138,0.74,1.891,2.973,2.005c2.233,0.112,2.263-1.096,2.065-1.464c-0.226-0.427-0.896-0.863-0.896-0.863s-0.899-0.575-1.092-0.847c-0.191-0.27,0.034-0.357,0.104-0.375c0.065-0.017,2.435-0.497,2.729-0.609l0.021-0.021l-0.562-0.171c0,0-0.119,0.134-0.789,0.313c-0.67,0.179-1.235,0.246-1.742,0.313c-0.506,0.066-0.506,0.239-0.506,0.239l-0.086,0.136c-0.025,0.075-0.067,0.321,0.375,0.642c0.528,0.387,1.172,0.75,1.438,1.04s0.586,0.783,0.012,1.137C76.48,4.576,76.27,4.64,75.977,4.671c0.002,0.008,0,0.012,0,0.012c-0.248,0.021-0.457,0.03-0.638,0.03c-0.049,0.002-0.102,0.006-0.155,0.009l-0.017-0.013c-0.506-0.024-0.746-0.142-1.067-0.302c-0.442-0.223-0.758-0.73-0.92-1.087s-0.521-0.662-0.521-0.662c-0.588-0.336-1.695-0.343-2.813-0.15c-1.115,0.193-2.656,0.707-2.925,0.812c-0.268,0.104-1.616,0.551-2.307,0.73c-0.693,0.178-1.222,0.357-1.646,0.47c-0.427,0.111-3.432,1.005-4.556,1.339c-1.126,0.334-1.849,0.46-1.849,0.46c-1.688,0.172-2.193-0.134-2.193-0.134c-0.296-0.124-0.261-0.526-0.261-0.526l0.009-1.147c0,0-0.027-0.433-0.357-0.611c-0.328-0.179-0.779-0.252-1.593-0.29c-0.811-0.038-1.683,0.044-2.093,0.134c-0.408,0.09-1.189,0.313-1.764,0.952c-0.572,0.641-0.481,1.139-0.481,1.139s0.004,0.079,0.01,0.201c0.154,0.245,0.416,0.524,0.862,0.739c1.015,0.485-1.137,1.342-1.137,1.342l0,0c-0.479,0.208-1.191,0.478-2.208,0.777c-2.21,0.647-3.684,0.774-3.684,0.774l0.679,0.254c0,0,5.468-1.016,7.148-2.616c0,0,0.625-0.293,0.021-0.88c-0.606-0.585-0.898-0.761-0.898-0.761s-0.41-0.223,0.02-0.772c0.428-0.546,0.922-0.794,1.352-0.933c0.428-0.135,1.754-0.249,2.925-0.093c0,0,0.491,0.042,0.457,0.407c-0.032,0.365-0.087,0.873-0.077,1.028c0.01,0.157,0.025,0.515,0.399,0.845c0.379,0.332,1.098,0.453,1.098,0.453s1.257,0.228,2.845-0.218c1.586-0.444,3.65-1.141,5.438-1.629L63.301,4.417L63.301,4.417z"/><linearGradient id="SVGID_37_" gradientUnits="userSpaceOnUse" x1="412.6152" y1="535.3994" x2="501.5865" y2="514.8846" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#FFF"/><stop offset=".6538" stop-color="#FFF"/><stop offset="1" stop-color="#CBCCCE"/></linearGradient><path style="&st56;" d="M91.596,12.992l0.271,0.486c-0.021,0.01-0.034,0.014-0.034,0.014c-1.114,0.515-3.999,1.32-4.271,1.398c-0.271,0.08-0.166,0.234-0.166,0.234c0.029,0.078,0.988,1.106,0.988,1.106c1.178,1.249,0.494,2.296,0.494,2.296s-0.852,1.347-2.844,1.12c-1.993-0.227-2.618-1.767-2.729-2.049c-0.111-0.282-0.197-0.768-0.22-1.095c-0.022-0.33-0.272-0.32-0.272-0.32c-1.069-0.03-2.887,1.089-2.887,1.089c-0.824,0.47-1.682,1.147-3.479,1.958c-0.879,0.396-1.694,0.716-2.287,0.936l-0.967-0.173l0.091,0.482c-0.042,0.013-0.063,0.021-0.063,0.021s-2.268,0.822-4.529,1.553c-2.268,0.732-3.65,0.8-3.65,0.8s-1.336,0.12-2.266-0.266c-0.93-0.386-0.959-0.94-0.939-1.415c0.021-0.476,0.483-1.404,0.623-1.958c0.139-0.555-0.683-0.644-0.683-0.644s-1.958-0.236-3.65,0.296c-1.69,0.535-2.54,1.159-2.54,1.159c-0.91,0.512-0.863,0.957-0.863,0.957s-0.012,0.467,0.551,0.869s1.072,0.505,1.736,0.654c0.668,0.149,0.791,0.311,0.791,0.311c1.08,0.894,0.416,1.785,0.047,2.434c-0.631,1.113-3.674,1.653-3.674,1.653c-3.276,0.758-5.12,1.08-7.827,1.452l-0.876-0.46l-0.276,0.615c-7.089,0.936-10.065,1.877-11.065,2.135c-1,0.257-1.261,0.784-1.369,0.904c-0.108,0.12,0.079,0.171,0.079,0.171c0.427,0.137,0.647,0.442,0.647,0.442s1.399,1.697-0.012,3.108c-1.359,1.36-2.785,1.674-2.785,1.674s-2.177,0.737-4.077,0.341c-1.899-0.399-2.439-1.889-2.416-2.395c0.024-0.505,0.176-1.075,0.694-1.661c0.517-0.585,0.654-0.667,0.654-0.667s0.358-0.279,0.421-0.415c0.079-0.172-0.404-0.233-0.404-0.233c-1.034-0.13-3.496-0.097-5.822,0.131c-1.439,0.14-2.769,0.374-3.578,0.518l-0.223-0.48l-0.543,0.625l-3.12,0.504c-1.514,0.222-2.576-0.028-2.576-0.028s-1.331-0.397-1.479-1.252c-0.147-0.852,0.359-1.87,0.49-2.177c0.13-0.307,0.5-0.934,0.5-0.934s0.068-0.13,0.068-0.34c0-0.211-0.233-0.536-0.233-0.536s-0.205-0.396-0.886-0.38c-0.682,0.018-0.866,0.131-1.144,0.364c-0.044,0.038-0.079,0.081-0.108,0.127c0.021-0.064,0.045-0.117,0.073-0.158c0.21-0.309,0.65-0.668,1.401-0.7c0.748-0.034,1.041,0.228,1.041,0.228c0.719,0.82,0.115,1.845-0.351,2.76c-0.057,0.095-0.155,0.271-0.229,0.483c-0.032,0.076-0.062,0.153-0.087,0.227c-0.358,1.06,0.292,1.565,0.668,1.661c0.376,0.097,1.141,0.57,4.269-0.031c3.13-0.603,3.587-0.731,3.587-0.731s6.145-1.087,8.96-0.425l0.023,0.004c0,0,1.297,0.367,0.331,1.334c-0.966,0.966-1.729,1.617-1.504,2.377c0.223,0.762,1.267,1.903,3.646,1.603c0,0,0.842-0.113,1.105-0.165c1.733-0.336,2.899-1.268,2.899-1.268s0.972-0.721,0.782-1.631c-0.187-0.908-1.017-1.189-1.017-1.189s-0.659-0.424-0.141-1.237c0,0,0.141-0.69,2.553-1.317c2.412-0.626,6.813-1.518,10.555-1.989c3.49-0.408,9.652-1.575,10.89-2.08c1.235-0.508,1.497-1.4,1.521-1.708c0.024-0.31,0.072-0.83-1.14-1.09c-1.213-0.259-1.758-0.655-1.931-0.79c-0.172-0.138-0.545-0.483-0.545-1.275c0-0.791,1.607-1.745,3.392-2.35c1.78-0.606,3.927-0.34,3.927-0.34c1.948,0.167,0.936,1.963,0.936,1.963c-0.074,0.322-0.946,1.785,0.5,2.169c1.541,0.409,4.175-0.347,5.188-0.669c0.829-0.261,3.141-1.074,4.688-1.62c0.352-0.122,0.66-0.231,0.908-0.318c1.656-0.577,3.019-1.219,3.785-1.681c0.771-0.462,2.144-1.297,2.144-1.297s1.351-0.744,1.799-0.808c0.451-0.064,1.619-0.346,1.771,0.771c0.155,1.115,0.631,1.899,1.4,2.271c0.771,0.371,2.064,0.538,3.246-0.231c1.182-0.771,0.359-1.901,0.359-1.901c-0.021-0.028-0.039-0.057-0.062-0.085c-0.151-0.27-0.369-0.518-0.487-0.646c-0.388-0.47-0.736-0.822-0.736-0.822c-0.295-0.436,0.053-0.692,0.425-0.834c0.373-0.141,2.351-0.758,2.351-0.758c1.155-0.383,1.592-0.551,2.053-0.988c0.445-0.419-0.189-1.34-0.239-1.412l0.004-0.002l0.608,0.256c0.136,0.182,0.27,0.362,0.4,0.547l-0.007,0.005c0,0,0.226,0.527,0.054,0.799c-0.095,0.149-0.343,0.293-0.545,0.395L91.596,12.992L91.596,12.992z"/><linearGradient id="SVGID_38_" gradientUnits="userSpaceOnUse" x1="390.042" y1="485.6797" x2="390.042" y2="485.6797" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#FFF"/><stop offset=".6538" stop-color="#FFF"/><stop offset="1" stop-color="#CBCCCE"/></linearGradient><path style="&st57;" d="M-16.122-14.641"/><path style="&st34;" d="M57.455,92.28c-0.034-0.042-0.042-0.034-0.012-0.063c0.021-0.021,0.086-0.082,0.115-0.137c0,0-1.17-0.063-2.141,0.077c-0.962,0.137-1.933,0.335-1.933,0.335l0.301,0.146c0,0,0.127-0.055,1.047-0.183c0.921-0.128,1.267-0.128,1.267-0.128s0.808-0.063,0.969-0.063c0.162,0,0.061,0.104,0.061,0.104s-0.078,0.136,0.366,0.124c0,0,0.663-0.027,1.313-0.188C58.809,92.309,57.678,92.544,57.455,92.28z"/><path style="&st34;" d="M54.469,92.691c0,0,0.146,0.266-2.923,0.394c0,0,1.788,0.052,3.31-0.198C55.963,92.707,54.469,92.691,54.469,92.691L54.469,92.691z"/><path style="&st34;" d="M13.114,9.856c0,0-0.005,0.096,0.019,0.131c0.024,0.034,0.107,0.112,0.107,0.112s0.328-0.5,0.194-0.438c0,0-0.119,0.187-0.164,0.23c0,0-0.029,0.04-0.088,0.077C13.123,10.006,13.117,9.991,13.114,9.856L13.114,9.856z"/><path style="&st34;" d="M10.595,12.501c0,0-0.388,0.46,0,0.52l0.528-0.527c0,0,0.139-0.234,0.139-0.398C11.263,12.095,10.752,12.343,10.595,12.501L10.595,12.501z"/><path style="&st12;" d="M-16.122-14.641"/><path style="&st31;" d="M21.093,23.707c1.227,0.146,1.499-0.132,1.527-0.172c0.294-0.003,1.475-0.034,2.865-0.207c1.685-0.21,3.564-0.891,3.564-0.891s1.554-0.568,2.096-1.18l0.016-0.002c0,0-0.693-0.6-1.057-1.122c0,0-0.286-0.557,0.027-1.035c0.316-0.479,0.836-1.008,2.344-1.643c1.506-0.636,2.356-0.514,2.356-0.514s0.594-0.006,1.007,0.45c0.415,0.458,0.649,1.006,0.649,1.006s0.029,0.38-0.115,0.63c-0.141,0.251-0.155,0.277-0.155,0.277s0.049,0.017,0.378-0.007c0.329-0.021,1.165-0.142,2.67-0.506c1.508-0.363,3.407-0.972,3.407-0.972s4.9-1.578,5.407-1.714c0.507-0.135,1.357-0.436,1.357-0.436l0.027,0.059c0,0,0.405,0.663,0.392,1.269V16.94c0,0-0.021,0.301-0.698,0.818c-0.674,0.517-1.226,0.479-1.678,0.442c-0.452-0.039-0.665-0.071-0.794-0.045l-0.72,0.04c-0.787,0.111-1.224,0.407-1.224,0.407c-1.804,1.065,0.731,9.287,0.731,9.287c-3.742,0.47-8.143,1.363-10.555,1.989c-2.412,0.627-2.553,1.317-2.553,1.317c-0.519,0.813,0.141,1.236,0.141,1.236s0.829,0.283,1.017,1.19c0.19,0.91-0.783,1.629-0.783,1.629s-1.159,0.97-2.898,1.268c-1.738,0.298-2.396,0.35-3.429-0.47c-0.91-0.721-0.297-1.864,0.312-2.301c0.612-0.438,0.909-0.91,0.988-1.113c0.079-0.203,0.032-0.376,0.032-0.376l-0.58-0.534c-2.005-1.33-9.884,0.063-9.884,0.063c-0.213-1.169-0.362-1.171-0.282-3.117c0.051-1.244,0.291-1.752,0.291-1.752l0.058-0.164c0,0,0.448-1.443,1.141-2.44c0,0,0.602-0.172,1.364-0.349C20.616,23.793,21.093,23.707,21.093,23.707L21.093,23.707z"/><linearGradient id="SVGID_39_" gradientUnits="userSpaceOnUse" x1="231.2324" y1="-407.8711" x2="263.6191" y2="-407.8711" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#696969"/><stop offset=".3702" stop-color="#2E2E2E"/><stop offset=".4554" stop-color="#424242"/><stop offset=".6014" stop-color="#303030"/><stop offset=".6947" stop-color="#4A4A4A"/><stop offset="1" stop-color="#666666"/></linearGradient><path style="&st26;" d="M49.855,16.94c0,0-4.085,1.326-3.891,1.254c0,0-0.39,0.075-0.686,0.161c-0.294,0.086-0.539,0.247-0.539,0.247s-3.288,1.222-6.438,1.848c-3.148,0.627-2.977-0.361-2.708-0.83c0.232-0.409,0.829-1.112-0.188-1.254c-1.019-0.14-1.788,0.251-2.21,0.439c-0.422,0.189-3.162,1.362-1.251,2.254c0,0,1.423,0.642-0.377,1.755c0,0-1.816,1.16-5.355,1.77c0,0-0.565,0.063-1.88,0.111c-1.316,0.046-2.558,0.213-4.12,0.658c-1.378,0.391-1.992,0.579-2.744,1.065l0.194-0.501l0.2-0.462c1.069-0.533,3.719-1.288,5.717-1.378c1.997-0.089,2.908-0.16,4.721-0.624c2.134-0.546,2.702-1.019,2.93-1.163c0.194-0.123,0.771-0.479,0.493-0.633c-0.359-0.199-0.895-0.729-0.895-0.729c-0.217-0.256-0.39-0.373-0.158-1.046c0.356-1.029,2.196-1.644,2.196-1.644s1.028-0.534,2.334-0.514c1.305,0.021,1.287,0.752,1.287,0.752s0.062,0.34-0.268,0.827c0,0-0.503,0.579-0.049,0.656c0.454,0.081,1.622,0.179,5.48-1.028c3.859-1.207,8.085-2.611,8.085-2.611S49.855,16.66,49.855,16.94L49.855,16.94z"/><linearGradient id="SVGID_40_" gradientUnits="userSpaceOnUse" x1="231.623" y1="-407.063" x2="263.4941" y2="-407.063" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#525252"/><stop offset=".1856" stop-color="#333333"/><stop offset=".354" stop-color="#AEAEAE"/><stop offset=".4199" stop-color="#ADADAD"/><stop offset=".4276" stop-color="#9D9D9D"/><stop offset=".4433" stop-color="#818181"/><stop offset=".4611" stop-color="#6A6A6A"/><stop offset=".4814" stop-color="#585858"/><stop offset=".506" stop-color="#4C4C4C"/><stop offset=".539" stop-color="#444444"/><stop offset=".6166" stop-color="#424242"/><stop offset=".6684" stop-color="#454545"/><stop offset="1" stop-color="#BDBDBD"/></linearGradient><path style="&st58;" d="M31.145,21.257c-0.541,0.612-2.096,1.18-2.096,1.18s-1.88,0.68-3.564,0.891c-1.608,0.201-2.777,0.209-2.777,0.209l-0.082-0.002c-0.175,0.145-0.483,0.188-0.728,0.21c-0.244,0.023-0.806-0.039-0.806-0.039s-2.156,0.432-2.603,0.616c0,0-0.253,0.392-0.331,0.539c-0.08,0.146-0.299,0.594-0.299,0.594c1.069-0.534,3.718-1.289,5.717-1.379c1.997-0.089,2.908-0.159,4.721-0.624c2.134-0.546,2.702-1.019,2.929-1.163c0.195-0.123,0.771-0.479,0.493-0.633c-0.358-0.199-0.894-0.729-0.894-0.729c-0.217-0.256-0.391-0.373-0.158-1.046c0.356-1.029,2.196-1.644,2.196-1.644s1.028-0.533,2.333-0.514c1.306,0.021,1.287,0.753,1.287,0.753s0.062,0.34-0.269,0.826c0,0-0.503,0.579-0.049,0.657c0.455,0.08,1.622,0.178,5.48-1.028c3.858-1.208,8.085-2.612,8.085-2.612c-0.098-0.29-0.296-0.652-0.296-0.652s-0.85,0.301-1.358,0.436c-0.506,0.136-5.407,1.714-5.407,1.714s-1.9,0.608-3.407,0.972c-1.506,0.364-2.342,0.485-2.671,0.508c-0.329,0.021-0.378,0.006-0.378,0.006s0.013-0.027,0.156-0.279c0.144-0.248,0.115-0.629,0.115-0.629s-0.235-0.548-0.649-1.006c-0.414-0.457-1.007-0.45-1.007-0.45s-0.849-0.121-2.355,0.514c-1.508,0.636-2.029,1.164-2.346,1.643c-0.312,0.478-0.026,1.035-0.026,1.035c0.365,0.521,1.057,1.122,1.057,1.122"/><linearGradient id="SVGID_41_" gradientUnits="userSpaceOnUse" x1="236.917" y1="-417.333" x2="235.8882" y2="-410.5272" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)"><stop offset="0" stop-color="#969696"/><stop offset="1" stop-color="#000"/></linearGradient><path style="&st24;" d="M21.606,31.241c0,0-0.064-0.328,0.172-0.939c0.234-0.611,0.908-0.595,1.362-0.503c0.455,0.095,0.846,0.298,1.472-0.124c0.627-0.423,0.47-1.583,0.046-2.852c-0.423-1.267-0.328-2.128-0.328-2.128l-0.608-0.649l-0.237,0.696c0.047,1.316,0.657,3.226,0.829,3.759c0.173,0.533,0.297,0.8-0.735,0.517c-1.034-0.282-1.519,0.125-1.519,0.125c-1.332,0.862-1.082,2.161-1.082,2.161L21.606,31.241z"/><path style="&st60;" d="M27.498,36.633c-0.264-1.763-0.917-2.749-0.917-2.749c-0.25,0.188-0.513,0.693-0.513,0.693s0.179,0.208,0.471,1.568c0,0,0.141,0.106,0.438,0.279C27.273,36.597,27.498,36.633,27.498,36.633z"/><path style="&st60;" d="M33.152,32.881c0,0-0.78,0.907-0.378,2.336c0,0,0.454-0.379,0.585-0.68c0,0-0.145-0.458,0.138-1.017C33.5,33.52,33.37,33.1,33.152,32.881L33.152,32.881z"/><linearGradient id="SVGID_42_" gradientUnits="userSpaceOnUse" x1="428.7803" y1="532.0527" x2="429.5303" y2="524.4692" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)"><stop offset="0" stop-color="#333333"/><stop offset=".431" stop-color="#000"/><stop offset="1" stop-color="#2E2E2E"/></linearGradient><path style="&st3;" d="M21.2,31.253c0.017-1.299,0.471-1.492,0.905-1.818c0.436-0.328,1.326-0.024,1.326-0.024s0.678,0.218,1.046-0.1c0.369-0.319-0.017-1.467-0.217-2.123c-0.202-0.653-0.41-1.599-0.445-2.262c-0.025-0.489-0.091-0.707-0.125-0.789l-0.205,0.604c0.047,1.316,0.657,3.226,0.829,3.759c0.173,0.533,0.297,0.8-0.735,0.517c-1.035-0.282-1.519,0.125-1.519,0.125c-1.332,0.862-1.082,2.162-1.082,2.162l0.259-0.027L21.2,31.253L21.2,31.253z"/><path style="&st21;" d="M26.239,34.29c0.045,0.06,0.421,0.597,0.736,2.113l0.005,0.025c0.294,0.17,0.519,0.205,0.519,0.205c-0.264-1.763-0.917-2.749-0.917-2.749C26.46,33.977,26.336,34.143,26.239,34.29L26.239,34.29z"/><path style="&st21;" d="M33.152,32.881c0,0-0.78,0.907-0.378,2.336c0,0,0.125-0.104,0.262-0.248l0.021-0.051c-0.304-1.033,0.283-1.763,0.283-1.763l0.004-0.003C33.291,33.053,33.225,32.957,33.152,32.881z"/><path style="&st10;" d="M17.159,8.189h0.117c-0.16,0.481-0.789,1.141-1.068,1.583c-0.156,0.248-0.252,0.572-0.474,0.751c0.038,0.043-0.003,0.003,0.04,0.04c0.088,0.052,0.813-0.139,0.95-0.236c0.082,0.015,0.076,0.011,0.12,0.039c0.042,0.07-0.481,0.991-0.595,1.109v0.04c0.196-0.023,0.502-0.056,0.634-0.16c0.383-0.299,0.47-0.937,0.75-1.346c-0.013-0.066-0.026-0.132-0.04-0.196c-0.222-0.04-0.681,0.02-0.87,0.157h-0.039c0.091-0.473,0.868-1.346,1.146-1.741c0.454-0.647,0.881-1.269,1.345-1.9c0.243-0.331,0.585-0.622,0.831-0.949c0.276-0.367,0.569-0.85,0.949-1.107V4.194h-0.158c-0.186,0.135-0.675,0.218-0.908,0.354c0.032,0.135,0.019,0.101,0.118,0.158c-0.139,0.386-0.598,0.673-0.832,0.991c-0.371,0.5-0.784,0.968-1.147,1.464c-0.123,0.164-0.205,0.421-0.356,0.553c-0.237,0.208-0.913,0.185-1.185,0.396h-0.08c0.056-0.332,0.907-1.392,1.147-1.622v-0.04c-0.408,0.057-0.724,0.273-0.989,0.473c0.044,0.091,0.037,0.073,0.12,0.12c-0.145,0.238-0.361,0.415-0.515,0.633c-0.197,0.275-0.305,0.602-0.514,0.871c0.014,0.04,0.028,0.077,0.04,0.118C15.948,8.641,17.001,8.307,17.159,8.189L17.159,8.189z M51.936,13.534c0.199,0.066,0.396,0.131,0.596,0.197c0.159,0.002,0.327-0.002,0.432-0.04c-0.009-0.654-0.364-0.913-0.593-1.345c-0.113-0.22-0.175-0.523-0.355-0.673c0.069-0.242,0.727-0.308,0.988-0.396c0.643-0.211,1.371-0.422,2.02-0.633c0.305-0.099,0.664-0.077,0.91-0.236c0.146,0.015,0.22,0.029,0.277,0.118c0.143,0.212,0.26,1.667,0.156,2.097c-0.398,1.663-0.896,2.963-1.938,3.958v0.039c0.385-0.062,0.568-0.436,0.83-0.632c1.051-0.794,1.762-1.972,2.137-3.444c0.221-0.865-0.14-1.713,0.199-2.452h-0.039c-0.074-0.188-1.082-0.553-1.388-0.555c-0.164,0.177-0.399,0.416-0.634,0.515c-0.357,0.152-0.838,0.109-1.146,0.315c-0.287-0.024-0.506-0.57-0.315-0.791c-0.011-0.09-0.009-0.112-0.04-0.158c-0.239-0.191-0.85-0.171-1.268-0.158c-0.133,0.125-0.252,0.15-0.314,0.358h-0.039c-0.021,0.076,0.02,0.05,0.039,0.078c0.025,0.016,0.163-0.007,0.236,0.04c0.449,0.047,0.438,0.566,0.675,0.831c-0.027,0.069-0.011,0.04-0.042,0.08c-0.155,0.123-1.301,0.453-1.543,0.515c-0.185,0.046-0.414-0.068-0.553-0.081c-0.336-0.028-0.633,0.16-0.831,0.277c0.107,0.157,0.434,0.118,0.554,0.276C51.368,12.193,51.556,12.913,51.936,13.534L51.936,13.534z M59.807,12.977c0.187,0.093,0.187,0.332,0.278,0.514c0.185,0.371,0.437,0.82,0.554,1.228v0.316c0.092,0.252,0.396,0.284,0.596,0.435c0.156-0.021,0.214-0.061,0.314-0.118c-0.066-0.753-0.525-1.378-0.791-1.979c-0.08-0.188-0.207-0.452-0.236-0.633c-0.021-0.109,0.063-0.169,0-0.276c-0.051-0.123-0.072-0.085-0.156-0.159c-0.059-0.04-0.031-0.016,0-0.078c0.068-0.144,0.213-0.287,0.275-0.436c0.133-0.313,0.127-0.576,0.396-0.751c-0.04-0.41-0.639-0.457-1.107-0.435c-0.057,0.042-0.156,0.064-0.24,0.077c0.05,0.103,0.082,0.124,0.199,0.157c0.113,1.161-0.699,2.225-1.229,2.928c-0.208,0.279-0.556,0.456-0.75,0.754h-0.04v0.038C58.395,14.473,59.54,13.383,59.807,12.977L59.807,12.977z M12.407,22.83c-0.081,0.017-0.076,0.009-0.117,0.039c-0.288,0.148-0.773,1.426-1.346,1.069c-0.292,0.002-0.319-0.055-0.476-0.16c0.02-0.376,0.659-1.063,0.913-1.226c0.031-0.604,0.187-1.252,0.118-1.819c-0.041-0.014-0.08-0.028-0.118-0.039c-0.14-0.046-0.25,0.168-0.357,0.276c-0.29,0.291-0.648,0.597-0.871,0.949c-0.337-0.003-1.414-0.013-1.623,0.119H8.411c0.099-0.256,0.86-1.096,0.633-1.464c-0.013-0.041-0.025-0.08-0.04-0.12c-0.722,0.002-1.592,1.287-1.82,1.82c-0.115,0.266-0.115,0.573-0.276,0.791v0.04c0.54-0.066,1.082-0.133,1.622-0.199c0.205-0.044,0.487,0.052,0.633-0.039h0.554c0.092-0.118,0.184-0.238,0.277-0.356c0.33-0.349,0.768-0.702,1.028-1.106h0.119v0.551c-0.303,0.273-0.773,0.695-0.91,1.108v0.316c-0.203,0.88,0.223,1.329,0.99,1.267c0.5-0.466,1.324-0.848,1.226-1.779C12.405,22.833,12.444,22.873,12.407,22.83L12.407,22.83z M7.819,22.118H7.58c0.109-0.436,0.537-0.935,1.069-0.95v0.197c-0.185,0.239-0.369,0.475-0.554,0.713C7.982,22.076,7.88,22.076,7.819,22.118z M93.044,22.315c-0.164-0.405-0.294-0.722-0.475-1.068c-0.3-0.574-0.613-1.414-1.464-1.425c-0.211,0.179-0.435,0.322-0.555,0.593c-0.777,1.762,0.819,3.747,1.543,4.71c0.256,0.339,0.557,0.712,0.948,0.908c-0.091,1.376-1.269,1.813-2.53,1.267c-0.899-0.386-1.617-1.237-2.179-1.979c-0.188-0.249-0.481-0.457-0.672-0.713c-0.177-0.239-0.304-0.507-0.515-0.713v-0.039h-0.078c0.107,0.426,0.354,0.815,0.514,1.188c0.669,1.538,1.52,2.614,2.811,3.521c0.608,0.428,1.621,1.104,2.494,0.475C94.412,27.942,93.669,23.851,93.044,22.315L93.044,22.315z M92.928,24.216c-0.104,0.1-0.539-0.419-0.635-0.515c-0.441-0.443-1.329-1.221-0.83-1.979h0.197c0.388,0.403,0.746,0.788,0.99,1.344c0.129,0.299,0.152,0.805,0.313,1.069C92.941,24.205,92.958,24.175,92.928,24.216L92.928,24.216z M66.693,32.128v-0.395c0.179-0.801,0.137-1.765,0.314-2.572c0.241-1.088-0.101-2.148,0.99-2.414c0.021-0.106,0.057-0.148,0-0.238c-0.025-0.067-0.009-0.039-0.04-0.079c-0.043-0.031-0.038-0.024-0.116-0.039c-0.305,0.222-1.131,0.373-1.543,0.474c-0.313,0.076-0.639,0.01-0.871,0.158c-0.039,0.013-0.078,0.027-0.119,0.04c0.014,0.079,0.025,0.159,0.041,0.237c0.451,0.147,0.867-0.031,1.067,0.356c0.13,0.252,0.112,1.157,0.039,1.504c-0.251,1.163-0.146,2.491-0.396,3.64c-0.086,0.397,0.022,1.171-0.157,1.463v0.08c-0.241-0.115-0.397-0.426-0.554-0.633c-0.354-0.467-0.875-0.84-1.229-1.305c-0.213-0.281-0.437-0.617-0.712-0.833c0.037-0.761,0.259-1.56,0.438-2.254c0.131-0.522,0.135-1.005,0.395-1.386c0.148-0.217,0.505-0.355,0.751-0.475c-0.002-0.1-0.004-0.146-0.04-0.198c-0.014-0.04-0.023-0.079-0.037-0.119c-0.543,0.081-1.003,0.341-1.505,0.475c-0.454,0.123-0.911,0.092-1.269,0.276c0.012,0.091,0.01,0.112,0.041,0.158c0.014,0.039,0.024,0.08,0.039,0.118c1.391-0.078,1.18,0.678,0.912,1.742c-0.084,0.326-0.029,0.775-0.199,1.028v0.079h-0.039c-0.285-0.433-0.713-0.852-1.067-1.227c-0.146-0.132-0.291-0.264-0.435-0.395c-0.104-0.137-0.16-0.312-0.278-0.436c0.024-0.437,0.38-0.549,0.713-0.672c-0.015-0.183-0.052-0.206-0.118-0.317c-1.031,0.151-1.927,0.73-3.086,0.791v0.041h-0.04c0.004,0.1,0.004,0.146,0.04,0.197v0.079c0.227,0.039,0.564-0.054,0.831,0.04c0.427,0.15,0.81,0.648,1.067,0.99c0.388,0.513,0.996,0.949,1.384,1.463c0.204,0.274,0.434,0.634,0.713,0.832c-0.038,0.696-0.229,1.428-0.396,2.058c-0.086,0.323-0.035,0.735-0.197,0.988c-0.025,0.069-0.01,0.039-0.041,0.08c-0.377-0.718-1.104-1.265-1.582-1.9c-0.918-1.22-1.938-2.319-2.889-3.521c0-0.167,0.01-0.268,0.08-0.356c0.073-0.229,0.359-0.443,0.633-0.476c0.015-0.12,0.033-0.135-0.039-0.238c-0.016-0.038-0.026-0.077-0.041-0.118c-0.803,0.123-1.521,0.497-2.293,0.714c-0.401,0.112-0.928,0.057-1.229,0.276c-0.04,0.013-0.08,0.026-0.117,0.04c0.021,0.152,0.061,0.176,0.117,0.277c0.314-0.005,0.646-0.092,0.949,0c0.793,0.241,1.361,1.137,1.818,1.742c0.201,0.266,0.513,0.483,0.713,0.751c0.849,1.129,1.808,2.146,2.65,3.285c0.328,0.442,0.771,0.825,1.066,1.304c0.179-0.004,0.216-0.025,0.316-0.079c0.213-0.929,0.332-1.866,0.596-2.81c0.119-0.432,0.269-0.942,0.314-1.424c0.327,0.117,0.592,0.607,0.793,0.871c0.618,0.821,1.491,1.502,2.057,2.373c0.164-0.007,0.182-0.026,0.277-0.078C66.352,34.819,66.521,33.473,66.693,32.128L66.693,32.128z M4.297,38.894c-0.013,2.467-0.142,6.269,1.781,6.806c0.7,0.193,1.087-0.271,1.306-0.595c0.786-1.17,0.565-3.446-0.199-4.43c-0.339,0.034-0.825,0.84-0.988,1.106c-0.082-0.016-0.075-0.011-0.119-0.04c-0.091-0.041-0.085-0.066-0.159-0.118c-0.06-0.933,0.127-1.802,0.159-2.691c1.044,0.102,1.941,0.696,3.007,0.751c-0.001-0.185,0-0.434,0.077-0.552c-0.009-0.092-0.007-0.112-0.04-0.16c-0.145-0.115-0.949-0.306-1.186-0.315v-0.04h-0.04c0.146-1.174-0.186-2.082-0.99-2.414c-0.449-0.08-0.897-0.16-1.346-0.239c-0.229-0.083-0.341-0.266-0.514-0.395c-0.058-0.38-0.133-0.806,0.159-1.029c-0.021-0.142-0.032-0.168-0.119-0.238v-0.039h-0.04c-0.133,0.228-0.245,0.493-0.315,0.792c-0.234,0.983,0.309,1.818,0.909,2.018c0.397,0.065,0.792,0.132,1.188,0.197c0.314,0.122,0.453,0.379,0.671,0.595c-0.009,0.512-0.5,0.568-0.91,0.435c-0.64-0.208-1.321-0.353-1.977-0.592c-0.172-0.064-0.333-0.17-0.555-0.199c-0.027,0.25-0.054,0.501-0.08,0.751c0.554,0.171,1.109,0.343,1.662,0.515c-0.023,1.398-0.574,3.074,0.119,4.153c0.084,0.021,0.143,0.037,0.198,0.08c0.78-0.054,0.943-0.68,1.345-1.108c0.342,0.82,0.086,2.253-0.671,2.453c-0.326,0.224-0.803-0.066-0.989-0.237c-0.648-0.599-0.785-1.511-1.027-2.532c-0.083-0.344,0.033-1.042-0.118-1.307c0.006-0.404,0.092-1.134-0.12-1.344v-0.039L4.297,38.894L4.297,38.894z M99.336,45.543c-0.143-0.666,0.055-1.478-0.08-2.097v-0.633c-0.097-0.453-0.059-1.056-0.156-1.502c-0.189-0.882-0.022-1.926-0.355-2.652c-0.197-0.047-0.393-0.084-0.671-0.08c-0.103,0.104-0.179,0.158-0.159,0.278c0.083,0.359,0.547,0.432,0.673,0.792c0.015,0.172,0.026,0.343,0.04,0.514c0.133,0.561,0.111,1.286,0.236,1.86v0.475c0.063,0.289,0.16,1.036,0.078,1.267c-0.139,0.41-0.584,0.78-0.868,1.068c-0.754,0.755-1.64,1.715-2.97,1.859c-0.025,0.068-0.01,0.039-0.041,0.08c0.022,0.494,0.476,0.396,0.793,0.594c0.08-0.014,0.158-0.028,0.236-0.042c0.122-0.074,0.191-0.242,0.276-0.356c0.2-0.261,0.563-0.399,0.751-0.671h0.04c0.002,1.205,0.028,2.561,0.04,3.718h0.117c0.272-1.172,0.252-2.61,0.238-4.039c0.521-0.486,0.853-1.19,1.385-1.66v-0.078h0.041c0.007,1.251,0.037,2.529,0.037,3.797c0,0.407-0.102,1.378,0,1.544v0.237h0.08c0.106-0.425,0.133-1.023,0.236-1.501v-0.674C99.451,47.107,99.45,46.078,99.336,45.543L99.336,45.543z M28.313,48.073c-0.347-0.144-0.776-0.461-0.989-0.751c-0.637-0.875-1.337-2.948-0.87-4.51c0.379-1.266,1.202-2.127,2.532-2.454c0.673-0.163,1.397,0.062,1.821,0.278c1.208,0.622,1.725,1.506,2.096,2.967c0.48,1.887-0.132,3.619-1.385,4.194c0.078,0.764,0.158,1.529,0.237,2.295c0.444-0.013,0.852-0.105,1.227-0.199c0.629-0.155,1.271-0.176,1.86-0.315c0.431-0.104,1.052-0.015,1.384-0.237c0.079-0.17-0.247-1.35-0.315-1.623c-0.057-0.229-0.009-0.461-0.119-0.633v-0.079c-0.091,0.012-0.185,0.025-0.277,0.039c0.018,1.195-0.834,1.032-1.781,1.267c-0.473,0.119-1.049,0.27-1.581,0.276c0-0.17,0-0.343-0.001-0.514c0.045-0.083,0.739-0.386,0.868-0.476c0.461-0.318,0.931-0.826,1.229-1.304c0.202-0.327,0.366-0.764,0.473-1.149c0.138-0.489,0.154-1.17,0.041-1.662c-0.079-0.338-0.048-0.603-0.158-0.91c-0.427-1.187-1.322-2.054-2.453-2.532c-0.513-0.216-1.093-0.224-1.7-0.356c-0.539-0.116-1.509,0.124-1.901,0.238c-1.905,0.562-3.198,1.48-3.799,3.323c-0.236,0.728-0.163,1.736,0.04,2.414c0.467,1.561,1.773,3.02,3.72,3.047v0.039c0.062,0.088,0.06,0.264,0.118,0.355c-0.024,0.067-0.009,0.039-0.04,0.08c-0.164,0.111-0.404,0.061-0.633,0.117c-0.47,0.118-1.986,0.486-2.334,0.158c-0.222-0.072-0.37-0.363-0.396-0.632c-0.099,0.004-0.146,0.004-0.197,0.039h-0.082c0.146,0.767,0.291,1.53,0.435,2.296h0.041v0.04c1.385-0.239,2.77-0.478,4.154-0.713c-0.198-0.728-0.395-1.451-0.593-2.179C28.873,48.139,28.517,48.159,28.313,48.073L28.313,48.073z M96.014,43.682c0.086,0.218,0.254,0.58,0.435,0.712c0.086,0.063,0.132,0.04,0.198,0.119c1.31,0.113,1.842-2.143,1.308-3.442c-0.095-0.225-0.517-0.885-0.911-0.633h-0.08c0.026-0.069,0.01-0.038,0.04-0.08c-0.001-0.188-0.021-0.25-0.077-0.356c-0.08-0.013-0.156-0.026-0.238-0.038c-0.039,0.031-0.01,0.014-0.078,0.038c0.027,0.24,0.111,0.247,0.119,0.514C96.09,41.099,95.545,42.497,96.014,43.682z M96.446,41.585c0.088-0.213,0.265-0.35,0.396-0.515c0.082-0.015,0.075-0.01,0.117-0.04c0.301-0.001,0.385,0.057,0.555,0.159c0.301,0.711,0.346,1.954-0.156,2.494c-0.077,0.085-0.229,0.116-0.315,0.197C96.214,43.817,96.146,42.305,96.446,41.585L96.446,41.585z M78.092,57.168c-0.445-0.273-0.507-1.675-0.673-2.294c-0.327-1.215-0.483-2.489-0.831-3.72c-0.223-0.788-0.523-1.605-0.435-2.572c0.139-0.138,0.231-0.32,0.396-0.436c0.223-0.154,0.58-0.229,0.752-0.436c0.027-0.051-0.019-0.128-0.041-0.238c-0.459,0.074-0.879,0.35-1.267,0.515c-0.792,0.337-1.567,0.536-2.373,0.87c-0.252,0.104-0.515,0.282-0.833,0.315v0.238c0.027,0.014,0.055,0.025,0.08,0.04c0.042,0.045,1.033-0.392,1.346-0.118c0.356,0.125,0.311,0.59,0.514,0.872c-0.061,0.614-0.672,1.558-0.912,2.097c-0.58,1.326-1.17,2.592-1.816,3.836c-0.248,0.477-0.543,1.334-0.871,1.701v0.039c-0.216-0.296-0.199-1.051-0.314-1.462c-0.353-1.235-0.578-2.591-0.951-3.798c-0.068-0.23-0.305-1.279-0.156-1.503c0.051-0.546,0.822-0.785,1.266-0.95c-0.012-0.092-0.024-0.186-0.039-0.277c-0.701,0.105-1.429,0.479-2.058,0.713c-0.595,0.223-1.14,0.313-1.741,0.516c-0.298,0.102-0.636,0.275-0.986,0.314v0.041h-0.041c0.015,0.112,0.025,0.172,0.078,0.237c0.162,0.107,1.03-0.352,1.386-0.077c0.557,0.19,0.573,1.075,0.752,1.66c0.481,1.579,0.728,3.327,1.187,4.947c0.115,0.404,0.391,1.686,0.119,2.018c-0.148,0.439-0.885,0.615-1.306,0.791c0.014,0.08,0.024,0.159,0.036,0.237c0.609-0.09,1.162-0.373,1.707-0.56c1.063-0.354,2.066-0.65,3.089-1.029c-0.017-0.092-0.027-0.186-0.041-0.275c-0.437,0.116-0.773,0.248-1.386,0.236c-0.08-0.068-0.157-0.133-0.235-0.199c-0.067-0.21-0.134-0.422-0.199-0.632c0.043-0.499,0.683-1.421,0.91-1.86c0.673-1.293,1.262-2.6,1.856-3.955c0.229-0.519,0.754-1.275,0.832-1.857c0.184,0.111,0.133,0.438,0.197,0.672c0.135,0.475,0.211,0.98,0.355,1.503c0.281,1,0.391,2.075,0.673,3.126c0.104,0.387,0.261,1.048,0.08,1.464c-0.179,0.404-0.841,0.673-1.267,0.83c0.017,0.084,0.037,0.183,0.08,0.238c0.004,0.007,0.906-0.288,1.064-0.354c1.104-0.471,2.236-0.959,3.361-1.386c-0.015-0.093-0.024-0.187-0.039-0.274C79.117,57.043,78.477,57.407,78.092,57.168L78.092,57.168z M96.803,60.498c-0.143,0.13-0.354,0.163-0.514,0.277c-0.501,0.359-1.025,0.962-1.385,1.463c-0.288,0.402-0.534,0.843-0.791,1.268c-0.112,0.188-0.137,0.402-0.277,0.553v0.08c0.346-0.059,0.549-0.283,0.792-0.436c0.659-0.408,1.249-0.781,1.858-1.225c0.295-0.217,0.515-0.551,0.83-0.754c0.029-0.473,0.125-0.844-0.077-1.188C97.115,60.512,96.975,60.496,96.803,60.498z M96.329,61.921c-0.239,0.177-0.47,0.423-0.712,0.595c-0.208,0.146-0.458,0.224-0.633,0.396h-0.04c0.13-0.408,0.817-1.107,1.146-1.344c0.17-0.124,0.383-0.157,0.557-0.279h0.156c0.036,0.046,0.034,0.044,0.08,0.08C96.846,61.667,96.523,61.774,96.329,61.921L96.329,61.921z M10.825,63.465c-0.166-0.502-0.278-0.99-0.435-1.465c-0.079-0.246-0.062-0.525-0.199-0.713v-0.118c0.269,0.097,0.679,0.087,0.911,0.238h0.201c-0.045-0.206-0.086-0.552-0.201-0.713c-0.12-0.195-0.886-0.197-1.106-0.354c-0.312-0.244-0.607-0.947-0.832-1.307c-0.56-0.887-1.302-1.832-2.137-2.453c-0.443-0.329-0.751-0.671-1.544-0.673c-0.092,0.065-0.185,0.132-0.276,0.198c-0.178,0.789,0.139,2.248,0.592,2.611v0.078c-0.189-0.051-0.393-0.152-0.514-0.275h-0.04c0.007,0.227,0.051,0.522,0.158,0.672c0.046,0.195,0.362,0.354,0.554,0.396c0.248,1.037,0.592,2.101,0.95,3.05c0.289,0.758,0.436,1.48,0.75,2.213c0.155,0.356,0.522,0.617,0.634,0.99h0.117c-0.089-0.334-0.271-0.646-0.394-0.949c-0.364-0.875-0.653-1.804-0.952-2.688C6.868,61.62,6.734,61.057,6.548,60.5c-0.069-0.21-0.049-0.427-0.158-0.595v-0.039c0.269,0.049,0.43,0.188,0.634,0.276c0.461,0.201,0.903,0.277,1.385,0.476c0.353,0.146,0.796,0.347,1.228,0.396c0.048,0.359,0.253,1.301,0.435,1.545v0.117c-0.602-0.412-0.589-1-1.663-0.91c-0.054,0.063-0.128,0.117-0.197,0.158c-0.098,0.244-0.104,0.646,0,0.909c0.257,0.646,1.072,1.991,1.741,2.179c0.257,0.184,0.634-0.043,0.75-0.24c0.242,0.127,0.293,0.682,0.395,0.951c0.212,0.558,0.522,1.289,1.031,1.543v0.041h0.083c-0.066-0.447-0.318-0.851-0.475-1.229C11.387,65.223,11.113,64.324,10.825,63.465L10.825,63.465z M9.678,60.26C9.26,60.23,8.905,60.067,8.57,59.945c-0.894-0.332-1.703-0.615-2.492-0.991c-0.095-0.358-0.76-1.644-0.396-2.095c0.026-0.04,0.053-0.081,0.079-0.12c0.081-0.019,0.077-0.011,0.119-0.039c1.219,0.146,2.442,1.629,3.046,2.452c0.236,0.32,0.43,0.799,0.752,1.029V60.26L9.678,60.26z M10.311,63.701c-0.12,0.146-0.237,0.291-0.356,0.436c-0.105,0.078-0.223,0.109-0.316,0.198c-0.68-0.021-0.704-0.686-0.989-1.108c0.005-0.389,0.152-0.39,0.315-0.594c0.092-0.007,0.112-0.007,0.158-0.037c0.614,0.004,0.753,0.278,1.109,0.515C10.29,63.344,10.327,63.445,10.311,63.701L10.311,63.701z M33.578,69.794c-0.165-0.271-0.49-0.342-0.713-0.554c-0.069-0.023-0.04-0.007-0.079-0.039c0.51-0.264,1.053-0.555,1.583-0.79c0.142,0.158,0.801,0.792,1.029,0.671c0.04-0.012,0.079-0.023,0.118-0.038c-0.013-0.224-0.025-0.448-0.04-0.673c-0.499-0.498-1.234-0.91-2.059-1.066v0.039h-0.039c0.093,0.273,0.398,0.534,0.636,0.672v0.119c-0.469,0.068-0.885,0.295-1.307,0.437c-0.289,0.093-0.638,0.08-0.873,0.235h-0.117c0.171-0.479,0.737-0.871,1.028-1.267c0.576-0.776,1.033-1.728,1.94-2.176c-0.024-0.365-1.076-1.12-1.464-0.871c-0.097,0.051-0.029-0.021-0.079,0.078c-0.059,0.144,0.137,0.321,0.079,0.554c-0.076,0.305-0.831,1.74-1.029,1.9v0.041c-0.408-0.139-0.718-0.523-1.107-0.713c0.069-0.364,0.375-0.644,0.554-0.91c0.453-0.684,0.816-1.335,1.503-1.782c-0.006-0.526-0.855-1.075-1.425-1.065c0.002,0.242,0.125,0.379,0.08,0.592c-0.14,0.646-0.435,1.297-0.672,1.861c-0.156,0.364-0.226,0.799-0.476,1.065c-0.054,0.03-0.492-0.006-0.594-0.077c-0.149-0.002-0.298,0.005-0.394,0.038v0.079c0.666,0.645,1.387,0.865,2.295,1.268c-0.126,0.655-0.786,1.092-1.108,1.584c-0.166,0-0.3-0.011-0.395-0.08c-0.091,0.017-0.098,0.021-0.158,0.041c0.016,0.582,0.5,1.077,0.987,1.188c0.327-0.366,0.737-0.543,1.228-0.751c0.449,0.468,0.578,1.137,0.751,1.897c0.075,0.332-0.047,0.697,0.04,0.988c0.152,0.514,0.426,0.667,0.672,1.027h0.277c0.174-0.93-0.253-1.832-0.475-2.571C33.71,70.43,33.644,70.111,33.578,69.794L33.578,69.794z M96.09,63.108c-0.238,0.202-0.57,0.296-0.83,0.475c-0.4,0.282-0.758,0.659-1.146,0.95c-0.177,0.134-0.435,0.253-0.556,0.436c-0.199,0.299-0.16,0.806-0.396,1.067v0.157c0.314-0.114,0.464-0.483,0.713-0.672c0.307-0.23,0.563-0.536,0.87-0.754c0.192-0.133,0.411-0.207,0.594-0.355c0.125,0.023,0.115,0.037,0.199,0.081c-0.021,1.005-0.549,1.714-0.871,2.454c-0.093,0.215-0.121,0.551-0.276,0.71c-0.074,0.076-0.229,0.094-0.314,0.157c-0.264,0.291-0.528,0.58-0.794,0.873c-0.25,0.344-0.365,0.803-0.632,1.146c-0.002,0.114-0.002,0.216,0.037,0.276c0.041,0.031,0.11,0.059,0.16,0.08c0.51-0.483,1.004-0.887,1.424-1.465c0.658-0.904,0.986-2.047,1.465-3.125c0.3-0.683,0.734-1.354,0.711-2.334c-0.047-0.045-0.084-0.102-0.117-0.158L96.09,63.108L96.09,63.108z M93.32,69.361V69.4h-0.04c0.069-0.475,0.43-0.606,0.596-0.952h0.079C93.904,68.842,93.605,69.194,93.32,69.361L93.32,69.361z M34.171,69.993c-0.08,0.342,0.76,1.106,1.027,1.308c0.133,0.1,0.312,0.328,0.515,0.235c0.104-0.008,0.136-0.019,0.199-0.04c0.046-0.105,0.115-0.24,0.039-0.354C35.93,70.645,34.64,70.088,34.171,69.993z M37.97,73.037c0.067,0.034,0.122,0.021,0.198-0.039c0.139-0.113,0.063-0.313,0.159-0.475c0.222-0.159,0.615-0.118,0.911-0.199c0.809-0.213,1.753-0.198,2.65-0.396c0.425-0.093,1.128,0.16,1.464-0.037c0.04-0.016,0.081-0.026,0.118-0.043c-0.019-0.517-1.009-0.737-1.545-0.588c-0.237,0.066-0.513,0.213-0.751,0.275c-0.185,0.014-0.37,0.027-0.555,0.038c-0.062-0.644-0.38-1.144-0.395-1.817c0.595-0.013,1.341-0.091,1.739-0.316c-0.008-0.2-0.045-0.2-0.118-0.314c-0.453-0.107-1.23-0.126-1.583,0.116c-0.1-0.004-0.147-0.004-0.197-0.039c-0.221-0.28-0.116-0.851-0.316-1.146v-0.158c0.426-0.092,1.122-0.168,1.345-0.475c0.031-0.041,0.014-0.011,0.039-0.078c-0.036-0.035-0.051-0.068-0.079-0.119c-0.619-0.156-0.887-0.049-1.423,0.158c-0.167-0.535,0.034-0.959-0.514-1.108c0.117-0.203,0.506-0.194,0.751-0.276c0.382-0.126,0.817-0.296,1.148-0.474c0.026-0.068,0.007-0.04,0.04-0.08c-0.022-0.2-0.078-0.193-0.159-0.316c-0.571-0.044-1.027,0.011-1.346,0.316h-0.076c0.047-0.295,0.231-0.718,0.394-0.949c0.112-0.162,0.318-0.14,0.396-0.356h0.04V64.1c-0.081-0.104-0.159-0.211-0.238-0.314c-0.186-0.13-0.454-0.143-0.632-0.279c-0.263-0.004-0.515-0.003-0.672,0.079c0.021,0.152,0.089,0.248,0.119,0.356c0.109,0.408-0.284,1.669-0.436,1.859c-0.123,0.154-1.551,0.672-1.939,0.555c-0.092-0.029-0.36-0.164-0.435-0.239c-0.032-0.039-0.015-0.008-0.04-0.077c0.561-0.527,0.965-1.702,1.741-1.939c0.014-0.064,0.027-0.131,0.041-0.196c-0.194-0.2-1.135-1.188-1.622-0.871c-0.04,0.014-0.079,0.022-0.117,0.038c0,0.338,0.168,0.593,0.078,0.949c-0.182,0.711-0.587,1.556-0.95,2.139c-0.218,0.35-0.693,0.729-0.712,1.229c0.646-0.064,0.802-0.731,1.304-0.912c0.146,0.135,0.29,0.267,0.436,0.396c0.207,0.311,0.168,0.778,0.276,1.186c0.185,0.856,0.371,1.715,0.554,2.571c0.025,0.425,0.052,0.845,0.08,1.269C37.246,72.28,37.561,72.945,37.97,73.037L37.97,73.037z M39.233,70.032c0.031,0.368,0.258,1.407,0.436,1.662c0.001,0.024,0.001,0.054,0.001,0.08c-0.477,0.102-0.973,0.239-1.504,0.237c-0.082-0.564-0.352-1.061-0.355-1.662C38.418,70.338,38.731,70.094,39.233,70.032z M36.939,66.75c0.063-0.107,1.113-0.273,1.228-0.199c0.42,0.195,0.27,0.813,0.514,1.188c-0.083,0.194-1.047,0.487-1.345,0.514C37.283,67.834,37.213,66.977,36.939,66.75L36.939,66.75z M38.76,68.253h0.04c0.076,0.36,0.119,0.978,0.317,1.267c-0.142,0.348-1.016,0.317-1.346,0.516c-0.138-0.083-0.32-1.076-0.316-1.346C37.757,68.662,38.541,68.402,38.76,68.253L38.76,68.253z M31.914,70.506c-0.06,0.135-0.053,0.354-0.117,0.514c-0.342,0.84-0.454,1.015,0.079,1.82c0.237,0,0.269-0.037,0.396-0.119C32.429,72.064,32.454,70.814,31.914,70.506L31.914,70.506z M77.023,70.744c-1.154-0.285-2.125,0.285-3.325,0.199c-0.114-0.121-0.2-0.19-0.275-0.356c-0.835,0.024-1.757,1.886-0.909,2.453c0.453,0.308,1.744,0.129,2.295,0c0.306-0.071,0.783-0.139,1.027,0.038c0.332,0.247,0.273,1.182,0.157,1.703c-0.132,0.975-0.265,1.951-0.396,2.929c-0.117,0.593-0.236,1.185-0.356,1.779c0.606-0.003,1.178-0.623,1.349-1.069c0.1-0.258,0.047-0.502,0.119-0.791c0.209-0.83,0.237-1.82,0.436-2.689c0.127-0.563,0.041-1.1,0.156-1.621c0.086-0.393,0.143-1.696,0.041-2.059C77.281,71.059,77.126,70.901,77.023,70.744z M22.857,82.695c-0.135-0.102-0.229-0.283-0.356-0.395c-0.473-0.42-1.029-0.826-1.543-1.188c-0.426-0.298-1.008-0.476-1.387-0.829c-0.01-0.086,0.123-0.296,0.041-0.516c-0.335-0.896-1.589-1.933-2.374-2.412c-0.363-0.225-0.972-0.328-1.305-0.555c-0.246-0.017-0.374-0.025-0.435,0.155c-0.097,0.218,0.209,0.521,0.315,0.675c0.271,0.381,0.581,0.826,0.95,1.104c0.276,0.209,0.591,0.392,0.83,0.635h0.119c-0.154-0.426-0.609-0.657-0.949-0.909c-0.311-0.229-0.449-0.632-0.712-0.909c0.021-0.125,0.035-0.115,0.08-0.199c1.093,0.009,1.802,1.012,2.294,1.662c0.22,0.291,0.571,0.461,0.594,0.951c-0.116,0-0.216,0-0.276-0.041h-0.119c0.188,0.522,0.824,0.479,1.267,0.754c0.888,0.549,1.603,1.409,2.373,2.094c0.262,0.234,0.719,0.466,0.791,0.873c-0.537-0.028-0.917-0.327-1.261-0.555c-0.614-0.4-1.597-1.1-2.019-1.662c-0.08-0.104-0.106-0.263-0.199-0.355c-0.109-0.111-0.261-0.145-0.355-0.275h-0.158c-0.039,0.41,0.407,0.705,0.671,0.948c0.819,0.75,1.696,1.442,2.73,1.979c0.373,0.191,1.053,0.521,1.465,0.275C23.874,83.434,23.227,82.975,22.857,82.695L22.857,82.695z M47.226,85.307c-2.014-1.379-4.985-2.775-8.427-2.689c-0.167,0.104-0.503,0.021-0.711,0.078c-0.288,0.076-0.464,0.223-0.672,0.355c-0.008,0.971,1.446,1.496,2.255,1.698c0.483,0.123,0.909-0.104,1.188-0.198c0.215-0.82-0.776-0.94-1.227-1.347h-0.081v-0.038c3.036-0.119,5.308,0.729,7.043,2.02c0.433,0.322,0.93,0.783,1.148,1.306c0.081,0.194,0.116,0.515,0,0.674c-0.159,0.44-0.685,0.401-1.188,0.515c-1.162,0.267-2.755-0.391-3.285-0.91c-0.108,0.189,0.049,0.48-0.118,0.674c-0.176,0.478-0.788,0.354-1.346,0.474c-0.917,0.199-2.353-0.271-2.888-0.632c-0.149-0.104-0.257-0.286-0.396-0.396c-0.007-0.103-0.018-0.136-0.041-0.199c0.081-0.073,0.177-0.187,0.237-0.275c1.139-0.085,1.718-0.027,2.376,0.596c-0.017,0.078-0.01,0.073-0.041,0.114c-0.074,0.152-0.245,0.17-0.474,0.161v0.074c0.417,0.004,0.593-0.059,0.83-0.197c0.013-0.079,0.027-0.159,0.04-0.236c-0.136-0.141-0.231-0.328-0.396-0.438c-0.65-0.426-1.991-0.641-2.729-0.156c-0.116,0.561,0.232,0.864,0.554,1.105c0.646,0.488,1.191,0.771,2.098,1.029c0.291,0.082,0.55,0.008,0.871,0.076c0.28,0.064,0.765,0.079,1.068,0c0.504-0.128,1.205-0.658,0.632-1.268v-0.037c0.299,0.109,0.544,0.402,0.831,0.556c0.761,0.397,2.021,0.726,3.167,0.476c0.562-0.125,1.143-0.125,1.303-0.635c0.179-0.277-0.068-0.668-0.156-0.826C48.322,86.151,47.836,85.721,47.226,85.307L47.226,85.307z M39.906,83.485c0.14,0.094,0.22,0.291,0.356,0.396c-0.003,0.1-0.004,0.148-0.04,0.199c-0.257,0.697-1.706,0.182-2.058-0.081c-0.11-0.08-0.153-0.248-0.236-0.354c0.015-0.082,0.01-0.076,0.041-0.116c0.108-0.306,0.417-0.203,0.671-0.354C39.142,83.174,39.596,83.274,39.906,83.485z M76.625,83.881h-0.396c-0.262,0.209-0.692,0.236-0.991,0.396c-0.263,0.141-0.581,0.332-0.829,0.515c-0.207,0.148-0.326,0.418-0.516,0.592c0.004,0.197,0.008,0.229,0.16,0.277c0.039,0.029,0.01,0.018,0.075,0.04c0.042-0.047,0.08-0.063,0.12-0.12c0.033-0.023-0.027-0.104-0.04-0.232c0.384-0.386,0.667-0.598,1.228-0.832c0.144-0.059,0.447-0.233,0.634-0.119h0.079c-0.026,0.391-0.916,1.591-1.188,1.781v0.115c0.729-0.188,1.215-1.643,1.702-2.174c-0.013-0.09-0.01-0.111-0.04-0.157L76.625,83.881L76.625,83.881z M73.459,86.809c-0.234,0.209-0.807,0.229-1.066,0.435h-0.041c0.104-0.149,0.291-0.213,0.396-0.354c0.076-0.104,0.107-0.226,0.197-0.315c-0.018-0.081-0.01-0.075-0.039-0.117v-0.08c-1.155-0.212-3.084,0.784-3.68,1.308c-0.155,0.135-0.248,0.336-0.396,0.477c0.003,0.111,0.016,0.168,0.039,0.236c0.701,0.047,2.016-0.383,2.174-0.949c0.031-0.025,0.012-0.002,0-0.04v-0.079c-0.479-0.027-1.124,0.075-1.422,0.355h-0.039c0.26-0.396,1.223-0.746,1.739-0.91c0.172-0.053,0.55-0.149,0.714-0.039c0.037,0.015,0.077,0.025,0.117,0.039c-0.094,0.396-0.657,0.838-1.029,0.949v0.08c0.607-0.141,1.163-0.416,1.7-0.634c0.368-0.149,0.786-0.188,1.108-0.396c0.229-0.149,1.008-1.207,1.068-1.504C74.086,85.409,74.012,86.313,73.459,86.809L73.459,86.809z M70.333,87.6v0.119c-0.075,0.049-0.129,0.156-0.198,0.196c-0.205,0.12-0.479,0.106-0.674,0.238c-0.09-0.011-0.109-0.009-0.156-0.041h-0.039C69.373,87.775,70.025,87.621,70.333,87.6L70.333,87.6z M53.835,91.317c0.015-0.037,0.025-0.078,0.039-0.117c-0.976-0.04-1.953-0.079-2.927-0.119c-0.123,0.082-0.312,0.035-0.475,0.079c-0.202,0.059-0.426,0.15-0.593,0.239c0.026,0.067,0.008,0.038,0.04,0.077c0.238,0.188,1.624,0.199,1.9,0h0.078v-0.077c-0.419-0.134-1.183,0.2-1.503,0h-0.041v-0.041c1.052-0.073,2.23-0.044,3.325-0.04c-0.105,0.072-0.328,0.051-0.436,0.119c-0.039,0.014-0.078,0.027-0.117,0.039v0.08c0.238,0.037,0.475,0.078,0.711,0.117c0.037,0.041-0.004,0.004,0.039,0.037c-0.35,0.233-1.254,0.139-1.581-0.037v-0.08c-0.178-0.082-0.991,0.084-1.148,0.117c-0.133,0.03-0.27-0.014-0.357,0.039c-0.165,0.01-0.181,0.029-0.276,0.079c0.022,0.128,0.035,0.115,0.08,0.198c0.255,0.06,0.696,0.064,0.987,0.156v-0.039h0.04v-0.039c-0.148-0.057-0.559-0.025-0.713-0.115h-0.079c0.132-0.104,1.425-0.278,1.663-0.119c0.067,0.023,0.039,0.007,0.079,0.039c-0.211,0.038-0.424,0.08-0.634,0.117c0.025,0.066,0.009,0.039,0.04,0.078c0.065,0.045,0.193,0.045,0.316,0.039c-0.04,0.074-0.054,0.109-0.119,0.158c0.013,0.023,0.027,0.051,0.04,0.078c0.561,0,1.031-0.057,1.502-0.156c0.28-0.062,0.624,0.052,0.831-0.08h0.317v-0.078c-0.539-0.002-1.885-0.055-2.215,0.158h-0.117c0.033-0.043-0.004-0.004,0.038-0.041c0.155-0.18,0.471-0.09,0.75-0.156c0.44-0.104,1.168-0.284,1.544,0c0.105,0.064,0.04-0.008,0.039,0.117c0.107-0.002,0.181-0.002,0.236-0.036h0.277v-0.081c-0.359-0.088-0.889-0.251-1.188-0.434C54.057,91.488,54.135,91.344,53.835,91.317L53.835,91.317z M13.635,18.359c-0.088,0.32-0.274,0.593-0.395,0.87c-0.268,0.613-0.507,1.225-0.751,1.822c-0.207,0.496-0.335,1.295-0.633,1.699v0.079c0.416-0.074,0.698-0.493,0.949-0.751c0.617-0.634,1.92-2.22,1.9-3.402c-0.062-0.061-0.119-0.162-0.159-0.237C14.3,18.38,13.982,18.353,13.635,18.359z M13.794,20.022c-0.181,0.298-0.281,0.592-0.476,0.871c-0.178,0.255-0.46,0.452-0.633,0.713h-0.041c0.051-0.302,0.214-0.546,0.319-0.792c0.235-0.561,0.396-1.118,0.671-1.621c0.152,0.003,0.268,0.015,0.356,0.078c0.095,0.052,0.028-0.018,0.079,0.08C14.15,19.548,13.89,19.862,13.794,20.022L13.794,20.022z M84.023,7.875c-0.414-0.416-0.729-0.938-1.147-1.346V6.49c-0.205,0.073-0.899,0.688-1.028,0.871c-0.25-0.095-0.391-0.365-0.594-0.514c-0.676-0.508-1.313-1.167-2.49-1.147c-0.148,0.115-0.367,0.118-0.556,0.197c-0.53,0.23-1.083,0.688-1.305,1.227c-0.249,0.602,0.004,1.491,0.196,1.939c0.392,0.904,1.03,1.667,1.582,2.414c0.457,0.615,0.973,1.252,1.819,1.464c0.956,0.238,1.422-0.884,1.781-1.308c0.37-0.435,1.182-0.539,1.464-1.107c0.104-0.207,0.034-0.615-0.039-0.791c-0.18-0.426-1.066-1.622-1.425-1.859c0.024-0.239,0.135-0.247,0.235-0.396c0.248,0.121,0.338,0.471,0.516,0.673c0.227,0.258,0.546,0.396,0.791,0.632c0.378,0.003,0.604-0.094,0.79-0.277h0.041C84.561,8.243,84.212,8.06,84.023,7.875L84.023,7.875z M81.77,12.148c-0.699,0.165-1.047-0.293-1.424-0.673c-0.938-0.935-1.57-2.093-2.298-3.244c-0.247-0.396-0.885-1.134-0.554-1.702h0.156c0.199,0.299,0.539,0.507,0.754,0.792c0.591,0.784,1.313,1.469,1.898,2.255c0.359,0.485,0.758,0.94,1.106,1.424c0.178,0.249,0.315,0.565,0.556,0.751C81.924,11.931,81.848,12.015,81.77,12.148L81.77,12.148z M82.361,9.339c0.32,0.439,0.755,0.688,0.751,1.463c-0.122,0.116-0.157,0.224-0.356,0.276c-0.039,0.032-0.011,0.015-0.078,0.041c-0.56-0.932-1.367-1.711-2.017-2.573c-0.212-0.282-0.541-0.511-0.752-0.791c-0.362-0.48-0.793-0.864-1.188-1.305c-0.113-0.131-0.168-0.257-0.313-0.357c0.033-0.086,0.031-0.057,0.076-0.118c0.672,0.006,0.994,0.458,1.386,0.753C80.837,7.453,81.648,8.354,82.361,9.339L82.361,9.339z"/><radialGradient id="SVGID_43_" cx="251.8086" cy="-408.3613" r="72.7509" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#000" stop-opacity="0"/><stop offset=".8022" stop-color="#000" stop-opacity=".08"/><stop offset="1" stop-color="#000" stop-opacity=".3882"/></radialGradient><path style="&st15;" d="M49.885,17.037c0.014-0.606-0.392-1.27-0.392-1.27l-0.025-0.058c0,0-0.487-0.949-1.302-1.228c-0.815-0.278-1.478,0.342-1.478,0.342s-0.114,0.131-0.429,0.494c-0.313,0.364-0.507,0.666-1.198,0.938c-0.692,0.271-1.379,0.204-1.743,0.033c-0.364-0.172-0.457-0.537-0.457-0.537s-0.229-0.722-0.313-1.049c-0.086-0.331-0.309-1.694-0.309-1.694s-0.491-2.747-0.534-3.304c0,0,1.475-0.126,3.687-0.775c2.299-0.673,3.043-1.206,3.043-1.206s-0.432-0.156-0.484-0.662c-0.051-0.507-0.089-1.19-0.089-1.19s-0.089-0.5,0.483-1.139c0.571-0.64,1.354-0.863,1.762-0.953c0.41-0.089,1.281-0.17,2.093-0.134c0.812,0.038,1.267,0.112,1.593,0.291c0.328,0.178,0.357,0.61,0.357,0.61l-0.008,1.146c0,0-0.037,0.402,0.261,0.529c0,0,0.505,0.305,2.196,0.133c0,0,0.72-0.126,1.846-0.46c1.125-0.335,4.129-1.229,4.554-1.341c0.425-0.111,0.953-0.291,1.646-0.469c0.691-0.179,2.039-0.626,2.308-0.73c0.271-0.104,1.812-0.618,2.927-0.81c1.115-0.195,2.227-0.186,2.813,0.149c0,0,0.357,0.304,0.521,0.662c0.163,0.358,0.478,0.863,0.923,1.088c0.454,0.227,0.752,0.371,1.875,0.273c0,0,0.022-0.096-0.395-0.37c-0.417-0.277-0.991-0.701-0.991-0.701S74.29,3.4,74.215,3.198c-0.072-0.202-0.043-0.306-0.043-0.306l0.877-1.406c0,0,0-0.172,0.508-0.238c0.506-0.067,1.071-0.134,1.74-0.313c0.67-0.178,0.788-0.312,0.788-0.312l0.58,0.178c0,0,3.546,2.853,4.655,3.583l0.199-0.239c0,0,0.436,0.018,0.826,0.172c0.394,0.154,0.979,0.562,0.979,0.562s1.612,1.31,2.071,2.2l0.223,0.679l-0.102,0.161c0,0,0.918,1.307,2.096,2.602c0,0,1.227,1.664,1.689,2.09c0,0-0.108-0.399-0.201-0.849l0.336-0.226l0.203-0.144l0.617,0.259c3.573,4.811,6.432,10.424,8.141,16.328l-0.12,0.484l0.395,0.501c1.128,4.212,1.728,8.643,1.728,13.211c0,1.122-0.036,2.236-0.107,3.339l-0.304,0.511l0.225,0.555c-2.231,26.1-24.124,46.584-50.801,46.584c-18.502,0-34.702-9.854-43.637-24.6L7.674,68.2l-0.205-0.153c-3.387-5.742-5.682-12.205-6.595-19.103l0.212-0.525L0.75,47.936c-0.213-1.892-0.322-3.812-0.322-5.756c0-2.985,0.255-5.909,0.748-8.755l0.25-0.562l-0.087-0.328c1.157-6.048,3.383-11.716,6.474-16.799l0.684-0.384l0.081,0.032c0,0,0.233-0.169,0.354-0.217l0.076-0.023c0,0,1.179-1.971,1.625-2.601c0,0,0.542-0.348,0.745-0.407c0,0,0.124-0.016,0.189,0.076c0,0,0.496-0.432,1.699-2.054c0.004-0.005,0.007-0.011,0.012-0.017c0,0-0.114-0.076-0.131-0.174c-0.018-0.097,0.108-0.591,0.173-0.717c0.065-0.126,0.108-0.156,0.108-0.156s1.722-2.032,3.151-3.238c0,0,0.26-0.202,0.678-0.25c0,0,1.472-0.613,3.264-2.184c0,0,0.051-0.289,0.478-0.858c0.428-0.57,1.456-1.163,2.222-1.337c0.764-0.174,0.896-0.038,0.896-0.038l0.064,0.065l0.515,0.766c0,0,0.565-0.316,1.413-0.604c0.847-0.289,0.979-0.262,0.979-0.262l0.825,1.336l-0.987,2c0,0-0.644,1.421-1.655,2.185c0,0-0.472,0.284-1.12,0.127c-0.648-0.157-1.072,0.333-1.072,0.333l-0.17,0.14c0,0,0.14-0.024,0.346-0.103c0,0,0.158,0.065,0.274,0.223c0.114,0.158,0.913,1.175,0.913,1.175s0.005,0.837-0.415,1.938c-0.419,1.1-1.467,2.891-1.467,2.891s-0.733,1.424-1.075,2.253c-0.342,0.829-0.515,1.765-0.488,2.262c0,0,0.187,0.062,0.707-0.202c0.655-0.332,1.083,0.027,1.083,0.027s0.719,0.53,1.041,0.881c0.262,0.289,0.802,1.765,0.209,3.224c0,0-0.402,1.008-1.377,1.724c0,0-0.216,0.332-1.529,0.174c-0.368-0.043-0.585-0.276-1.372-0.2c-0.785,0.077-1.231,0.815-1.231,0.815l0.013-0.024c-0.692,0.999-1.154,2.458-1.154,2.458l-0.057,0.165c0,0-0.241,0.509-0.292,1.752c-0.053,1.284,0.284,3.109,0.284,3.109s7.876-1.387,9.88-0.055l0.58,0.532c0,0,0.046,0.174-0.031,0.376c-0.08,0.204-0.375,0.673-0.987,1.113c-0.611,0.438-1.222,1.583-0.313,2.304c1.034,0.818,1.691,0.766,3.43,0.468c1.74-0.297,2.898-1.269,2.898-1.269s0.972-0.72,0.783-1.628c-0.188-0.908-1.017-1.189-1.017-1.189s-0.658-0.423-0.141-1.238c0,0,0.141-0.689,2.553-1.316c2.414-0.626,6.812-1.52,10.556-1.989c0,0-2.539-8.223-0.737-9.289c0,0,0.438-0.296,1.224-0.408l0.721-0.037c0.131-0.027,0.344,0.005,0.796,0.045c0.452,0.038,1.001,0.076,1.678-0.441c0.676-0.519,0.697-0.819,0.697-0.819"/></svg>
\ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" version="1.1" x="0px" y="0px" width="103px" height="94px" viewBox="0 0 103 94">
+  <title>Wikipedia logo version 2</title>
+  <radialGradient id="SVGID_1_" cx="244.5713" cy="-427.1392" r="68.6868" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)" gradientUnits="userSpaceOnUse">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".4835" stop-color="#EAEAEB"></stop>
+    <stop offset=".9451" stop-color="#A9ABAE"></stop>
+    <stop offset="1" stop-color="#999B9E"></stop>
+  </radialGradient>
+  <path style="&st16;" d="M49.85,17.003c0.014-0.606-0.392-1.27-0.392-1.27l-0.025-0.058c0,0-0.487-0.949-1.301-1.228c-0.815-0.278-1.478,0.342-1.478,0.342s-0.114,0.131-0.428,0.494c-0.313,0.364-0.507,0.666-1.198,0.938c-0.692,0.271-1.379,0.204-1.743,0.033c-0.365-0.172-0.457-0.537-0.457-0.537s-0.229-0.722-0.313-1.049c-0.086-0.331-0.308-1.694-0.308-1.694s-0.492-2.747-0.535-3.304c0,0,1.475-0.126,3.686-0.775c2.3-0.673,3.043-1.206,3.043-1.206s-0.432-0.156-0.484-0.662c-0.051-0.507-0.089-1.19-0.089-1.19s-0.089-0.5,0.483-1.139c0.572-0.64,1.354-0.863,1.763-0.953c0.41-0.089,1.281-0.17,2.092-0.134c0.813,0.038,1.266,0.112,1.594,0.291c0.327,0.178,0.356,0.61,0.356,0.61l-0.009,1.146c0,0-0.035,0.402,0.262,0.529c0,0,0.505,0.305,2.197,0.133c0,0,0.719-0.126,1.845-0.46c1.125-0.335,4.129-1.229,4.554-1.341c0.426-0.111,0.953-0.291,1.645-0.469c0.693-0.179,2.041-0.626,2.309-0.73c0.27-0.104,1.811-0.618,2.928-0.81c1.114-0.195,2.226-0.186,2.813,0.149c0,0,0.357,0.304,0.521,0.662c0.162,0.358,0.476,0.863,0.92,1.088c0.457,0.227,0.754,0.371,1.877,0.273c0,0,0.021-0.096-0.396-0.37c-0.417-0.277-0.991-0.701-0.991-0.701s-0.334-0.245-0.408-0.447c-0.072-0.202-0.043-0.306-0.043-0.306l0.877-1.406c0,0,0-0.172,0.506-0.238c0.506-0.067,1.074-0.134,1.742-0.313c0.67-0.178,0.789-0.312,0.789-0.312l0.578,0.178c0,0,3.547,2.853,4.655,3.583l0.198-0.239c0,0,0.437,0.018,0.828,0.172c0.393,0.154,0.979,0.562,0.979,0.562s1.613,1.31,2.072,2.2l0.222,0.679l-0.102,0.161c0,0,0.919,1.307,2.096,2.602c0,0,1.229,1.664,1.689,2.09c0,0-0.108-0.399-0.203-0.849l0.339-0.226l0.2-0.144l0.617,0.259c3.575,4.811,6.435,10.424,8.144,16.328l-0.121,0.484l0.396,0.501c1.129,4.212,1.729,8.643,1.729,13.211c0,1.122-0.038,2.236-0.109,3.339l-0.304,0.511l0.226,0.555C99.95,72.645,78.057,93.131,51.38,93.131c-18.502,0-34.702-9.854-43.637-24.601l-0.102-0.365l-0.205-0.151c-3.387-5.742-5.682-12.205-6.595-19.104l0.212-0.524l-0.336-0.482c-0.213-1.892-0.322-3.812-0.322-5.758c0-2.985,0.255-5.909,0.748-8.755l0.25-0.562l-0.087-0.328C2.463,26.451,4.689,20.783,7.78,15.7l0.684-0.384l0.081,0.032c0,0,0.233-0.169,0.354-0.217l0.076-0.023c0,0,1.179-1.971,1.625-2.601c0,0,0.542-0.348,0.745-0.407c0,0,0.124-0.016,0.189,0.076c0,0,0.496-0.432,1.699-2.054c0.004-0.005,0.007-0.011,0.012-0.017c0,0-0.114-0.076-0.131-0.174c-0.018-0.097,0.108-0.591,0.173-0.717s0.108-0.156,0.108-0.156s1.722-2.032,3.151-3.238c0,0,0.26-0.202,0.678-0.25c0,0,1.472-0.613,3.264-2.184c0,0,0.051-0.289,0.478-0.858c0.428-0.57,1.456-1.163,2.222-1.337c0.764-0.174,0.896-0.038,0.896-0.038l0.064,0.065l0.515,0.766c0,0,0.565-0.316,1.413-0.604c0.847-0.289,0.979-0.262,0.979-0.262l0.823,1.336l-0.987,2c0,0-0.644,1.421-1.655,2.185c0,0-0.472,0.284-1.12,0.127c-0.648-0.157-1.072,0.333-1.072,0.333l-0.17,0.14c0,0,0.14-0.024,0.346-0.103c0,0,0.158,0.065,0.274,0.223c0.114,0.158,0.913,1.175,0.913,1.175s0.005,0.837-0.415,1.938c-0.419,1.1-1.467,2.891-1.467,2.891s-0.733,1.424-1.075,2.253c-0.342,0.829-0.515,1.765-0.488,2.262c0,0,0.187,0.062,0.707-0.202c0.655-0.332,1.083,0.027,1.083,0.027s0.719,0.53,1.041,0.881c0.262,0.289,0.802,1.765,0.209,3.224c0,0-0.402,1.008-1.377,1.724c0,0-0.216,0.332-1.529,0.174c-0.368-0.043-0.585-0.276-1.372-0.2c-0.785,0.077-1.231,0.815-1.231,0.815l0.013-0.024c-0.692,0.999-1.154,2.458-1.154,2.458l-0.057,0.165c0,0-0.241,0.509-0.292,1.752c-0.053,1.284,0.284,3.109,0.284,3.109s7.876-1.387,9.88-0.055l0.58,0.532c0,0,0.046,0.174-0.031,0.376c-0.08,0.204-0.375,0.673-0.987,1.113c-0.611,0.438-1.222,1.583-0.313,2.304c1.034,0.818,1.691,0.766,3.43,0.468c1.74-0.297,2.898-1.269,2.898-1.269s0.972-0.72,0.783-1.628c-0.188-0.908-1.017-1.189-1.017-1.189s-0.658-0.423-0.141-1.238c0,0,0.141-0.689,2.553-1.316c2.414-0.626,6.812-1.52,10.557-1.989c0,0-2.539-8.223-0.738-9.289c0,0,0.438-0.296,1.224-0.408l0.72-0.037c0.131-0.027,0.343,0.005,0.796,0.045c0.453,0.038,1.001,0.076,1.678-0.441c0.676-0.519,0.697-0.819,0.697-0.819"></path>
+  <radialGradient id="SVGID_2_" cx="197.6416" cy="-371.8613" r="0" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)" gradientUnits="userSpaceOnUse">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".4835" stop-color="#EAEAEB"></stop>
+    <stop offset=".9451" stop-color="#A9ABAE"></stop>
+    <stop offset="1" stop-color="#999B9E"></stop>
+  </radialGradient>
+  <path style="&st17;" d="M-16.122-14.641"></path>
+  <linearGradient id="SVGID_3_" gradientUnits="userSpaceOnUse" x1="456.2744" y1="510.1602" x2="502.7757" y2="582.9122" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#8A8A8A"></stop>
+    <stop offset=".5687" stop-color="#606060"></stop>
+    <stop offset=".5914" stop-color="#FFF"></stop>
+    <stop offset=".6116" stop-color="#585858"></stop>
+    <stop offset="1" stop-color="#303030"></stop>
+  </linearGradient>
+  <path style="&st25;" d="M82.447,79.307l0.451,0.168c-0.032,0.061-0.067,0.119-0.101,0.179c-0.864,1.573-0.936,1.927-1.36,2.253c-0.424,0.326-0.58,0.539-1.443,0.695c-0.865,0.156-1.771,1.175-1.771,1.175s-0.875,0.835-0.48,0.866c0.396,0.025,1.177-0.609,1.177-0.609s0.497-0.354,0.625-0.255c0.125,0.099-0.539,0.751-0.539,0.751s-1.161,1.176-2.479,1.982l-0.396,0.213c0,0,1.869-1.392,2.764-2.395c0,0,0.185-0.271-0.185,0.057c-0.369,0.325-1.332,0.821-1.741,0.821c-0.41,0,0.016-0.563,0.016-0.563s0.596-0.768,1.812-1.758c1.22-0.994,2.239-0.922,2.239-0.922s0.227,0.043,0.253-0.271c0.028-0.313,0.312-0.863,0.312-0.863s0.453-1.032,1.674-2.749c1.218-1.714,1.77-2.195,1.77-2.195s1.588-1.629,2.423-3.259c0,0,0.581-1.006-0.364-0.255c-0.951,0.753-2.211,1.7-3.44,2.014c-1.233,0.312-1.349-0.836-1.349-0.836s-0.271-1.884,1.049-3.344c1.188-1.316,2.492-1.273,3.684-1.415c1.188-0.144,2.21-1.571,2.21-1.571s0.82-0.922,1.289-3.797c0.218-1.337,0.067-2.244-0.144-2.818l0.021-0.647l-0.368-0.021c-0.078-0.106-0.135-0.153-0.135-0.153s-0.297-0.299-0.607-0.524c-0.313-0.227-0.692-0.649-1.063-1.457c0,0-1.019-2.11-0.792-5.156c0.227-3.047,2.762-2.762,2.762-2.762s1.475,0.143,1.76-0.298c0.283-0.438,0.553-0.993,0.649-2.223c0.101-1.233-0.396-2.408-0.396-2.408s-0.186-0.609-1.049-0.708c-0.863-0.1-1.051,0.169-1.051,0.169s-0.255,0.072-0.723,1.09c-0.471,1.021-0.75,1.488-1.602,1.702c-0.849,0.211-2.023,0.099-2.549-0.992c-0.515-1.072-1.757-3.693-2.351-5.772l0.084-0.735l-0.325-0.471c-0.063-0.396-0.117-0.846-0.13-1.236c-0.026-0.992-0.312-2.097-0.638-3.2c-0.326-1.106-1.459-2.972-1.672-3.399c-0.324-0.667-0.621-0.949-1.021-0.893c-0.396,0.056-0.339,0.056-0.513,0.056c-0.168,0-0.266,0.212-0.311,0.425c-0.042,0.212-0.375,1.315-1.104,1.812c-0.752,0.51-1.147,0.737-2.438,0.85c0,0-1.487,0.099-2.661-2.097C71,29.36,71.301,27.96,71.398,27.45c0.099-0.51,0.539-1.247,1.229-1.388c0.693-0.141,1.119-0.184,1.119-0.184s1.122-0.01,1.603-0.071c0.551-0.071,0.283-1.006,0.283-1.006s-0.361-2.371-2.348-4.318l-0.096-0.487l-0.756-0.381c-1.881-2.04-4.354-5.354-4.354-5.354s-1.105-1.048-0.17-2.493c0,0,0.438-0.963,1.742-0.792c0.419,0.081,0.457,0.123,0.818,0.199c0.481,0.099,1.008,0.225,1.478-0.398c0,0,0.438-0.792-0.271-1.812s-0.923-1.742-1.785-1.687c0,0-0.397-0.055-0.793,0.384C68.702,8.1,67.682,8.378,67.086,8.44c-0.679,0.071-2.252-0.528-3.156-2.082c-0.513-0.874-0.752-1.35-0.865-1.595l0.231-0.34l0.72,0.08c0.186,0.358,0.549,1.056,0.978,1.812c0.552,0.978,1.048,1.118,1.373,1.261c0.325,0.141,1.049-0.071,1.431-0.283c0.385-0.213,0.766-0.695,1.43-0.738c0.668-0.043,1.629,0.071,2.311,0.793c0.682,0.723,1.531,2.224,1.459,2.932c-0.068,0.708-0.111,1.403-1.035,1.699c-0.921,0.298-1.303,0.043-1.884-0.084c-0.581-0.128-0.864-0.072-1.104,0.211c-0.242,0.284-0.512,0.892-0.412,1.162c0.102,0.27,0.186,0.454,0.75,1.262c0.566,0.806,3.474,4.25,4.031,4.766l-0.152,0.698l0.968,0.176c0.625,0.724,1.358,1.668,1.687,2.263c0.493,0.907,0.752,2.337,0.779,3.002c0.025,0.666-0.299,0.963-0.299,0.963s-0.313,0.524-2.012,0.524c-1.517,0-1.614,0.297-1.614,0.297s-0.47,0.467-0.369,1.615c0.103,1.146,0.616,2.344,1.56,3.37c0.778,0.851,2.252-0.006,2.748-0.295c0.51-0.299,0.822-1.264,0.822-1.264s0.102-1.217,1.432-1.217c1.377,0,1.969,1.318,1.969,1.318s0.963,1.295,1.941,4.121c0.481,1.389,0.566,2.281,0.566,2.281l0.126,1.668l-0.513,0.892l0.732,0.312c0.135,0.541,0.348,1.24,0.686,2.044c0,0,1.008,2.381,1.443,3.286c0.44,0.906,0.896,0.766,1.264,0.808c0,0,0.85,0.113,1.16-0.282c0.313-0.398,0.61-1.092,0.61-1.092s0.663-1.812,2.138-1.657c1.475,0.156,1.84,1.092,2.096,2.168c0.256,1.074,0.313,3.229-0.479,4.545c-0.798,1.318-1.688,1.135-1.688,1.135s-1.813-0.17-2.225,0.213c-0.41,0.382-0.623,0.724-0.681,1.613c-0.058,0.894,0.026,2.336,0.751,4.08c0.631,1.523,1.146,1.361,1.432,1.728c0.284,0.368,1.188,1.204,1.104,3.272c-0.082,2.067-0.791,4.149-1.586,5.439c-0.793,1.288-1.997,2.053-1.997,2.053s-0.338,0.211-1.076,0.311c-0.735,0.102-1.784,0.086-2.534,0.513c-0.75,0.426-0.992,1.501-1.063,1.971c-0.069,0.468-0.112,1.529,0.921,1.413c1.034-0.109,2.577-1.4,2.577-1.4s1.486-1.104,1.685-0.752c0.199,0.354-0.636,1.784-0.636,1.784s-1.035,1.562-1.898,2.523c-0.709,0.791-1.641,1.868-2.431,3.102L82.447,79.307L82.447,79.307z"></path>
+  <linearGradient id="SVGID_4_" gradientUnits="userSpaceOnUse" x1="425.2861" y1="502.9512" x2="445.7861" y2="598.6606" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#A8A9AB"></stop>
+    <stop offset="1" stop-color="#636668"></stop>
+  </linearGradient>
+  <path style="&st18;" d="M23.441,22.732c-0.007,0.008-0.013,0.018-0.021,0.025C23.428,22.75,23.434,22.74,23.441,22.732z"></path>
+  <linearGradient id="SVGID_5_" gradientUnits="userSpaceOnUse" x1="421.0684" y1="504.3701" x2="441.068" y2="597.7433" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#A8A9AB"></stop>
+    <stop offset="1" stop-color="#636668"></stop>
+  </linearGradient>
+  <path style="&st19;" d="M38.188,89.707l0.163,0.01c-0.163-0.147-0.37-0.281-0.62-0.414c-0.699-0.371-3.731-2.375-4.669-3.009c-0.936-0.633-2.575-1.7-3.297-1.478c-0.554,0.172-0.475,0.394-0.804,0.556c-0.331,0.157-0.687,0.052-1.504-0.384c-0.818-0.434-1.424-0.725-3.02-2.239c-1.245-1.185,1.595-0.118,1.595-0.118s0.619,0.262,0.343-0.385c-0.277-0.646-1.676-2.333-2.994-3.691c-1.068-1.098-1.674-1.86-2.435-2.97l-0.566-0.661l0.007-0.166c-0.018-0.024-0.035-0.054-0.052-0.078c0,0-1.874-3.377-1.676-4.617c0,0,0.068-0.828,1.241-1.188c0.817-0.251,0.71,0.542,3.112,0.975c0,0,2.07,0.559,2.611-0.945c0.539-1.505-0.567-3.393-0.567-3.393s-1.449-2.656-3.244-2.758c-0.95-0.053-0.692,0.586-2.125,0.926c0,0-1.913,0.184-2.519-0.963c-0.734-1.389-1.04-2.969-1.015-4.022c0.022-1,0.054-1.079,0.119-1.371c0.045-0.206,0.192-0.582,0.254-1.128l-0.134-0.667l0.204-0.501c0.002-0.26-0.004-0.535-0.021-0.83c-0.091-1.66-0.213-4.221-0.437-5.71c-0.223-1.491-0.633-3.798-1.991-3.866c0,0-0.671-0.051-1.634,0.885c-0.884,0.856-1.684-0.708-1.728-1.663c-0.053-1.121,0.131-2.254,0.409-2.795c0.277-0.541,1.042-0.567,1.186-0.555c0.146,0.013,0.555,0.171,1.043,0.474c0.488,0.305,1.16,0.305,1.557-0.092c0.395-0.394,0.947-0.856,1.173-2.598c0.224-1.741,0.224-3.547,0.013-5.71l-0.225-0.484l1.339-0.289c-0.001-0.017-0.021-0.126-0.032-0.193c0-0.002,0-0.003,0-0.005c-0.002-0.017-0.005-0.032-0.007-0.049c-0.001-0.003-0.001-0.005-0.001-0.007c-0.003-0.019-0.007-0.038-0.009-0.057c0-0.001-0.001-0.001-0.001-0.003c-0.003-0.02-0.006-0.042-0.009-0.062c0-0.001,0-0.001,0-0.001c-0.004-0.023-0.007-0.045-0.011-0.068c0-0.004-0.001-0.006-0.001-0.008c-0.002-0.022-0.006-0.045-0.008-0.066c-0.001-0.006-0.001-0.01-0.003-0.017c-0.002-0.023-0.005-0.046-0.009-0.069c-0.001-0.004-0.001-0.007-0.002-0.014c-0.003-0.026-0.007-0.05-0.011-0.076c0-0.002,0-0.004,0-0.005c-0.004-0.024-0.008-0.05-0.011-0.076c-0.001-0.007-0.001-0.01-0.003-0.017c-0.002-0.025-0.006-0.052-0.009-0.08c-0.001-0.005-0.001-0.011-0.002-0.021c-0.005-0.027-0.007-0.053-0.011-0.081c-0.001-0.006-0.002-0.013-0.002-0.019c-0.002-0.029-0.006-0.058-0.01-0.087c0-0.004-0.001-0.008-0.003-0.014c-0.002-0.028-0.005-0.056-0.009-0.082c-0.001-0.006-0.001-0.011-0.002-0.016c-0.002-0.031-0.006-0.06-0.01-0.09c0-0.006-0.001-0.015-0.002-0.021c-0.004-0.03-0.006-0.061-0.011-0.09c0-0.007-0.001-0.015-0.002-0.022c-0.003-0.03-0.006-0.062-0.01-0.093c0-0.006-0.002-0.012-0.002-0.019c-0.003-0.032-0.005-0.063-0.009-0.094c0-0.002,0-0.005,0-0.009c-0.004-0.032-0.005-0.066-0.01-0.1c0-0.005,0-0.012-0.001-0.02c-0.002-0.033-0.005-0.065-0.007-0.098c-0.001-0.007-0.001-0.014-0.001-0.021c-0.004-0.033-0.006-0.067-0.008-0.099c0-0.005,0-0.012-0.001-0.02c-0.002-0.033-0.006-0.069-0.007-0.102c0-0.003,0-0.007-0.001-0.01c-0.002-0.033-0.004-0.066-0.006-0.1c-0.001-0.006-0.001-0.011-0.001-0.017c-0.001-0.032-0.003-0.068-0.005-0.1c0-0.008,0-0.014-0.001-0.021c-0.001-0.031-0.002-0.065-0.003-0.099c-0.001-0.006-0.001-0.013-0.001-0.021c-0.002-0.033-0.003-0.066-0.004-0.1c0-0.005,0-0.009,0-0.014c-0.001-0.032-0.001-0.066-0.002-0.099c0-0.003,0-0.005,0-0.009c0-0.034,0-0.067-0.001-0.101c0-0.005,0-0.013,0-0.017c0-0.033,0-0.067,0-0.098c0-0.005,0.001-0.012,0.001-0.019c0-0.032,0-0.065,0.001-0.095c0-0.005,0-0.009,0-0.015c0.001-0.033,0.001-0.065,0.003-0.099c0.052-1.244,0.292-1.752,0.292-1.752l0.057-0.164c0,0,0.46-1.459,1.154-2.459l-0.013,0.024c0,0,0.446-0.738,1.231-0.814c0.785-0.079,1.004,0.156,1.371,0.2c0.04,0.004,0.078,0.008,0.115,0.013c0.013,0.002,0.025,0.002,0.037,0.004c0.025,0.002,0.051,0.004,0.075,0.006c0.014,0.001,0.027,0.003,0.041,0.003c0.021,0.001,0.043,0.003,0.064,0.004c0.014,0.001,0.028,0.002,0.041,0.003c0.02,0.001,0.04,0.001,0.058,0.003c0.014,0,0.026,0,0.042,0c0.019,0.001,0.036,0.002,0.055,0.002c0.013,0.001,0.026,0.001,0.038,0.002c0.017,0,0.034,0,0.051,0c0.011,0,0.023,0,0.034,0c0.017,0,0.033,0,0.05,0c0.011,0,0.02-0.001,0.032-0.001c0.016-0.001,0.031-0.001,0.046-0.001c0.011-0.001,0.02-0.001,0.03-0.002c0.016,0,0.03-0.002,0.045-0.002c0.009,0,0.019,0,0.026-0.001c0.016-0.001,0.03-0.002,0.044-0.004c0.006,0,0.016-0.001,0.023-0.002c0.015-0.001,0.029-0.001,0.044-0.003c0.006-0.001,0.013-0.002,0.02-0.002c0.016-0.002,0.03-0.004,0.045-0.008c0.004,0,0.008,0,0.013-0.001c0.019-0.002,0.036-0.005,0.052-0.008l0,0c0.035-0.006,0.068-0.014,0.098-0.021c0,0,0,0,0.002-0.002c0.012-0.002,0.026-0.005,0.039-0.01c0.002,0,0.004,0,0.008-0.001c0.009-0.003,0.019-0.006,0.028-0.009c0.004,0,0.007-0.002,0.01-0.003c0.009-0.003,0.019-0.007,0.026-0.009c0.002-0.001,0.005-0.002,0.008-0.003c0.008-0.003,0.015-0.006,0.021-0.009c0.004-0.001,0.006-0.003,0.009-0.003c0.007-0.004,0.014-0.005,0.02-0.009c0.003-0.001,0.006-0.003,0.008-0.004c0.005-0.002,0.012-0.005,0.019-0.007c0.001-0.001,0.003-0.002,0.005-0.004c0.005-0.003,0.01-0.005,0.016-0.007c0.002-0.002,0.003-0.002,0.006-0.004c0.004-0.001,0.008-0.005,0.012-0.007c0.002-0.001,0.004-0.001,0.005-0.003c0.004-0.002,0.008-0.006,0.012-0.008c0.001,0,0.003-0.002,0.004-0.003c0.003-0.003,0.007-0.004,0.011-0.007c0.001-0.001,0.001-0.001,0.002-0.003c0.004-0.001,0.006-0.005,0.008-0.008h0.002c0.003-0.002,0.005-0.005,0.008-0.007l0.001-0.001c0.002-0.002,0.004-0.004,0.006-0.006s0.002-0.002,0.003-0.003c0.001,0,0.002-0.002,0.002-0.002c0.001-0.003,0.001-0.003,0.003-0.005c0.003-0.001,0.003-0.004,0.003-0.004c0.328-0.241,0.593-0.516,0.797-0.775c0.014-0.017,0.026-0.033,0.04-0.05c0.002-0.004,0.005-0.009,0.008-0.012c0.013-0.015,0.026-0.031,0.038-0.048c0.002-0.004,0.006-0.008,0.007-0.011c0.012-0.018,0.025-0.033,0.038-0.05c0.001,0,0.001,0,0.001-0.001c0.039-0.054,0.075-0.109,0.109-0.159c0-0.002,0.002-0.004,0.003-0.007c0.01-0.016,0.02-0.029,0.03-0.044c0.001-0.004,0.005-0.007,0.007-0.011c0.008-0.014,0.017-0.029,0.024-0.042c0.003-0.004,0.005-0.009,0.009-0.013c0.008-0.014,0.017-0.028,0.024-0.042l0.001-0.002c0.017-0.027,0.032-0.055,0.046-0.079c0.002-0.003,0.004-0.008,0.007-0.012c0.005-0.009,0.01-0.021,0.016-0.03c0.003-0.007,0.007-0.014,0.012-0.02c0.004-0.008,0.01-0.017,0.014-0.024c0.002-0.008,0.006-0.017,0.011-0.023c0.004-0.007,0.009-0.016,0.012-0.022c0.004-0.007,0.007-0.014,0.011-0.021c0.002-0.006,0.007-0.011,0.01-0.018c0.066-0.13,0.097-0.207,0.097-0.207c0.594-1.459,0.052-2.935-0.21-3.224c-0.32-0.354-1.041-0.883-1.041-0.883s-0.426-0.357-1.08-0.025c-0.521,0.264-0.711,0.201-0.711,0.201c-0.024-0.498,0.149-1.432,0.491-2.263c0.341-0.829,1.075-2.253,1.075-2.253s1.047-1.792,1.467-2.89c0.418-1.102,0.415-1.94,0.415-1.94s-0.795-1.019-0.91-1.177c-0.115-0.158-0.272-0.223-0.272-0.223c-0.205,0.078-0.345,0.103-0.345,0.103l0.169-0.14c0,0,0.424-0.492,1.073-0.334c0.648,0.158,1.119-0.126,1.119-0.126c1.011-0.764,1.654-2.187,1.654-2.187l0.988-1.997L27.059,1.12c0,0-0.131-0.028-0.979,0.259c0,0-0.773,1.905-1.725,3.087c0,0-0.374,0.552-0.664,0.416c-0.289-0.134-0.629-0.144-0.91-0.085c-0.281,0.06-1.156,0.349-1.948,1.413c-0.79,1.064-0.883,1.863-0.458,2.101c0.425,0.238,1.045-0.076,1.42-0.407c0.375-0.333,0.638-0.485,0.757-0.47c0.118,0.02,0.093,0.315,0.068,0.41c-0.026,0.094-0.154,1.364-1.625,3.913c-0.369,0.64-0.845,1.589-1.069,2.046l0.608,0.447l-0.999,0.503c-0.266,0.758-0.605,1.945-0.471,2.61c0.204,1.011,1.158,1.011,1.158,1.011s0.621,0.027,1.344-0.348c0.722-0.375,0.892,1.242,0.892,1.505c0,0.264-0.007,1.726-1.02,2.442c0,0-0.409,0.313-0.985,0.313c-0.579,0-0.954-0.169-0.954-0.169s-0.984-0.272-1.59,0.083c-0.604,0.358-1.326,1.098-1.897,2.17c-0.569,1.072-1.045,2.986-1.019,4.397c0.02,1.111,0.18,2.162,0.291,2.879l0.667,0.435l-0.543,0.623c0.079,1.136,0.168,3.363,0.155,4.109c-0.018,0.911-0.154,2.84-1.115,3.292c0,0-0.324,0.171-0.868-0.238s-1.132-0.426-1.37-0.435c-0.238-0.007-1.285,0.162-1.692,1.311c-0.408,1.145-0.375,2.688-0.29,3.597c0.086,0.91,0.876,2.458,1.872,2.458c0,0,0.484,0.035,1.055-0.553c0.568-0.586,0.902-0.637,1.156-0.629c0.255,0.009,0.749,0.272,1.072,2.851c0.307,2.442,0.437,4.442,0.414,6.668l0.605,0.392l-0.758,0.768c-0.042,0.199-0.089,0.417-0.142,0.626c-0.169,0.682-0.187,1.364-0.024,2.569c0.161,1.21,0.811,3.72,1.754,4.375c1.252,0.871,2.071,0.856,2.916,0.791c0.842-0.067,1.424-0.712,1.424-0.712s0.331-0.342,0.685-0.237c0.356,0.104,1.346,0.66,2.058,2.084c0.713,1.425,0.871,2.992-0.316,3.272c-1.187,0.272-3.231-0.846-3.231-0.846s-1.161-0.647-2.109,0.064c-0.951,0.713-0.977,1.807-0.502,3.15c0.261,0.738,0.782,1.938,1.513,3.188l0.721,0.302l-0.193,0.551c0.492,0.748,1.055,1.479,1.678,2.105c0,0,2.466,2.729,3.838,4.457c0,0,0.08,0.157-0.158,0.016c-0.238-0.146-1.266-0.621-1.82-0.566c-0.555,0.054-0.45,0.395-0.45,0.395s0.238,1.254,4.01,3.365c0,0,1.359,0.766,2.216,0.766c0,0,0.277,0.039,0.619-0.346c0.346-0.381,0.45-0.341,0.688-0.262c0.237,0.076,0.553,0.249,1.741,1.105c1.188,0.857,3.496,2.176,4.325,2.731c0.83,0.555,0.793,0.657,0.621,1.054c-0.171,0.396,0.593,0.619,0.593,0.619s1.899,0.855,2.928,0.846c1.029-0.016,1.464-0.119,2.097,0.012c0.632,0.133,1.28,0.291,1.345,0.346c0.066,0.053-0.315,0.038-0.315,0.038s-2.362-0.276-2.494-0.21c-0.13,0.066,0.014,0.184,0.99,0.436v0.006c1.245,0.217,2.507,0.387,3.782,0.51c-0.489-0.061-2.52-0.322-3.823-0.713c0,0-0.131-0.078,0.173-0.014c0.303,0.065,2.018,0.225,2.466,0.157c0.448-0.065-0.092-0.274-0.092-0.274s-0.897-0.425-2.889-0.582c0,0-0.803-0.055-1.503,0.014c-0.699,0.066-1.41-0.264-1.41-0.264s-1.028-0.317-0.78-0.646c0.126-0.165,0.137-0.336,0.065-0.502L38.188,89.707L38.188,89.707z"></path>
+  <linearGradient id="SVGID_6_" gradientUnits="userSpaceOnUse" x1="444.7598" y1="550.8145" x2="473.8418" y2="550.8145" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#231F20"></stop>
+    <stop offset="1" stop-color="#474747"></stop>
+  </linearGradient>
+  <path style="&st5;" d="M62.342,86.16l-0.438,0.646c0.096,0.655-0.104,0.875-0.254,1.119c-0.156,0.26-0.59,0.592-0.375,1.185c0.218,0.59,1.687,0.401,2.535,0.144c0.852-0.26,1.569-0.49,2.134-0.693c0.562-0.198,0.719,0.086,0.719,0.086s0.246,0.175-0.574,0.606c-0.821,0.433-2.336,0.634-3.055,0.72c-0.724,0.086-0.506-0.043-1.586,0.33c-1.082,0.377-0.07,0.707-0.07,0.707c2.435,0.635,4.188,0.115,4.188,0.115l0.332-0.097c0.27-0.077,0.535-0.161,0.803-0.244c-2.021,0.456-3.326,0.149-3.739,0.095c-0.431-0.058-0.778-0.145-0.88-0.361c-0.102-0.215,0.479-0.2,0.479-0.2s1.683-0.188,3.24-0.69c1.557-0.506,1.932-0.98,1.871-1.341c-0.057-0.358-0.848-0.332-1.785-0.028c-0.937,0.305-2.334,0.75-2.662,0.821c-0.334,0.07-0.691,0.06-0.812-0.146c-0.114-0.203-0.216-0.53,0.146-0.905c0.36-0.376,0.402-0.694,0.402-0.694c0.055-0.254,0.057-0.523,0.021-0.8L62.342,86.16l0.545-0.118c-0.298-1.124-1.052-2.218-1.279-2.577c-0.314-0.507-0.836-0.793-2.393-0.535c-1.556,0.26-3.386,1.035-3.386,1.035s-1.888,0.997-3.298,0.812c-1.413-0.188-1.703-1.614-1.688-2.063c0.015-0.447,0.304-0.835,1.039-1.123c0.735-0.289,2.465-0.678,4.985-0.808s3.458-1.771,3.458-1.771c0.33-0.478,0.922-1.543-0.489-2.336c-1.41-0.791-1.441-0.593-2.119-1.107c-0.678-0.52-1.412-1.153-1.701-2.033c-0.025-0.084-0.066-0.197-0.104-0.292l-0.849-0.558l0.41-0.371c-0.34-0.67-0.693-1.271-0.693-1.271s-1.281-1.902-0.246-3.703c1.038-1.803,2.736-2.348,2.736-2.348s1.5-0.332,2.996,0.016c1.418,0.324,2.133-0.219,2.133-0.219s0.865-0.374,1.123-2.102c0.264-1.729-0.014-4.943-2.316-5.578c-2.307-0.633-3.527,0.563-3.527,0.563s-1.24,1.369-1.644,1.57c-0.404,0.201-1.022,0.563-1.022,0.563s-0.867,0.519-1.453,0.648c-0.393,0.086-1.791-1.771-1.933-3.201c-0.133-1.316-0.401-2.388-0.306-5.096l-0.485-0.63l0.537-0.533c0.101-2.999-0.417-5.116-1.188-6.461c-0.807-1.412-2.119-2.161-2.766-2.478c-0.648-0.318-2.147-0.462-3.17-0.086c-1.023,0.374-1.239,0.678-1.613,1.326c-0.376,0.648-0.836,0.605-0.836,0.605s-1.427,0.043-1.888-2.133c-0.646-3.049,0.359-4.035,0.359-4.035s0.318-0.476,1.369-0.619c1.053-0.144,1.73,0.115,2.537,0.315c0.806,0.202,1.41,0.116,2.419-0.374c1.008-0.491,1.442-1.297,1.238-2.739c-0.195-1.393-0.255-1.742-1.483-5.964l-0.875-0.46l0.583-0.579C44.87,23.032,44.8,20.35,44.8,20.35c-0.106-0.977,0.049-1.292,0.598-1.54c0.576-0.259,1.116-0.175,1.934-0.123c0.818,0.053,1.425-0.079,1.847-0.316c0.422-0.237,1.581-0.87,1.504-2.162c-0.08-1.292-1.109-2.824-1.953-3.351c-0.843-0.528-1.953-0.316-2.558,0.131c-0.606,0.45-0.845,1.029-1.214,1.214c-0.369,0.183-0.895,0.605-1.45,0.474c-0.554-0.132-0.581-1.371-0.818-2.77c-0.153-0.907-0.271-1.611-0.338-1.989l-0.678-0.254c0.044,0.557,0.535,3.304,0.535,3.304s0.223,1.364,0.308,1.692c0.086,0.329,0.314,1.05,0.314,1.05s0.092,0.364,0.456,0.535c0.365,0.172,1.052,0.24,1.743-0.032c0.692-0.271,0.886-0.572,1.199-0.938c0.315-0.364,0.429-0.493,0.429-0.493s0.663-0.622,1.478-0.343c0.813,0.278,1.299,1.228,1.299,1.228l0.028,0.06c0,0,0.403,0.661,0.392,1.269v-0.057c0,0-0.022,0.301-0.697,0.818c-0.676,0.519-1.226,0.479-1.678,0.442c-0.454-0.04-0.666-0.072-0.797-0.045l-0.719,0.038C45.178,18.303,44.74,18.6,44.74,18.6c-1.8,1.064,0.736,9.288,0.736,9.288l0,0L45.2,28.501c0,0,0.514,2.052,0.904,3.378c0.388,1.326,0.562,2.637,0.62,2.91c0.058,0.274,0.044,0.762-0.317,1.051c-0.359,0.289-1.239,0.534-1.425,0.562c-0.187,0.029-0.535-0.042-0.996-0.201c-0.462-0.158-0.922-0.187-0.922-0.187s-1.11-0.188-2.291,0.173c-1.182,0.359-2.076,1.569-2.148,3.083c-0.071,1.513-0.057,2.278,0.535,3.617c0.59,1.34,1.657,2.104,2.463,2.118c0.808,0.014,1.469-0.403,1.931-1.051c0.459-0.65,0.59-0.751,0.59-0.751c0.548-0.302,1.944-0.433,2.651-0.172c0.708,0.258,2.007,1.073,2.723,2.679c0.579,1.298,0.76,2.75,0.729,5.363l0.584,0.448l-0.61,0.703c-0.007,0.246-0.016,0.498-0.026,0.761c-0.143,3.876,0.72,5.864,0.72,5.864c0.491,1.065,1.023,1.83,1.745,2.318c0.719,0.489,1.699,0.156,2.018,0c0.315-0.158,1.194-0.563,1.943-1.037c0.749-0.477,0.725-0.679,1.629-1.515c0.907-0.833,2.076-0.604,2.076-0.604s1.107,0.028,1.74,1.313c0.637,1.283,0.509,3.109,0.347,3.773c-0.158,0.662-0.444,1.097-1.063,0.979c-0.621-0.114-1.645-0.217-2.019-0.231c-0.375-0.014-1.433-0.049-2.394,0.203c-0.938,0.244-2.205,0.92-3.414,2.883c-0.938,1.52-0.478,3.013-0.262,3.603c0.17,0.462,0.635,1.104,1.043,1.896l0.756,0.252l-0.35,0.656c0.398,0.963,0.701,1.493,1.305,2.151c0.648,0.705,1.672,1.251,2.881,1.714c1.213,0.462,0.662,1.282,0.662,1.282c-0.69,1.497-2.75,1.557-3.354,1.628c-0.604,0.07-1.872,0.188-3.058,0.447c-1.182,0.261-2.291,0.418-2.954,1.182c-0.661,0.764-0.402,1.557-0.013,2.393c0.388,0.834,1.427,1.28,2.853,1.226c1.426-0.058,2.35-0.476,3.214-0.851s2.362-0.809,2.81-0.937c0.445-0.129,1.051-0.39,1.498,0.26c0.482,0.701,0.994,1.697,1.229,2.45L62.342,86.16L62.342,86.16z"></path>
+  <path style="&st34;" d="M101.566,51.162c0,0,0.347-3.236,0.457-4.392c0.018-0.173,0.031-0.343,0.045-0.513l-0.098-0.241c-0.459,5.815-0.938,7.727-0.938,7.727s0.013-0.037,0.031-0.101c0.189-0.817,0.357-1.646,0.51-2.48C101.568,51.162,101.566,51.162,101.566,51.162L101.566,51.162z"></path>
+  <path style="&st34;" d="M91.268,12.579l0.328,0.413l0.279,0.5c1.021,1.648,2.846,4.295,2.846,4.295s0.739,0.958,0.905,0.968c0.164,0.01-0.084-0.741-0.084-0.741s-0.271-0.979,0.517,0.298c0.73,1.19,1.207,2.359,1.317,2.72c0.114,0.361,0.042,0.411,0.042,0.411s-0.146,0.072-0.146,0.33c0,0.505,0.496,1.659,0.496,1.659s1.545,3.926,2.07,5.563c0.526,1.641,1.133,4.368,1.133,4.368s0.187,0.804,0.207,1.174c0.021,0.371-0.063,0.381-0.229,0.186c-0.164-0.196-0.329-0.072-0.329-0.072s-0.175,0.072-0.175,0.773c0,0.7,0.085,2.346,0.217,3.615c0.152,1.505,0.616,2.133,0.616,2.133s0.413,0.618,0.526-0.577c0.096-1.028,0.185-0.228,0.185-0.228c0.196,1.557,0.062,4.544,0.03,4.955c-0.019,0.218-0.032,0.433-0.049,0.64l0.133-0.221c0.201-2.971,0.06-5.359,0.06-5.359s-0.144-1.323-0.3-1.311c-0.155,0.01-0.211,0.701-0.211,0.701s-0.065,0.467-0.156,0.456c-0.088-0.011-0.369,0.022-0.566-1.412c-0.199-1.436-0.156-2.949-0.156-2.949s-0.043-0.155,0.048-0.189c0.09-0.034,0.188,0.1,0.188,0.1s0.133,0.189,0.287,0.033c0.156-0.154,0.19-0.622-0.301-3.08c-0.288-1.454-0.711-2.896-1.006-3.832l-0.294-0.333l-0.058-0.718c0,0-0.311-0.913-1.033-2.737c-0.723-1.824-0.846-2.458-0.846-2.458s-0.045-0.2,0.066-0.234c0.111-0.032,0.091-0.178,0.091-0.178s-0.013-0.245-0.278-0.99c-0.268-0.746-0.426-1.281-1.356-2.86c-0.869-1.468-1.124-1.558-1.124-1.558s-0.426-0.234-0.112,0.591c0.313,0.823-0.075,0.232-0.075,0.232c-0.925-1.177-2.838-4.292-2.838-4.292l-0.537-0.373l-0.508-1.261l-0.015,0.01"></path>
+  <path style="&st34;" d="M1.547,32.696l-0.183,0.37c-0.472,2.495-0.625,5.135-0.62,5.31c0.01,0.208-0.036,0.211-0.075,0.178c-0.042-0.035,0.03-0.16-0.048-0.16c-0.079,0-0.108,0.211-0.108,0.211L0.512,38.6c-0.021,0.288-0.038,0.574-0.053,0.861l0.016-0.003c0.068-0.098,0.097-0.028,0.097-0.028s-0.029,0.126,0.097,0.056c0.126-0.07,0.21-1.379,0.21-1.379s0.014-2.813,0.836-6.14c0.819-3.327,1.281-4.259,1.281-4.259s0.154-0.418,0.138-0.083C3.12,27.958,3.33,27.986,3.33,27.986c0.375-0.054,0.821-1.125,1.267-2.493c0.445-1.363,0.668-2.589,0.668-2.7c0-0.11-0.055-0.194-0.152-0.138c-0.098,0.056-0.125,0.014-0.125,0.014c-0.014-0.208,0.361-1.127,0.361-1.127c1.253-3.202,3.104-5.694,3.104-5.694l0.09-0.504c-0.164,0.254-0.27,0.419-0.421,0.661c-0.056,0.089-0.042,0.297-0.001,0.32c-0.201,0.191-0.365,0.35-0.476,0.456c-2.707,4.473-3.059,6.556-3.059,6.556c-0.017,0.214,0.004,0.311,0.111,0.306c0.065-0.003,0.251-0.349,0.116,0.354c-0.09,0.468-0.524,1.708-0.693,2.212c-0.171,0.505-0.358,0.85-0.495,0.826C3.49,27.01,3.49,26.801,3.49,26.801s-0.042-0.546-0.398,0.245c-0.356,0.791-0.713,1.859-1.425,4.65c-0.031,0.125-0.063,0.251-0.092,0.38L1.547,32.696L1.547,32.696z"></path>
+  <path style="&st34;" d="M4.841,62.626c-0.15-0.401-0.264-0.722-0.179-0.581c0.085,0.143,0.198,0.432,0.462,0.725c0.263,0.291,0.442-0.226-0.622-3.104s-1.354-3.04-1.402-3.095c-0.046-0.058-0.215-0.237-0.167,0.167c0.045,0.404,0.018,0.656-0.51-1.146c-0.528-1.806-0.996-4.844-1.025-5.089c-0.027-0.243-0.169-1.778-0.396-3.594c-0.226-1.814-0.246-3.743-0.207-4.28c0.039-0.537-0.066-0.828-0.123-0.837c-0.056-0.008-0.094,0.047-0.131,0.284c-0.038,0.234-0.01,0.395-0.067,0.385c-0.057-0.009-0.076-0.471-0.076-0.471H0.391c0,0.05,0,0.1,0,0.151c0,0.174,0.001,0.345,0.002,0.519l0.039,0.402c0.033,0.597,0.129,0.354,0.135,0.246c0.006-0.109,0.03-0.329,0.03-0.329s0.103-0.884,0.084,0.02c-0.019,0.904,0.236,4.563,0.236,4.563c0.019,0.236,0.041,0.479,0.068,0.729l0.063,0.092l-0.042,0.104c0.265,2.425,0.795,5.086,0.795,5.086c0.507,2.417,1.11,3.846,1.308,4.25c0.198,0.405,0.236,0.085,0.17-0.271c-0.066-0.357,0.546,0.688,0.873,1.674c0.332,0.99,0.556,1.815,0.556,1.815s0.254,0.781,0.142,0.828c-0.113,0.046-0.292-0.293-0.292-0.293s-0.473-0.835-0.274-0.228c0.398,1.231,1.6,3.822,1.6,3.822l1.396,2.471C6.282,65.836,4.982,63.004,4.841,62.626L4.841,62.626z"></path>
+  <path style="&st34;" d="M7.281,67.639c0.069,0.125,0.136,0.246,0.202,0.359L7.281,67.639z"></path>
+  <path style="&st34;" d="M20.722,2.874C20.51,3.216,20.48,3.388,20.48,3.388s0.112-0.118,0.183-0.237C20.733,3.033,20.722,2.874,20.722,2.874z"></path>
+  <path style="&st34;" d="M17.216,5.572c-0.417,0.048-0.677,0.25-0.677,0.25S16.889,5.761,17.216,5.572z"></path>
+  <path style="&st34;" d="M84.005,4.909c0,0,0.247-0.03,0.347,0.02c0.101,0.049,0.066-0.158,0.066-0.158s-0.287-0.406-0.322-0.556c-0.32-0.089-0.611-0.1-0.611-0.1l-0.028,0.034c-0.01,0.075-0.036,0.188-0.012,0.297C83.441,4.448,83.917,4.811,84.005,4.909L84.005,4.909z"></path>
+  <path style="&st34;" d="M87.487,7.88l0.095-0.152l-0.223-0.679c-0.004-0.012-0.012-0.022-0.021-0.036c-0.007,0.066-0.049,0.125-0.172,0.115c0,0-0.099-0.03-0.011,0.198C87.219,7.469,87.355,7.699,87.487,7.88L87.487,7.88z"></path>
+  <path style="&st34;" d="M101.566,51.162c0,0,0.347-3.236,0.457-4.392c0.018-0.173,0.031-0.343,0.045-0.513l-0.098-0.241c-0.459,5.815-0.938,7.727-0.938,7.727s0.013-0.037,0.031-0.101c0.189-0.817,0.357-1.646,0.51-2.48C101.568,51.162,101.566,51.162,101.566,51.162L101.566,51.162z"></path>
+  <linearGradient id="SVGID_7_" gradientUnits="userSpaceOnUse" x1="266.4922" y1="-395.2783" x2="295.9644" y2="-485.0349" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".0094" stop-color="#FCFCFC"></stop>
+    <stop offset=".0655" stop-color="#EEEEEE"></stop>
+    <stop offset=".1342" stop-color="#E5E5E5"></stop>
+    <stop offset=".2515" stop-color="#E3E3E3"></stop>
+    <stop offset=".3357" stop-color="#8A8A8A"></stop>
+    <stop offset=".4422" stop-color="#B8B8B8"></stop>
+    <stop offset="1" stop-color="#3B3B3B"></stop>
+  </linearGradient>
+  <path style="&st33;" d="M79.003,84.528c0,0,0.667-0.653,0.539-0.752c-0.128-0.101-0.623,0.256-0.623,0.256s-0.073,0.062-0.185,0.142l0.393-0.252c0,0-0.038,0.238-0.355,0.555c0,0-0.094,0.094-0.258,0.248c-0.957,0.938-2.386,1.998-2.386,1.998l0.396-0.211C77.844,85.703,79.003,84.528,79.003,84.528z"></path>
+  <linearGradient id="SVGID_8_" gradientUnits="userSpaceOnUse" x1="460.4629" y1="512.5557" x2="509.5884" y2="573.3062" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EFF0F0"></stop>
+    <stop offset=".5914" stop-color="#F0F1F2"></stop>
+    <stop offset=".599" stop-color="#787878"></stop>
+    <stop offset=".6456" stop-color="#EEEFF0"></stop>
+    <stop offset="1" stop-color="#D8D9DB"></stop>
+  </linearGradient>
+  <path style="&st30;" d="M82.984,39.705l0.51-0.891l-0.127-1.667c0,0-0.085-0.893-0.566-2.28c-0.979-2.828-1.941-4.122-1.941-4.122s-0.592-1.318-1.969-1.318c-1.33,0-1.431,1.219-1.431,1.219s-0.312,0.963-0.821,1.261c-0.5,0.29-1.972,1.146-2.748,0.296c-0.941-1.026-1.461-2.225-1.56-3.372c-0.101-1.146,0.367-1.615,0.367-1.615s0.1-0.296,1.614-0.296c1.703,0,2.014-0.525,2.014-0.525s0.324-0.296,0.297-0.963s-0.284-2.097-0.779-3.001c-0.324-0.596-1.058-1.537-1.684-2.261l-0.967-0.178l0.15-0.699c-0.56-0.514-3.465-3.959-4.031-4.766c-0.564-0.808-0.65-0.993-0.75-1.262c-0.099-0.269,0.17-0.877,0.412-1.161c0.238-0.283,0.521-0.341,1.104-0.212c0.58,0.127,0.961,0.381,1.886,0.084c0.919-0.297,0.962-0.992,1.033-1.699c0.071-0.708-0.78-2.209-1.458-2.932c-0.684-0.721-1.645-0.836-2.311-0.792c-0.664,0.042-1.047,0.523-1.433,0.737c-0.382,0.213-1.103,0.425-1.429,0.284c-0.326-0.142-0.823-0.284-1.375-1.261c-0.43-0.76-0.794-1.459-0.979-1.817L63.299,4.42v0.012c0,0,0.633,1.654,1.633,2.811c0.998,1.157,2.266,0.919,2.266,0.919s0.82-0.089,1.533-0.772c0.711-0.683,1.761-0.148,2.024,0.04c0.269,0.189,0.853,0.911,1.478,2.127c0.621,1.216-0.355,2.058-0.355,2.058s-0.555,0.535-1.691,0.088c-1.14-0.443-1.813,0.259-1.986,0.614c-0.182,0.357-0.508,0.99,0.076,1.73c0.584,0.742,3.178,4.273,4.916,5.945c1.74,1.672,2.314,3.047,2.682,4.342c0.365,1.297,0.079,1.899-0.521,2.018c-0.604,0.118-1.148,0.021-2.086,0.187c-0.94,0.17-1.349,0.367-1.543,1.653c-0.199,1.286,0.562,3.373,1.67,4.361c1.106,0.989,2.334,0.386,2.76,0.228c0.427-0.159,1.352-0.653,1.681-2.027c0.188-0.783,0.851-0.721,0.851-0.721s0.563-0.071,0.854,0.117c0.287,0.19,0.633,0.525,1.402,1.87c0.772,1.346,1.453,3.146,1.724,4.738C82.924,38.35,82.729,38.576,82.984,39.705c0.256,1.128,1.078,3.245,1.466,4.074c0.383,0.832,0.78,1.662,0.989,2.107c0.205,0.445,0.531,0.782,1.443,0.802c0.908,0.02,1.273-0.228,1.541-0.662c0.27-0.435,0.612-1.088,0.713-1.316c0.1-0.228,0.467-0.911,1.146-1.02c0.685-0.108,1.762,0.01,2.106,1.198c0.313,1.071,0.76,2.622-0.158,4.5c-0.65,1.334-1.129,0.859-2.451,0.948c0,0-1.165-0.01-1.781,0.921c-0.611,0.93-0.416,2.61-0.286,3.877s0.988,3.113,1.621,3.563c0.636,0.443,0.86,0.849,1.08,1.256c0.216,0.404,0.534,1.205,0.216,3.313c-0.313,2.106-0.979,3.74-1.867,4.521c-0.024,0.021-0.05,0.043-0.07,0.063c-0.067,0.065-0.141,0.135-0.219,0.201c-0.537,0.521-0.371,0.543-0.889,0.793c-0.594,0.289-0.988,0.207-1.958,0.365c-0.97,0.16-1.583,0.327-2.088,0.821c-0.503,0.495-1.243,1.409-0.979,3.187c0.148,0.986,1.318,0.584,2.229,0.111c-0.274,0.125-0.553,0.221-0.798,0.246c-1.033,0.113-0.991-0.949-0.921-1.415c0.069-0.47,0.313-1.544,1.063-1.97s1.799-0.41,2.533-0.512c0.738-0.101,1.076-0.313,1.076-0.313s1.205-0.766,1.997-2.055c0.793-1.289,1.502-3.371,1.587-5.438c0.084-2.068-0.821-2.902-1.104-3.271c-0.283-0.366-0.799-0.203-1.431-1.729c-0.724-1.74-0.81-3.188-0.751-4.079c0.057-0.892,0.27-1.231,0.682-1.612c0.41-0.383,2.223-0.213,2.223-0.213s0.893,0.185,1.686-1.134c0.793-1.317,0.738-3.471,0.481-4.546c-0.253-1.076-0.623-2.013-2.097-2.168c-1.471-0.152-2.138,1.66-2.138,1.66s-0.297,0.693-0.608,1.092c-0.312,0.395-1.16,0.285-1.16,0.285c-0.37-0.044-0.821,0.099-1.264-0.81c-0.438-0.906-1.442-3.286-1.442-3.286c-0.339-0.809-0.556-1.512-0.688-2.055L82.984,39.705L82.984,39.705z"></path>
+  <linearGradient id="SVGID_9_" gradientUnits="userSpaceOnUse" x1="272.8721" y1="-392.8257" x2="302.4699" y2="-482.9646" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st9;" d="M82.487,79.313l0.801-0.479c0.788-1.231,1.72-2.31,2.431-3.103c0.863-0.961,1.896-2.522,1.896-2.522s0.835-1.431,0.638-1.784c-0.13-0.23-0.704,0.02-1.687,0.752c-3.674,2.746-3.971,1.246-3.971,1.246c0.422,2.438,4.729-1.633,5.108-1.375c-0.063,0.563-0.457,1.172-1.25,2.25c0,0-0.388,0.555-0.78,0.953c-0.369,0.432-0.578,0.641-0.578,0.641s-0.088,0.09-0.125,0.125c-0.471,0.541-1.024,1.242-1.595,2.018c-0.019,0.021-0.104,0.113-0.125,0.143c-0.295,0.416-0.491,0.73-0.702,1.062c-0.014,0.022-0.064,0.011-0.076,0.034c0-0.002-0.013,0.014-0.025,0.037c-0.245,0.39-0.373,0.713-0.508,0.959c-0.012,0.029-0.021,0.065-0.03,0.095c0,0-0.319,0.665-0.457,1.067c-0.14,0.405-0.12,0.547-0.623,0.625c-0.504,0.078-0.276-0.053-1.021,0.196c0,0-0.403,0.199-0.938,0.571c-0.027,0.021-0.057,0.042-0.082,0.063c-0.736,0.604-1.247,1.119-1.534,1.436c-0.051,0.063-0.099,0.13-0.146,0.195c0,0-0.157,0.168,0.051,0.188c0.206,0.021,0.633-0.01,1.008-0.169l0.088-0.057c-0.186,0.103-0.373,0.174-0.513,0.162c-0.396-0.026,0.479-0.864,0.479-0.864s0.906-1.019,1.771-1.175c0.862-0.156,1.021-0.371,1.444-0.693c0.426-0.327,0.494-0.682,1.359-2.254c0.03-0.059,0.064-0.115,0.098-0.176L82.487,79.313L82.487,79.313z"></path>
+  <linearGradient id="SVGID_10_" gradientUnits="userSpaceOnUse" x1="444.6943" y1="510.9561" x2="469.7246" y2="592.0699" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".7473" stop-color="#F9F9F9"></stop>
+    <stop offset="1" stop-color="#D5D7D8"></stop>
+  </linearGradient>
+  <path style="&st27;" d="M55.064,72.686l0.408-0.377c-0.34-0.668-0.693-1.269-0.693-1.269s-1.282-1.901-0.245-3.703c1.036-1.803,2.737-2.348,2.737-2.348s1.5-0.332,2.996,0.017c1.418,0.323,2.133-0.22,2.133-0.22s0.865-0.376,1.123-2.104c0.261-1.729-0.014-4.94-2.317-5.576c-2.306-0.633-3.528,0.563-3.528,0.563s-1.242,1.369-1.644,1.57c-0.404,0.2-1.024,0.562-1.024,0.562s-0.865,0.52-1.453,0.648c-0.39,0.087-1.788-1.771-1.931-3.2c-0.133-1.313-0.4-2.385-0.305-5.084c0-0.005,0-0.01,0-0.017l-0.486-0.629l0.537-0.526c0.102-3-0.418-5.119-1.188-6.463c-0.805-1.414-2.118-2.163-2.766-2.479c-0.647-0.317-2.146-0.461-3.169-0.086c-1.022,0.375-1.237,0.677-1.613,1.325c-0.376,0.65-0.835,0.606-0.835,0.606s-1.427,0.044-1.89-2.132c-0.644-3.049,0.36-4.036,0.36-4.036s0.318-0.475,1.369-0.619c1.053-0.144,1.73,0.115,2.536,0.317c0.807,0.2,1.41,0.114,2.42-0.374c1.009-0.49,1.442-1.298,1.24-2.738c-0.196-1.397-0.249-1.727-1.484-5.966l-0.874-0.458l0.582-0.579c-1.182-4.271-1.257-6.961-1.257-6.961c-0.105-0.975,0.049-1.29,0.598-1.537c0.576-0.261,1.117-0.177,1.934-0.125c0.819,0.052,1.425-0.079,1.847-0.316c0.423-0.236,1.583-0.87,1.503-2.163c-0.078-1.292-1.108-2.823-1.951-3.35c-0.844-0.528-1.952-0.317-2.56,0.132c-0.606,0.447-0.843,1.028-1.213,1.212c-0.368,0.185-0.896,0.607-1.45,0.474c-0.554-0.132-0.581-1.372-0.818-2.77c-0.155-0.915-0.276-1.614-0.342-1.989l-0.674-0.254c0.043,0.557,0.535,3.304,0.535,3.304l0.294,1.624c0,0,0,0.007,0,0.02c0.006,0.018,0.009,0.036,0.013,0.05c0.019,0.079,0.049,0.18,0.082,0.289c0.114,0.215,0.37,0.456,0.942,0.502c1.076,0.089,1.772-0.468,2.025-0.709c0.254-0.239,0.86-0.911,0.86-0.911s0.329-0.632,1.253-0.494c0.922,0.14,1.238,0.773,1.403,1.013c0.167,0.242,1.57,1.961,0.672,2.861c-0.039,0.039-0.093,0.089-0.152,0.146c-0.104,0.111-0.245,0.246-0.446,0.399c-0.675,0.517-1.226,0.48-1.678,0.442c-0.453-0.039-0.665-0.07-0.795-0.043l-0.72,0.038c-0.787,0.11-1.224,0.408-1.224,0.408c-1.8,1.065,0.735,9.287,0.735,9.287s0.671,2.029,0.973,2.979c0.301,0.949,0.496,1.625,0.791,3.264c0.295,1.639-0.231,2.092-0.525,2.251c-0.294,0.158-0.984,0.568-1.77,0.604c-0.783,0.034-1.027-0.368-2.371-0.432c-1.345-0.065-2.246,0.345-2.661,0.906c-0.417,0.561-0.913,1.862-0.675,3.4c0.237,1.541,0.504,2.064,1.092,2.748c0.59,0.681,1.165,0.97,1.978,0.696c0.792-0.266,0.764-0.756,1.173-1.164c0.517-0.519,0.855-0.792,2.063-0.821c1.208-0.029,1.979,0.288,2.594,0.784c0.619,0.496,1.934,1.646,2.294,3.356c0.359,1.713,0.424,2.268,0.424,3.676s-0.101,2.978-0.064,4.381c0.036,1.4,0.187,2.841,0.577,3.795c0.386,0.955,0.926,1.755,1.4,2.18c0.475,0.426,0.896,0.438,1.373,0.252c0.475-0.188,1.511-0.771,2.373-1.324c0.861-0.555,0.797-0.99,1.576-1.502c0.875-0.576,1.799-0.605,2.457-0.486c0.661,0.112,1.676,0.631,2.092,1.889C63.059,60.58,63,61.998,63,61.998s0.035,1.186-0.445,1.876c-0.48,0.688-1.272,0.616-1.625,0.545c-0.354-0.071-1.094-0.136-1.094-0.136s-1.451-0.165-2.563,0.094c-1.105,0.258-2.077,1.085-2.73,1.896c-0.652,0.813-0.941,1.57-0.984,2.158c-0.043,0.59,0.027,1.595,0.642,2.572c0.612,0.979,0.712,1.432,1.409,2.827c0.695,1.396,2.15,2.17,2.201,2.206c0.05,0.037,1.388,0.523,1.89,0.949c0.505,0.425,0.555,0.826,0.411,1.208c-0.145,0.381-0.438,1.094-1.604,1.604c-1.166,0.512-2.591,0.523-3.496,0.617c-0.906,0.094-2.651,0.332-3.697,0.834c-1.043,0.503-0.97,1.454-0.97,1.454s-0.028,1.556,1.337,1.983c1.365,0.434,2.64,0,3.201-0.237c0.562-0.238,1.487-0.583,1.487-0.583s1.791-0.631,2.752-0.848c0.965-0.217,1.533-0.323,2.188,0.832c0.652,1.158,1.014,1.886,1.078,2.625c0.064,0.74-0.209,1.148-0.461,1.432c-0.25,0.279-0.217,0.46-0.217,0.46c-0.105,0.873,1.182,0.763,1.182,0.763s0.041-0.004,0.11-0.018c-0.26,0.021-0.499-0.021-0.59-0.178c-0.116-0.202-0.217-0.531,0.146-0.906c0.359-0.374,0.402-0.693,0.402-0.693c0.305-1.439-1.038-3.371-1.354-3.875c-0.315-0.503-0.836-0.791-2.394-0.531c-1.556,0.26-3.386,1.037-3.386,1.037s-1.891,0.995-3.299,0.809c-1.413-0.188-1.701-1.614-1.687-2.063c0.016-0.444,0.304-0.836,1.038-1.122c0.733-0.289,2.464-0.679,4.984-0.809c2.522-0.128,3.458-1.771,3.458-1.771c0.331-0.478,0.923-1.543-0.489-2.338c-1.412-0.789-1.44-0.589-2.116-1.104c-0.68-0.521-1.412-1.153-1.701-2.034c-0.026-0.084-0.07-0.198-0.108-0.291L55.064,72.686L55.064,72.686z"></path>
+  <linearGradient id="SVGID_11_" gradientUnits="userSpaceOnUse" x1="390.042" y1="485.6797" x2="390.042" y2="485.6797" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".7473" stop-color="#F9F9F9"></stop>
+    <stop offset="1" stop-color="#D5D7D8"></stop>
+  </linearGradient>
+  <path style="&st35;" d="M-16.122-14.641"></path>
+  <linearGradient id="SVGID_12_" gradientUnits="userSpaceOnUse" x1="390.042" y1="485.6797" x2="390.042" y2="485.6797" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".7473" stop-color="#F9F9F9"></stop>
+    <stop offset="1" stop-color="#D5D7D8"></stop>
+  </linearGradient>
+  <path style="&st36;" d="M-16.122-14.641"></path>
+  <path style="&st34;" d="M92.002,13.363c1.021,1.649,2.844,4.295,2.844,4.295s0.256,0.332,0.5,0.605l0.01-0.011c0.011-0.375-0.051-0.571-0.06-0.621l-0.091-0.274c-0.021-0.367,0.438,0.095,0.611,0.288c-0.498-0.754-0.659-0.811-0.659-0.811s-0.423-0.234-0.111,0.59c0.312,0.824-0.075,0.233-0.075,0.233c-0.924-1.177-2.838-4.293-2.838-4.293l-0.553-0.383L92.002,13.363L92.002,13.363z"></path>
+  <path style="&st34;" d="M101.539,51.352c0.014-0.063,0.023-0.125,0.034-0.188c-0.004,0-0.009,0-0.009,0s0.005-0.03,0.013-0.089C101.563,51.17,101.551,51.262,101.539,51.352L101.539,51.352z"></path>
+  <path style="&st34;" d="M100.822,35.679c0.088-0.033,0.188,0.1,0.188,0.1s0.135,0.189,0.289,0.033c0.107-0.104,0.155-0.353,0.025-1.257c-0.004,0.229-0.053,0.409-0.137,0.59c-0.139,0.296-0.314,0.02-0.441-0.081c-0.129-0.098-0.168,0.07-0.168,0.07l-0.004,0.162c0,0.7,0.087,2.346,0.217,3.617c0.063,0.605,0.173,1.071,0.287,1.408l0.041,0.076c0.089,0.148,0.188,0.343,0.307,0.255c0.116-0.089,0.274-0.582,0.274-0.582l0.128-0.591c0.191,0.113,0.291,0.529,0.341,0.962c-0.002-0.037-0.004-0.056-0.004-0.056s-0.144-1.324-0.3-1.313c-0.155,0.01-0.21,0.701-0.21,0.701s-0.066,0.468-0.157,0.456c-0.088-0.011-0.365,0.022-0.564-1.412c-0.201-1.436-0.158-2.949-0.158-2.949S100.732,35.713,100.822,35.679L100.822,35.679z"></path>
+  <path style="&st34;" d="M99.696,28.566l0.298,0.341c0.479,1.494,0.996,3.843,1.092,4.264c-0.027-0.139-0.056-0.286-0.088-0.441c-0.288-1.454-0.711-2.896-1.006-3.832L99.696,28.566l-0.05-0.702c-0.004-0.01-0.006-0.016-0.006-0.016s-0.312-0.913-1.033-2.737c-0.725-1.824-0.848-2.458-0.848-2.458s-0.043-0.2,0.066-0.234c0.109-0.032,0.09-0.178,0.09-0.178s-0.013-0.245-0.277-0.99c-0.182-0.503-0.312-0.911-0.662-1.607c0.281,0.585,0.463,1.052,0.524,1.259l0.028,0.068c0,0,0.099,0.148,0.066,0.552c-0.027,0.403-0.146,0.452-0.146,0.452l0.022,0.14c0.141,0.538,0.418,1.187,0.418,1.187s1.065,2.709,1.748,4.54L99.696,28.566L99.696,28.566z"></path>
+  <linearGradient id="SVGID_13_" gradientUnits="userSpaceOnUse" x1="274.5342" y1="-396.1577" x2="255.2091" y2="-490.1944" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st37;" d="M42.958,91.536c0.938,0.177,1.723,0.358,1.889,0.396C44.514,91.822,43.895,91.653,42.958,91.536z"></path>
+  <linearGradient id="SVGID_14_" gradientUnits="userSpaceOnUse" x1="422.5586" y1="518.7568" x2="427.2878" y2="578.1768" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset=".2198" stop-color="#989A9C"></stop>
+    <stop offset=".2527" stop-color="#FFF"></stop>
+  </linearGradient>
+  <path style="&st38;" d="M20.381,74.92l0.007-0.164l-0.052-0.08c0,0-1.874-3.375-1.676-4.615c0,0,0.069-0.827,1.241-1.187c0.817-0.25,0.71,0.538,3.112,0.976c0,0,2.07,0.557,2.611-0.946c0.539-1.507-0.566-3.394-0.566-3.394s-1.45-2.656-3.244-2.756c-0.95-0.055-0.692,0.583-2.125,0.924c0,0-1.913,0.185-2.519-0.963c-0.733-1.389-1.015-2.968-1.015-4.021c0-1.058,0.045-1.001,0.126-1.405c0.045-0.219,0.186-0.548,0.248-1.09l-0.134-0.675l0.204-0.499c0.002-0.26-0.004-0.535-0.021-0.83c-0.092-1.661-0.211-4.221-0.436-5.711c-0.223-1.491-0.633-3.799-1.991-3.865c0,0-0.671-0.052-1.636,0.885c-0.882,0.856-1.682-0.708-1.726-1.663c-0.052-1.121,0.131-2.255,0.409-2.795c0.277-0.541,1.042-0.566,1.186-0.554c0.146,0.012,0.555,0.17,1.042,0.474c0.489,0.304,1.161,0.304,1.558-0.092c0.395-0.395,0.948-0.856,1.173-2.598c0.225-1.741,0.225-3.547,0.013-5.71l-0.224-0.485l1.339-0.288c-0.001-0.016-0.021-0.125-0.032-0.195c0,0,0-0.001-0.001-0.002c-0.001-0.017-0.004-0.033-0.007-0.052c0-0.002,0-0.004,0-0.005c-0.003-0.019-0.007-0.038-0.009-0.057c-0.001-0.001-0.001-0.001-0.001-0.003c-0.003-0.022-0.006-0.042-0.009-0.062c0-0.001,0-0.001,0-0.001c-0.004-0.022-0.006-0.045-0.011-0.067c0-0.003-0.001-0.006-0.001-0.007c-0.002-0.022-0.006-0.045-0.008-0.068c-0.001-0.005-0.001-0.01-0.003-0.015c-0.002-0.023-0.005-0.047-0.009-0.069c-0.001-0.004-0.002-0.01-0.002-0.014c-0.003-0.026-0.007-0.05-0.012-0.076c0-0.002,0-0.005,0-0.005c-0.004-0.025-0.008-0.05-0.012-0.076c0-0.007-0.001-0.012-0.002-0.018c-0.002-0.024-0.006-0.052-0.009-0.079c-0.001-0.005-0.001-0.011-0.003-0.021c-0.004-0.027-0.006-0.053-0.01-0.081c-0.001-0.007-0.002-0.013-0.003-0.02c-0.003-0.028-0.005-0.057-0.009-0.087c-0.001-0.003-0.001-0.008-0.003-0.013c-0.003-0.028-0.005-0.056-0.009-0.082c-0.001-0.006-0.001-0.011-0.002-0.017c-0.003-0.031-0.006-0.06-0.01-0.091c-0.001-0.007-0.001-0.014-0.002-0.02c-0.004-0.03-0.006-0.062-0.011-0.09c0-0.007-0.001-0.015-0.002-0.022c-0.003-0.031-0.006-0.063-0.01-0.094c-0.001-0.006-0.002-0.011-0.002-0.018c-0.003-0.032-0.005-0.063-0.009-0.094c0-0.003,0-0.005-0.001-0.009c-0.003-0.033-0.004-0.068-0.009-0.1c0-0.005,0-0.013-0.001-0.02c-0.002-0.035-0.005-0.065-0.007-0.099c-0.001-0.008-0.001-0.015-0.002-0.021c-0.003-0.032-0.005-0.066-0.007-0.099c0-0.005-0.001-0.011-0.001-0.02c-0.002-0.033-0.006-0.068-0.007-0.101c0-0.003-0.001-0.008-0.001-0.009c-0.002-0.033-0.004-0.066-0.007-0.1c0-0.006,0-0.012,0-0.017c-0.001-0.032-0.003-0.068-0.005-0.1c0-0.008,0-0.014-0.001-0.022c-0.001-0.033-0.004-0.067-0.005-0.098c0-0.006,0-0.013,0-0.021c-0.002-0.033-0.003-0.066-0.004-0.101c0-0.005,0-0.008,0-0.013c-0.001-0.032-0.002-0.066-0.002-0.099c0-0.003,0-0.005,0-0.009c0-0.034-0.001-0.067-0.001-0.102c0-0.005,0-0.012,0-0.016c0-0.033,0-0.067,0-0.098c0-0.005,0-0.012,0-0.019c0.001-0.032,0.001-0.065,0.001-0.096c0-0.004,0.001-0.009,0.001-0.014c0-0.033,0.001-0.066,0.003-0.1c0.052-1.243,0.291-1.751,0.291-1.751l0.058-0.166c0,0,0.46-1.458,1.152-2.458l-0.011,0.024c0,0,0.446-0.738,1.231-0.816c0.785-0.077,1.003,0.158,1.371,0.202c0.04,0.004,0.078,0.008,0.115,0.013c0.013,0.001,0.025,0.002,0.037,0.004c0.025,0.002,0.051,0.003,0.074,0.006c0.014,0.001,0.028,0.002,0.042,0.003c0.021,0.001,0.043,0.002,0.064,0.004c0.014,0.001,0.028,0.001,0.041,0.002c0.02,0.001,0.04,0.002,0.058,0.002c0.013,0.002,0.026,0.002,0.04,0.002c0.021,0.001,0.037,0.002,0.055,0.002c0.014,0,0.026,0.001,0.039,0.001c0.016,0,0.034,0.001,0.051,0.001c0.011,0,0.023,0,0.034,0c0.017,0,0.032,0,0.05-0.001c0.01,0,0.02,0,0.032-0.001c0.016,0,0.031,0,0.046-0.001c0.011,0,0.02-0.001,0.03-0.001c0.015,0,0.03-0.002,0.045-0.002c0.009,0,0.017-0.001,0.026-0.001c0.015-0.001,0.03-0.003,0.044-0.004c0.006-0.001,0.016-0.002,0.022-0.002c0.016-0.001,0.03-0.002,0.044-0.004c0.007-0.001,0.014-0.001,0.019-0.002c0.016-0.002,0.03-0.004,0.045-0.007c0.006,0,0.009,0,0.014-0.001c0.019-0.002,0.036-0.006,0.052-0.008l0,0c0.035-0.008,0.068-0.014,0.098-0.021c0,0,0-0.002,0.002-0.002c0.012-0.002,0.025-0.005,0.039-0.01c0.002,0,0.004-0.001,0.007-0.001c0.01-0.003,0.02-0.006,0.029-0.009c0.003-0.001,0.007-0.002,0.01-0.004c0.009-0.002,0.018-0.006,0.026-0.008c0.002-0.002,0.005-0.003,0.008-0.003c0.008-0.003,0.015-0.006,0.021-0.009c0.003-0.001,0.006-0.003,0.009-0.004c0.006-0.003,0.014-0.004,0.02-0.009c0.003-0.001,0.006-0.002,0.008-0.003c0.005-0.002,0.012-0.005,0.019-0.007c0.001-0.002,0.003-0.003,0.005-0.004c0.005-0.004,0.01-0.005,0.015-0.008c0.003-0.001,0.004-0.003,0.008-0.003c0.004-0.002,0.008-0.005,0.012-0.008c0,0,0.004,0,0.005-0.002c0.004-0.003,0.008-0.006,0.012-0.008c0.001-0.002,0.002-0.002,0.004-0.003c0.003-0.003,0.007-0.004,0.01-0.008c0.002,0,0.002,0,0.002-0.002c0.003-0.001,0.007-0.005,0.011-0.008c0,0,0.001,0,0.001-0.001c0.004-0.002,0.005-0.004,0.009-0.007h0.001c0.002-0.002,0.004-0.004,0.006-0.007c0.001-0.001,0.002-0.001,0.003-0.002s0.002-0.002,0.002-0.002c0.001-0.003,0.001-0.003,0.003-0.005c0.002-0.002,0.004-0.004,0.004-0.004c0.328-0.241,0.591-0.516,0.797-0.775c0.014-0.017,0.026-0.034,0.04-0.05c0.002-0.004,0.005-0.009,0.008-0.012c0.013-0.016,0.026-0.032,0.038-0.05c0.002-0.003,0.006-0.006,0.007-0.01c0.012-0.018,0.025-0.032,0.038-0.05c0,0,0,0,0.001,0c0.039-0.055,0.075-0.109,0.109-0.159c0-0.003,0.002-0.006,0.003-0.008c0.01-0.015,0.021-0.028,0.03-0.044c0.001-0.003,0.004-0.007,0.007-0.01c0.008-0.016,0.017-0.029,0.024-0.042c0.002-0.004,0.005-0.009,0.009-0.013c0.008-0.014,0.017-0.028,0.023-0.042c0.001-0.001,0.001-0.002,0.002-0.002c0.017-0.028,0.032-0.055,0.046-0.079c0.002-0.003,0.004-0.008,0.006-0.013c0.006-0.01,0.01-0.021,0.017-0.029c0.003-0.007,0.007-0.014,0.012-0.02c0.004-0.008,0.009-0.017,0.014-0.024c0.002-0.008,0.006-0.017,0.01-0.023c0.004-0.007,0.009-0.016,0.012-0.023c0.004-0.006,0.006-0.014,0.011-0.021c0.002-0.006,0.007-0.013,0.01-0.021c0.066-0.128,0.097-0.205,0.097-0.205c0.593-1.459,0.052-2.936-0.21-3.225c-0.32-0.353-1.041-0.882-1.041-0.882s-0.288-0.241-0.751-0.144c0.349-0.049,0.791,0.091,0.966,0.558c0.277,0.734,0.376,1.335,0.212,2.33c0,0-0.26,1.387-1.384,2.233c-1.125,0.848-1.923,0.096-2.885,0.13c-0.962,0.032-1.516,0.701-1.809,1.157c-0.293,0.457-1.417,2.2-1.319,5.067c0.097,2.868,0.291,4.301,0.325,5.558c0.033,1.205,0.178,3.976-0.635,5.278c-0.815,1.303-1.628,0.65-2.2,0.309c-0.571-0.341-1.223-0.245-1.744,0.131c-0.521,0.375-0.833,1.124-0.848,3.324c-0.016,2.364,1.532,2.606,1.532,2.606s0.293,0.146,0.945-0.537c0,0,0.651-0.685,1.253-0.603c0.604,0.082,0.995,0.716,1.255,1.808c0.261,1.092,0.796,5.621,0.717,8.668c-0.034,1.271-0.62,1.286-0.36,3.617c0,0,0.409,3.13,1.401,4.089c0.995,0.962,2.378,0.781,2.706,0.75c0.324-0.032,0.7-0.26,0.7-0.26s0.309-0.197,0.537-0.374c0.23-0.182,0.522-0.428,1.011-0.277c0.489,0.146,1.645,0.896,2.557,2.571c0.915,1.678,0.496,3.317-0.26,3.521c-0.668,0.182-0.848,0.229-1.971-0.05c-1.124-0.274-1.451-0.567-1.957-0.766c-0.504-0.196-1.043-0.263-1.547,0.114c-0.505,0.373-1.345,1.057-0.343,3.32c0.961,2.174,1.692,3.797,3.518,5.623c-0.522-0.607-0.956-1.188-1.427-1.871L20.381,74.92L20.381,74.92z"></path>
+  <linearGradient id="SVGID_15_" gradientUnits="userSpaceOnUse" x1="237.3721" y1="-388.3604" x2="218.8474" y2="-478.5023" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st39;" d="M22.494,7.387l-0.05,0.025C22.45,7.41,22.469,7.401,22.494,7.387z"></path>
+  <linearGradient id="SVGID_16_" gradientUnits="userSpaceOnUse" x1="259.9063" y1="-479.3379" x2="259.8987" y2="-479.3752" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st40;" d="M46.028,92.85c0.13,0.014,0.26,0.026,0.391,0.041c-0.114-0.016-0.31-0.039-0.561-0.074C45.916,92.828,45.972,92.838,46.028,92.85L46.028,92.85z"></path>
+  <path style="&st29;" d="M24.089,5.489c-0.649,0.36-0.7-0.016-1.141,0.017c-0.439,0.033-1.026,0.212-1.596,1.043c-0.571,0.831-0.586,1.89,0.326,1.417c0,0,0.436-0.428,0.815-0.579c0.081-0.043,0.24-0.126,0.406-0.174l0.144-0.117c0,0,0.424-0.491,1.073-0.333c0.648,0.156,1.119-0.129,1.119-0.129c1.01-0.761,1.655-2.184,1.655-2.184l0.987-1.998l-0.826-1.336c0,0-0.698,1.489-1.383,2.53C24.986,4.692,25.033,4.972,24.089,5.489L24.089,5.489z"></path>
+  <path style="&st29;" d="M19.871,16.292c0,0-0.424,1.89,1.156,1.597c0,0,0.006-0.002,0.022-0.007c-0.062,0.003-0.089-0.006-0.089-0.006c-0.025-0.497,0.149-1.432,0.49-2.261c0.341-0.83,1.075-2.254,1.075-2.254s1.047-1.791,1.467-2.89c0.42-1.102,0.416-1.939,0.416-1.939s-0.8-1.019-0.915-1.176c-0.115-0.157-0.272-0.223-0.272-0.223c-0.054,0.019-0.103,0.036-0.146,0.051c0.115-0.007,0.221,0.021,0.283,0.114c0.213,0.31-0.39,2.036-0.39,2.036s-0.522,1.238-1.548,3.03C20.393,14.157,19.871,16.292,19.871,16.292L19.871,16.292z"></path>
+  <linearGradient id="SVGID_17_" gradientUnits="userSpaceOnUse" x1="268.9033" y1="-394.6382" x2="249.4966" y2="-489.0725" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st41;" d="M38.185,89.697l0.166,0.02c-0.134-0.119-0.305-0.236-0.497-0.347c0.184,0.113,0.489,0.358,0.312,0.665c-0.245,0.424-0.048,0.457,0.489,0.863c0,0,0.022,0.013,0.061,0.033c-0.156-0.107-0.253-0.234-0.156-0.362c0.125-0.166,0.136-0.334,0.065-0.499L38.185,89.697L38.185,89.697z"></path>
+  <linearGradient id="SVGID_18_" gradientUnits="userSpaceOnUse" x1="254.4561" y1="-391.5991" x2="235.0337" y2="-486.1104" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st42;" d="M24.437,82.749c-1.245-1.185,1.595-0.118,1.595-0.118s0.619,0.262,0.343-0.385c-0.246-0.57-1.373-1.963-2.543-3.219l0.144,0.156c0,0,2.346,2.721,2.183,2.966c-0.164,0.245-1.108-0.325-1.108-0.325s-1.401-0.539-1.206,0.13c0.143,0.491,1.059,1.271,1.536,1.649C25.109,83.372,24.798,83.09,24.437,82.749z"></path>
+  <linearGradient id="SVGID_19_" gradientUnits="userSpaceOnUse" x1="262.6514" y1="-392.9692" x2="243.1559" y2="-487.8355" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st43;" d="M26.977,84.73c0.424,0.256,0.903,0.513,1.365,0.676c0,0,0.831,0.293,1.337-0.114c0.503-0.403,0.503-0.601,1.238-0.243c0,0,1.388,0.696,2.382,1.444c0.98,0.735,3.557,2.336,4.396,2.791c-0.764-0.417-3.712-2.365-4.633-2.99c-0.936-0.633-2.574-1.698-3.297-1.476c-0.554,0.172-0.474,0.396-0.804,0.555c-0.331,0.158-0.688,0.055-1.504-0.383C27.291,84.9,27.134,84.818,26.977,84.73L26.977,84.73z"></path>
+  <linearGradient id="SVGID_20_" gradientUnits="userSpaceOnUse" x1="271.5479" y1="-390.9575" x2="251.1904" y2="-490.0176" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st44;" d="M39.903,91.448c0.37,0.104,0.672,0.06,0.787,0.035c-0.678,0.04-1.35-0.269-1.35-0.269s-0.067-0.021-0.165-0.061C39.413,91.268,39.689,91.385,39.903,91.448z"></path>
+  <linearGradient id="SVGID_21_" gradientUnits="userSpaceOnUse" x1="274.6582" y1="-395.8442" x2="255.2559" y2="-490.2569" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#E4E5E6"></stop>
+  </linearGradient>
+  <path style="&st45;" d="M44.293,92.169c0,0-1.344-0.163-1.939-0.163c0,0-0.037,0.064,0.226,0.158c-0.021-0.021-0.031-0.049,0.189,0c0.304,0.064,2.018,0.225,2.465,0.158c0.448-0.068-0.091-0.278-0.091-0.278s-0.088-0.041-0.267-0.102C44.553,92.201,44.293,92.169,44.293,92.169L44.293,92.169z"></path>
+  <path style="&st34;" d="M4.662,62.045c0.085,0.143,0.198,0.432,0.462,0.725c0.146,0.16,0.266,0.072,0.162-0.525c-0.253-0.182-0.407-0.318-0.464-0.371c-0.113-0.013-0.263-0.297-0.263-0.297s-0.471-0.835-0.274-0.227c0.398,1.23,1.6,3.821,1.6,3.821l1.396,2.47c-0.999-1.803-2.299-4.633-2.44-5.013C4.691,62.223,4.577,61.904,4.662,62.045L4.662,62.045z"></path>
+  <path style="&st34;" d="M1.547,32.696l0.032-0.643c0.027-0.122,0.057-0.241,0.087-0.36c0.712-2.789,1.069-3.858,1.425-4.649c0.356-0.792,0.398-0.245,0.398-0.245s0,0.209,0.136,0.234c0.135,0.023,0.324-0.32,0.494-0.826c0.17-0.504,0.604-1.745,0.693-2.212c0.135-0.704-0.051-0.358-0.115-0.354c-0.108,0.005-0.126-0.091-0.113-0.306c0,0,0.382-2.122,3.064-6.563c0.18-0.17,0.321-0.307,0.47-0.449c-0.055-0.052-0.033-0.265,0.001-0.32c0.136-0.214,0.275-0.435,0.422-0.661l-0.09,0.504c0,0-1.85,2.492-3.104,5.694c0,0-0.342,0.835-0.359,1.094c-0.025,0.154-0.104,0.739,0.152,0.582l0.065-0.048c-0.093,0.484-0.295,1.37-0.607,2.325c-0.288,0.878-0.573,1.633-0.841,2.078l-0.002-0.004c-0.08,0.067-0.098-0.016-0.146-0.21c-0.048-0.198-0.113-0.198-0.113-0.198c-0.179,0-0.324,0.202-0.41,0.359c-0.04,0.056-0.089,0.19-0.089,0.19s-0.461,0.934-1.281,4.26c-0.822,3.328-0.836,6.14-0.836,6.14s-0.084,1.309-0.21,1.379c-0.126,0.07-0.097-0.056-0.097-0.056s-0.045-0.052-0.114,0.045c0,0,0.02-0.76,0.044-0.875c0,0,0.041-0.206,0.119-0.206s0.006,0.125,0.048,0.16c0.039,0.032,0.084,0.03,0.075-0.178c-0.005-0.176,0.147-2.816,0.621-5.312L1.547,32.696L1.547,32.696z"></path>
+  <path style="&st34;" d="M0.392,41.994c0-0.002,0-0.004,0-0.005c0,0,0.018,0.217,0.042,0.356l-0.003,0.01c0.078,0.357,0.187,0.357,0.187,0.357c0.008-0.096,0.087-0.273,0.183-0.458c0.007,0.106,0.007,0.231-0.004,0.375c-0.037,0.537-0.019,2.466,0.209,4.28c0.225,1.815,0.367,3.349,0.394,3.594c0.029,0.245,0.498,3.283,1.025,5.089c0.285,0.967,0.422,1.344,0.483,1.424l0.008,0.049c0,0,0.097,0.184,0.348,0.32c0,0,0.111-0.097,0.112-0.412c0.018,0.031,0.037,0.065,0.057,0.105c-0.083,0.262-0.105,0.426-0.105,0.426l0,0c-0.042-0.043-0.06-0.031-0.046,0.045c0.067,0.357,0.027,0.68-0.169,0.272c-0.198-0.403-0.8-1.832-1.307-4.251c0,0-0.531-2.659-0.795-5.084l0.042-0.105L0.989,48.29c-0.027-0.248-0.048-0.491-0.067-0.729c0,0-0.255-3.657-0.237-4.562c0.019-0.904-0.085-0.02-0.085-0.02s-0.021,0.219-0.028,0.329c-0.008,0.109-0.103,0.352-0.136-0.246C0.459,43.322,0.392,42.261,0.392,41.994L0.392,41.994z"></path>
+  <path style="&st34;" d="M24.672,1.974l-0.53-0.753l-0.066-0.065c0,0-0.131-0.136-0.896,0.038l-0.11,0.022c0,0,0.38-0.094,0.168,0.191c-0.212,0.285-0.56,0.496-1.172,1.15c-0.612,0.655-0.411,0.803-0.01,0.668c0.401-0.138,1.188-0.499,2.606-1.243L24.672,1.974z"></path>
+  <path style="&st34;" d="M23.106,1.21c-0.022,0.003-0.036,0.006-0.036,0.006L23.106,1.21z"></path>
+  <path style="&st34;" d="M23.637,1.811c0.31-0.272,0.429-0.484,0.454-0.641l-0.015-0.014c0,0-0.131-0.136-0.896,0.038l-0.075,0.015c0.09-0.015,0.303-0.032,0.133,0.198c-0.212,0.285-0.56,0.496-1.172,1.151c-0.612,0.656-0.411,0.803-0.01,0.667c0.106-0.038,0.239-0.088,0.4-0.157C22.599,2.755,23.046,2.325,23.637,1.811z"></path>
+  <path style="&st34;" d="M83.482,4.115l-0.2,0.235c0,0,0.136,0.081,0.208,0.141c0.008,0.005,0.014,0.01,0.021,0.012c-0.043-0.034-0.07-0.056-0.07-0.056c-0.023-0.109,0.004-0.223,0.014-0.297L83.482,4.115c0,0,0.055,0.002,0.143,0.011L83.482,4.115L83.482,4.115z"></path>
+  <path style="&st6;" d="M62.559,90.319c0,0,1.686-0.187,3.239-0.691c1.558-0.504,1.935-0.981,1.874-1.341c-0.037-0.238-0.396-0.305-0.906-0.238c0.271-0.021,0.514,0.032,0.354,0.332c-0.313,0.582-0.861,0.782-0.861,0.782s-0.267,0.19-0.89,0.371c-0.806,0.245-1.794,0.375-2.335,0.438c-0.691,0.082-0.521-0.033-1.465,0.291c-0.023,0.016-0.047,0.025-0.065,0.043c-0.289,0.237,1.071,0.514,1.071,0.514s1.302,0.361,3.257,0.23l0.067-0.021c0.267-0.078,0.533-0.161,0.8-0.245c-2.021,0.457-3.324,0.149-3.737,0.095c-0.434-0.058-0.777-0.144-0.88-0.359C61.98,90.305,62.559,90.319,62.559,90.319L62.559,90.319z"></path>
+  <path style="&st34;" d="M63.301,4.417l0.717,0.076c1.429-0.405,2.651-0.776,2.651-0.776s1.265-0.41,1.901-0.636c0.635-0.227,1.09-0.313,1.656-0.41c0.563-0.096,1.309-0.14,1.709-0.131c0.398,0.007,0.528,0.122,0.528,0.122s0.166,0.131,0.245,0.27c0.077,0.138,0.74,1.891,2.975,2.005c2.231,0.112,2.261-1.096,2.063-1.464c-0.226-0.427-0.896-0.863-0.896-0.863s-0.898-0.575-1.09-0.846c-0.192-0.271,0.033-0.358,0.104-0.376c0.066-0.018,2.433-0.497,2.729-0.608l0.021-0.02l-0.563-0.174c0,0-0.119,0.134-0.789,0.312c-0.67,0.179-1.233,0.246-1.742,0.313c-0.504,0.067-0.504,0.239-0.504,0.239l-0.879,1.406c0,0-0.029,0.104,0.043,0.305c0.073,0.202,0.41,0.448,0.41,0.448s0.573,0.424,0.99,0.699c0.418,0.275,0.395,0.373,0.395,0.373c-1.123,0.095-1.418-0.048-1.875-0.276c-0.445-0.223-0.76-0.729-0.922-1.086c-0.162-0.357-0.521-0.663-0.521-0.663c-0.589-0.336-1.696-0.343-2.813-0.15c-1.115,0.194-2.656,0.707-2.923,0.812c-0.271,0.104-1.616,0.551-2.309,0.729c-0.694,0.179-1.224,0.357-1.646,0.47c-0.426,0.11-3.431,1.005-4.557,1.339c-1.124,0.333-1.848,0.461-1.848,0.461c-1.688,0.171-2.193-0.134-2.193-0.134c-0.297-0.125-0.262-0.527-0.262-0.527l0.009-1.147c0,0-0.029-0.433-0.356-0.611c-0.328-0.179-0.779-0.252-1.593-0.29c-0.811-0.038-1.683,0.044-2.093,0.134c-0.408,0.09-1.19,0.313-1.764,0.952c-0.573,0.641-0.481,1.141-0.481,1.141s0.035,0.683,0.088,1.19c0.052,0.504,0.484,0.662,0.484,0.662s-0.744,0.532-3.045,1.206c-2.209,0.648-3.682,0.774-3.682,0.774l0.678,0.254c0,0,5.468-1.015,7.148-2.616c0,0,0.624-0.293,0.02-0.88c-0.606-0.585-0.897-0.761-0.897-0.761s-0.41-0.224,0.019-0.771c0.43-0.547,0.923-0.795,1.352-0.933c0.429-0.138,1.753-0.25,2.925-0.093c0,0,0.491,0.041,0.459,0.408c-0.034,0.366-0.088,0.872-0.077,1.028c0.008,0.158,0.023,0.515,0.398,0.845c0.378,0.332,1.099,0.453,1.099,0.453s1.257,0.228,2.843-0.217c1.584-0.445,3.642-1.14,5.431-1.629L63.301,4.417L63.301,4.417z"></path>
+  <path style="&st12;" d="M-16.122-14.641"></path>
+  <path style="&st22;" d="M48.462,6.628c0.31,0.207,0.476,0.221,0.5,0.421c0.055,0.339-0.56,0.64-0.56,0.64l-0.006-0.011c0,0-0.431-0.157-0.484-0.664c-0.052-0.505-0.088-1.19-0.088-1.19s0.001,0.2,0.046,0.26C48.004,6.256,48.087,6.378,48.462,6.628z"></path>
+  <path style="&st20;" d="M82.447,79.307l0.451,0.17c1.104-0.617,1.496-0.809,1.759-0.963c1.183-0.703,2.592-1.563,2.963-1.855c0,0,0.761-0.518,0.116,0.195s-0.969,1.007-0.969,1.007s-0.625,0.626-0.471,0.782c0,0,0.166,0.246,1.193-0.687c1.023-0.929,2.15-2.258,2.275-2.44c0.127-0.188,0.146-0.293,0.146-0.293s0.107-0.215,0.273-0.393c0.145-0.15,1.377-1.496,1.994-2.121c0,0,0.002,0.001,0.006,0.003c0.273-0.362,0.541-0.729,0.806-1.102c-0.358,0.379-1.724,1.829-2.483,2.684c0,0-0.713,0.763-0.938,1.056s-0.225,0.47-0.225,0.47s-0.117,0.196-0.392,0.519s-1.24,1.186-1.24,1.186s-0.577,0.47-0.754,0.478c-0.179,0.011,0.431-0.538,0.431-0.538s0.588-0.625,0.967-1.123c0.382-0.498,0.137-0.47,0.137-0.47s-0.186-0.049-0.986,0.459c-0.8,0.508-1.367,0.858-1.367,0.858s-1.722,0.986-2.814,1.623c-1.096,0.636-3.6,1.908-5.021,2.492c-1.43,0.588-2.162,0.715-2.035,0.527c0.127-0.186,0.461-0.459,0.461-0.459s0.399-0.4,0.399-0.803c0,0,0.128-0.586-1.604-0.223c-1.729,0.36-3.293,1.213-3.293,1.213s-2.571,1.182-1.965,1.887c0,0,0.117,0.186,0.635,0.352c0.52,0.166-0.92,0.606-0.92,0.606c-1.365,0.448-2.413,0.651-3.74,0.926c-1.963,0.403-3.564,0.761-4.165,0.894c-0.165,0.035-0.253,0.059-0.253,0.059s-1.212,0.292-3.229,1.072c-2.015,0.783-5.972,1.43-5.972,1.43s-2.542,0.293-2.777,0.627c-0.234,0.331,0.177,0.499,0.177,0.499s0.362,0.224,1.671,0.283c0,0,0.451,0,0.471,0.036c0.018,0.039,0.046,0.068-0.235,0.156c-0.286,0.088-0.854,0.314-2.778,0.558c-1.936,0.245-1.896-0.067-1.896-0.067s-0.01-0.076,0.078-0.216c0.087-0.134,0.009-0.369-0.293-0.535c0,0-0.419-0.272-1.829-0.262c-1.408,0.009-4.212,0.017-6.833-0.14c-2.374-0.143-5.59-0.551-6.099-0.664c0,0-0.117-0.029-0.206-0.117c-0.088-0.09-0.646-0.422-1.164-0.733c-0.517-0.313-2.073-0.907-2.073-0.907s-2.011-0.783-1.945-0.521c0.015,0.063,0.13,0.153,0.268,0.246c0.351,0.188,0.704,0.375,1.06,0.56l0.002-0.002c0,0-0.743-0.402-0.538-0.402s0.438,0.109,0.438,0.109s1.213,0.332,1.966,0.686c0.753,0.353,1.407,0.83,1.407,0.83s0.929,0.549,2.319,0.732c1.346,0.182,3.174,0.389,3.777,0.448l0.594-0.272l0.433,0.354c1.106,0.068,2.575,0.146,2.575,0.146s2.976,0.111,4.605-0.019c0.733-0.063,0.507,0.317,0.507,0.317s-0.214,0.354,0.206,0.529c0,0,0.771,0.439,3.343,0.157c2.573-0.286,3.138-0.862,3.138-0.862s0.299-0.275-0.351-0.398c-0.513-0.1-0.513-0.051-1.175-0.117c-0.665-0.067-0.998-0.205-0.557-0.323c0.441-0.114,1.174-0.175,1.174-0.175s2.249-0.313,4.066-0.783c0,0,1.938-0.458,3.861-1.134c0.756-0.265,1.395-0.459,1.887-0.599l0.438-0.644l0.644,0.372c0.065-0.014,0.103-0.021,0.103-0.021s2.306-0.539,3.274-0.703c0.966-0.168,3.154-0.637,4.087-1.086c0.928-0.448,1.396-0.805,1.505-1.075c0.107-0.272-0.393-0.431-0.393-0.431s-0.588-0.138-0.508-0.34c0.075-0.205,0.293-0.382,1.213-0.793c0.918-0.41,2.07-0.859,3.227-1.144c1.154-0.282,0.732,0.194,0.732,0.194s-0.692,0.705-0.783,0.979c-0.086,0.273,0.029,0.285,0.119,0.333c0.088,0.05,0.646,0.028,1.022-0.067c0.383-0.099,3.464-1.271,5.341-2.347c0.049-0.026,0.094-0.054,0.139-0.08L82.447,79.307z"></path>
+  <linearGradient id="SVGID_22_" gradientUnits="userSpaceOnUse" x1="221.1826" y1="-454.5649" x2="221.373" y2="-454.5649" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#808080"></stop>
+    <stop offset=".0868" stop-color="#7A7A7A"></stop>
+    <stop offset=".36" stop-color="#6B6B6B"></stop>
+    <stop offset=".5192" stop-color="#686868"></stop>
+    <stop offset=".6377" stop-color="#5F5F5F"></stop>
+    <stop offset=".7431" stop-color="#4E4E4E"></stop>
+    <stop offset=".8408" stop-color="#383838"></stop>
+    <stop offset=".9324" stop-color="#1B1B1B"></stop>
+    <stop offset="1" stop-color="#000"></stop>
+  </linearGradient>
+  <path style="&st13;" d="M7.432,68.01l0.178,0.131c-0.105-0.099-0.167-0.155-0.167-0.155s-0.008,0.003-0.024,0.003C7.423,67.993,7.429,68.002,7.432,68.01L7.432,68.01z"></path>
+  <linearGradient id="SVGID_23_" gradientUnits="userSpaceOnUse" x1="221.4043" y1="-449.8027" x2="316.0254" y2="-449.8027" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#808080"></stop>
+    <stop offset=".0868" stop-color="#7A7A7A"></stop>
+    <stop offset=".3817" stop-color="#828282"></stop>
+    <stop offset=".5185" stop-color="#808080"></stop>
+    <stop offset=".5677" stop-color="#797979"></stop>
+    <stop offset=".6027" stop-color="#6E6E6E"></stop>
+    <stop offset=".608" stop-color="#6B6B6B"></stop>
+    <stop offset="1" stop-color="#4E4E4E"></stop>
+  </linearGradient>
+  <path style="&st7;" d="M90.412,59.954l-0.371-0.021c-1.807,1.463-2.342,1.938-3.781,2.955c0,0-1.644,1.228-3.44,2.196c-1.804,0.97-3.919,0.853-3.919,0.853s-0.573-0.062-0.295-0.477c0.275-0.415,0.634-1.425,0.771-1.938c0.141-0.516,0.415-1.82-0.633-2.235s-3.018,0.196-3.018,0.196s-4.525,1.205-6.371,3.23c-1.793,1.969-0.286,2.846-0.286,2.846s0.702,0.49,1.386,0.73c0.645,0.229,0.516,0.436,0.354,0.555c0,0-2.021,1.287-4.408,1.974c0,0-3.268,1.001-6.441,1.206c-3.154,0.203-4.495,0.283-4.495,0.283l-0.399,0.379l-0.758-0.252c-1.283,0.154-3.455,0.463-5.596,0.979c-2.622,0.64-5.123,1.147-6.903,2.296c0,0-1.333,0.887-0.566,1.746c0.765,0.862,1.565,1.23,1.565,1.23s0.472,0.207,0.464,0.336c-0.009,0.164-0.015,0.309-0.367,0.512c0,0-1.394,0.814-4.108,0.859c-2.714,0.045-3.911,0.021-4.707-0.695c-0.658-0.591,0.165-1.844,0.165-1.844s0.33-0.612,0-1.453c-0.33-0.84-2.218-0.854-2.218-0.854l-2.615-0.134c-3.095-0.081-7.182-0.427-9.001-0.653c0,0-0.012-0.002-0.033-0.006l-0.006,0.166l-0.721-0.303c-1.139-0.221-3.243-0.668-4.075-1.084c-0.759-0.38-1.167-0.313-1.066-1.102c0.101-0.769-0.753-1.836-0.753-1.836s-1.188-1.287-2.257-2.086c-1.069-0.804-1.523-0.564-1.523-0.564s-0.484,0.258-0.049,1.296c0.436,1.04,0.86,1.403,0.86,1.403s0.208,0.22,0.089,0.279c-0.118,0.06-0.484-0.219-0.789-0.478c-0.253-0.21-1.885-1.742-2.456-2.276l0.105,0.356c0.019,0.028,0.035,0.062,0.052,0.086c0.184,0.291,0.855,1.269,2.155,2.28c1.549,1.213,1.559,0.729,1.559,0.729s0.061-0.399-0.297-0.84c-0.359-0.44-0.934-1.373-0.791-1.715c0.144-0.339,0.309-0.31,0.309-0.31s0.133-0.051,0.596,0.299c0.462,0.351,1.498,1.076,2.011,1.703c0.513,0.623,0.48,1.181,0.48,1.181s-0.102,0.563,0.453,1.17c0.553,0.604,1.733,1.714,5.859,2.351c0.025,0.004,0.034,0.006,0.059,0.01l0.193-0.551l0.573,0.663c3.598,0.521,5.555,0.563,5.555,0.563s4.709,0.162,5.982,0.162c1.272,0,1.035,0.666,1.035,0.666s-0.072,0.359-0.225,0.646c-0.155,0.287-0.524,1.365-0.144,1.939c0,0,0.585,1.427,4.381,1.527c0,0,3.324,0.268,5.643-0.688c2.319-0.954,0.226-2.275,0.226-2.275s-0.794-0.481-1.13-0.739c-0.308-0.234-0.184-0.481-0.121-0.646c0.06-0.162,0.297-0.359,0.563-0.492c0.266-0.134,1.239-0.654,5.365-1.722c4.124-1.069,6.587-1.183,6.587-1.183s0.02-0.002,0.055-0.004l0.338-0.656l0.854,0.556c0.732-0.06,1.681-0.129,2.526-0.171c1.691-0.082,4.341-0.471,5.879-0.807c1.54-0.343,3.869-1.062,5.592-1.951c1.725-0.895,1.809-1.519,1.809-1.519s0.328-0.475-0.392-0.995c-0.719-0.523-1.036-0.382-1.673-1.027c-0.637-0.646,0.557-1.62,0.557-1.62s0.612-0.861,4.021-2.175c3.403-1.313,3.979-0.873,4.153-0.729s0.195,0.615,0.123,0.935c-0.069,0.317-0.494,1.455-0.721,2.053c-0.227,0.594-0.316,1.406,0.605,1.601c0.923,0.194,2.215-0.008,3.428-0.442c2.893-1.033,3.756-2.295,8.534-5.764c0.012-0.008,0.021-0.017,0.03-0.021L90.412,59.954l0.689,0.108c1.978-1.573,3.869-3.571,3.869-3.571s1.258-1.261,1.889-2.356c0.595-1.026,0.027,0.89,0.027,0.89s-0.32,1.516,0.19,2.077c0.405,0.445,1.563-0.795,1.563-0.795s0.688-0.789,0.965-2.061c0.408-1.875,0.185-2.248,0.185-2.248s-0.246-0.389-0.093-0.852c0.154-0.459,1.158-3.047,1.98-4.01l0.502-0.563c0-0.008,0.002-0.02,0.002-0.027l-0.224-0.557l0.304-0.512c0,0-0.279,0.322-1.404,2.177c-1.266,2.087-1.467,3.771-1.467,3.771s-0.119,0.653-0.054,1.034c0.063,0.355,0.188,0.519,0.192,0.622c0.009,0.104-0.073,0.959-0.508,1.773c-0.438,0.814-0.815,1.031-0.815,1.031s-0.756,0.545-0.86,0.157c-0.104-0.39-0.074-0.72-0.035-0.966c0.035-0.248,0.289-1.579,0.252-2.072c-0.035-0.494-0.479-0.098-0.479-0.098s-0.104,0.119-0.298,0.366s-1.288,1.637-1.705,2.125c-0.988,1.157-1.886,1.989-4.292,3.93c-0.007,0.003-0.015,0.011-0.019,0.015L90.412,59.954L90.412,59.954z"></path>
+  <linearGradient id="SVGID_24_" gradientUnits="userSpaceOnUse" x1="214.5928" y1="-431.356" x2="314.4043" y2="-431.356" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#555555"></stop>
+    <stop offset="1" stop-color="#231F20"></stop>
+  </linearGradient>
+  <path style="&st11;" d="M1.193,48.543l0.104,0.975c0.235,0.285,0.577,0.677,1.071,1.187c1.084,1.121,1.427,0.416,1.427,0.416s0.253-0.705-0.054-1.985C3.434,47.85,3.488,47.09,4.066,47.11c0.578,0.019,2.258,0.361,2.782,1.752c0,0,0.217,0.649,0.018,1.479c-0.2,0.834-0.162,1.57-0.018,2.295c0.145,0.725,0.759,1.407,1.464,1.971c0.706,0.562,2.746,1.535,4.734,1.66c1.987,0.127,2.601,0.021,2.601,0.021l0.746-0.767l0.134,0.668c0.812-0.09,2.116-0.229,3.62-0.393c2.528-0.271,4.227-0.544,5.798-0.308c1.573,0.235,2.079,1.462,2.079,1.462s0.108,0.314-0.253,0.544c-0.349,0.224-0.749,0.547-0.948,0.89c-0.199,0.346-0.411,1.068,0.16,2.035c0.572,0.963,2.142,1.592,3.478,1.432c1.335-0.155,3.335-0.67,4.52-0.979c1.287-0.337,2.424-0.971,2.572-1.98c0.147-1.008-1.534-2.295-1.534-2.295s-0.812-0.609-0.91-0.75c-0.1-0.139,0.099-0.197,0.099-0.197s0.949-0.229,1.357-0.414c0.404-0.189,1.522-0.646,3.353-1.219s5.608-1.248,5.608-1.248s2.084-0.332,4.685-0.543l0.622-0.702l0.485,0.625c0.409-0.024,0.825-0.046,1.243-0.063c3.572-0.129,5.344-0.554,7.242-0.979c1.897-0.427,4.568-1.978,4.965-2.276c0.396-0.295,1.229-0.66,1.396-1.957c0.168-1.295-1.364-2.157-1.364-2.157s-1.218-0.644-1.475-0.93c-0.258-0.287-0.02-0.562-0.02-0.562s0.689-1.485,2.896-2.354c2.205-0.872,3.689-1.107,4.618-1.208c0.932-0.099,1.245,0.237,1.374,0.396c0.128,0.157,0.128,0.485,0.021,0.821c-0.102,0.308-0.444,1.038-0.645,1.395c-0.197,0.356-0.523,1.216-0.316,1.622c0.208,0.405,0.843,0.593,1.662,0.445c0.821-0.149,2.988-0.761,4.888-1.553c1.9-0.792,5.073-2.345,5.073-2.345s0.009-0.004,0.022-0.012l0.086-0.729l0.729,0.295c1.02-0.562,2.764-1.58,4.01-2.631c1.871-1.573,3.699-3.225,4.166-3.639c0.465-0.417,0.892-0.752,1.307-0.732c0.414,0.021,0.732,0.317,0.988,1.434c0.258,1.118,0.308,2.038,0.426,2.582c0.117,0.543,0.285,1.175,0.931,1.304c0.646,0.129,1.513-0.434,1.838-0.713c0.33-0.276,0.92-1.176,0.882-2.382c0,0,0.068-1.604-0.761-3.127c0,0-0.351-0.614-0.479-0.782c-0.088-0.118-0.155-0.238-0.01-0.525c0.148-0.286,0.467-0.821,1.062-1.156c0.448-0.256,0.88-0.316,1.128-0.396c0,0,0.275-0.067,0.626-0.261l-0.126-0.412l0.289,0.316c0.404-0.239,0.652,0.045,0.652,0.045l-0.392-0.501l0.119-0.484c0,0-0.304-0.163-0.685-0.088c-0.383,0.078-0.42,0.362-1.014,0.458c-0.593,0.096-1.275,0.306-1.945,1.319c-0.67,1.011,0,2.271,0,2.271s0.359,0.592,0.533,0.896c0.172,0.306,1.066,2.215,0.037,3.608c0,0-0.552,0.643-1.525,0.86c-0.86,0.19-0.642-0.816-0.729-1.355c0,0-0.129-2.281-1.237-3.588c-0.976-1.146-2.746,0.888-3.629,1.566c-0.822,0.629-3.228,3.112-6.684,4.925l-0.51,0.892l-0.324-0.472c-1.658,0.827-5.418,2.656-7.87,3.514c0,0-1.875,0.762-2.64,0.782c0,0-0.17,0.006-0.034-0.179c0.133-0.185,0.276-0.322,0.507-0.737c0.23-0.418,0.646-1.357,0.646-2.327c0-0.969-1.119-1.917-2.68-1.748c-1.561,0.167-3.052,0.6-4.849,1.292c-1.796,0.692-3.343,2.159-3.55,3.375c-0.209,1.216,1.105,1.92,1.105,1.92s1.484,0.751,1.674,1.157c0.188,0.406,0.049,0.783,0.049,0.783s-0.129,0.406-0.783,0.782c-0.649,0.377-2.42,1.287-2.42,1.287s-2.207,1.217-5.562,1.512c0,0-3.058,0.26-4.817,0.348c-0.022,0.002-0.046,0.002-0.069,0.003l-0.541,0.53l-0.587-0.449c-1.64,0.136-3.54,0.359-3.54,0.359s-4.29,0.609-8.219,1.822c-3.336,1.027-3.971,1.594-3.971,1.594s-0.711,0.596-0.118,1.453c0,0,0.341,0.539,1.03,1.08c0.685,0.541,1.237,1.057,1.159,1.317c-0.08,0.265-0.29,0.382-0.29,0.382s-0.961,0.856-4.894,1.518c0,0-2.531,0.488-3.31-0.275c-0.778-0.766-0.435-1.279-0.435-1.279s0.052-0.173,0.62-0.527c0,0,0.951-0.596,0.647-1.727c-0.303-1.135-2.129-2.545-4.101-2.533c0,0-2.303,0.006-8.196,0.762c0,0-0.008,0-0.027,0.005l-0.204,0.501l-0.605-0.393c-0.324,0.039-0.713,0.076-1.114,0.107c-1.012,0.07-2.477-0.057-4.3-0.596c-1.824-0.543-2.963-1.535-3.126-2.133c-0.114-0.412,0.035-0.996,0.035-0.996S7.62,50.2,7.405,49.115c-0.22-1.083-0.582-1.934-1.631-2.567c-1.048-0.632-1.643-0.522-1.643-0.522s-0.939-0.144-0.904,1.446c0.037,1.59,0.235,2.256,0.235,2.256s0.181,0.562-0.108,0.635c-0.288,0.072-0.74-0.361-0.74-0.361s-1.021-0.924-1.619-1.688l0.048,0.066l-0.214,0.525c0,0,0.067,0.115,0.266,0.375L1.193,48.543L1.193,48.543z"></path>
+  <linearGradient id="SVGID_25_" gradientUnits="userSpaceOnUse" x1="214.8965" y1="-410.8818" x2="306.501" y2="-410.8818" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#A0A0A0"></stop>
+    <stop offset=".0769" stop-color="#656767"></stop>
+    <stop offset="1" stop-color="#717375"></stop>
+  </linearGradient>
+  <path style="&st1;" d="M91.075,11.742l0.183,0.846c0,0-0.563,0.313-1.678,0.613c-1.113,0.3-2.188,0.801-2.188,0.801s-0.89,0.289-0.551,1.013c0.338,0.726,0.838,1.076,0.838,1.076s0.711,0.69,0.736,1.213c0.021,0.526-0.199,0.765-0.764,1.076c-0.563,0.313-1.075,0.375-1.075,0.375s-1.338,0.24-2.001-0.387c-0.663-0.626-0.787-1.663-0.787-1.663s-0.05-0.876-1.148-1.251c-1.102-0.375-2.453,0.425-2.453,0.425s-1.063,0.563-2.2,1.3c-1.14,0.738-3.291,1.64-4.642,2.114l-0.152,0.699l-0.758-0.382c-1.422,0.489-3.271,1.109-3.271,1.109S66.652,21.645,65,21.871c-1.648,0.224-2.016-0.014-2.238-0.238c-0.228-0.224,0.039-1.012,0.039-1.012s0.674-1.376,0.348-2.09c-0.324-0.714-2.451-0.9-2.486-0.9c-0.04,0-2.318-0.265-4.451,0.932c-1.895,1.062-2.143,1.642-2.143,1.642s-0.604,0.938,0.113,1.867c0.807,1.051,1.879,1.146,1.879,1.146s1.021,0.079,1.258,0.38c0.24,0.299,0.076,0.626,0.076,0.626s-0.336,0.925-2.228,1.312c0,0-3.181,0.933-9.113,1.776l-0.582,0.579c-3.743,0.47-8.143,1.363-10.555,1.989c-2.412,0.627-2.553,1.317-2.553,1.317c-0.519,0.813,0.141,1.236,0.141,1.236s0.829,0.283,1.017,1.19c0.19,0.91-0.783,1.629-0.783,1.629s-1.159,0.972-2.898,1.269c-1.739,0.297-2.396,0.35-3.429-0.47c-0.91-0.721-0.298-1.863,0.312-2.301c0.612-0.438,0.909-0.91,0.988-1.112c0.079-0.204,0.032-0.377,0.032-0.377l-0.58-0.534c-2.005-1.33-9.883,0.063-9.883,0.063s0,0.002,0,0l-1.341,0.289l-0.673-0.435c0,0-2.291,0.604-4.876,0.589c-2.712-0.014-1.27-2.128-1.27-2.128s0.638-1.118,0.75-1.764c0,0,0.224-1.745-1.42-1.631c-1.645,0.112-1.968,1.546-1.968,1.546s-0.112,0.801-0.089,1.392c0.021,0.594-0.002,1.847-0.742,2.56c-0.737,0.713-1.529,0.495-1.529,0.495s-1.331-0.582-1.595-0.718c-0.274-0.142-0.517-0.151-0.751-0.066c-0.02,0.007-0.039,0.018-0.057,0.029l-0.175,0.75l0.025-0.623c-0.156,0.176-0.271,0.42-0.271,0.42l0.088,0.327l-0.248,0.565c-0.002,0.012-0.005,0.023-0.006,0.035l0.008,0.003c0,0,0.087-0.196,0.222-0.357l0.182-0.369L1.493,32.94c0.055-0.044,0.112-0.07,0.172-0.074c0.281-0.017,0.629,0.158,0.629,0.158s1.135,0.611,1.642,0.716s0.875-0.065,1.205-0.292c0.527-0.365,1.143-1.121,1.4-1.839c0.229-0.646,0.279-2.394,0.279-2.394l0.004,0.014c0,0,0-0.421-0.011-0.518c-0.012-0.098-0.075-0.553,0.204-0.783c0.278-0.234,0.459-0.347,1.144-0.364c0.68-0.017,0.886,0.38,0.886,0.38S9.28,28.269,9.28,28.48c0,0.21-0.068,0.34-0.068,0.34s-0.371,0.626-0.5,0.934c-0.13,0.307-0.636,1.323-0.489,2.177c0.148,0.852,1.479,1.251,1.479,1.251s1.062,0.25,2.575,0.027l3.12-0.503l0.543-0.624l0.218,0.474c0.805-0.147,2.14-0.369,3.583-0.511c2.326-0.228,4.787-0.262,5.821-0.132c0,0,0.484,0.061,0.405,0.234c-0.062,0.136-0.421,0.415-0.421,0.415s-0.135,0.081-0.654,0.667s-0.671,1.155-0.693,1.661c-0.024,0.505,0.516,1.995,2.415,2.394c1.901,0.397,4.077-0.341,4.077-0.341s1.427-0.314,2.786-1.674c1.41-1.411,0.012-3.108,0.012-3.108s-0.22-0.304-0.647-0.442c0,0-0.187-0.05-0.079-0.17c0.108-0.12,0.37-0.647,1.37-0.905c1-0.257,3.977-1.198,11.065-2.135l0.274-0.617l0.874,0.459c2.708-0.373,4.551-0.694,7.827-1.454c0,0,3.04-0.539,3.675-1.651c0.368-0.65,1.032-1.539-0.045-2.434c0,0-0.125-0.161-0.791-0.312c-0.666-0.151-1.179-0.252-1.738-0.653c-0.563-0.403-0.551-0.869-0.551-0.869s-0.047-0.445,0.862-0.958c0,0,0.851-0.623,2.54-1.158c1.691-0.533,3.648-0.296,3.648-0.296s0.82,0.089,0.685,0.643c-0.14,0.555-0.604,1.482-0.622,1.959c-0.021,0.474,0.012,1.027,0.938,1.414c0.931,0.386,2.267,0.266,2.267,0.266s1.386-0.069,3.647-0.801c2.265-0.731,4.594-1.573,4.594-1.573l-0.094-0.483l0.971,0.173c0.591-0.22,1.406-0.539,2.285-0.936c1.801-0.81,2.656-1.488,3.48-1.958c0,0,1.815-1.118,2.887-1.088c0,0,0.25-0.009,0.272,0.32c0.022,0.329,0.104,0.814,0.218,1.096c0.111,0.281,0.734,1.822,2.729,2.048c1.993,0.228,2.846-1.118,2.846-1.118s0.683-1.049-0.493-2.296c0,0-0.961-1.028-0.99-1.107c0,0-0.104-0.155,0.168-0.233c0.269-0.078,3.152-0.884,4.268-1.398c0,0,0.012-0.005,0.036-0.015l-0.271-0.485l0.535,0.365c0.202-0.101,0.449-0.246,0.544-0.395c0.171-0.271-0.054-0.797-0.054-0.797l0.006-0.007c-0.133-0.184-0.266-0.366-0.4-0.546l-0.606-0.256l-0.06,0.033L91.075,11.742"></path>
+  <path style="&st34;" d="M8.12,16.324l-0.291,0.435c0.134-0.023,0.244-0.056,0.244-0.056s0.404-0.066,1.112,0.12c0.917,0.244,2.067-0.496,2.067-0.496s0.872-0.523,1.274-1.381l0.361-1.005c0,0,0.291-0.972,1.105-1.281c0,0,0.317-0.171,0.831-0.177c0.513-0.005,0.392,0.354,0.392,0.354s-0.056,0.233-0.586,0.912c-0.529,0.677-0.671,1.196-0.671,1.196s-0.318,0.722,0.193,0.957c0,0,0.24,0.141,0.939-0.091c0.7-0.233,3.02-0.843,4.438-1.06l0.993-0.506c-0.313-0.23-0.602-0.444-0.602-0.444c-2.388,0.442-4.168,0.995-4.775,1.226c-0.609,0.23-0.62,0.082-0.62,0.082c-0.211-0.248,0.435-1.022,0.435-1.022s0.655-0.894,0.913-1.715c0.259-0.82-0.892-0.707-0.892-0.707c-0.758,0.121-1.303,0.48-1.93,1.236c-0.486,0.586-0.782,1.389-0.893,1.814c-0.071,0.267-0.193,0.515-0.193,0.515s-0.091,0.165-0.45,0.322c-0.416,0.182-1.228,0.396-1.228,0.396s-0.54,0.045-0.931-0.15c-0.24-0.118-0.901,0.05-0.901,0.05l0.091-0.504l-0.082-0.032l-0.683,0.383c-0.115,0.189-0.228,0.379-0.341,0.569c-0.063,0.146-0.179,0.475,0.044,0.51c0.05,0.008,0.113,0.008,0.164,0.008L8.12,16.324L8.12,16.324z"></path>
+  <path style="&st34;" d="M13.295,9.19c0,0,0.342-0.271,0.342-0.026c0,0.248-0.333,0.69-0.333,0.69s-0.107,0.134-0.172,0.134C13.068,9.987,13.15,9.516,13.295,9.19L13.295,9.19z"></path>
+  <path style="&st12;" d="M-16.122-14.641"></path>
+  <path style="&st34;" d="M10.595,12.501c0,0-0.388,0.46,0,0.52l0.838-0.828c-0.037-0.037-0.054-0.055-0.157-0.112C11.277,12.081,10.752,12.343,10.595,12.501L10.595,12.501z"></path>
+  <path style="&st28;" d="M77.807,85.745c0,0-1.75,0.806-3.396,1.603c-1.647,0.803-2.25,1.14-2.25,1.14s-0.088,0.049-0.031,0.082c0.056,0.028-0.008,0.063-0.008,0.063c-0.847,0.409-2.703,1.07-2.703,1.07s-0.167,0.057-0.157,0.02c0.006-0.041,0.199-0.105,0.199-0.105s0.381-0.146,0.283-0.217c-0.098-0.072-1.21,0.305-1.21,0.305s-1.949,0.621-2.894,0.887c-0.941,0.269-2.75,0.735-2.75,0.735c-1.201,0.298-3.75,0.798-4.315,0.901c-0.562,0.104-0.417,0.064-0.417,0.064s0.492-0.189-0.411-0.226c-1.146-0.05-2.362,0.112-3.612,0.288c-1.25,0.18-1.178,0.324-1.178,0.324s-0.04,0.154,0.708,0.14c0.752-0.019,0.534,0.046,0.534,0.046c-1.396,0.202-4.2,0.238-4.2,0.238l0.005,0.01c0.456,0.014,0.916,0.018,1.376,0.018c0.608,0,1.218-0.01,1.822-0.031c0.241-0.021,1.483-0.15,1.925-0.244c0.486-0.105-0.093-0.105-0.093-0.105l-0.74-0.023c0,0-0.776-0.026-0.052-0.158c0.729-0.133,1.834-0.192,2.388-0.252c0.83-0.094,0.541,0.105,0.541,0.105l-0.04,0.023c0,0,0.014,0.094,0.528,0.066c0.515-0.024,4.721-0.804,7.069-1.487c2.347-0.688,4.102-1.255,4.102-1.255s0.157-0.055,0.066,0.025c-0.094,0.078-0.463,0.2-0.463,0.2s-0.595,0.3,0.844-0.108c0.066-0.02,0.134-0.039,0.197-0.06c1.234-0.469,2.446-0.983,3.635-1.543c0.029-0.028,0.131-0.146-0.021-0.101c-0.172,0.056,0-0.065,0.187-0.131c0.184-0.066,2.267-1.162,3.363-1.608l0.002,0.003c0.394-0.227,0.781-0.455,1.17-0.691L77.807,85.745L77.807,85.745z"></path>
+  <linearGradient id="SVGID_26_" gradientUnits="userSpaceOnUse" x1="217.6563" y1="-436.751" x2="217.6563" y2="-436.751" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".078" stop-color="#F4F4F4"></stop>
+    <stop offset=".3807" stop-color="#CECECE"></stop>
+    <stop offset=".5396" stop-color="#BFBFBF"></stop>
+    <stop offset=".8357" stop-color="#7C7C7C"></stop>
+    <stop offset=".8996" stop-color="#A8A8A8"></stop>
+    <stop offset=".9093" stop-color="#9A9A9A"></stop>
+    <stop offset=".9327" stop-color="#7D7D7D"></stop>
+    <stop offset=".9558" stop-color="#686868"></stop>
+    <stop offset=".9785" stop-color="#5B5B5B"></stop>
+    <stop offset="1" stop-color="#575757"></stop>
+  </linearGradient>
+  <path style="&st46;" d="M3.893,50.249L3.893,50.249L3.893,50.249z"></path>
+  <linearGradient id="SVGID_27_" gradientUnits="userSpaceOnUse" x1="214.3262" y1="-436.5107" x2="322.8717" y2="-424.4851" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".078" stop-color="#F4F4F4"></stop>
+    <stop offset=".3807" stop-color="#CECECE"></stop>
+    <stop offset=".5396" stop-color="#BFBFBF"></stop>
+    <stop offset=".8357" stop-color="#7C7C7C"></stop>
+    <stop offset=".8996" stop-color="#A8A8A8"></stop>
+    <stop offset=".9093" stop-color="#9A9A9A"></stop>
+    <stop offset=".9327" stop-color="#7D7D7D"></stop>
+    <stop offset=".9558" stop-color="#686868"></stop>
+    <stop offset=".9785" stop-color="#5B5B5B"></stop>
+    <stop offset="1" stop-color="#575757"></stop>
+  </linearGradient>
+  <path style="&st47;" d="M1.193,48.543l0.107,0.98c0.236,0.285,0.579,0.675,1.067,1.181c1.084,1.121,1.428,0.416,1.428,0.416s0.072-0.201,0.092-0.586c-0.042,0.216-0.209,0.403-0.788-0.056c-0.298-0.237-0.688-0.627-1.076-1.049c-0.326-0.326-0.723-0.742-1.021-1.117l0.048,0.065l-0.213,0.526c0,0,0.069,0.115,0.268,0.376L1.193,48.543L1.193,48.543z"></path>
+  <linearGradient id="SVGID_28_" gradientUnits="userSpaceOnUse" x1="213.8887" y1="-436.4771" x2="336.8819" y2="-422.851" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".078" stop-color="#F4F4F4"></stop>
+    <stop offset=".3807" stop-color="#CECECE"></stop>
+    <stop offset=".5396" stop-color="#BFBFBF"></stop>
+    <stop offset=".8357" stop-color="#7C7C7C"></stop>
+    <stop offset=".8996" stop-color="#A8A8A8"></stop>
+    <stop offset=".9093" stop-color="#9A9A9A"></stop>
+    <stop offset=".9327" stop-color="#7D7D7D"></stop>
+    <stop offset=".9558" stop-color="#686868"></stop>
+    <stop offset=".9785" stop-color="#5B5B5B"></stop>
+    <stop offset="1" stop-color="#575757"></stop>
+  </linearGradient>
+  <path style="&st48;" d="M3.741,49.133c-0.006-0.027-0.013-0.054-0.02-0.078c0.012,0.088,0.028,0.179,0.043,0.272c0,0,0.094,0.394,0.12,0.753C3.87,49.813,3.83,49.498,3.741,49.133z"></path>
+  <linearGradient id="SVGID_29_" gradientUnits="userSpaceOnUse" x1="411.2539" y1="557.002" x2="507.2363" y2="537.6277" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset="1" stop-color="#BFC0C2"></stop>
+  </linearGradient>
+  <path style="&st49;" d="M99.696,28.566l0.29,0.316c0.406-0.238,0.654,0.045,0.654,0.045l-0.386-0.494c-0.383,0.082-1.093,0.256-2.233,0.61c0,0-1.353,0.594-1.59,1.532c-0.016,0.332,0.049,0.641,0.116,0.876c0.753,1.429,0.913,1.666,0.913,1.666c0.499,1.091,0.422,1.993,0.25,2.824c-0.188,0.921-1.188,1.668-1.794,1.842c-0.608,0.171-0.953,0-1.14-0.483c-0.188-0.485-0.157-0.845-0.438-2.34c-0.28-1.499-0.653-2.2-1.216-2.372c-0.563-0.173-1.313,0.468-1.749,0.811c-0.438,0.344-2.417,2.152-2.417,2.152s-2.324,2.091-5.743,4.026c-3.417,1.935-7.69,3.418-7.69,3.418s-2.842,1.092-3.525,0.998c-0.688-0.093-0.516-0.718-0.297-1.076c0.219-0.36,0.516-0.937,0.766-1.545c0.313-0.756,0.201-1.654-0.359-1.967c-0.562-0.311-1.248-0.468-3.523,0.094s-3.197,1.076-3.838,1.451c-0.643,0.376-1.576,1.233-1.842,1.716c-0.266,0.484-0.563,0.983,0.126,1.576c0.688,0.594,0.763,0.485,1.56,1.046c0.797,0.562,0.922,0.64,0.969,1.466c0.045,0.827-0.766,1.341-0.766,1.341s-3.123,2.082-6.602,2.777c-2.34,0.469-6.833,0.639-6.833,0.639s-1.327-0.045-5.384,0.547c-4.056,0.594-6.787,1.514-7.738,1.856c-0.952,0.343-2.34,0.81-3.136,1.17c-0.795,0.358-0.375,0.89-0.203,1.015c0.172,0.123,1.185,1.074,1.498,1.354c0.312,0.282,1.466,1.279,0.108,2.108c-1.356,0.826-3.603,1.264-3.603,1.264s-3.121,0.654-4.119,0.234c-1.001-0.422-1.2-0.778-1.452-1.358c-0.313-0.718-0.016-1.263,0.794-1.78c0.77-0.486,0.469-1.199,0.469-1.199c-0.983-1.9-3.058-2.058-4.774-1.936c-2.72,0.198-5.358,0.471-7.425,0.734c-3.059,0.39-4.541-0.063-5.992-0.516c-1.452-0.453-2.997-1.529-3.154-2.498c-0.027-0.097-0.039-0.199-0.042-0.307c-0.078-0.758,0.074-1.146,0.184-1.784c0.108-0.625-0.043-1.532-0.084-1.758c-0.008-0.026-0.016-0.058-0.023-0.085c-0.507-1.304-1.725-1.903-2.668-2.058c-0.953-0.157-0.983,0.857-0.983,0.857s-0.002,0.012-0.006,0.04c0.078-0.208,0.222-0.315,0.444-0.31c0.577,0.02,2.259,0.361,2.781,1.754c0,0,0.218,0.648,0.019,1.479c-0.199,0.832-0.162,1.571-0.019,2.295c0.145,0.725,0.759,1.408,1.465,1.969c0.704,0.562,2.745,1.535,4.734,1.66c1.814,0.117,2.483,0.037,2.587,0.023l0.759-0.768l0.135,0.666c0.81-0.088,2.115-0.229,3.619-0.394c2.529-0.271,4.227-0.541,5.8-0.306c1.572,0.232,2.078,1.463,2.078,1.463s0.107,0.313-0.253,0.542c-0.349,0.226-0.75,0.548-0.947,0.891c-0.2,0.345-0.411,1.066,0.159,2.033c0.572,0.965,2.142,1.595,3.478,1.435c1.336-0.158,3.336-0.672,4.521-0.98c1.286-0.336,2.424-0.969,2.572-1.979c0.148-1.009-1.534-2.297-1.534-2.297s-0.81-0.611-0.909-0.75c-0.1-0.14,0.099-0.197,0.099-0.197s0.95-0.229,1.356-0.416c0.403-0.19,1.523-0.644,3.353-1.217c1.831-0.572,5.609-1.248,5.609-1.248s2.09-0.332,4.694-0.543l0.612-0.705l0.493,0.627c0.406-0.023,0.819-0.045,1.235-0.061c3.572-0.129,5.343-0.555,7.24-0.979c1.897-0.426,4.569-1.979,4.965-2.276c0.396-0.296,1.229-0.662,1.395-1.958c0.17-1.294-1.363-2.157-1.363-2.157s-1.218-0.643-1.476-0.929c-0.256-0.288-0.019-0.562-0.019-0.562s0.689-1.485,2.896-2.355c2.207-0.872,3.69-1.106,4.619-1.207c0.931-0.099,1.247,0.237,1.375,0.395c0.128,0.158,0.128,0.485,0.021,0.821c-0.101,0.308-0.443,1.039-0.644,1.396c-0.199,0.356-0.522,1.216-0.317,1.622c0.211,0.405,0.842,0.592,1.662,0.444c0.822-0.147,2.987-0.761,4.889-1.553c1.897-0.793,5.074-2.344,5.074-2.344s0.01-0.005,0.021-0.013l0.086-0.729l0.729,0.296c1.021-0.562,2.764-1.582,4.01-2.63c1.871-1.574,3.699-3.225,4.166-3.641c0.465-0.415,0.89-0.751,1.305-0.732c0.416,0.021,0.734,0.318,0.99,1.434c0.258,1.119,0.306,2.038,0.426,2.583c0.117,0.542,0.285,1.176,0.929,1.305c0.644,0.128,1.513-0.436,1.841-0.713c0.326-0.277,0.918-1.176,0.879-2.383c0,0,0.068-1.605-0.762-3.127c0,0-0.348-0.614-0.477-0.782c-0.088-0.119-0.155-0.238-0.008-0.523c0.146-0.288,0.465-0.821,1.059-1.157c0.449-0.256,0.881-0.316,1.128-0.396c0,0,0.28-0.069,0.636-0.266L99.696,28.566L99.696,28.566z"></path>
+  <path style="&st23;" d="M1.547,32.696l-0.05,0.239c0.053-0.041,0.111-0.068,0.17-0.072c0.281-0.018,0.629,0.158,0.629,0.158s1.135,0.61,1.642,0.716c0.507,0.105,0.875-0.065,1.205-0.292c0.25-0.174,0.521-0.434,0.763-0.737c-0.29,0.29-0.725,0.548-1.365,0.595c0,0-0.571-0.049-1.419-0.587c-0.846-0.537-1.327-0.231-1.729,0.105l0,0l-0.249,0.564c-0.001,0.013-0.003,0.023-0.006,0.036l0.009,0.001c0,0,0.087-0.192,0.217-0.352L1.547,32.696L1.547,32.696z"></path>
+  <path style="&st4;" d="M6.387,31.961c0.059-0.12,0.112-0.238,0.155-0.354c0.05-0.14,0.09-0.334,0.125-0.55c-0.096,0.44-0.223,0.73-0.223,0.73S6.428,31.853,6.387,31.961L6.387,31.961z"></path>
+  <path style="&st4;" d="M6.807,28.654c-0.038,0.439-0.038,0.987-0.038,1.493l0,0c0.04-0.509,0.052-0.935,0.052-0.935l0.004,0.014c0,0,0-0.421-0.011-0.518C6.811,28.695,6.809,28.677,6.807,28.654z"></path>
+  <linearGradient id="SVGID_30_" gradientUnits="userSpaceOnUse" x1="413.5137" y1="516.4121" x2="417.999" y2="516.4121" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#E2E3E4"></stop>
+    <stop offset=".5055" stop-color="#FFF"></stop>
+  </linearGradient>
+  <path style="&st50;" d="M8.12,16.324l-0.291,0.435c0.087-0.014,0.089-0.007,0.244-0.056c0,0,0.404-0.066,1.112,0.12c0.917,0.244,2.067-0.496,2.067-0.496s0.272-0.164,0.583-0.458c-0.4,0.24-1.385,0.762-2.132,0.585c-0.961-0.229-0.994-0.017-1.336-0.049c-0.292-0.028-0.292-0.11-0.275-0.314c0.002-0.032,0.023-0.054,0.03-0.092c0.05-0.261,0.339-0.689,0.339-0.689l-0.684,0.383c-0.115,0.189-0.227,0.378-0.34,0.569c-0.063,0.146-0.179,0.475,0.044,0.509c0.055,0.009,0.11,0.011,0.163,0.009L8.12,16.324L8.12,16.324z"></path>
+  <path style="&st23;" d="M17.019,15.036c-1.027,0.278-1.972,0.734-2.494,0.604c-0.438-0.109-0.084-0.76,0.042-0.965c0.069-0.146,0.167-0.292,0.249-0.406c0.31-0.506,0.833-1.351,0.833-1.351s0,0,0-0.002c0.017-0.032,0.033-0.067,0.049-0.102c0.067-0.18,0.12-0.537-0.472-0.597c-0.799-0.082-1.613,0.619-1.613,0.619s-0.547,0.301-0.838,1.417l0.112-0.313c0,0,0.291-0.973,1.105-1.281c0,0,0.317-0.171,0.831-0.177c0.513-0.005,0.392,0.354,0.392,0.354s-0.056,0.233-0.586,0.912c-0.529,0.677-0.671,1.195-0.671,1.195s-0.319,0.723,0.193,0.957c0,0,0.238,0.141,0.939-0.091c0.7-0.233,3.02-0.843,4.438-1.06l0.993-0.506C19.849,14.299,18.012,14.769,17.019,15.036L17.019,15.036z"></path>
+  <path style="&st23;" d="M-16.122-14.641"></path>
+  <path style="&st23;" d="M-16.122-14.641"></path>
+  <path style="&st59;" d="M90.578,74.217c0.006-0.004,0.009-0.008,0.014-0.013C90.587,74.21,90.582,74.213,90.578,74.217z"></path>
+  <path style="&st59;" d="M90.821,73.951c0.001-0.002,0.002-0.004,0.003-0.004C90.823,73.947,90.822,73.949,90.821,73.951z"></path>
+  <path style="&st59;" d="M90.738,74.042c0.003-0.004,0.008-0.009,0.011-0.013C90.746,74.033,90.741,74.038,90.738,74.042z"></path>
+  <path style="&st8;" d="M87.9,76.68l-0.164,0.174c-0.646,0.716-0.969,1.007-0.969,1.007l-0.242,0.269l0,0c-0.002,0.006-0.006,0.012-0.012,0.014c0.008,0.002,0.018,0.004,0.041,0.004c0,0,0.229-0.002,0.588-0.219c-0.072,0.041-0.139,0.068-0.181,0.07c-0.177,0.012,0.431-0.538,0.431-0.538s0.588-0.626,0.967-1.125c0.382-0.497,0.138-0.468,0.138-0.468s-0.087-0.021-0.397,0.125C88.125,76.018,88.268,76.223,87.9,76.68L87.9,76.68z"></path>
+  <path style="&st59;" d="M90.657,74.133c0.003-0.006,0.007-0.013,0.013-0.017C90.664,74.121,90.662,74.128,90.657,74.133z"></path>
+  <path style="&st0;" d="M72.945,81.21c0,0,2.215-0.931,3.402-0.79c0,0,0.611,0.059-0.062,0.711c-0.672,0.652-0.892,1.168,0.396,0.752c0,0,0.094-0.033,0.244-0.086c-0.504,0.143-0.734,0.143-0.658,0.033c0.127-0.188,0.461-0.461,0.461-0.461s0.399-0.4,0.399-0.801c0,0,0.128-0.588-1.604-0.225c-1.01,0.209-1.969,0.59-2.588,0.867C72.941,81.211,72.943,81.211,72.945,81.21L72.945,81.21z"></path>
+  <path style="&st59;" d="M72.234,81.555l-0.004,0.002C72.232,81.557,72.234,81.555,72.234,81.555z"></path>
+  <path style="&st59;" d="M88.057,76.014c-0.014,0.008-0.026,0.015-0.043,0.021C88.028,76.027,88.043,76.021,88.057,76.014z"></path>
+  <path style="&st14;" d="M78.305,81.299c1.448-0.521,3.93-1.854,5.023-2.492C82.232,79.442,79.73,80.717,78.305,81.299z"></path>
+  <path style="&st59;" d="M87.512,76.325c0.026-0.017,0.055-0.032,0.08-0.049C87.566,76.293,87.538,76.309,87.512,76.325z"></path>
+  <path style="&st59;" d="M91.618,73.084c0.008-0.011,0.019-0.023,0.026-0.029C91.637,73.061,91.626,73.073,91.618,73.084z"></path>
+  <path style="&st59;" d="M91.434,73.281c0.01-0.012,0.019-0.02,0.023-0.027C91.451,73.264,91.441,73.27,91.434,73.281z"></path>
+  <path style="&st59;" d="M91.521,73.188c0.01-0.012,0.02-0.022,0.029-0.032C91.539,73.165,91.53,73.178,91.521,73.188z"></path>
+  <path style="&st59;" d="M90.9,73.863c0.003-0.006,0.007-0.01,0.012-0.016C90.907,73.854,90.903,73.857,90.9,73.863z"></path>
+  <path style="&st59;" d="M71.246,82.088c-0.014,0.008-0.023,0.018-0.036,0.022C71.221,82.105,71.232,82.096,71.246,82.088z"></path>
+  <path style="&st59;" d="M91.346,73.375c0.008-0.008,0.014-0.015,0.021-0.021C91.359,73.36,91.354,73.367,91.346,73.375z"></path>
+  <path style="&st59;" d="M91.077,73.669c0.006-0.006,0.01-0.011,0.013-0.015C91.087,73.659,91.083,73.663,91.077,73.669z"></path>
+  <path style="&st59;" d="M91.251,73.479c0.007-0.006,0.015-0.012,0.021-0.021C91.266,73.463,91.255,73.473,91.251,73.479z"></path>
+  <path style="&st59;" d="M90.987,73.768c0.004-0.006,0.01-0.012,0.016-0.021C90.997,73.756,90.991,73.762,90.987,73.768z"></path>
+  <path style="&st59;" d="M91.161,73.578c0.006-0.006,0.011-0.014,0.017-0.02C91.172,73.564,91.165,73.572,91.161,73.578z"></path>
+  <path style="&st59;" d="M71.816,81.764c-0.008,0.006-0.02,0.012-0.026,0.017C71.799,81.774,71.809,81.77,71.816,81.764z"></path>
+  <path style="&st59;" d="M72.044,81.646c-0.006,0.004-0.012,0.006-0.019,0.009C72.032,81.652,72.038,81.649,72.044,81.646z"></path>
+  <path style="&st59;" d="M71.305,82.055c-0.011,0.008-0.021,0.014-0.031,0.021C71.283,82.067,71.294,82.063,71.305,82.055z"></path>
+  <path style="&st59;" d="M71.969,81.684c-0.008,0.004-0.015,0.009-0.021,0.014C71.953,81.692,71.961,81.688,71.969,81.684z"></path>
+  <path style="&st59;" d="M72.109,81.613c-0.004,0.005-0.01,0.006-0.016,0.01C72.102,81.619,72.105,81.617,72.109,81.613z"></path>
+  <path style="&st59;" d="M72.008,81.665c-0.007,0.004-0.013,0.007-0.021,0.011C71.994,81.672,72.001,81.669,72.008,81.665z"></path>
+  <path style="&st59;" d="M72.206,81.566c-0.003,0.002-0.005,0.003-0.007,0.004C72.201,81.569,72.203,81.568,72.206,81.566z"></path>
+  <path style="&st59;" d="M72.148,81.594c-0.008,0.004-0.016,0.01-0.024,0.015C72.135,81.604,72.141,81.598,72.148,81.594z"></path>
+  <path style="&st59;" d="M71.928,81.707c-0.01,0.003-0.02,0.01-0.027,0.016C71.908,81.715,71.918,81.711,71.928,81.707z"></path>
+  <path style="&st59;" d="M72.193,81.575c-0.006,0.001-0.01,0.003-0.014,0.004C72.184,81.578,72.189,81.575,72.193,81.575z"></path>
+  <path style="&st59;" d="M72.172,81.585c-0.004,0.003-0.008,0.005-0.015,0.007C72.164,81.59,72.168,81.585,72.172,81.585z"></path>
+  <path style="&st59;" d="M72.079,81.631c-0.008,0.002-0.013,0.004-0.019,0.006C72.066,81.635,72.071,81.633,72.079,81.631z"></path>
+  <path style="&st59;" d="M71.866,81.739c-0.011,0.002-0.021,0.008-0.027,0.013C71.848,81.747,71.857,81.741,71.866,81.739z"></path>
+  <path style="&st59;" d="M71.555,81.906c-0.018,0.012-0.033,0.02-0.052,0.029C71.521,81.924,71.539,81.918,71.555,81.906z"></path>
+  <path style="&st59;" d="M72.219,81.561c0,0.002-0.002,0.002-0.002,0.004C72.217,81.563,72.219,81.563,72.219,81.561z"></path>
+  <path style="&st59;" d="M71.421,81.986c-0.011,0.006-0.022,0.01-0.03,0.02C71.398,81.996,71.41,81.992,71.421,81.986z"></path>
+  <path style="&st59;" d="M71.363,82.02c-0.012,0.005-0.021,0.013-0.032,0.021C71.342,82.032,71.354,82.024,71.363,82.02z"></path>
+  <path style="&st59;" d="M71.482,81.948c-0.014,0.006-0.023,0.015-0.036,0.021C71.458,81.963,71.471,81.956,71.482,81.948z"></path>
+  <path style="&st59;" d="M71.768,81.79c-0.01,0.004-0.018,0.009-0.024,0.017C71.751,81.799,71.758,81.794,71.768,81.79z"></path>
+  <path style="&st59;" d="M91.708,72.984c0.013-0.014,0.024-0.028,0.04-0.042C91.734,72.956,91.723,72.971,91.708,72.984z"></path>
+  <path style="&st59;" d="M71.717,81.818c-0.01,0.004-0.019,0.009-0.027,0.015C71.698,81.827,71.707,81.822,71.717,81.818z"></path>
+  <path style="&st59;" d="M71.665,81.846c-0.011,0.006-0.021,0.014-0.028,0.018C71.645,81.858,71.654,81.852,71.665,81.846z"></path>
+  <path style="&st59;" d="M71.609,81.875c-0.01,0.008-0.021,0.014-0.03,0.02C71.589,81.889,71.602,81.881,71.609,81.875z"></path>
+  <path style="&st59;" d="M92.443,72.197c-0.021,0.021-0.037,0.039-0.055,0.061C92.406,72.238,92.423,72.217,92.443,72.197z"></path>
+  <path style="&st59;" d="M92.359,72.287c-0.021,0.021-0.041,0.044-0.061,0.066C92.318,72.331,92.339,72.309,92.359,72.287z"></path>
+  <path style="&st59;" d="M92.501,72.135c-0.011,0.015-0.021,0.023-0.028,0.033C92.48,72.157,92.49,72.148,92.501,72.135z"></path>
+  <path style="&st59;" d="M91.786,72.899c0.022-0.021,0.042-0.042,0.062-0.063C91.828,72.857,91.807,72.878,91.786,72.899z"></path>
+  <path style="&st59;" d="M92.273,72.379c-0.014,0.016-0.025,0.029-0.041,0.045C92.245,72.409,92.26,72.395,92.273,72.379z"></path>
+  <path style="&st59;" d="M92.579,72.053c-0.013,0.012-0.021,0.021-0.03,0.032C92.559,72.074,92.569,72.063,92.579,72.053z"></path>
+  <path style="&st59;" d="M92.762,71.858c-0.002,0-0.002,0-0.002,0.001C92.76,71.858,92.76,71.858,92.762,71.858z"></path>
+  <path style="&st59;" d="M92.703,71.919c-0.004,0.005-0.01,0.013-0.014,0.017C92.693,71.928,92.699,71.924,92.703,71.919z"></path>
+  <path style="&st59;" d="M87.938,76.074c-0.017,0.008-0.027,0.016-0.043,0.023C87.907,76.09,87.922,76.082,87.938,76.074z"></path>
+  <path style="&st59;" d="M92.648,71.98c-0.012,0.011-0.021,0.021-0.029,0.029C92.629,72,92.637,71.991,92.648,71.98z"></path>
+  <path style="&st59;" d="M87.666,76.229c-0.018,0.013-0.031,0.021-0.051,0.031C87.633,76.25,87.648,76.241,87.666,76.229z"></path>
+  <path style="&st59;" d="M48.204,91.026c0,0-0.001,0-0.003,0.002C48.203,91.026,48.203,91.026,48.204,91.026z"></path>
+  <path style="&st59;" d="M48.073,91.03c-0.007,0-0.013,0-0.021,0C48.06,91.03,48.066,91.03,48.073,91.03z"></path>
+  <path style="&st59;" d="M48.136,91.028c-0.007,0.002-0.013,0.002-0.02,0.002C48.123,91.03,48.129,91.03,48.136,91.028z"></path>
+  <path style="&st59;" d="M48.005,91.032c-0.003,0-0.007,0-0.012,0C47.998,91.032,48.002,91.032,48.005,91.032z"></path>
+  <path style="&st59;" d="M47.94,91.035c-0.004,0-0.009,0-0.015,0C47.93,91.035,47.935,91.035,47.94,91.035z"></path>
+  <path style="&st59;" d="M87.869,76.111c-0.014,0.006-0.021,0.015-0.039,0.021C87.848,76.126,87.857,76.117,87.869,76.111z"></path>
+  <path style="&st59;" d="M87.807,76.146c-0.014,0.009-0.025,0.018-0.041,0.021C87.779,76.164,87.793,76.155,87.807,76.146z"></path>
+  <path style="&st59;" d="M87.738,76.186c-0.016,0.011-0.028,0.018-0.044,0.024C87.71,76.203,87.723,76.196,87.738,76.186z"></path>
+  <path style="&st59;" d="M87.951,76.067c0.02-0.011,0.035-0.021,0.054-0.027C87.986,76.047,87.971,76.057,87.951,76.067z"></path>
+  <path style="&st59;" d="M87.719,77.52c0,0-0.082,0.064-0.193,0.147C87.592,77.621,87.664,77.566,87.719,77.52z"></path>
+  <path style="&st59;" d="M92.205,72.452c-0.021,0.022-0.042,0.046-0.064,0.067C92.163,72.498,92.186,72.475,92.205,72.452z"></path>
+  <path style="&st59;" d="M28.05,87.006c-0.019-0.007-0.037-0.016-0.056-0.021C28.013,86.991,28.031,86.999,28.05,87.006z"></path>
+  <path style="&st59;" d="M71.051,82.217c-0.013,0.008-0.021,0.017-0.037,0.021C71.027,82.23,71.038,82.223,71.051,82.217z"></path>
+  <path style="&st59;" d="M45.854,89.871c-0.024,0-0.049-0.004-0.074-0.004C45.806,89.867,45.831,89.871,45.854,89.871z"></path>
+  <path style="&st59;" d="M27.678,86.883c0.02,0.004,0.039,0.01,0.058,0.019C27.717,86.893,27.696,86.887,27.678,86.883z"></path>
+  <path style="&st59;" d="M27.555,86.844c0.038,0.012,0.078,0.022,0.119,0.037C27.633,86.869,27.593,86.854,27.555,86.844z"></path>
+  <path style="&st59;" d="M27.862,86.941c-0.041-0.014-0.082-0.026-0.123-0.04C27.78,86.913,27.82,86.928,27.862,86.941z"></path>
+  <path style="&st59;" d="M91.88,72.799c0.015-0.016,0.026-0.027,0.04-0.043C91.907,72.771,91.895,72.784,91.88,72.799z"></path>
+  <path style="&st59;" d="M28.058,87.006c0.04,0.016,0.079,0.026,0.116,0.041C28.137,87.033,28.098,87.021,28.058,87.006z"></path>
+  <path style="&st59;" d="M91.977,72.695c0.014-0.016,0.026-0.027,0.041-0.044C92.002,72.668,91.989,72.68,91.977,72.695z"></path>
+  <path style="&st59;" d="M92.057,72.612c0.021-0.022,0.039-0.041,0.061-0.062C92.096,72.571,92.076,72.59,92.057,72.612z"></path>
+  <path style="&st59;" d="M27.991,86.985c-0.021-0.009-0.041-0.017-0.062-0.022C27.95,86.969,27.97,86.977,27.991,86.985z"></path>
+  <path style="&st59;" d="M71.133,82.162c0.021-0.013,0.037-0.021,0.055-0.036C71.17,82.139,71.152,82.149,71.133,82.162z"></path>
+  <path style="&st59;" d="M46.361,89.926c-0.016-0.002-0.035-0.004-0.053-0.008C46.325,89.922,46.345,89.924,46.361,89.926z"></path>
+  <path style="&st59;" d="M71.122,82.17c-0.019,0.012-0.034,0.021-0.049,0.032C71.088,82.191,71.105,82.18,71.122,82.17z"></path>
+  <path style="&st59;" d="M46.439,89.939c-0.017-0.002-0.032-0.006-0.049-0.008C46.407,89.934,46.422,89.938,46.439,89.939z"></path>
+  <path style="&st59;" d="M46.513,89.953c-0.016-0.005-0.031-0.007-0.047-0.011C46.481,89.946,46.497,89.948,46.513,89.953z"></path>
+  <path style="&st59;" d="M46.096,89.891c-0.031-0.002-0.065-0.006-0.1-0.01C46.031,89.885,46.065,89.889,46.096,89.891z"></path>
+  <path style="&st59;" d="M46.278,89.913c-0.02-0.002-0.041-0.004-0.061-0.006C46.237,89.909,46.259,89.911,46.278,89.913z"></path>
+  <path style="&st59;" d="M46.19,89.904c-0.028-0.005-0.054-0.008-0.084-0.012C46.136,89.896,46.163,89.899,46.19,89.904z"></path>
+  <path style="&st59;" d="M45.971,89.881c-0.025-0.005-0.051-0.006-0.079-0.009C45.92,89.875,45.945,89.877,45.971,89.881z"></path>
+  <path style="&st59;" d="M27.926,86.961c-0.02-0.004-0.039-0.011-0.058-0.019C27.888,86.953,27.908,86.957,27.926,86.961z"></path>
+  <path style="&st59;" d="M47.395,90.975c0.001,0.004,0.003,0.004,0.004,0.004C47.398,90.979,47.396,90.979,47.395,90.975z"></path>
+  <path style="&st59;" d="M47.375,90.969c0.001,0,0.002,0,0.003,0C47.378,90.969,47.376,90.969,47.375,90.969z"></path>
+  <path style="&st59;" d="M47.418,90.982L47.418,90.982L47.418,90.982z"></path>
+  <path style="&st59;" d="M47.465,90.996c0.003,0,0.004,0.001,0.006,0.001C47.47,90.997,47.468,90.996,47.465,90.996z"></path>
+  <path style="&st59;" d="M47.442,90.988c0.001,0,0.002,0.002,0.002,0.002L47.442,90.988z"></path>
+  <path style="&st59;" d="M47.342,90.955L47.342,90.955L47.342,90.955z"></path>
+  <path style="&st59;" d="M46.589,89.967c-0.02-0.002-0.036-0.009-0.054-0.013C46.553,89.959,46.57,89.965,46.589,89.967z"></path>
+  <path style="&st59;" d="M46.764,90.012c-0.011-0.004-0.021-0.008-0.033-0.012C46.742,90.006,46.753,90.008,46.764,90.012z"></path>
+  <path style="&st59;" d="M49.049,90.953c-0.01,0.001-0.021,0.002-0.032,0.004C49.028,90.955,49.038,90.954,49.049,90.953z"></path>
+  <path style="&st59;" d="M46.659,89.984c-0.021-0.006-0.039-0.01-0.061-0.014C46.619,89.975,46.639,89.979,46.659,89.984z"></path>
+  <path style="&st59;" d="M47.767,91.028c0.003,0,0.003,0,0.006,0C47.773,91.028,47.77,91.028,47.767,91.028z"></path>
+  <path style="&st59;" d="M47.821,91.03c0.003,0,0.007,0,0.013,0C47.828,91.03,47.824,91.03,47.821,91.03z"></path>
+  <path style="&st59;" d="M45.214,89.855c0.047,0,0.092,0,0.139,0C45.308,89.855,45.261,89.855,45.214,89.855z"></path>
+  <path style="&st59;" d="M47.716,91.026c0.005,0,0.009,0,0.013,0.002C47.727,91.026,47.722,91.026,47.716,91.026z"></path>
+  <path style="&st32;" d="M46.772,90.014c0,0,0.158,0.021,0.059,0.338c-0.1,0.313,0.06,0.451,0.475,0.594l0.023,0.006c0,0,0.001,0,0.002,0c-0.079-0.04-0.074-0.08-0.074-0.08s-0.01-0.078,0.078-0.216c0.087-0.136,0.009-0.369-0.293-0.536C47.044,90.118,46.967,90.07,46.772,90.014L46.772,90.014L46.772,90.014z"></path>
+  <path style="&st59;" d="M47.869,91.032c0.006,0,0.012,0,0.018,0C47.881,91.032,47.875,91.032,47.869,91.032z"></path>
+  <path style="&st59;" d="M47.495,91.002c0.001,0,0.004,0,0.004,0C47.497,91.002,47.496,91.002,47.495,91.002z"></path>
+  <path style="&st59;" d="M47.677,91.024c0.004,0,0.007,0,0.01,0C47.684,91.024,47.681,91.024,47.677,91.024z"></path>
+  <path style="&st59;" d="M47.529,91.006c0.001,0,0.002,0,0.002,0S47.53,91.006,47.529,91.006z"></path>
+  <path style="&st59;" d="M47.559,91.01c0.002,0.002,0.005,0.002,0.008,0.002C47.566,91.012,47.561,91.01,47.559,91.01z"></path>
+  <path style="&st59;" d="M47.594,91.016c0.002,0,0.004,0,0.008,0C47.599,91.016,47.596,91.016,47.594,91.016z"></path>
+  <path style="&st59;" d="M46.711,89.997c-0.011-0.002-0.026-0.007-0.043-0.011C46.685,89.99,46.701,89.995,46.711,89.997z"></path>
+  <path style="&st59;" d="M28.748,87.256c-0.008-0.003-0.018-0.007-0.025-0.01C28.731,87.25,28.739,87.254,28.748,87.256z"></path>
+  <path style="&st59;" d="M28.776,87.27c-0.007-0.002-0.016-0.008-0.024-0.012C28.761,87.262,28.769,87.268,28.776,87.27z"></path>
+  <path style="&st59;" d="M28.681,87.23c-0.008-0.004-0.021-0.006-0.03-0.01C28.66,87.225,28.672,87.227,28.681,87.23z"></path>
+  <path style="&st59;" d="M45.66,89.862c0.026,0,0.052,0.005,0.074,0.005C45.711,89.865,45.686,89.865,45.66,89.862z"></path>
+  <path style="&st59;" d="M28.797,87.275c-0.005-0.002-0.013-0.006-0.021-0.008C28.784,87.271,28.792,87.273,28.797,87.275z"></path>
+  <path style="&st59;" d="M27.38,86.794c0.038,0.011,0.074,0.022,0.112,0.031C27.453,86.816,27.417,86.803,27.38,86.794z"></path>
+  <path style="&st59;" d="M27.492,86.825c0.021,0.004,0.039,0.011,0.058,0.019C27.53,86.836,27.512,86.829,27.492,86.825z"></path>
+  <path style="&st59;" d="M28.814,87.284c-0.003-0.002-0.006-0.005-0.012-0.007C28.808,87.279,28.811,87.282,28.814,87.284z"></path>
+  <path style="&st59;" d="M28.64,87.219c-0.011-0.006-0.021-0.01-0.03-0.014C28.621,87.21,28.629,87.213,28.64,87.219z"></path>
+  <path style="&st61;" d="M27.687,87.239c-0.014-0.013-0.023-0.022-0.03-0.032c-0.263-0.197-0.343-0.418-0.343-0.418l0.009-0.011c0.017,0.007,0.034,0.011,0.052,0.014c-0.291-0.075-0.5-0.104-0.479-0.018c0.015,0.061,0.131,0.153,0.269,0.245c0.174,0.094,0.346,0.187,0.522,0.279V87.239L27.687,87.239z"></path>
+  <path style="&st59;" d="M28.716,87.245c-0.009-0.003-0.018-0.006-0.027-0.01C28.699,87.239,28.708,87.242,28.716,87.245z"></path>
+  <path style="&st59;" d="M28.602,87.202c-0.013-0.004-0.027-0.01-0.042-0.015C28.574,87.192,28.59,87.198,28.602,87.202z"></path>
+  <path style="&st59;" d="M28.841,87.293c0,0-0.006-0.004-0.022-0.009C28.835,87.29,28.841,87.293,28.841,87.293z"></path>
+  <path style="&st59;" d="M28.232,87.068c-0.018-0.006-0.031-0.014-0.047-0.018C28.201,87.055,28.215,87.063,28.232,87.068z"></path>
+  <path style="&st59;" d="M45.534,89.858c0.028,0,0.054,0.002,0.081,0.002C45.587,89.858,45.562,89.858,45.534,89.858z"></path>
+  <path style="&st59;" d="M45.397,89.855c0.029,0,0.06,0,0.089,0C45.457,89.855,45.427,89.855,45.397,89.855z"></path>
+  <path style="&st59;" d="M28.348,87.11c-0.034-0.013-0.07-0.025-0.106-0.039C28.278,87.085,28.314,87.098,28.348,87.11z"></path>
+  <path style="&st59;" d="M28.506,87.168c-0.013-0.005-0.025-0.01-0.037-0.015C28.481,87.158,28.494,87.162,28.506,87.168z"></path>
+  <path style="&st59;" d="M28.401,87.129c-0.013-0.006-0.028-0.011-0.043-0.015C28.373,87.118,28.388,87.123,28.401,87.129z"></path>
+  <path style="&st59;" d="M28.557,87.186c-0.013-0.004-0.025-0.012-0.04-0.017C28.531,87.176,28.544,87.182,28.557,87.186z"></path>
+  <path style="&st59;" d="M28.458,87.15c-0.016-0.008-0.031-0.014-0.047-0.018C28.427,87.137,28.442,87.143,28.458,87.15z"></path>
+  <path style="&st59;" d="M70.02,84.389c0.004,0,0.006,0,0.006,0C70.023,84.389,70.023,84.389,70.02,84.389z"></path>
+  <path style="&st59;" d="M70.789,84.087c-0.001,0.003-0.003,0.005-0.004,0.005C70.786,84.09,70.788,84.088,70.789,84.087z"></path>
+  <path style="&st59;" d="M70.762,84.102c-0.002,0.002-0.004,0.002-0.006,0.004C70.758,84.104,70.76,84.104,70.762,84.102z"></path>
+  <path style="&st59;" d="M70.732,84.116c-0.002,0.002-0.004,0.003-0.006,0.004C70.729,84.119,70.73,84.118,70.732,84.116z"></path>
+  <path style="&st59;" d="M70.697,84.133c-0.001,0-0.002,0-0.002,0.002C70.695,84.133,70.696,84.133,70.697,84.133z"></path>
+  <path style="&st59;" d="M70.417,84.249c0.006-0.001,0.009-0.002,0.011-0.002C70.426,84.247,70.423,84.248,70.417,84.249z"></path>
+  <path style="&st59;" d="M70.389,84.262h0.002H70.389z"></path>
+  <path style="&st59;" d="M70.662,84.146L70.662,84.146L70.662,84.146z"></path>
+  <path style="&st59;" d="M70.316,84.289c0.002-0.002,0.007-0.003,0.009-0.003C70.321,84.286,70.318,84.287,70.316,84.289z"></path>
+  <path style="&st59;" d="M70.285,84.299c0.003,0,0.005-0.002,0.009-0.004C70.29,84.297,70.288,84.299,70.285,84.299z"></path>
+  <path style="&st59;" d="M70.354,84.275c0.002-0.002,0.002-0.002,0.002-0.002S70.354,84.273,70.354,84.275z"></path>
+  <path style="&st59;" d="M48.946,90.965c-0.011,0-0.022,0.004-0.033,0.004C48.924,90.969,48.935,90.965,48.946,90.965z"></path>
+  <path style="&st59;" d="M70.256,84.311c0.002,0,0.004,0,0.006-0.002C70.26,84.311,70.258,84.311,70.256,84.311z"></path>
+  <path style="&st59;" d="M70.78,82.41c0.013-0.009,0.026-0.021,0.042-0.032C70.809,82.389,70.793,82.401,70.78,82.41z"></path>
+  <path style="&st59;" d="M70.956,82.279c0.013-0.008,0.022-0.016,0.033-0.023C70.98,82.264,70.969,82.271,70.956,82.279z"></path>
+  <path style="&st59;" d="M70.898,82.321c0.012-0.009,0.022-0.017,0.033-0.022C70.923,82.305,70.91,82.313,70.898,82.321z"></path>
+  <path style="&st2;" d="M70.903,83.794c0.2,0.064,0.106,0.171-0.087,0.278l0.089-0.033c0,0,0.793-0.436,0.458-0.633c-0.338-0.198-1.129-0.275-0.613-0.969l0.02-0.02c-0.442,0.344-0.756,0.727-0.498,1.021C70.27,83.443,70.387,83.629,70.903,83.794z"></path>
+  <path style="&st59;" d="M70.566,84.191c-0.002,0-0.006,0.002-0.008,0.003C70.561,84.193,70.564,84.191,70.566,84.191z"></path>
+  <path style="&st59;" d="M70.631,84.163c-0.002,0.001-0.004,0.002-0.006,0.003C70.627,84.165,70.629,84.164,70.631,84.163z"></path>
+  <path style="&st59;" d="M70.598,84.176c0,0.002-0.004,0.004-0.006,0.004C70.594,84.18,70.598,84.178,70.598,84.176z"></path>
+  <path style="&st59;" d="M70.493,84.223c-0.003,0-0.003,0-0.007,0.002C70.49,84.223,70.49,84.223,70.493,84.223z"></path>
+  <path style="&st59;" d="M70.459,84.233c-0.002,0-0.004,0.002-0.008,0.004C70.455,84.235,70.457,84.233,70.459,84.233z"></path>
+  <path style="&st59;" d="M70.842,82.363c0.012-0.01,0.024-0.018,0.034-0.025C70.866,82.346,70.854,82.354,70.842,82.363z"></path>
+  <path style="&st59;" d="M48.293,91.024c-0.007,0-0.016,0-0.023,0C48.277,91.024,48.286,91.024,48.293,91.024z"></path>
+  <path style="&st59;" d="M48.444,91.014c-0.004,0-0.011,0-0.017,0C48.434,91.014,48.44,91.014,48.444,91.014z"></path>
+  <path style="&st59;" d="M48.369,91.02c-0.009,0-0.017,0-0.027,0C48.352,91.02,48.359,91.02,48.369,91.02z"></path>
+  <path style="&st8;" d="M50.023,89.904c0,0,0.362,0.225,1.673,0.285c0,0,0.45,0,0.468,0.035c0.016,0.028,0.036,0.056-0.068,0.102l0,0c0.06-0.021,0.793-0.254,0.476-0.391c-0.04-0.019-0.063-0.024-0.074-0.028c-0.006,0-0.013,0-0.019-0.003l0,0c-0.008-0.002-0.016-0.002-0.021-0.004c-0.007,0-0.009-0.001-0.013-0.001c-0.006-0.003-0.014-0.003-0.021-0.005c-0.004,0-0.009,0-0.015-0.002c-0.005,0-0.011-0.002-0.017-0.002c-0.004,0-0.011-0.002-0.017-0.004c-0.005,0-0.012,0-0.015,0c-0.008-0.002-0.014-0.002-0.018-0.004c-0.004,0-0.012,0-0.016-0.002c-0.005,0-0.012,0-0.018,0c-0.002-0.004-0.008-0.004-0.01-0.004c-0.011-0.002-0.015-0.003-0.021-0.003c-0.004,0-0.006-0.001-0.008-0.001c-0.01-0.001-0.021-0.001-0.028-0.002c0,0-0.002,0-0.004-0.002c-0.007,0-0.015,0-0.021,0c-0.005-0.002-0.007-0.002-0.012-0.002s-0.011,0-0.02,0c-0.004,0-0.006,0-0.008,0c-0.008-0.002-0.013-0.002-0.019-0.002c-0.003,0-0.007,0-0.009,0c-0.008,0-0.014-0.002-0.018-0.002c-0.005,0-0.009-0.003-0.012-0.003c-0.006,0-0.014,0-0.021,0c-0.004,0-0.004,0-0.006,0c-0.039-0.004-0.08-0.004-0.128-0.009c-0.002,0-0.004,0-0.004,0c-0.009,0-0.015-0.002-0.022-0.002c-0.002,0-0.007,0-0.011,0c-0.007,0-0.013,0-0.021,0c-0.004-0.002-0.01-0.002-0.014-0.002c-0.006,0-0.013-0.002-0.02-0.002c-0.005,0-0.011,0-0.016-0.002c-0.005,0-0.011,0-0.019,0c-0.01,0-0.02-0.002-0.028-0.004c-0.006,0-0.012,0-0.019,0c-0.008,0-0.018-0.002-0.028-0.002c-0.005,0-0.011-0.002-0.016-0.002c-0.008,0-0.019,0-0.024-0.001c-0.006-0.001-0.013-0.001-0.021-0.001c-0.007-0.002-0.018-0.002-0.026-0.004c-0.005,0-0.013-0.001-0.02-0.001c-0.008-0.001-0.018-0.003-0.026-0.003c-0.007-0.001-0.016-0.001-0.023-0.003c-0.01,0-0.019-0.002-0.027-0.002c-0.007-0.002-0.014-0.002-0.021-0.002c-0.012,0-0.024-0.003-0.035-0.005c-0.007,0-0.014,0-0.02,0c-0.02-0.004-0.037-0.006-0.057-0.006c-0.142-0.019-0.271-0.033-0.378-0.055l-0.187-0.025c0,0-0.47-0.014-0.653-0.316c-0.118-0.197,0.457-0.318,0.457-0.318s0.956-0.193,1.917-0.321c0,0-2.542,0.294-2.777,0.626C49.613,89.737,50.023,89.904,50.023,89.904L50.023,89.904z"></path>
+  <path style="&st59;" d="M69.986,84.401L69.986,84.401L69.986,84.401z"></path>
+  <path style="&st59;" d="M69.989,84.399c0.001,0,0.001,0,0.004,0C69.991,84.399,69.99,84.399,69.989,84.399z"></path>
+  <path style="&st59;" d="M48.839,90.979c-0.007,0-0.015,0.002-0.021,0.002C48.824,90.98,48.831,90.979,48.839,90.979z"></path>
+  <path style="&st59;" d="M48.727,90.988c-0.009,0-0.017,0.002-0.026,0.002C48.71,90.99,48.718,90.988,48.727,90.988z"></path>
+  <path style="&st59;" d="M48.637,90.998c-0.009,0-0.019,0.001-0.029,0.003C48.619,90.999,48.628,90.998,48.637,90.998z"></path>
+  <path style="&st59;" d="M48.55,91.004c-0.009,0-0.018,0.002-0.025,0.004C48.532,91.006,48.54,91.004,48.55,91.004z"></path>
+  <path style="&st59;" d="M70.139,84.35c0.004,0,0.006-0.002,0.01-0.004C70.145,84.348,70.143,84.35,70.139,84.35z"></path>
+  <path style="&st59;" d="M70.116,84.359c0.002-0.002,0.004-0.002,0.007-0.002C70.12,84.357,70.118,84.357,70.116,84.359z"></path>
+  <path style="&st59;" d="M70.193,84.332c0.002,0,0.002-0.002,0.004-0.002C70.195,84.33,70.195,84.332,70.193,84.332z"></path>
+  <path style="&st59;" d="M70.068,84.374c0.003,0,0.004,0,0.007-0.002C70.072,84.374,70.07,84.374,70.068,84.374z"></path>
+  <path style="&st59;" d="M70.163,84.342c0.005,0,0.007-0.002,0.011-0.005C70.17,84.34,70.167,84.342,70.163,84.342z"></path>
+  <path style="&st59;" d="M69.996,84.395c0.002,0,0.002,0,0.004,0C69.998,84.395,69.998,84.395,69.996,84.395z"></path>
+  <path style="&st59;" d="M70.004,84.395c0.002,0,0.006-0.002,0.008-0.002C70.006,84.395,70.006,84.395,70.004,84.395z"></path>
+  <path style="&st59;" d="M72.23,81.559c-0.002,0-0.004,0-0.004,0S72.229,81.559,72.23,81.559z"></path>
+  <path style="&st59;" d="M70.053,84.379c0.002,0,0.002,0,0.004,0C70.053,84.379,70.053,84.379,70.053,84.379z"></path>
+  <path style="&st59;" d="M70.036,84.385c0.001,0,0.003,0,0.004-0.002C70.039,84.385,70.037,84.385,70.036,84.385z"></path>
+  <linearGradient id="SVGID_31_" gradientUnits="userSpaceOnUse" x1="414.2451" y1="568.2656" x2="509.0055" y2="545.7273" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset=".9505" stop-color="#B2B4B6"></stop>
+  </linearGradient>
+  <path style="&st51;" d="M7.61,68.141c-0.065-0.062-0.112-0.105-0.139-0.131L7.45,68.021L7.61,68.141L7.61,68.141z"></path>
+  <linearGradient id="SVGID_32_" gradientUnits="userSpaceOnUse" x1="416.6992" y1="578.5645" x2="511.8228" y2="555.9398" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset=".9505" stop-color="#B2B4B6"></stop>
+  </linearGradient>
+  <path style="&st62;" d="M90.412,59.954l0.689,0.108c1.976-1.573,3.869-3.571,3.869-3.571s1.258-1.261,1.889-2.356c0.22-0.381,0.281-0.356,0.271-0.177c0.023-0.09,0.103-0.456-0.038-0.714c-0.094-0.176-0.381,0.288-0.83,0.861c0,0-2.013,2.926-5.798,5.816c-3.786,2.891-4.776,3.771-8.083,5.655c0,0-2.309,1.021-3.914,0.669c-1.246-0.271,0-1.547,0.271-2.699c0.271-1.146,0.063-1.58-0.225-1.807c-0.287-0.225-0.91-0.385-2.142-0.109c0,0-4.709,1.264-6.819,3.307c-1.918,1.854,0.478,2.619,1.021,2.875c0,0,0.78,0.338,0.719,0.672c-0.063,0.336-0.496,0.623-0.733,0.783c-0.239,0.16-3.338,1.977-8.324,2.764c-4.039,0.641-3.26,0.255-7.143,0.654c-3.881,0.399-4.952,0.72-8.068,1.453c-3.116,0.734-4.945,1.537-5.352,2.349c-0.336,0.671,0.479,1.103,0.991,1.407c0.511,0.304,1.423,0.781,1.119,1.293c-0.305,0.512-1.631,1.277-4.874,1.391c-3.243,0.114-4.569-0.336-5.16-1.04c-0.548-0.649-0.08-1.323,0.096-1.946c0,0,0.382-0.814,0.16-1.215c-0.224-0.398-0.737-0.494-1.278-0.559c-0.544-0.064-3.245-0.158-5.337-0.271c-2.372-0.127-5.208-0.211-8.611-0.928c0,0-2.237-0.441-3.69-1.262c-0.096-0.055-0.18-0.107-0.25-0.156c-0.11-0.059-0.194-0.122-0.25-0.209c-0.41-0.432-0.047-0.748-0.186-1.168c-0.121-0.359-0.352-0.878-0.896-1.501c-0.176-0.183-0.428-0.437-0.72-0.713c-0.08-0.069-0.165-0.144-0.254-0.214c-1.276-1.037-1.422-1.149-1.964-1.166c-0.542-0.019-0.235,0.895-0.129,1.246c0.041,0.136,0.139,0.328,0.242,0.508c-0.2-0.364-0.336-0.729-0.257-0.915c0.144-0.337,0.309-0.308,0.309-0.308s0.133-0.053,0.595,0.297c0.463,0.35,1.499,1.078,2.012,1.705c0.512,0.625,0.481,1.18,0.481,1.18s-0.103,0.563,0.451,1.17c0.555,0.604,1.733,1.714,5.859,2.349c0.021,0.005,0.041,0.005,0.06,0.009l0.193-0.549l0.568,0.663c0.006,0,0.01,0.001,0.016,0.002c3.592,0.519,5.544,0.563,5.544,0.563s4.709,0.164,5.982,0.164c1.271,0,1.035,0.664,1.035,0.664s-0.072,0.361-0.225,0.647c-0.153,0.288-0.524,1.365-0.144,1.94c0,0,0.585,1.426,4.382,1.527c0,0,3.324,0.267,5.643-0.688c2.317-0.954,0.224-2.277,0.224-2.277s-0.794-0.483-1.129-0.737c-0.308-0.233-0.184-0.48-0.122-0.646c0.061-0.163,0.297-0.355,0.564-0.492c0.265-0.134,1.241-0.652,5.365-1.722c4.124-1.067,6.587-1.183,6.587-1.183s0.021-0.004,0.062-0.006l0.334-0.656l0.845,0.559c0.732-0.061,1.686-0.129,2.537-0.17c1.691-0.083,4.341-0.475,5.879-0.811c1.539-0.342,3.869-1.059,5.591-1.951c1.724-0.891,1.808-1.519,1.808-1.519s0.328-0.472-0.391-0.995c-0.719-0.521-1.037-0.38-1.672-1.024c-0.638-0.646,0.553-1.619,0.553-1.619s0.615-0.865,4.021-2.177c3.408-1.313,3.98-0.873,4.156-0.728c0.175,0.142,0.195,0.613,0.123,0.933c-0.072,0.316-0.494,1.455-0.721,2.055c-0.227,0.592-0.316,1.402,0.604,1.6c0.924,0.193,2.215-0.009,3.427-0.443c2.893-1.033,3.757-2.295,8.535-5.764c0.011-0.01,0.021-0.016,0.029-0.023L90.412,59.954L90.412,59.954z"></path>
+  <linearGradient id="SVGID_33_" gradientUnits="userSpaceOnUse" x1="415.4736" y1="573.4199" x2="510.5869" y2="550.7977" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset=".9505" stop-color="#B2B4B6"></stop>
+  </linearGradient>
+  <path style="&st52;" d="M100.895,47.596c-0.635,1.186-1.164,2.608-1.443,3.5c-0.045,0.213-0.061,0.33-0.061,0.33s-0.119,0.654-0.054,1.036c0.028,0.161,0.069,0.279,0.106,0.375c0.021,0.052,0.039,0.095,0.055,0.134c0.02,0.045,0.031,0.082,0.033,0.111c0.007,0.082-0.044,0.614-0.27,1.23l0,0c0,0,0,0,0,0.002c-0.063,0.176-0.143,0.359-0.24,0.539c-0.024,0.05-0.053,0.095-0.074,0.139c-0.458,0.814-1.098,1.457-1.604,1.532c-0.324,0.049-0.484-0.117-0.557-0.386c0.014,0.369,0.086,0.738,0.289,0.963c0.406,0.441,1.563-0.795,1.563-0.795s0.688-0.789,0.965-2.062c0.406-1.875,0.187-2.248,0.187-2.248s-0.247-0.389-0.093-0.853c0.152-0.461,1.156-3.047,1.979-4.01l0.502-0.562c0-0.009,0.002-0.02,0.002-0.029l-0.211-0.521c-0.129,0.13-0.259,0.284-0.385,0.454C101.405,46.763,101.178,47.129,100.895,47.596L100.895,47.596z"></path>
+  <linearGradient id="SVGID_34_" gradientUnits="userSpaceOnUse" x1="414.7754" y1="570.4785" x2="509.8697" y2="547.861" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset=".9505" stop-color="#B2B4B6"></stop>
+  </linearGradient>
+  <path style="&st53;" d="M10.564,70.807L10.564,70.807c-0.146-0.092-0.315-0.229-0.469-0.356c-0.133-0.112-0.641-0.585-1.18-1.086c-0.212-0.194-0.411-0.382-0.589-0.55c-0.277-0.262-0.524-0.493-0.688-0.646l0.107,0.358c0.017,0.028,0.034,0.06,0.052,0.089c0.183,0.29,0.854,1.264,2.153,2.277c1.549,1.213,1.559,0.729,1.559,0.729s0.062-0.4-0.296-0.84c-0.063-0.076-0.131-0.167-0.202-0.27v0.002C11.011,70.516,12.023,71.998,10.564,70.807z"></path>
+  <linearGradient id="SVGID_35_" gradientUnits="userSpaceOnUse" x1="414.915" y1="571.0664" x2="510.04" y2="548.4415" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset=".9505" stop-color="#B2B4B6"></stop>
+  </linearGradient>
+  <path style="&st54;" d="M10.678,69.98c0.103,0.186,0.219,0.371,0.333,0.533C11,70.501,10.833,70.253,10.678,69.98z"></path>
+  <linearGradient id="SVGID_36_" gradientUnits="userSpaceOnUse" x1="416.1035" y1="576.0654" x2="511.2286" y2="553.4405" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#EDEDEE"></stop>
+    <stop offset=".4176" stop-color="#FFF"></stop>
+    <stop offset=".6264" stop-color="#F8F9F9"></stop>
+    <stop offset=".9505" stop-color="#B2B4B6"></stop>
+  </linearGradient>
+  <path style="&st55;" d="M96.887,55.023c0,0,0.227-0.76,0.243-1.066c-0.003,0.014-0.005,0.021-0.005,0.021s-0.513,1.443-0.333,2.16C96.771,55.579,96.887,55.023,96.887,55.023z"></path>
+  <path style="&st34;" d="M63.301,4.417l0.728,0.072c1.426-0.402,2.643-0.772,2.643-0.772s1.265-0.41,1.901-0.637c0.635-0.226,1.09-0.313,1.654-0.409c0.565-0.096,1.311-0.14,1.709-0.131c0.4,0.007,0.531,0.122,0.531,0.122s0.166,0.131,0.244,0.27c0.077,0.138,0.74,1.891,2.973,2.005c2.233,0.112,2.263-1.096,2.065-1.464c-0.226-0.427-0.896-0.863-0.896-0.863s-0.899-0.575-1.092-0.847c-0.191-0.27,0.034-0.357,0.104-0.375c0.065-0.017,2.435-0.497,2.729-0.609l0.021-0.021l-0.562-0.171c0,0-0.119,0.134-0.789,0.313c-0.67,0.179-1.235,0.246-1.742,0.313c-0.506,0.066-0.506,0.239-0.506,0.239l-0.086,0.136c-0.025,0.075-0.067,0.321,0.375,0.642c0.528,0.387,1.172,0.75,1.438,1.04s0.586,0.783,0.012,1.137C76.48,4.576,76.27,4.64,75.977,4.671c0.002,0.008,0,0.012,0,0.012c-0.248,0.021-0.457,0.03-0.638,0.03c-0.049,0.002-0.102,0.006-0.155,0.009l-0.017-0.013c-0.506-0.024-0.746-0.142-1.067-0.302c-0.442-0.223-0.758-0.73-0.92-1.087s-0.521-0.662-0.521-0.662c-0.588-0.336-1.695-0.343-2.813-0.15c-1.115,0.193-2.656,0.707-2.925,0.812c-0.268,0.104-1.616,0.551-2.307,0.73c-0.693,0.178-1.222,0.357-1.646,0.47c-0.427,0.111-3.432,1.005-4.556,1.339c-1.126,0.334-1.849,0.46-1.849,0.46c-1.688,0.172-2.193-0.134-2.193-0.134c-0.296-0.124-0.261-0.526-0.261-0.526l0.009-1.147c0,0-0.027-0.433-0.357-0.611c-0.328-0.179-0.779-0.252-1.593-0.29c-0.811-0.038-1.683,0.044-2.093,0.134c-0.408,0.09-1.189,0.313-1.764,0.952c-0.572,0.641-0.481,1.139-0.481,1.139s0.004,0.079,0.01,0.201c0.154,0.245,0.416,0.524,0.862,0.739c1.015,0.485-1.137,1.342-1.137,1.342l0,0c-0.479,0.208-1.191,0.478-2.208,0.777c-2.21,0.647-3.684,0.774-3.684,0.774l0.679,0.254c0,0,5.468-1.016,7.148-2.616c0,0,0.625-0.293,0.021-0.88c-0.606-0.585-0.898-0.761-0.898-0.761s-0.41-0.223,0.02-0.772c0.428-0.546,0.922-0.794,1.352-0.933c0.428-0.135,1.754-0.249,2.925-0.093c0,0,0.491,0.042,0.457,0.407c-0.032,0.365-0.087,0.873-0.077,1.028c0.01,0.157,0.025,0.515,0.399,0.845c0.379,0.332,1.098,0.453,1.098,0.453s1.257,0.228,2.845-0.218c1.586-0.444,3.65-1.141,5.438-1.629L63.301,4.417L63.301,4.417z"></path>
+  <linearGradient id="SVGID_37_" gradientUnits="userSpaceOnUse" x1="412.6152" y1="535.3994" x2="501.5865" y2="514.8846" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".6538" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#CBCCCE"></stop>
+  </linearGradient>
+  <path style="&st56;" d="M91.596,12.992l0.271,0.486c-0.021,0.01-0.034,0.014-0.034,0.014c-1.114,0.515-3.999,1.32-4.271,1.398c-0.271,0.08-0.166,0.234-0.166,0.234c0.029,0.078,0.988,1.106,0.988,1.106c1.178,1.249,0.494,2.296,0.494,2.296s-0.852,1.347-2.844,1.12c-1.993-0.227-2.618-1.767-2.729-2.049c-0.111-0.282-0.197-0.768-0.22-1.095c-0.022-0.33-0.272-0.32-0.272-0.32c-1.069-0.03-2.887,1.089-2.887,1.089c-0.824,0.47-1.682,1.147-3.479,1.958c-0.879,0.396-1.694,0.716-2.287,0.936l-0.967-0.173l0.091,0.482c-0.042,0.013-0.063,0.021-0.063,0.021s-2.268,0.822-4.529,1.553c-2.268,0.732-3.65,0.8-3.65,0.8s-1.336,0.12-2.266-0.266c-0.93-0.386-0.959-0.94-0.939-1.415c0.021-0.476,0.483-1.404,0.623-1.958c0.139-0.555-0.683-0.644-0.683-0.644s-1.958-0.236-3.65,0.296c-1.69,0.535-2.54,1.159-2.54,1.159c-0.91,0.512-0.863,0.957-0.863,0.957s-0.012,0.467,0.551,0.869s1.072,0.505,1.736,0.654c0.668,0.149,0.791,0.311,0.791,0.311c1.08,0.894,0.416,1.785,0.047,2.434c-0.631,1.113-3.674,1.653-3.674,1.653c-3.276,0.758-5.12,1.08-7.827,1.452l-0.876-0.46l-0.276,0.615c-7.089,0.936-10.065,1.877-11.065,2.135c-1,0.257-1.261,0.784-1.369,0.904c-0.108,0.12,0.079,0.171,0.079,0.171c0.427,0.137,0.647,0.442,0.647,0.442s1.399,1.697-0.012,3.108c-1.359,1.36-2.785,1.674-2.785,1.674s-2.177,0.737-4.077,0.341c-1.899-0.399-2.439-1.889-2.416-2.395c0.024-0.505,0.176-1.075,0.694-1.661c0.517-0.585,0.654-0.667,0.654-0.667s0.358-0.279,0.421-0.415c0.079-0.172-0.404-0.233-0.404-0.233c-1.034-0.13-3.496-0.097-5.822,0.131c-1.439,0.14-2.769,0.374-3.578,0.518l-0.223-0.48l-0.543,0.625l-3.12,0.504c-1.514,0.222-2.576-0.028-2.576-0.028s-1.331-0.397-1.479-1.252c-0.147-0.852,0.359-1.87,0.49-2.177c0.13-0.307,0.5-0.934,0.5-0.934s0.068-0.13,0.068-0.34c0-0.211-0.233-0.536-0.233-0.536s-0.205-0.396-0.886-0.38c-0.682,0.018-0.866,0.131-1.144,0.364c-0.044,0.038-0.079,0.081-0.108,0.127c0.021-0.064,0.045-0.117,0.073-0.158c0.21-0.309,0.65-0.668,1.401-0.7c0.748-0.034,1.041,0.228,1.041,0.228c0.719,0.82,0.115,1.845-0.351,2.76c-0.057,0.095-0.155,0.271-0.229,0.483c-0.032,0.076-0.062,0.153-0.087,0.227c-0.358,1.06,0.292,1.565,0.668,1.661c0.376,0.097,1.141,0.57,4.269-0.031c3.13-0.603,3.587-0.731,3.587-0.731s6.145-1.087,8.96-0.425l0.023,0.004c0,0,1.297,0.367,0.331,1.334c-0.966,0.966-1.729,1.617-1.504,2.377c0.223,0.762,1.267,1.903,3.646,1.603c0,0,0.842-0.113,1.105-0.165c1.733-0.336,2.899-1.268,2.899-1.268s0.972-0.721,0.782-1.631c-0.187-0.908-1.017-1.189-1.017-1.189s-0.659-0.424-0.141-1.237c0,0,0.141-0.69,2.553-1.317c2.412-0.626,6.813-1.518,10.555-1.989c3.49-0.408,9.652-1.575,10.89-2.08c1.235-0.508,1.497-1.4,1.521-1.708c0.024-0.31,0.072-0.83-1.14-1.09c-1.213-0.259-1.758-0.655-1.931-0.79c-0.172-0.138-0.545-0.483-0.545-1.275c0-0.791,1.607-1.745,3.392-2.35c1.78-0.606,3.927-0.34,3.927-0.34c1.948,0.167,0.936,1.963,0.936,1.963c-0.074,0.322-0.946,1.785,0.5,2.169c1.541,0.409,4.175-0.347,5.188-0.669c0.829-0.261,3.141-1.074,4.688-1.62c0.352-0.122,0.66-0.231,0.908-0.318c1.656-0.577,3.019-1.219,3.785-1.681c0.771-0.462,2.144-1.297,2.144-1.297s1.351-0.744,1.799-0.808c0.451-0.064,1.619-0.346,1.771,0.771c0.155,1.115,0.631,1.899,1.4,2.271c0.771,0.371,2.064,0.538,3.246-0.231c1.182-0.771,0.359-1.901,0.359-1.901c-0.021-0.028-0.039-0.057-0.062-0.085c-0.151-0.27-0.369-0.518-0.487-0.646c-0.388-0.47-0.736-0.822-0.736-0.822c-0.295-0.436,0.053-0.692,0.425-0.834c0.373-0.141,2.351-0.758,2.351-0.758c1.155-0.383,1.592-0.551,2.053-0.988c0.445-0.419-0.189-1.34-0.239-1.412l0.004-0.002l0.608,0.256c0.136,0.182,0.27,0.362,0.4,0.547l-0.007,0.005c0,0,0.226,0.527,0.054,0.799c-0.095,0.149-0.343,0.293-0.545,0.395L91.596,12.992L91.596,12.992z"></path>
+  <linearGradient id="SVGID_38_" gradientUnits="userSpaceOnUse" x1="390.042" y1="485.6797" x2="390.042" y2="485.6797" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#FFF"></stop>
+    <stop offset=".6538" stop-color="#FFF"></stop>
+    <stop offset="1" stop-color="#CBCCCE"></stop>
+  </linearGradient>
+  <path style="&st57;" d="M-16.122-14.641"></path>
+  <path style="&st34;" d="M57.455,92.28c-0.034-0.042-0.042-0.034-0.012-0.063c0.021-0.021,0.086-0.082,0.115-0.137c0,0-1.17-0.063-2.141,0.077c-0.962,0.137-1.933,0.335-1.933,0.335l0.301,0.146c0,0,0.127-0.055,1.047-0.183c0.921-0.128,1.267-0.128,1.267-0.128s0.808-0.063,0.969-0.063c0.162,0,0.061,0.104,0.061,0.104s-0.078,0.136,0.366,0.124c0,0,0.663-0.027,1.313-0.188C58.809,92.309,57.678,92.544,57.455,92.28z"></path>
+  <path style="&st34;" d="M54.469,92.691c0,0,0.146,0.266-2.923,0.394c0,0,1.788,0.052,3.31-0.198C55.963,92.707,54.469,92.691,54.469,92.691L54.469,92.691z"></path>
+  <path style="&st34;" d="M13.114,9.856c0,0-0.005,0.096,0.019,0.131c0.024,0.034,0.107,0.112,0.107,0.112s0.328-0.5,0.194-0.438c0,0-0.119,0.187-0.164,0.23c0,0-0.029,0.04-0.088,0.077C13.123,10.006,13.117,9.991,13.114,9.856L13.114,9.856z"></path>
+  <path style="&st34;" d="M10.595,12.501c0,0-0.388,0.46,0,0.52l0.528-0.527c0,0,0.139-0.234,0.139-0.398C11.263,12.095,10.752,12.343,10.595,12.501L10.595,12.501z"></path>
+  <path style="&st12;" d="M-16.122-14.641"></path>
+  <path style="&st31;" d="M21.093,23.707c1.227,0.146,1.499-0.132,1.527-0.172c0.294-0.003,1.475-0.034,2.865-0.207c1.685-0.21,3.564-0.891,3.564-0.891s1.554-0.568,2.096-1.18l0.016-0.002c0,0-0.693-0.6-1.057-1.122c0,0-0.286-0.557,0.027-1.035c0.316-0.479,0.836-1.008,2.344-1.643c1.506-0.636,2.356-0.514,2.356-0.514s0.594-0.006,1.007,0.45c0.415,0.458,0.649,1.006,0.649,1.006s0.029,0.38-0.115,0.63c-0.141,0.251-0.155,0.277-0.155,0.277s0.049,0.017,0.378-0.007c0.329-0.021,1.165-0.142,2.67-0.506c1.508-0.363,3.407-0.972,3.407-0.972s4.9-1.578,5.407-1.714c0.507-0.135,1.357-0.436,1.357-0.436l0.027,0.059c0,0,0.405,0.663,0.392,1.269V16.94c0,0-0.021,0.301-0.698,0.818c-0.674,0.517-1.226,0.479-1.678,0.442c-0.452-0.039-0.665-0.071-0.794-0.045l-0.72,0.04c-0.787,0.111-1.224,0.407-1.224,0.407c-1.804,1.065,0.731,9.287,0.731,9.287c-3.742,0.47-8.143,1.363-10.555,1.989c-2.412,0.627-2.553,1.317-2.553,1.317c-0.519,0.813,0.141,1.236,0.141,1.236s0.829,0.283,1.017,1.19c0.19,0.91-0.783,1.629-0.783,1.629s-1.159,0.97-2.898,1.268c-1.738,0.298-2.396,0.35-3.429-0.47c-0.91-0.721-0.297-1.864,0.312-2.301c0.612-0.438,0.909-0.91,0.988-1.113c0.079-0.203,0.032-0.376,0.032-0.376l-0.58-0.534c-2.005-1.33-9.884,0.063-9.884,0.063c-0.213-1.169-0.362-1.171-0.282-3.117c0.051-1.244,0.291-1.752,0.291-1.752l0.058-0.164c0,0,0.448-1.443,1.141-2.44c0,0,0.602-0.172,1.364-0.349C20.616,23.793,21.093,23.707,21.093,23.707L21.093,23.707z"></path>
+  <linearGradient id="SVGID_39_" gradientUnits="userSpaceOnUse" x1="231.2324" y1="-407.8711" x2="263.6191" y2="-407.8711" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#696969"></stop>
+    <stop offset=".3702" stop-color="#2E2E2E"></stop>
+    <stop offset=".4554" stop-color="#424242"></stop>
+    <stop offset=".6014" stop-color="#303030"></stop>
+    <stop offset=".6947" stop-color="#4A4A4A"></stop>
+    <stop offset="1" stop-color="#666666"></stop>
+  </linearGradient>
+  <path style="&st26;" d="M49.855,16.94c0,0-4.085,1.326-3.891,1.254c0,0-0.39,0.075-0.686,0.161c-0.294,0.086-0.539,0.247-0.539,0.247s-3.288,1.222-6.438,1.848c-3.148,0.627-2.977-0.361-2.708-0.83c0.232-0.409,0.829-1.112-0.188-1.254c-1.019-0.14-1.788,0.251-2.21,0.439c-0.422,0.189-3.162,1.362-1.251,2.254c0,0,1.423,0.642-0.377,1.755c0,0-1.816,1.16-5.355,1.77c0,0-0.565,0.063-1.88,0.111c-1.316,0.046-2.558,0.213-4.12,0.658c-1.378,0.391-1.992,0.579-2.744,1.065l0.194-0.501l0.2-0.462c1.069-0.533,3.719-1.288,5.717-1.378c1.997-0.089,2.908-0.16,4.721-0.624c2.134-0.546,2.702-1.019,2.93-1.163c0.194-0.123,0.771-0.479,0.493-0.633c-0.359-0.199-0.895-0.729-0.895-0.729c-0.217-0.256-0.39-0.373-0.158-1.046c0.356-1.029,2.196-1.644,2.196-1.644s1.028-0.534,2.334-0.514c1.305,0.021,1.287,0.752,1.287,0.752s0.062,0.34-0.268,0.827c0,0-0.503,0.579-0.049,0.656c0.454,0.081,1.622,0.179,5.48-1.028c3.859-1.207,8.085-2.611,8.085-2.611S49.855,16.66,49.855,16.94L49.855,16.94z"></path>
+  <linearGradient id="SVGID_40_" gradientUnits="userSpaceOnUse" x1="231.623" y1="-407.063" x2="263.4941" y2="-407.063" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#525252"></stop>
+    <stop offset=".1856" stop-color="#333333"></stop>
+    <stop offset=".354" stop-color="#AEAEAE"></stop>
+    <stop offset=".4199" stop-color="#ADADAD"></stop>
+    <stop offset=".4276" stop-color="#9D9D9D"></stop>
+    <stop offset=".4433" stop-color="#818181"></stop>
+    <stop offset=".4611" stop-color="#6A6A6A"></stop>
+    <stop offset=".4814" stop-color="#585858"></stop>
+    <stop offset=".506" stop-color="#4C4C4C"></stop>
+    <stop offset=".539" stop-color="#444444"></stop>
+    <stop offset=".6166" stop-color="#424242"></stop>
+    <stop offset=".6684" stop-color="#454545"></stop>
+    <stop offset="1" stop-color="#BDBDBD"></stop>
+  </linearGradient>
+  <path style="&st58;" d="M31.145,21.257c-0.541,0.612-2.096,1.18-2.096,1.18s-1.88,0.68-3.564,0.891c-1.608,0.201-2.777,0.209-2.777,0.209l-0.082-0.002c-0.175,0.145-0.483,0.188-0.728,0.21c-0.244,0.023-0.806-0.039-0.806-0.039s-2.156,0.432-2.603,0.616c0,0-0.253,0.392-0.331,0.539c-0.08,0.146-0.299,0.594-0.299,0.594c1.069-0.534,3.718-1.289,5.717-1.379c1.997-0.089,2.908-0.159,4.721-0.624c2.134-0.546,2.702-1.019,2.929-1.163c0.195-0.123,0.771-0.479,0.493-0.633c-0.358-0.199-0.894-0.729-0.894-0.729c-0.217-0.256-0.391-0.373-0.158-1.046c0.356-1.029,2.196-1.644,2.196-1.644s1.028-0.533,2.333-0.514c1.306,0.021,1.287,0.753,1.287,0.753s0.062,0.34-0.269,0.826c0,0-0.503,0.579-0.049,0.657c0.455,0.08,1.622,0.178,5.48-1.028c3.858-1.208,8.085-2.612,8.085-2.612c-0.098-0.29-0.296-0.652-0.296-0.652s-0.85,0.301-1.358,0.436c-0.506,0.136-5.407,1.714-5.407,1.714s-1.9,0.608-3.407,0.972c-1.506,0.364-2.342,0.485-2.671,0.508c-0.329,0.021-0.378,0.006-0.378,0.006s0.013-0.027,0.156-0.279c0.144-0.248,0.115-0.629,0.115-0.629s-0.235-0.548-0.649-1.006c-0.414-0.457-1.007-0.45-1.007-0.45s-0.849-0.121-2.355,0.514c-1.508,0.636-2.029,1.164-2.346,1.643c-0.312,0.478-0.026,1.035-0.026,1.035c0.365,0.521,1.057,1.122,1.057,1.122"></path>
+  <linearGradient id="SVGID_41_" gradientUnits="userSpaceOnUse" x1="236.917" y1="-417.333" x2="235.8882" y2="-410.5272" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)">
+    <stop offset="0" stop-color="#969696"></stop>
+    <stop offset="1" stop-color="#000"></stop>
+  </linearGradient>
+  <path style="&st24;" d="M21.606,31.241c0,0-0.064-0.328,0.172-0.939c0.234-0.611,0.908-0.595,1.362-0.503c0.455,0.095,0.846,0.298,1.472-0.124c0.627-0.423,0.47-1.583,0.046-2.852c-0.423-1.267-0.328-2.128-0.328-2.128l-0.608-0.649l-0.237,0.696c0.047,1.316,0.657,3.226,0.829,3.759c0.173,0.533,0.297,0.8-0.735,0.517c-1.034-0.282-1.519,0.125-1.519,0.125c-1.332,0.862-1.082,2.161-1.082,2.161L21.606,31.241z"></path>
+  <path style="&st60;" d="M27.498,36.633c-0.264-1.763-0.917-2.749-0.917-2.749c-0.25,0.188-0.513,0.693-0.513,0.693s0.179,0.208,0.471,1.568c0,0,0.141,0.106,0.438,0.279C27.273,36.597,27.498,36.633,27.498,36.633z"></path>
+  <path style="&st60;" d="M33.152,32.881c0,0-0.78,0.907-0.378,2.336c0,0,0.454-0.379,0.585-0.68c0,0-0.145-0.458,0.138-1.017C33.5,33.52,33.37,33.1,33.152,32.881L33.152,32.881z"></path>
+  <linearGradient id="SVGID_42_" gradientUnits="userSpaceOnUse" x1="428.7803" y1="532.0527" x2="429.5303" y2="524.4692" gradientTransform="matrix(1 0 0 1 -406.1641 -500.3203)">
+    <stop offset="0" stop-color="#333333"></stop>
+    <stop offset=".431" stop-color="#000"></stop>
+    <stop offset="1" stop-color="#2E2E2E"></stop>
+  </linearGradient>
+  <path style="&st3;" d="M21.2,31.253c0.017-1.299,0.471-1.492,0.905-1.818c0.436-0.328,1.326-0.024,1.326-0.024s0.678,0.218,1.046-0.1c0.369-0.319-0.017-1.467-0.217-2.123c-0.202-0.653-0.41-1.599-0.445-2.262c-0.025-0.489-0.091-0.707-0.125-0.789l-0.205,0.604c0.047,1.316,0.657,3.226,0.829,3.759c0.173,0.533,0.297,0.8-0.735,0.517c-1.035-0.282-1.519,0.125-1.519,0.125c-1.332,0.862-1.082,2.162-1.082,2.162l0.259-0.027L21.2,31.253L21.2,31.253z"></path>
+  <path style="&st21;" d="M26.239,34.29c0.045,0.06,0.421,0.597,0.736,2.113l0.005,0.025c0.294,0.17,0.519,0.205,0.519,0.205c-0.264-1.763-0.917-2.749-0.917-2.749C26.46,33.977,26.336,34.143,26.239,34.29L26.239,34.29z"></path>
+  <path style="&st21;" d="M33.152,32.881c0,0-0.78,0.907-0.378,2.336c0,0,0.125-0.104,0.262-0.248l0.021-0.051c-0.304-1.033,0.283-1.763,0.283-1.763l0.004-0.003C33.291,33.053,33.225,32.957,33.152,32.881z"></path>
+  <path style="&st10;" d="M17.159,8.189h0.117c-0.16,0.481-0.789,1.141-1.068,1.583c-0.156,0.248-0.252,0.572-0.474,0.751c0.038,0.043-0.003,0.003,0.04,0.04c0.088,0.052,0.813-0.139,0.95-0.236c0.082,0.015,0.076,0.011,0.12,0.039c0.042,0.07-0.481,0.991-0.595,1.109v0.04c0.196-0.023,0.502-0.056,0.634-0.16c0.383-0.299,0.47-0.937,0.75-1.346c-0.013-0.066-0.026-0.132-0.04-0.196c-0.222-0.04-0.681,0.02-0.87,0.157h-0.039c0.091-0.473,0.868-1.346,1.146-1.741c0.454-0.647,0.881-1.269,1.345-1.9c0.243-0.331,0.585-0.622,0.831-0.949c0.276-0.367,0.569-0.85,0.949-1.107V4.194h-0.158c-0.186,0.135-0.675,0.218-0.908,0.354c0.032,0.135,0.019,0.101,0.118,0.158c-0.139,0.386-0.598,0.673-0.832,0.991c-0.371,0.5-0.784,0.968-1.147,1.464c-0.123,0.164-0.205,0.421-0.356,0.553c-0.237,0.208-0.913,0.185-1.185,0.396h-0.08c0.056-0.332,0.907-1.392,1.147-1.622v-0.04c-0.408,0.057-0.724,0.273-0.989,0.473c0.044,0.091,0.037,0.073,0.12,0.12c-0.145,0.238-0.361,0.415-0.515,0.633c-0.197,0.275-0.305,0.602-0.514,0.871c0.014,0.04,0.028,0.077,0.04,0.118C15.948,8.641,17.001,8.307,17.159,8.189L17.159,8.189z M51.936,13.534c0.199,0.066,0.396,0.131,0.596,0.197c0.159,0.002,0.327-0.002,0.432-0.04c-0.009-0.654-0.364-0.913-0.593-1.345c-0.113-0.22-0.175-0.523-0.355-0.673c0.069-0.242,0.727-0.308,0.988-0.396c0.643-0.211,1.371-0.422,2.02-0.633c0.305-0.099,0.664-0.077,0.91-0.236c0.146,0.015,0.22,0.029,0.277,0.118c0.143,0.212,0.26,1.667,0.156,2.097c-0.398,1.663-0.896,2.963-1.938,3.958v0.039c0.385-0.062,0.568-0.436,0.83-0.632c1.051-0.794,1.762-1.972,2.137-3.444c0.221-0.865-0.14-1.713,0.199-2.452h-0.039c-0.074-0.188-1.082-0.553-1.388-0.555c-0.164,0.177-0.399,0.416-0.634,0.515c-0.357,0.152-0.838,0.109-1.146,0.315c-0.287-0.024-0.506-0.57-0.315-0.791c-0.011-0.09-0.009-0.112-0.04-0.158c-0.239-0.191-0.85-0.171-1.268-0.158c-0.133,0.125-0.252,0.15-0.314,0.358h-0.039c-0.021,0.076,0.02,0.05,0.039,0.078c0.025,0.016,0.163-0.007,0.236,0.04c0.449,0.047,0.438,0.566,0.675,0.831c-0.027,0.069-0.011,0.04-0.042,0.08c-0.155,0.123-1.301,0.453-1.543,0.515c-0.185,0.046-0.414-0.068-0.553-0.081c-0.336-0.028-0.633,0.16-0.831,0.277c0.107,0.157,0.434,0.118,0.554,0.276C51.368,12.193,51.556,12.913,51.936,13.534L51.936,13.534z M59.807,12.977c0.187,0.093,0.187,0.332,0.278,0.514c0.185,0.371,0.437,0.82,0.554,1.228v0.316c0.092,0.252,0.396,0.284,0.596,0.435c0.156-0.021,0.214-0.061,0.314-0.118c-0.066-0.753-0.525-1.378-0.791-1.979c-0.08-0.188-0.207-0.452-0.236-0.633c-0.021-0.109,0.063-0.169,0-0.276c-0.051-0.123-0.072-0.085-0.156-0.159c-0.059-0.04-0.031-0.016,0-0.078c0.068-0.144,0.213-0.287,0.275-0.436c0.133-0.313,0.127-0.576,0.396-0.751c-0.04-0.41-0.639-0.457-1.107-0.435c-0.057,0.042-0.156,0.064-0.24,0.077c0.05,0.103,0.082,0.124,0.199,0.157c0.113,1.161-0.699,2.225-1.229,2.928c-0.208,0.279-0.556,0.456-0.75,0.754h-0.04v0.038C58.395,14.473,59.54,13.383,59.807,12.977L59.807,12.977z M12.407,22.83c-0.081,0.017-0.076,0.009-0.117,0.039c-0.288,0.148-0.773,1.426-1.346,1.069c-0.292,0.002-0.319-0.055-0.476-0.16c0.02-0.376,0.659-1.063,0.913-1.226c0.031-0.604,0.187-1.252,0.118-1.819c-0.041-0.014-0.08-0.028-0.118-0.039c-0.14-0.046-0.25,0.168-0.357,0.276c-0.29,0.291-0.648,0.597-0.871,0.949c-0.337-0.003-1.414-0.013-1.623,0.119H8.411c0.099-0.256,0.86-1.096,0.633-1.464c-0.013-0.041-0.025-0.08-0.04-0.12c-0.722,0.002-1.592,1.287-1.82,1.82c-0.115,0.266-0.115,0.573-0.276,0.791v0.04c0.54-0.066,1.082-0.133,1.622-0.199c0.205-0.044,0.487,0.052,0.633-0.039h0.554c0.092-0.118,0.184-0.238,0.277-0.356c0.33-0.349,0.768-0.702,1.028-1.106h0.119v0.551c-0.303,0.273-0.773,0.695-0.91,1.108v0.316c-0.203,0.88,0.223,1.329,0.99,1.267c0.5-0.466,1.324-0.848,1.226-1.779C12.405,22.833,12.444,22.873,12.407,22.83L12.407,22.83z M7.819,22.118H7.58c0.109-0.436,0.537-0.935,1.069-0.95v0.197c-0.185,0.239-0.369,0.475-0.554,0.713C7.982,22.076,7.88,22.076,7.819,22.118z M93.044,22.315c-0.164-0.405-0.294-0.722-0.475-1.068c-0.3-0.574-0.613-1.414-1.464-1.425c-0.211,0.179-0.435,0.322-0.555,0.593c-0.777,1.762,0.819,3.747,1.543,4.71c0.256,0.339,0.557,0.712,0.948,0.908c-0.091,1.376-1.269,1.813-2.53,1.267c-0.899-0.386-1.617-1.237-2.179-1.979c-0.188-0.249-0.481-0.457-0.672-0.713c-0.177-0.239-0.304-0.507-0.515-0.713v-0.039h-0.078c0.107,0.426,0.354,0.815,0.514,1.188c0.669,1.538,1.52,2.614,2.811,3.521c0.608,0.428,1.621,1.104,2.494,0.475C94.412,27.942,93.669,23.851,93.044,22.315L93.044,22.315z M92.928,24.216c-0.104,0.1-0.539-0.419-0.635-0.515c-0.441-0.443-1.329-1.221-0.83-1.979h0.197c0.388,0.403,0.746,0.788,0.99,1.344c0.129,0.299,0.152,0.805,0.313,1.069C92.941,24.205,92.958,24.175,92.928,24.216L92.928,24.216z M66.693,32.128v-0.395c0.179-0.801,0.137-1.765,0.314-2.572c0.241-1.088-0.101-2.148,0.99-2.414c0.021-0.106,0.057-0.148,0-0.238c-0.025-0.067-0.009-0.039-0.04-0.079c-0.043-0.031-0.038-0.024-0.116-0.039c-0.305,0.222-1.131,0.373-1.543,0.474c-0.313,0.076-0.639,0.01-0.871,0.158c-0.039,0.013-0.078,0.027-0.119,0.04c0.014,0.079,0.025,0.159,0.041,0.237c0.451,0.147,0.867-0.031,1.067,0.356c0.13,0.252,0.112,1.157,0.039,1.504c-0.251,1.163-0.146,2.491-0.396,3.64c-0.086,0.397,0.022,1.171-0.157,1.463v0.08c-0.241-0.115-0.397-0.426-0.554-0.633c-0.354-0.467-0.875-0.84-1.229-1.305c-0.213-0.281-0.437-0.617-0.712-0.833c0.037-0.761,0.259-1.56,0.438-2.254c0.131-0.522,0.135-1.005,0.395-1.386c0.148-0.217,0.505-0.355,0.751-0.475c-0.002-0.1-0.004-0.146-0.04-0.198c-0.014-0.04-0.023-0.079-0.037-0.119c-0.543,0.081-1.003,0.341-1.505,0.475c-0.454,0.123-0.911,0.092-1.269,0.276c0.012,0.091,0.01,0.112,0.041,0.158c0.014,0.039,0.024,0.08,0.039,0.118c1.391-0.078,1.18,0.678,0.912,1.742c-0.084,0.326-0.029,0.775-0.199,1.028v0.079h-0.039c-0.285-0.433-0.713-0.852-1.067-1.227c-0.146-0.132-0.291-0.264-0.435-0.395c-0.104-0.137-0.16-0.312-0.278-0.436c0.024-0.437,0.38-0.549,0.713-0.672c-0.015-0.183-0.052-0.206-0.118-0.317c-1.031,0.151-1.927,0.73-3.086,0.791v0.041h-0.04c0.004,0.1,0.004,0.146,0.04,0.197v0.079c0.227,0.039,0.564-0.054,0.831,0.04c0.427,0.15,0.81,0.648,1.067,0.99c0.388,0.513,0.996,0.949,1.384,1.463c0.204,0.274,0.434,0.634,0.713,0.832c-0.038,0.696-0.229,1.428-0.396,2.058c-0.086,0.323-0.035,0.735-0.197,0.988c-0.025,0.069-0.01,0.039-0.041,0.08c-0.377-0.718-1.104-1.265-1.582-1.9c-0.918-1.22-1.938-2.319-2.889-3.521c0-0.167,0.01-0.268,0.08-0.356c0.073-0.229,0.359-0.443,0.633-0.476c0.015-0.12,0.033-0.135-0.039-0.238c-0.016-0.038-0.026-0.077-0.041-0.118c-0.803,0.123-1.521,0.497-2.293,0.714c-0.401,0.112-0.928,0.057-1.229,0.276c-0.04,0.013-0.08,0.026-0.117,0.04c0.021,0.152,0.061,0.176,0.117,0.277c0.314-0.005,0.646-0.092,0.949,0c0.793,0.241,1.361,1.137,1.818,1.742c0.201,0.266,0.513,0.483,0.713,0.751c0.849,1.129,1.808,2.146,2.65,3.285c0.328,0.442,0.771,0.825,1.066,1.304c0.179-0.004,0.216-0.025,0.316-0.079c0.213-0.929,0.332-1.866,0.596-2.81c0.119-0.432,0.269-0.942,0.314-1.424c0.327,0.117,0.592,0.607,0.793,0.871c0.618,0.821,1.491,1.502,2.057,2.373c0.164-0.007,0.182-0.026,0.277-0.078C66.352,34.819,66.521,33.473,66.693,32.128L66.693,32.128z M4.297,38.894c-0.013,2.467-0.142,6.269,1.781,6.806c0.7,0.193,1.087-0.271,1.306-0.595c0.786-1.17,0.565-3.446-0.199-4.43c-0.339,0.034-0.825,0.84-0.988,1.106c-0.082-0.016-0.075-0.011-0.119-0.04c-0.091-0.041-0.085-0.066-0.159-0.118c-0.06-0.933,0.127-1.802,0.159-2.691c1.044,0.102,1.941,0.696,3.007,0.751c-0.001-0.185,0-0.434,0.077-0.552c-0.009-0.092-0.007-0.112-0.04-0.16c-0.145-0.115-0.949-0.306-1.186-0.315v-0.04h-0.04c0.146-1.174-0.186-2.082-0.99-2.414c-0.449-0.08-0.897-0.16-1.346-0.239c-0.229-0.083-0.341-0.266-0.514-0.395c-0.058-0.38-0.133-0.806,0.159-1.029c-0.021-0.142-0.032-0.168-0.119-0.238v-0.039h-0.04c-0.133,0.228-0.245,0.493-0.315,0.792c-0.234,0.983,0.309,1.818,0.909,2.018c0.397,0.065,0.792,0.132,1.188,0.197c0.314,0.122,0.453,0.379,0.671,0.595c-0.009,0.512-0.5,0.568-0.91,0.435c-0.64-0.208-1.321-0.353-1.977-0.592c-0.172-0.064-0.333-0.17-0.555-0.199c-0.027,0.25-0.054,0.501-0.08,0.751c0.554,0.171,1.109,0.343,1.662,0.515c-0.023,1.398-0.574,3.074,0.119,4.153c0.084,0.021,0.143,0.037,0.198,0.08c0.78-0.054,0.943-0.68,1.345-1.108c0.342,0.82,0.086,2.253-0.671,2.453c-0.326,0.224-0.803-0.066-0.989-0.237c-0.648-0.599-0.785-1.511-1.027-2.532c-0.083-0.344,0.033-1.042-0.118-1.307c0.006-0.404,0.092-1.134-0.12-1.344v-0.039L4.297,38.894L4.297,38.894z M99.336,45.543c-0.143-0.666,0.055-1.478-0.08-2.097v-0.633c-0.097-0.453-0.059-1.056-0.156-1.502c-0.189-0.882-0.022-1.926-0.355-2.652c-0.197-0.047-0.393-0.084-0.671-0.08c-0.103,0.104-0.179,0.158-0.159,0.278c0.083,0.359,0.547,0.432,0.673,0.792c0.015,0.172,0.026,0.343,0.04,0.514c0.133,0.561,0.111,1.286,0.236,1.86v0.475c0.063,0.289,0.16,1.036,0.078,1.267c-0.139,0.41-0.584,0.78-0.868,1.068c-0.754,0.755-1.64,1.715-2.97,1.859c-0.025,0.068-0.01,0.039-0.041,0.08c0.022,0.494,0.476,0.396,0.793,0.594c0.08-0.014,0.158-0.028,0.236-0.042c0.122-0.074,0.191-0.242,0.276-0.356c0.2-0.261,0.563-0.399,0.751-0.671h0.04c0.002,1.205,0.028,2.561,0.04,3.718h0.117c0.272-1.172,0.252-2.61,0.238-4.039c0.521-0.486,0.853-1.19,1.385-1.66v-0.078h0.041c0.007,1.251,0.037,2.529,0.037,3.797c0,0.407-0.102,1.378,0,1.544v0.237h0.08c0.106-0.425,0.133-1.023,0.236-1.501v-0.674C99.451,47.107,99.45,46.078,99.336,45.543L99.336,45.543z M28.313,48.073c-0.347-0.144-0.776-0.461-0.989-0.751c-0.637-0.875-1.337-2.948-0.87-4.51c0.379-1.266,1.202-2.127,2.532-2.454c0.673-0.163,1.397,0.062,1.821,0.278c1.208,0.622,1.725,1.506,2.096,2.967c0.48,1.887-0.132,3.619-1.385,4.194c0.078,0.764,0.158,1.529,0.237,2.295c0.444-0.013,0.852-0.105,1.227-0.199c0.629-0.155,1.271-0.176,1.86-0.315c0.431-0.104,1.052-0.015,1.384-0.237c0.079-0.17-0.247-1.35-0.315-1.623c-0.057-0.229-0.009-0.461-0.119-0.633v-0.079c-0.091,0.012-0.185,0.025-0.277,0.039c0.018,1.195-0.834,1.032-1.781,1.267c-0.473,0.119-1.049,0.27-1.581,0.276c0-0.17,0-0.343-0.001-0.514c0.045-0.083,0.739-0.386,0.868-0.476c0.461-0.318,0.931-0.826,1.229-1.304c0.202-0.327,0.366-0.764,0.473-1.149c0.138-0.489,0.154-1.17,0.041-1.662c-0.079-0.338-0.048-0.603-0.158-0.91c-0.427-1.187-1.322-2.054-2.453-2.532c-0.513-0.216-1.093-0.224-1.7-0.356c-0.539-0.116-1.509,0.124-1.901,0.238c-1.905,0.562-3.198,1.48-3.799,3.323c-0.236,0.728-0.163,1.736,0.04,2.414c0.467,1.561,1.773,3.02,3.72,3.047v0.039c0.062,0.088,0.06,0.264,0.118,0.355c-0.024,0.067-0.009,0.039-0.04,0.08c-0.164,0.111-0.404,0.061-0.633,0.117c-0.47,0.118-1.986,0.486-2.334,0.158c-0.222-0.072-0.37-0.363-0.396-0.632c-0.099,0.004-0.146,0.004-0.197,0.039h-0.082c0.146,0.767,0.291,1.53,0.435,2.296h0.041v0.04c1.385-0.239,2.77-0.478,4.154-0.713c-0.198-0.728-0.395-1.451-0.593-2.179C28.873,48.139,28.517,48.159,28.313,48.073L28.313,48.073z M96.014,43.682c0.086,0.218,0.254,0.58,0.435,0.712c0.086,0.063,0.132,0.04,0.198,0.119c1.31,0.113,1.842-2.143,1.308-3.442c-0.095-0.225-0.517-0.885-0.911-0.633h-0.08c0.026-0.069,0.01-0.038,0.04-0.08c-0.001-0.188-0.021-0.25-0.077-0.356c-0.08-0.013-0.156-0.026-0.238-0.038c-0.039,0.031-0.01,0.014-0.078,0.038c0.027,0.24,0.111,0.247,0.119,0.514C96.09,41.099,95.545,42.497,96.014,43.682z M96.446,41.585c0.088-0.213,0.265-0.35,0.396-0.515c0.082-0.015,0.075-0.01,0.117-0.04c0.301-0.001,0.385,0.057,0.555,0.159c0.301,0.711,0.346,1.954-0.156,2.494c-0.077,0.085-0.229,0.116-0.315,0.197C96.214,43.817,96.146,42.305,96.446,41.585L96.446,41.585z M78.092,57.168c-0.445-0.273-0.507-1.675-0.673-2.294c-0.327-1.215-0.483-2.489-0.831-3.72c-0.223-0.788-0.523-1.605-0.435-2.572c0.139-0.138,0.231-0.32,0.396-0.436c0.223-0.154,0.58-0.229,0.752-0.436c0.027-0.051-0.019-0.128-0.041-0.238c-0.459,0.074-0.879,0.35-1.267,0.515c-0.792,0.337-1.567,0.536-2.373,0.87c-0.252,0.104-0.515,0.282-0.833,0.315v0.238c0.027,0.014,0.055,0.025,0.08,0.04c0.042,0.045,1.033-0.392,1.346-0.118c0.356,0.125,0.311,0.59,0.514,0.872c-0.061,0.614-0.672,1.558-0.912,2.097c-0.58,1.326-1.17,2.592-1.816,3.836c-0.248,0.477-0.543,1.334-0.871,1.701v0.039c-0.216-0.296-0.199-1.051-0.314-1.462c-0.353-1.235-0.578-2.591-0.951-3.798c-0.068-0.23-0.305-1.279-0.156-1.503c0.051-0.546,0.822-0.785,1.266-0.95c-0.012-0.092-0.024-0.186-0.039-0.277c-0.701,0.105-1.429,0.479-2.058,0.713c-0.595,0.223-1.14,0.313-1.741,0.516c-0.298,0.102-0.636,0.275-0.986,0.314v0.041h-0.041c0.015,0.112,0.025,0.172,0.078,0.237c0.162,0.107,1.03-0.352,1.386-0.077c0.557,0.19,0.573,1.075,0.752,1.66c0.481,1.579,0.728,3.327,1.187,4.947c0.115,0.404,0.391,1.686,0.119,2.018c-0.148,0.439-0.885,0.615-1.306,0.791c0.014,0.08,0.024,0.159,0.036,0.237c0.609-0.09,1.162-0.373,1.707-0.56c1.063-0.354,2.066-0.65,3.089-1.029c-0.017-0.092-0.027-0.186-0.041-0.275c-0.437,0.116-0.773,0.248-1.386,0.236c-0.08-0.068-0.157-0.133-0.235-0.199c-0.067-0.21-0.134-0.422-0.199-0.632c0.043-0.499,0.683-1.421,0.91-1.86c0.673-1.293,1.262-2.6,1.856-3.955c0.229-0.519,0.754-1.275,0.832-1.857c0.184,0.111,0.133,0.438,0.197,0.672c0.135,0.475,0.211,0.98,0.355,1.503c0.281,1,0.391,2.075,0.673,3.126c0.104,0.387,0.261,1.048,0.08,1.464c-0.179,0.404-0.841,0.673-1.267,0.83c0.017,0.084,0.037,0.183,0.08,0.238c0.004,0.007,0.906-0.288,1.064-0.354c1.104-0.471,2.236-0.959,3.361-1.386c-0.015-0.093-0.024-0.187-0.039-0.274C79.117,57.043,78.477,57.407,78.092,57.168L78.092,57.168z M96.803,60.498c-0.143,0.13-0.354,0.163-0.514,0.277c-0.501,0.359-1.025,0.962-1.385,1.463c-0.288,0.402-0.534,0.843-0.791,1.268c-0.112,0.188-0.137,0.402-0.277,0.553v0.08c0.346-0.059,0.549-0.283,0.792-0.436c0.659-0.408,1.249-0.781,1.858-1.225c0.295-0.217,0.515-0.551,0.83-0.754c0.029-0.473,0.125-0.844-0.077-1.188C97.115,60.512,96.975,60.496,96.803,60.498z M96.329,61.921c-0.239,0.177-0.47,0.423-0.712,0.595c-0.208,0.146-0.458,0.224-0.633,0.396h-0.04c0.13-0.408,0.817-1.107,1.146-1.344c0.17-0.124,0.383-0.157,0.557-0.279h0.156c0.036,0.046,0.034,0.044,0.08,0.08C96.846,61.667,96.523,61.774,96.329,61.921L96.329,61.921z M10.825,63.465c-0.166-0.502-0.278-0.99-0.435-1.465c-0.079-0.246-0.062-0.525-0.199-0.713v-0.118c0.269,0.097,0.679,0.087,0.911,0.238h0.201c-0.045-0.206-0.086-0.552-0.201-0.713c-0.12-0.195-0.886-0.197-1.106-0.354c-0.312-0.244-0.607-0.947-0.832-1.307c-0.56-0.887-1.302-1.832-2.137-2.453c-0.443-0.329-0.751-0.671-1.544-0.673c-0.092,0.065-0.185,0.132-0.276,0.198c-0.178,0.789,0.139,2.248,0.592,2.611v0.078c-0.189-0.051-0.393-0.152-0.514-0.275h-0.04c0.007,0.227,0.051,0.522,0.158,0.672c0.046,0.195,0.362,0.354,0.554,0.396c0.248,1.037,0.592,2.101,0.95,3.05c0.289,0.758,0.436,1.48,0.75,2.213c0.155,0.356,0.522,0.617,0.634,0.99h0.117c-0.089-0.334-0.271-0.646-0.394-0.949c-0.364-0.875-0.653-1.804-0.952-2.688C6.868,61.62,6.734,61.057,6.548,60.5c-0.069-0.21-0.049-0.427-0.158-0.595v-0.039c0.269,0.049,0.43,0.188,0.634,0.276c0.461,0.201,0.903,0.277,1.385,0.476c0.353,0.146,0.796,0.347,1.228,0.396c0.048,0.359,0.253,1.301,0.435,1.545v0.117c-0.602-0.412-0.589-1-1.663-0.91c-0.054,0.063-0.128,0.117-0.197,0.158c-0.098,0.244-0.104,0.646,0,0.909c0.257,0.646,1.072,1.991,1.741,2.179c0.257,0.184,0.634-0.043,0.75-0.24c0.242,0.127,0.293,0.682,0.395,0.951c0.212,0.558,0.522,1.289,1.031,1.543v0.041h0.083c-0.066-0.447-0.318-0.851-0.475-1.229C11.387,65.223,11.113,64.324,10.825,63.465L10.825,63.465z M9.678,60.26C9.26,60.23,8.905,60.067,8.57,59.945c-0.894-0.332-1.703-0.615-2.492-0.991c-0.095-0.358-0.76-1.644-0.396-2.095c0.026-0.04,0.053-0.081,0.079-0.12c0.081-0.019,0.077-0.011,0.119-0.039c1.219,0.146,2.442,1.629,3.046,2.452c0.236,0.32,0.43,0.799,0.752,1.029V60.26L9.678,60.26z M10.311,63.701c-0.12,0.146-0.237,0.291-0.356,0.436c-0.105,0.078-0.223,0.109-0.316,0.198c-0.68-0.021-0.704-0.686-0.989-1.108c0.005-0.389,0.152-0.39,0.315-0.594c0.092-0.007,0.112-0.007,0.158-0.037c0.614,0.004,0.753,0.278,1.109,0.515C10.29,63.344,10.327,63.445,10.311,63.701L10.311,63.701z M33.578,69.794c-0.165-0.271-0.49-0.342-0.713-0.554c-0.069-0.023-0.04-0.007-0.079-0.039c0.51-0.264,1.053-0.555,1.583-0.79c0.142,0.158,0.801,0.792,1.029,0.671c0.04-0.012,0.079-0.023,0.118-0.038c-0.013-0.224-0.025-0.448-0.04-0.673c-0.499-0.498-1.234-0.91-2.059-1.066v0.039h-0.039c0.093,0.273,0.398,0.534,0.636,0.672v0.119c-0.469,0.068-0.885,0.295-1.307,0.437c-0.289,0.093-0.638,0.08-0.873,0.235h-0.117c0.171-0.479,0.737-0.871,1.028-1.267c0.576-0.776,1.033-1.728,1.94-2.176c-0.024-0.365-1.076-1.12-1.464-0.871c-0.097,0.051-0.029-0.021-0.079,0.078c-0.059,0.144,0.137,0.321,0.079,0.554c-0.076,0.305-0.831,1.74-1.029,1.9v0.041c-0.408-0.139-0.718-0.523-1.107-0.713c0.069-0.364,0.375-0.644,0.554-0.91c0.453-0.684,0.816-1.335,1.503-1.782c-0.006-0.526-0.855-1.075-1.425-1.065c0.002,0.242,0.125,0.379,0.08,0.592c-0.14,0.646-0.435,1.297-0.672,1.861c-0.156,0.364-0.226,0.799-0.476,1.065c-0.054,0.03-0.492-0.006-0.594-0.077c-0.149-0.002-0.298,0.005-0.394,0.038v0.079c0.666,0.645,1.387,0.865,2.295,1.268c-0.126,0.655-0.786,1.092-1.108,1.584c-0.166,0-0.3-0.011-0.395-0.08c-0.091,0.017-0.098,0.021-0.158,0.041c0.016,0.582,0.5,1.077,0.987,1.188c0.327-0.366,0.737-0.543,1.228-0.751c0.449,0.468,0.578,1.137,0.751,1.897c0.075,0.332-0.047,0.697,0.04,0.988c0.152,0.514,0.426,0.667,0.672,1.027h0.277c0.174-0.93-0.253-1.832-0.475-2.571C33.71,70.43,33.644,70.111,33.578,69.794L33.578,69.794z M96.09,63.108c-0.238,0.202-0.57,0.296-0.83,0.475c-0.4,0.282-0.758,0.659-1.146,0.95c-0.177,0.134-0.435,0.253-0.556,0.436c-0.199,0.299-0.16,0.806-0.396,1.067v0.157c0.314-0.114,0.464-0.483,0.713-0.672c0.307-0.23,0.563-0.536,0.87-0.754c0.192-0.133,0.411-0.207,0.594-0.355c0.125,0.023,0.115,0.037,0.199,0.081c-0.021,1.005-0.549,1.714-0.871,2.454c-0.093,0.215-0.121,0.551-0.276,0.71c-0.074,0.076-0.229,0.094-0.314,0.157c-0.264,0.291-0.528,0.58-0.794,0.873c-0.25,0.344-0.365,0.803-0.632,1.146c-0.002,0.114-0.002,0.216,0.037,0.276c0.041,0.031,0.11,0.059,0.16,0.08c0.51-0.483,1.004-0.887,1.424-1.465c0.658-0.904,0.986-2.047,1.465-3.125c0.3-0.683,0.734-1.354,0.711-2.334c-0.047-0.045-0.084-0.102-0.117-0.158L96.09,63.108L96.09,63.108z M93.32,69.361V69.4h-0.04c0.069-0.475,0.43-0.606,0.596-0.952h0.079C93.904,68.842,93.605,69.194,93.32,69.361L93.32,69.361z M34.171,69.993c-0.08,0.342,0.76,1.106,1.027,1.308c0.133,0.1,0.312,0.328,0.515,0.235c0.104-0.008,0.136-0.019,0.199-0.04c0.046-0.105,0.115-0.24,0.039-0.354C35.93,70.645,34.64,70.088,34.171,69.993z M37.97,73.037c0.067,0.034,0.122,0.021,0.198-0.039c0.139-0.113,0.063-0.313,0.159-0.475c0.222-0.159,0.615-0.118,0.911-0.199c0.809-0.213,1.753-0.198,2.65-0.396c0.425-0.093,1.128,0.16,1.464-0.037c0.04-0.016,0.081-0.026,0.118-0.043c-0.019-0.517-1.009-0.737-1.545-0.588c-0.237,0.066-0.513,0.213-0.751,0.275c-0.185,0.014-0.37,0.027-0.555,0.038c-0.062-0.644-0.38-1.144-0.395-1.817c0.595-0.013,1.341-0.091,1.739-0.316c-0.008-0.2-0.045-0.2-0.118-0.314c-0.453-0.107-1.23-0.126-1.583,0.116c-0.1-0.004-0.147-0.004-0.197-0.039c-0.221-0.28-0.116-0.851-0.316-1.146v-0.158c0.426-0.092,1.122-0.168,1.345-0.475c0.031-0.041,0.014-0.011,0.039-0.078c-0.036-0.035-0.051-0.068-0.079-0.119c-0.619-0.156-0.887-0.049-1.423,0.158c-0.167-0.535,0.034-0.959-0.514-1.108c0.117-0.203,0.506-0.194,0.751-0.276c0.382-0.126,0.817-0.296,1.148-0.474c0.026-0.068,0.007-0.04,0.04-0.08c-0.022-0.2-0.078-0.193-0.159-0.316c-0.571-0.044-1.027,0.011-1.346,0.316h-0.076c0.047-0.295,0.231-0.718,0.394-0.949c0.112-0.162,0.318-0.14,0.396-0.356h0.04V64.1c-0.081-0.104-0.159-0.211-0.238-0.314c-0.186-0.13-0.454-0.143-0.632-0.279c-0.263-0.004-0.515-0.003-0.672,0.079c0.021,0.152,0.089,0.248,0.119,0.356c0.109,0.408-0.284,1.669-0.436,1.859c-0.123,0.154-1.551,0.672-1.939,0.555c-0.092-0.029-0.36-0.164-0.435-0.239c-0.032-0.039-0.015-0.008-0.04-0.077c0.561-0.527,0.965-1.702,1.741-1.939c0.014-0.064,0.027-0.131,0.041-0.196c-0.194-0.2-1.135-1.188-1.622-0.871c-0.04,0.014-0.079,0.022-0.117,0.038c0,0.338,0.168,0.593,0.078,0.949c-0.182,0.711-0.587,1.556-0.95,2.139c-0.218,0.35-0.693,0.729-0.712,1.229c0.646-0.064,0.802-0.731,1.304-0.912c0.146,0.135,0.29,0.267,0.436,0.396c0.207,0.311,0.168,0.778,0.276,1.186c0.185,0.856,0.371,1.715,0.554,2.571c0.025,0.425,0.052,0.845,0.08,1.269C37.246,72.28,37.561,72.945,37.97,73.037L37.97,73.037z M39.233,70.032c0.031,0.368,0.258,1.407,0.436,1.662c0.001,0.024,0.001,0.054,0.001,0.08c-0.477,0.102-0.973,0.239-1.504,0.237c-0.082-0.564-0.352-1.061-0.355-1.662C38.418,70.338,38.731,70.094,39.233,70.032z M36.939,66.75c0.063-0.107,1.113-0.273,1.228-0.199c0.42,0.195,0.27,0.813,0.514,1.188c-0.083,0.194-1.047,0.487-1.345,0.514C37.283,67.834,37.213,66.977,36.939,66.75L36.939,66.75z M38.76,68.253h0.04c0.076,0.36,0.119,0.978,0.317,1.267c-0.142,0.348-1.016,0.317-1.346,0.516c-0.138-0.083-0.32-1.076-0.316-1.346C37.757,68.662,38.541,68.402,38.76,68.253L38.76,68.253z M31.914,70.506c-0.06,0.135-0.053,0.354-0.117,0.514c-0.342,0.84-0.454,1.015,0.079,1.82c0.237,0,0.269-0.037,0.396-0.119C32.429,72.064,32.454,70.814,31.914,70.506L31.914,70.506z M77.023,70.744c-1.154-0.285-2.125,0.285-3.325,0.199c-0.114-0.121-0.2-0.19-0.275-0.356c-0.835,0.024-1.757,1.886-0.909,2.453c0.453,0.308,1.744,0.129,2.295,0c0.306-0.071,0.783-0.139,1.027,0.038c0.332,0.247,0.273,1.182,0.157,1.703c-0.132,0.975-0.265,1.951-0.396,2.929c-0.117,0.593-0.236,1.185-0.356,1.779c0.606-0.003,1.178-0.623,1.349-1.069c0.1-0.258,0.047-0.502,0.119-0.791c0.209-0.83,0.237-1.82,0.436-2.689c0.127-0.563,0.041-1.1,0.156-1.621c0.086-0.393,0.143-1.696,0.041-2.059C77.281,71.059,77.126,70.901,77.023,70.744z M22.857,82.695c-0.135-0.102-0.229-0.283-0.356-0.395c-0.473-0.42-1.029-0.826-1.543-1.188c-0.426-0.298-1.008-0.476-1.387-0.829c-0.01-0.086,0.123-0.296,0.041-0.516c-0.335-0.896-1.589-1.933-2.374-2.412c-0.363-0.225-0.972-0.328-1.305-0.555c-0.246-0.017-0.374-0.025-0.435,0.155c-0.097,0.218,0.209,0.521,0.315,0.675c0.271,0.381,0.581,0.826,0.95,1.104c0.276,0.209,0.591,0.392,0.83,0.635h0.119c-0.154-0.426-0.609-0.657-0.949-0.909c-0.311-0.229-0.449-0.632-0.712-0.909c0.021-0.125,0.035-0.115,0.08-0.199c1.093,0.009,1.802,1.012,2.294,1.662c0.22,0.291,0.571,0.461,0.594,0.951c-0.116,0-0.216,0-0.276-0.041h-0.119c0.188,0.522,0.824,0.479,1.267,0.754c0.888,0.549,1.603,1.409,2.373,2.094c0.262,0.234,0.719,0.466,0.791,0.873c-0.537-0.028-0.917-0.327-1.261-0.555c-0.614-0.4-1.597-1.1-2.019-1.662c-0.08-0.104-0.106-0.263-0.199-0.355c-0.109-0.111-0.261-0.145-0.355-0.275h-0.158c-0.039,0.41,0.407,0.705,0.671,0.948c0.819,0.75,1.696,1.442,2.73,1.979c0.373,0.191,1.053,0.521,1.465,0.275C23.874,83.434,23.227,82.975,22.857,82.695L22.857,82.695z M47.226,85.307c-2.014-1.379-4.985-2.775-8.427-2.689c-0.167,0.104-0.503,0.021-0.711,0.078c-0.288,0.076-0.464,0.223-0.672,0.355c-0.008,0.971,1.446,1.496,2.255,1.698c0.483,0.123,0.909-0.104,1.188-0.198c0.215-0.82-0.776-0.94-1.227-1.347h-0.081v-0.038c3.036-0.119,5.308,0.729,7.043,2.02c0.433,0.322,0.93,0.783,1.148,1.306c0.081,0.194,0.116,0.515,0,0.674c-0.159,0.44-0.685,0.401-1.188,0.515c-1.162,0.267-2.755-0.391-3.285-0.91c-0.108,0.189,0.049,0.48-0.118,0.674c-0.176,0.478-0.788,0.354-1.346,0.474c-0.917,0.199-2.353-0.271-2.888-0.632c-0.149-0.104-0.257-0.286-0.396-0.396c-0.007-0.103-0.018-0.136-0.041-0.199c0.081-0.073,0.177-0.187,0.237-0.275c1.139-0.085,1.718-0.027,2.376,0.596c-0.017,0.078-0.01,0.073-0.041,0.114c-0.074,0.152-0.245,0.17-0.474,0.161v0.074c0.417,0.004,0.593-0.059,0.83-0.197c0.013-0.079,0.027-0.159,0.04-0.236c-0.136-0.141-0.231-0.328-0.396-0.438c-0.65-0.426-1.991-0.641-2.729-0.156c-0.116,0.561,0.232,0.864,0.554,1.105c0.646,0.488,1.191,0.771,2.098,1.029c0.291,0.082,0.55,0.008,0.871,0.076c0.28,0.064,0.765,0.079,1.068,0c0.504-0.128,1.205-0.658,0.632-1.268v-0.037c0.299,0.109,0.544,0.402,0.831,0.556c0.761,0.397,2.021,0.726,3.167,0.476c0.562-0.125,1.143-0.125,1.303-0.635c0.179-0.277-0.068-0.668-0.156-0.826C48.322,86.151,47.836,85.721,47.226,85.307L47.226,85.307z M39.906,83.485c0.14,0.094,0.22,0.291,0.356,0.396c-0.003,0.1-0.004,0.148-0.04,0.199c-0.257,0.697-1.706,0.182-2.058-0.081c-0.11-0.08-0.153-0.248-0.236-0.354c0.015-0.082,0.01-0.076,0.041-0.116c0.108-0.306,0.417-0.203,0.671-0.354C39.142,83.174,39.596,83.274,39.906,83.485z M76.625,83.881h-0.396c-0.262,0.209-0.692,0.236-0.991,0.396c-0.263,0.141-0.581,0.332-0.829,0.515c-0.207,0.148-0.326,0.418-0.516,0.592c0.004,0.197,0.008,0.229,0.16,0.277c0.039,0.029,0.01,0.018,0.075,0.04c0.042-0.047,0.08-0.063,0.12-0.12c0.033-0.023-0.027-0.104-0.04-0.232c0.384-0.386,0.667-0.598,1.228-0.832c0.144-0.059,0.447-0.233,0.634-0.119h0.079c-0.026,0.391-0.916,1.591-1.188,1.781v0.115c0.729-0.188,1.215-1.643,1.702-2.174c-0.013-0.09-0.01-0.111-0.04-0.157L76.625,83.881L76.625,83.881z M73.459,86.809c-0.234,0.209-0.807,0.229-1.066,0.435h-0.041c0.104-0.149,0.291-0.213,0.396-0.354c0.076-0.104,0.107-0.226,0.197-0.315c-0.018-0.081-0.01-0.075-0.039-0.117v-0.08c-1.155-0.212-3.084,0.784-3.68,1.308c-0.155,0.135-0.248,0.336-0.396,0.477c0.003,0.111,0.016,0.168,0.039,0.236c0.701,0.047,2.016-0.383,2.174-0.949c0.031-0.025,0.012-0.002,0-0.04v-0.079c-0.479-0.027-1.124,0.075-1.422,0.355h-0.039c0.26-0.396,1.223-0.746,1.739-0.91c0.172-0.053,0.55-0.149,0.714-0.039c0.037,0.015,0.077,0.025,0.117,0.039c-0.094,0.396-0.657,0.838-1.029,0.949v0.08c0.607-0.141,1.163-0.416,1.7-0.634c0.368-0.149,0.786-0.188,1.108-0.396c0.229-0.149,1.008-1.207,1.068-1.504C74.086,85.409,74.012,86.313,73.459,86.809L73.459,86.809z M70.333,87.6v0.119c-0.075,0.049-0.129,0.156-0.198,0.196c-0.205,0.12-0.479,0.106-0.674,0.238c-0.09-0.011-0.109-0.009-0.156-0.041h-0.039C69.373,87.775,70.025,87.621,70.333,87.6L70.333,87.6z M53.835,91.317c0.015-0.037,0.025-0.078,0.039-0.117c-0.976-0.04-1.953-0.079-2.927-0.119c-0.123,0.082-0.312,0.035-0.475,0.079c-0.202,0.059-0.426,0.15-0.593,0.239c0.026,0.067,0.008,0.038,0.04,0.077c0.238,0.188,1.624,0.199,1.9,0h0.078v-0.077c-0.419-0.134-1.183,0.2-1.503,0h-0.041v-0.041c1.052-0.073,2.23-0.044,3.325-0.04c-0.105,0.072-0.328,0.051-0.436,0.119c-0.039,0.014-0.078,0.027-0.117,0.039v0.08c0.238,0.037,0.475,0.078,0.711,0.117c0.037,0.041-0.004,0.004,0.039,0.037c-0.35,0.233-1.254,0.139-1.581-0.037v-0.08c-0.178-0.082-0.991,0.084-1.148,0.117c-0.133,0.03-0.27-0.014-0.357,0.039c-0.165,0.01-0.181,0.029-0.276,0.079c0.022,0.128,0.035,0.115,0.08,0.198c0.255,0.06,0.696,0.064,0.987,0.156v-0.039h0.04v-0.039c-0.148-0.057-0.559-0.025-0.713-0.115h-0.079c0.132-0.104,1.425-0.278,1.663-0.119c0.067,0.023,0.039,0.007,0.079,0.039c-0.211,0.038-0.424,0.08-0.634,0.117c0.025,0.066,0.009,0.039,0.04,0.078c0.065,0.045,0.193,0.045,0.316,0.039c-0.04,0.074-0.054,0.109-0.119,0.158c0.013,0.023,0.027,0.051,0.04,0.078c0.561,0,1.031-0.057,1.502-0.156c0.28-0.062,0.624,0.052,0.831-0.08h0.317v-0.078c-0.539-0.002-1.885-0.055-2.215,0.158h-0.117c0.033-0.043-0.004-0.004,0.038-0.041c0.155-0.18,0.471-0.09,0.75-0.156c0.44-0.104,1.168-0.284,1.544,0c0.105,0.064,0.04-0.008,0.039,0.117c0.107-0.002,0.181-0.002,0.236-0.036h0.277v-0.081c-0.359-0.088-0.889-0.251-1.188-0.434C54.057,91.488,54.135,91.344,53.835,91.317L53.835,91.317z M13.635,18.359c-0.088,0.32-0.274,0.593-0.395,0.87c-0.268,0.613-0.507,1.225-0.751,1.822c-0.207,0.496-0.335,1.295-0.633,1.699v0.079c0.416-0.074,0.698-0.493,0.949-0.751c0.617-0.634,1.92-2.22,1.9-3.402c-0.062-0.061-0.119-0.162-0.159-0.237C14.3,18.38,13.982,18.353,13.635,18.359z M13.794,20.022c-0.181,0.298-0.281,0.592-0.476,0.871c-0.178,0.255-0.46,0.452-0.633,0.713h-0.041c0.051-0.302,0.214-0.546,0.319-0.792c0.235-0.561,0.396-1.118,0.671-1.621c0.152,0.003,0.268,0.015,0.356,0.078c0.095,0.052,0.028-0.018,0.079,0.08C14.15,19.548,13.89,19.862,13.794,20.022L13.794,20.022z M84.023,7.875c-0.414-0.416-0.729-0.938-1.147-1.346V6.49c-0.205,0.073-0.899,0.688-1.028,0.871c-0.25-0.095-0.391-0.365-0.594-0.514c-0.676-0.508-1.313-1.167-2.49-1.147c-0.148,0.115-0.367,0.118-0.556,0.197c-0.53,0.23-1.083,0.688-1.305,1.227c-0.249,0.602,0.004,1.491,0.196,1.939c0.392,0.904,1.03,1.667,1.582,2.414c0.457,0.615,0.973,1.252,1.819,1.464c0.956,0.238,1.422-0.884,1.781-1.308c0.37-0.435,1.182-0.539,1.464-1.107c0.104-0.207,0.034-0.615-0.039-0.791c-0.18-0.426-1.066-1.622-1.425-1.859c0.024-0.239,0.135-0.247,0.235-0.396c0.248,0.121,0.338,0.471,0.516,0.673c0.227,0.258,0.546,0.396,0.791,0.632c0.378,0.003,0.604-0.094,0.79-0.277h0.041C84.561,8.243,84.212,8.06,84.023,7.875L84.023,7.875z M81.77,12.148c-0.699,0.165-1.047-0.293-1.424-0.673c-0.938-0.935-1.57-2.093-2.298-3.244c-0.247-0.396-0.885-1.134-0.554-1.702h0.156c0.199,0.299,0.539,0.507,0.754,0.792c0.591,0.784,1.313,1.469,1.898,2.255c0.359,0.485,0.758,0.94,1.106,1.424c0.178,0.249,0.315,0.565,0.556,0.751C81.924,11.931,81.848,12.015,81.77,12.148L81.77,12.148z M82.361,9.339c0.32,0.439,0.755,0.688,0.751,1.463c-0.122,0.116-0.157,0.224-0.356,0.276c-0.039,0.032-0.011,0.015-0.078,0.041c-0.56-0.932-1.367-1.711-2.017-2.573c-0.212-0.282-0.541-0.511-0.752-0.791c-0.362-0.48-0.793-0.864-1.188-1.305c-0.113-0.131-0.168-0.257-0.313-0.357c0.033-0.086,0.031-0.057,0.076-0.118c0.672,0.006,0.994,0.458,1.386,0.753C80.837,7.453,81.648,8.354,82.361,9.339L82.361,9.339z"></path>
+  <radialGradient id="SVGID_43_" cx="251.8086" cy="-408.3613" r="72.7509" gradientTransform="matrix(1 0 0 -1 -213.7637 -386.502)" gradientUnits="userSpaceOnUse">
+    <stop offset="0" stop-color="#000" stop-opacity="0"></stop>
+    <stop offset=".8022" stop-color="#000" stop-opacity=".08"></stop>
+    <stop offset="1" stop-color="#000" stop-opacity=".3882"></stop>
+  </radialGradient>
+  <path style="&st15;" d="M49.885,17.037c0.014-0.606-0.392-1.27-0.392-1.27l-0.025-0.058c0,0-0.487-0.949-1.302-1.228c-0.815-0.278-1.478,0.342-1.478,0.342s-0.114,0.131-0.429,0.494c-0.313,0.364-0.507,0.666-1.198,0.938c-0.692,0.271-1.379,0.204-1.743,0.033c-0.364-0.172-0.457-0.537-0.457-0.537s-0.229-0.722-0.313-1.049c-0.086-0.331-0.309-1.694-0.309-1.694s-0.491-2.747-0.534-3.304c0,0,1.475-0.126,3.687-0.775c2.299-0.673,3.043-1.206,3.043-1.206s-0.432-0.156-0.484-0.662c-0.051-0.507-0.089-1.19-0.089-1.19s-0.089-0.5,0.483-1.139c0.571-0.64,1.354-0.863,1.762-0.953c0.41-0.089,1.281-0.17,2.093-0.134c0.812,0.038,1.267,0.112,1.593,0.291c0.328,0.178,0.357,0.61,0.357,0.61l-0.008,1.146c0,0-0.037,0.402,0.261,0.529c0,0,0.505,0.305,2.196,0.133c0,0,0.72-0.126,1.846-0.46c1.125-0.335,4.129-1.229,4.554-1.341c0.425-0.111,0.953-0.291,1.646-0.469c0.691-0.179,2.039-0.626,2.308-0.73c0.271-0.104,1.812-0.618,2.927-0.81c1.115-0.195,2.227-0.186,2.813,0.149c0,0,0.357,0.304,0.521,0.662c0.163,0.358,0.478,0.863,0.923,1.088c0.454,0.227,0.752,0.371,1.875,0.273c0,0,0.022-0.096-0.395-0.37c-0.417-0.277-0.991-0.701-0.991-0.701S74.29,3.4,74.215,3.198c-0.072-0.202-0.043-0.306-0.043-0.306l0.877-1.406c0,0,0-0.172,0.508-0.238c0.506-0.067,1.071-0.134,1.74-0.313c0.67-0.178,0.788-0.312,0.788-0.312l0.58,0.178c0,0,3.546,2.853,4.655,3.583l0.199-0.239c0,0,0.436,0.018,0.826,0.172c0.394,0.154,0.979,0.562,0.979,0.562s1.612,1.31,2.071,2.2l0.223,0.679l-0.102,0.161c0,0,0.918,1.307,2.096,2.602c0,0,1.227,1.664,1.689,2.09c0,0-0.108-0.399-0.201-0.849l0.336-0.226l0.203-0.144l0.617,0.259c3.573,4.811,6.432,10.424,8.141,16.328l-0.12,0.484l0.395,0.501c1.128,4.212,1.728,8.643,1.728,13.211c0,1.122-0.036,2.236-0.107,3.339l-0.304,0.511l0.225,0.555c-2.231,26.1-24.124,46.584-50.801,46.584c-18.502,0-34.702-9.854-43.637-24.6L7.674,68.2l-0.205-0.153c-3.387-5.742-5.682-12.205-6.595-19.103l0.212-0.525L0.75,47.936c-0.213-1.892-0.322-3.812-0.322-5.756c0-2.985,0.255-5.909,0.748-8.755l0.25-0.562l-0.087-0.328c1.157-6.048,3.383-11.716,6.474-16.799l0.684-0.384l0.081,0.032c0,0,0.233-0.169,0.354-0.217l0.076-0.023c0,0,1.179-1.971,1.625-2.601c0,0,0.542-0.348,0.745-0.407c0,0,0.124-0.016,0.189,0.076c0,0,0.496-0.432,1.699-2.054c0.004-0.005,0.007-0.011,0.012-0.017c0,0-0.114-0.076-0.131-0.174c-0.018-0.097,0.108-0.591,0.173-0.717c0.065-0.126,0.108-0.156,0.108-0.156s1.722-2.032,3.151-3.238c0,0,0.26-0.202,0.678-0.25c0,0,1.472-0.613,3.264-2.184c0,0,0.051-0.289,0.478-0.858c0.428-0.57,1.456-1.163,2.222-1.337c0.764-0.174,0.896-0.038,0.896-0.038l0.064,0.065l0.515,0.766c0,0,0.565-0.316,1.413-0.604c0.847-0.289,0.979-0.262,0.979-0.262l0.825,1.336l-0.987,2c0,0-0.644,1.421-1.655,2.185c0,0-0.472,0.284-1.12,0.127c-0.648-0.157-1.072,0.333-1.072,0.333l-0.17,0.14c0,0,0.14-0.024,0.346-0.103c0,0,0.158,0.065,0.274,0.223c0.114,0.158,0.913,1.175,0.913,1.175s0.005,0.837-0.415,1.938c-0.419,1.1-1.467,2.891-1.467,2.891s-0.733,1.424-1.075,2.253c-0.342,0.829-0.515,1.765-0.488,2.262c0,0,0.187,0.062,0.707-0.202c0.655-0.332,1.083,0.027,1.083,0.027s0.719,0.53,1.041,0.881c0.262,0.289,0.802,1.765,0.209,3.224c0,0-0.402,1.008-1.377,1.724c0,0-0.216,0.332-1.529,0.174c-0.368-0.043-0.585-0.276-1.372-0.2c-0.785,0.077-1.231,0.815-1.231,0.815l0.013-0.024c-0.692,0.999-1.154,2.458-1.154,2.458l-0.057,0.165c0,0-0.241,0.509-0.292,1.752c-0.053,1.284,0.284,3.109,0.284,3.109s7.876-1.387,9.88-0.055l0.58,0.532c0,0,0.046,0.174-0.031,0.376c-0.08,0.204-0.375,0.673-0.987,1.113c-0.611,0.438-1.222,1.583-0.313,2.304c1.034,0.818,1.691,0.766,3.43,0.468c1.74-0.297,2.898-1.269,2.898-1.269s0.972-0.72,0.783-1.628c-0.188-0.908-1.017-1.189-1.017-1.189s-0.658-0.423-0.141-1.238c0,0,0.141-0.689,2.553-1.316c2.414-0.626,6.812-1.52,10.556-1.989c0,0-2.539-8.223-0.737-9.289c0,0,0.438-0.296,1.224-0.408l0.721-0.037c0.131-0.027,0.344,0.005,0.796,0.045c0.452,0.038,1.001,0.076,1.678-0.441c0.676-0.519,0.697-0.819,0.697-0.819"></path>
+</svg>
diff --git a/web/src/app/admin/assistants/[id]/page.tsx b/web/src/app/admin/assistants/[id]/page.tsx
index bacfa625149..249ded66d65 100644
--- a/web/src/app/admin/assistants/[id]/page.tsx
+++ b/web/src/app/admin/assistants/[id]/page.tsx
@@ -10,7 +10,8 @@ import { AdminPageTitle } from "@/components/admin/Title";
 import CardSection from "@/components/admin/CardSection";
 import Title from "@/components/ui/title";
 
-export default async function Page({ params }: { params: { id: string } }) {
+export default async function Page(props: { params: Promise<{ id: string }> }) {
+  const params = await props.params;
   const [values, error] = await fetchAssistantEditorInfoSS(params.id);
 
   let body;
diff --git a/web/src/app/admin/bot/[id]/page.tsx b/web/src/app/admin/bot/[id]/page.tsx
index f0ae638f980..cdffb4e8ff6 100644
--- a/web/src/app/admin/bot/[id]/page.tsx
+++ b/web/src/app/admin/bot/[id]/page.tsx
@@ -13,7 +13,8 @@ import {
 } from "@/lib/assistants/fetchAssistantsSS";
 import { getStandardAnswerCategoriesIfEE } from "@/components/standardAnswers/getStandardAnswerCategoriesIfEE";
 
-async function Page({ params }: { params: { id: string } }) {
+async function Page(props: { params: Promise<{ id: string }> }) {
+  const params = await props.params;
   const tasks = [
     fetchSS("/manage/admin/slack-bot/config"),
     fetchSS("/manage/document-set"),
diff --git a/web/src/app/admin/connector/[ccPairId]/page.tsx b/web/src/app/admin/connector/[ccPairId]/page.tsx
index b703d75109d..9e6deefc1c4 100644
--- a/web/src/app/admin/connector/[ccPairId]/page.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/page.tsx
@@ -16,7 +16,7 @@ import { Button } from "@/components/ui/button";
 import Title from "@/components/ui/title";
 import { Separator } from "@/components/ui/separator";
 import { useRouter } from "next/navigation";
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useRef, useState, use } from "react";
 import useSWR, { mutate } from "swr";
 import { AdvancedConfigDisplay, ConfigDisplay } from "./ConfigDisplay";
 import { DeletionButton } from "./DeletionButton";
@@ -278,7 +278,8 @@ function Main({ ccPairId }: { ccPairId: number }) {
   );
 }
 
-export default function Page({ params }: { params: { ccPairId: string } }) {
+export default function Page(props: { params: Promise<{ ccPairId: string }> }) {
+  const params = use(props.params);
   const ccPairId = parseInt(params.ccPairId);
 
   return (
diff --git a/web/src/app/admin/connectors/[connector]/auth/callback/route.ts b/web/src/app/admin/connectors/[connector]/auth/callback/route.ts
index 9d80e1b2fd2..79d831cfba6 100644
--- a/web/src/app/admin/connectors/[connector]/auth/callback/route.ts
+++ b/web/src/app/admin/connectors/[connector]/auth/callback/route.ts
@@ -9,6 +9,7 @@ import {
 import { processCookies } from "@/lib/userSS";
 
 export const GET = async (request: NextRequest) => {
+  const requestCookies = await cookies();
   const connector = request.url.includes("gmail") ? "gmail" : "google-drive";
   const callbackEndpoint = `/manage/connector/${connector}/callback`;
   const url = new URL(buildUrl(callbackEndpoint));
@@ -16,7 +17,7 @@ export const GET = async (request: NextRequest) => {
 
   const response = await fetch(url.toString(), {
     headers: {
-      cookie: processCookies(cookies()),
+      cookie: processCookies(requestCookies),
     },
   });
 
@@ -33,7 +34,7 @@ export const GET = async (request: NextRequest) => {
       ? GMAIL_AUTH_IS_ADMIN_COOKIE_NAME
       : GOOGLE_DRIVE_AUTH_IS_ADMIN_COOKIE_NAME;
 
-  if (cookies().get(authCookieName)?.value?.toLowerCase() === "true") {
+  if (requestCookies.get(authCookieName)?.value?.toLowerCase() === "true") {
     return NextResponse.redirect(
       new URL(`/admin/connectors/${connector}`, getDomain(request))
     );
diff --git a/web/src/app/admin/connectors/[connector]/page.tsx b/web/src/app/admin/connectors/[connector]/page.tsx
index 1e5a690f93d..f9f737a0e3a 100644
--- a/web/src/app/admin/connectors/[connector]/page.tsx
+++ b/web/src/app/admin/connectors/[connector]/page.tsx
@@ -1,11 +1,10 @@
 import { ConfigurableSources } from "@/lib/types";
 import ConnectorWrapper from "./ConnectorWrapper";
 
-export default async function Page({
-  params,
-}: {
-  params: { connector: string };
+export default async function Page(props: {
+  params: Promise<{ connector: string }>;
 }) {
+  const params = await props.params;
   return (
     <ConnectorWrapper
       connector={params.connector.replace("-", "_") as ConfigurableSources}
diff --git a/web/src/app/admin/documents/explorer/page.tsx b/web/src/app/admin/documents/explorer/page.tsx
index 3b422beeab3..8c727005c04 100644
--- a/web/src/app/admin/documents/explorer/page.tsx
+++ b/web/src/app/admin/documents/explorer/page.tsx
@@ -3,11 +3,10 @@ import { ZoomInIcon } from "@/components/icons/icons";
 import { Explorer } from "./Explorer";
 import { fetchValidFilterInfo } from "@/lib/search/utilsSS";
 
-const Page = async ({
-  searchParams,
-}: {
-  searchParams: { [key: string]: string };
+const Page = async (props: {
+  searchParams: Promise<{ [key: string]: string }>;
 }) => {
+  const searchParams = await props.searchParams;
   const { connectors, documentSets } = await fetchValidFilterInfo();
 
   return (
diff --git a/web/src/app/admin/documents/sets/[documentSetId]/page.tsx b/web/src/app/admin/documents/sets/[documentSetId]/page.tsx
index ebce7d2027a..cfbdc6b58f7 100644
--- a/web/src/app/admin/documents/sets/[documentSetId]/page.tsx
+++ b/web/src/app/admin/documents/sets/[documentSetId]/page.tsx
@@ -1,4 +1,5 @@
 "use client";
+import { use } from "react";
 
 import { ErrorCallout } from "@/components/ErrorCallout";
 import { refreshDocumentSets, useDocumentSets } from "../hooks";
@@ -93,11 +94,10 @@ function Main({ documentSetId }: { documentSetId: number }) {
   );
 }
 
-export default function Page({
-  params,
-}: {
-  params: { documentSetId: string };
+export default function Page(props: {
+  params: Promise<{ documentSetId: string }>;
 }) {
+  const params = use(props.params);
   const documentSetId = parseInt(params.documentSetId);
 
   return (
diff --git a/web/src/app/admin/indexing/[id]/page.tsx b/web/src/app/admin/indexing/[id]/page.tsx
index a5fa79abe42..75aa482a00f 100644
--- a/web/src/app/admin/indexing/[id]/page.tsx
+++ b/web/src/app/admin/indexing/[id]/page.tsx
@@ -1,4 +1,5 @@
 "use client";
+import { use } from "react";
 
 import { BackButton } from "@/components/BackButton";
 import { ErrorCallout } from "@/components/ErrorCallout";
@@ -46,7 +47,8 @@ function Main({ id }: { id: number }) {
   );
 }
 
-export default function Page({ params }: { params: { id: string } }) {
+export default function Page(props: { params: Promise<{ id: string }> }) {
+  const params = use(props.params);
   const id = parseInt(params.id);
 
   return (
diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
index 118b9157860..7ade9b0cfd7 100644
--- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
+++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
@@ -5,6 +5,7 @@ import {
   TableHead,
   TableBody,
   TableCell,
+  TableHeader,
 } from "@/components/ui/table";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
@@ -363,52 +364,54 @@ export function CCPairIndexingStatusTable({
 
   return (
     <Table>
-      <ConnectorRow
-        invisible
-        ccPairsIndexingStatus={{
-          cc_pair_id: 1,
-          name: "Sample File Connector",
-          cc_pair_status: ConnectorCredentialPairStatus.ACTIVE,
-          last_status: "success",
-          connector: {
+      <TableHeader>
+        <ConnectorRow
+          invisible
+          ccPairsIndexingStatus={{
+            cc_pair_id: 1,
             name: "Sample File Connector",
-            source: "file",
-            input_type: "poll",
-            connector_specific_config: {
-              file_locations: ["/path/to/sample/file.txt"],
+            cc_pair_status: ConnectorCredentialPairStatus.ACTIVE,
+            last_status: "success",
+            connector: {
+              name: "Sample File Connector",
+              source: "file",
+              input_type: "poll",
+              connector_specific_config: {
+                file_locations: ["/path/to/sample/file.txt"],
+              },
+              refresh_freq: 86400,
+              prune_freq: null,
+              indexing_start: new Date("2023-07-01T12:00:00Z"),
+              id: 1,
+              credential_ids: [],
+              time_created: "2023-07-01T12:00:00Z",
+              time_updated: "2023-07-01T12:00:00Z",
             },
-            refresh_freq: 86400,
-            prune_freq: null,
-            indexing_start: new Date("2023-07-01T12:00:00Z"),
-            id: 1,
-            credential_ids: [],
-            time_created: "2023-07-01T12:00:00Z",
-            time_updated: "2023-07-01T12:00:00Z",
-          },
-          credential: {
-            id: 1,
-            name: "Sample Credential",
-            source: "file",
-            user_id: "1",
-            time_created: "2023-07-01T12:00:00Z",
-            time_updated: "2023-07-01T12:00:00Z",
-            credential_json: {},
-            admin_public: false,
-          },
-          access_type: "public",
-          docs_indexed: 1000,
-          last_success: "2023-07-01T12:00:00Z",
-          last_finished_status: "success",
-          latest_index_attempt: null,
-          owner: "1",
-          error_msg: "",
-          deletion_attempt: null,
-          is_deletable: true,
-          in_progress: false,
-          groups: [], // Add this line
-        }}
-        isEditable={false}
-      />
+            credential: {
+              id: 1,
+              name: "Sample Credential",
+              source: "file",
+              user_id: "1",
+              time_created: "2023-07-01T12:00:00Z",
+              time_updated: "2023-07-01T12:00:00Z",
+              credential_json: {},
+              admin_public: false,
+            },
+            access_type: "public",
+            docs_indexed: 1000,
+            last_success: "2023-07-01T12:00:00Z",
+            last_finished_status: "success",
+            latest_index_attempt: null,
+            owner: "1",
+            error_msg: "",
+            deletion_attempt: null,
+            is_deletable: true,
+            in_progress: false,
+            groups: [], // Add this line
+          }}
+          isEditable={false}
+        />
+      </TableHeader>
       <div className="flex -mt-12 items-center w-0 m4 gap-x-2">
         <input
           type="text"
diff --git a/web/src/app/admin/tools/edit/[toolId]/page.tsx b/web/src/app/admin/tools/edit/[toolId]/page.tsx
index 44d24ccfb81..c88ca3862e6 100644
--- a/web/src/app/admin/tools/edit/[toolId]/page.tsx
+++ b/web/src/app/admin/tools/edit/[toolId]/page.tsx
@@ -9,7 +9,10 @@ import { AdminPageTitle } from "@/components/admin/Title";
 import { BackButton } from "@/components/BackButton";
 import { ToolIcon } from "@/components/icons/icons";
 
-export default async function Page({ params }: { params: { toolId: string } }) {
+export default async function Page(props: {
+  params: Promise<{ toolId: string }>;
+}) {
+  const params = await props.params;
   const tool = await fetchToolByIdSS(params.toolId);
 
   let body;
diff --git a/web/src/app/api/[...path]/route.ts b/web/src/app/api/[...path]/route.ts
index 6ca13aba146..0ebd8210a81 100644
--- a/web/src/app/api/[...path]/route.ts
+++ b/web/src/app/api/[...path]/route.ts
@@ -6,50 +6,57 @@ each request type >:( */
 
 export async function GET(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
 export async function POST(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
 export async function PUT(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
 export async function PATCH(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
 export async function DELETE(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
 export async function HEAD(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
 export async function OPTIONS(
   request: NextRequest,
-  { params }: { params: { path: string[] } }
+  props: { params: Promise<{ path: string[] }> }
 ) {
+  const params = await props.params;
   return handleRequest(request, params.path);
 }
 
diff --git a/web/src/app/assistants/edit/[id]/page.tsx b/web/src/app/assistants/edit/[id]/page.tsx
index e57a32acc0c..e8dedb80a8e 100644
--- a/web/src/app/assistants/edit/[id]/page.tsx
+++ b/web/src/app/assistants/edit/[id]/page.tsx
@@ -9,7 +9,8 @@ import { DeletePersonaButton } from "@/app/admin/assistants/[id]/DeletePersonaBu
 import { LargeBackButton } from "../../LargeBackButton";
 import Title from "@/components/ui/title";
 
-export default async function Page({ params }: { params: { id: string } }) {
+export default async function Page(props: { params: Promise<{ id: string }> }) {
+  const params = await props.params;
   const [values, error] = await fetchAssistantEditorInfoSS(params.id);
 
   let body;
diff --git a/web/src/app/assistants/gallery/page.tsx b/web/src/app/assistants/gallery/page.tsx
index fd261fbcae8..538ab5d60a9 100644
--- a/web/src/app/assistants/gallery/page.tsx
+++ b/web/src/app/assistants/gallery/page.tsx
@@ -5,14 +5,15 @@ import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
 import WrappedAssistantsGallery from "./WrappedAssistantsGallery";
 import { AssistantsProvider } from "@/components/context/AssistantsContext";
+import { cookies } from "next/headers";
 
-export default async function GalleryPage({
-  searchParams,
-}: {
-  searchParams: { [key: string]: string };
+export default async function GalleryPage(props: {
+  searchParams: Promise<{ [key: string]: string }>;
 }) {
   noStore();
 
+  const searchParams = await props.searchParams;
+  const requestCookies = await cookies();
   const data = await fetchChatData(searchParams);
 
   if ("redirect" in data) {
@@ -30,7 +31,9 @@ export default async function GalleryPage({
 
   return (
     <>
-      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
+      {shouldShowWelcomeModal && (
+        <WelcomeModal user={user} requestCookies={requestCookies} />
+      )}
 
       <InstantSSRAutoRefresh />
 
diff --git a/web/src/app/assistants/mine/page.tsx b/web/src/app/assistants/mine/page.tsx
index 5761a8c48e5..7058e7d3db2 100644
--- a/web/src/app/assistants/mine/page.tsx
+++ b/web/src/app/assistants/mine/page.tsx
@@ -5,13 +5,14 @@ import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
 import WrappedAssistantsMine from "./WrappedAssistantsMine";
 import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
+import { cookies } from "next/headers";
 
-export default async function GalleryPage({
-  searchParams,
-}: {
-  searchParams: { [key: string]: string };
+export default async function GalleryPage(props: {
+  searchParams: Promise<{ [key: string]: string }>;
 }) {
   noStore();
+  const requestCookies = await cookies();
+  const searchParams = await props.searchParams;
 
   const data = await fetchChatData(searchParams);
 
@@ -30,7 +31,9 @@ export default async function GalleryPage({
 
   return (
     <>
-      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
+      {shouldShowWelcomeModal && (
+        <WelcomeModal user={user} requestCookies={requestCookies} />
+      )}
 
       <InstantSSRAutoRefresh />
       <WrappedAssistantsMine
diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index 3b5ff2bdf70..fc72c41719f 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -17,11 +17,10 @@ import { getSecondsUntilExpiration } from "@/lib/time";
 import AuthFlowContainer from "@/components/auth/AuthFlowContainer";
 import CardSection from "@/components/admin/CardSection";
 
-const Page = async ({
-  searchParams,
-}: {
-  searchParams?: { [key: string]: string | string[] | undefined };
+const Page = async (props: {
+  searchParams?: Promise<{ [key: string]: string | string[] | undefined }>;
 }) => {
+  const searchParams = await props.searchParams;
   const autoRedirectDisabled = searchParams?.disableAutoRedirect === "true";
 
   // catch cases where the backend is completely unreachable here
diff --git a/web/src/app/chat/page.tsx b/web/src/app/chat/page.tsx
index 05207ea9282..7894ce651fc 100644
--- a/web/src/app/chat/page.tsx
+++ b/web/src/app/chat/page.tsx
@@ -5,14 +5,14 @@ import { WelcomeModal } from "@/components/initialSetup/welcome/WelcomeModalWrap
 import { ChatProvider } from "@/components/context/ChatContext";
 import { fetchChatData } from "@/lib/chat/fetchChatData";
 import WrappedChat from "./WrappedChat";
+import { cookies } from "next/headers";
 
-export default async function Page({
-  searchParams,
-}: {
-  searchParams: { [key: string]: string };
+export default async function Page(props: {
+  searchParams: Promise<{ [key: string]: string }>;
 }) {
+  const searchParams = await props.searchParams;
   noStore();
-
+  const requestCookies = await cookies();
   const data = await fetchChatData(searchParams);
 
   if ("redirect" in data) {
@@ -37,7 +37,9 @@ export default async function Page({
   return (
     <>
       <InstantSSRAutoRefresh />
-      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
+      {shouldShowWelcomeModal && (
+        <WelcomeModal user={user} requestCookies={requestCookies} />
+      )}
       <ChatProvider
         value={{
           chatSessions,
diff --git a/web/src/app/chat/shared/[chatId]/page.tsx b/web/src/app/chat/shared/[chatId]/page.tsx
index 27728123262..11e624ff687 100644
--- a/web/src/app/chat/shared/[chatId]/page.tsx
+++ b/web/src/app/chat/shared/[chatId]/page.tsx
@@ -26,7 +26,10 @@ async function getSharedChat(chatId: string) {
   return null;
 }
 
-export default async function Page({ params }: { params: { chatId: string } }) {
+export default async function Page(props: {
+  params: Promise<{ chatId: string }>;
+}) {
+  const params = await props.params;
   const tasks = [
     getAuthTypeMetadataSS(),
     getCurrentUserSS(),
@@ -58,6 +61,7 @@ export default async function Page({ params }: { params: { chatId: string } }) {
   if (user && !user.is_verified && authTypeMetadata?.requiresVerification) {
     return redirect("/auth/waiting-on-verification");
   }
+  // prettier-ignore
   const persona: Persona =
     chatSession?.persona_id && availableAssistants?.length
       ? (availableAssistants.find((p) => p.id === chatSession.persona_id) ??
diff --git a/web/src/app/ee/admin/groups/[groupId]/page.tsx b/web/src/app/ee/admin/groups/[groupId]/page.tsx
index 82e9d747118..f378473f597 100644
--- a/web/src/app/ee/admin/groups/[groupId]/page.tsx
+++ b/web/src/app/ee/admin/groups/[groupId]/page.tsx
@@ -1,4 +1,5 @@
 "use client";
+import { use } from "react";
 
 import { GroupsIcon } from "@/components/icons/icons";
 import { GroupDisplay } from "./GroupDisplay";
@@ -9,7 +10,8 @@ import { useRouter } from "next/navigation";
 import { BackButton } from "@/components/BackButton";
 import { AdminPageTitle } from "@/components/admin/Title";
 
-const Page = ({ params }: { params: { groupId: string } }) => {
+const Page = (props: { params: Promise<{ groupId: string }> }) => {
+  const params = use(props.params);
   const router = useRouter();
 
   const {
diff --git a/web/src/app/ee/admin/performance/query-history/[id]/page.tsx b/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
index 3f924267cb2..ee5a03f28f6 100644
--- a/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
+++ b/web/src/app/ee/admin/performance/query-history/[id]/page.tsx
@@ -1,4 +1,5 @@
 "use client";
+import { use } from "react";
 
 import Text from "@/components/ui/text";
 import Title from "@/components/ui/title";
@@ -63,7 +64,8 @@ function MessageDisplay({ message }: { message: MessageSnapshot }) {
   );
 }
 
-export default function QueryPage({ params }: { params: { id: string } }) {
+export default function QueryPage(props: { params: Promise<{ id: string }> }) {
+  const params = use(props.params);
   const {
     data: chatSessionSnapshot,
     isLoading,
diff --git a/web/src/app/ee/admin/standard-answer/[id]/page.tsx b/web/src/app/ee/admin/standard-answer/[id]/page.tsx
index 772b1507d33..edacd785b31 100644
--- a/web/src/app/ee/admin/standard-answer/[id]/page.tsx
+++ b/web/src/app/ee/admin/standard-answer/[id]/page.tsx
@@ -6,7 +6,8 @@ import { BackButton } from "@/components/BackButton";
 import { ClipboardIcon } from "@/components/icons/icons";
 import { StandardAnswer, StandardAnswerCategory } from "@/lib/types";
 
-async function Page({ params }: { params: { id: string } }) {
+async function Page(props: { params: Promise<{ id: string }> }) {
+  const params = await props.params;
   const tasks = [
     fetchSS("/manage/admin/standard-answer"),
     fetchSS(`/manage/admin/standard-answer/category`),
diff --git a/web/src/app/layout.tsx b/web/src/app/layout.tsx
index 07592b2fda6..8918aa5a636 100644
--- a/web/src/app/layout.tsx
+++ b/web/src/app/layout.tsx
@@ -17,13 +17,10 @@ import { Logo } from "@/components/Logo";
 import { fetchAssistantData } from "@/lib/chat/fetchAssistantdata";
 import { AppProvider } from "@/components/context/AppProvider";
 import { PHProvider } from "./providers";
-import { default as dynamicImport } from "next/dynamic";
 import { getCurrentUserSS } from "@/lib/userSS";
 import CardSection from "@/components/admin/CardSection";
-
-const PostHogPageView = dynamicImport(() => import("./PostHogPageView"), {
-  ssr: false,
-});
+import { Suspense } from "react";
+import PostHogPageView from "./PostHogPageView";
 
 const inter = Inter({
   subsets: ["latin"],
@@ -181,7 +178,9 @@ export default async function RootLayout({
       hasAnyConnectors={hasAnyConnectors}
       hasImageCompatibleModel={hasImageCompatibleModel}
     >
-      <PostHogPageView />
+      <Suspense fallback={null}>
+        <PostHogPageView />
+      </Suspense>
       {children}
     </AppProvider>
   );
diff --git a/web/src/app/prompts/page.tsx b/web/src/app/prompts/page.tsx
index 20c9038ac2e..5591e26b7b3 100644
--- a/web/src/app/prompts/page.tsx
+++ b/web/src/app/prompts/page.tsx
@@ -3,11 +3,10 @@ import { unstable_noStore as noStore } from "next/cache";
 import { redirect } from "next/navigation";
 import WrappedPrompts from "../assistants/mine/WrappedInputPrompts";
 
-export default async function GalleryPage({
-  searchParams,
-}: {
-  searchParams: { [key: string]: string };
+export default async function GalleryPage(props: {
+  searchParams: Promise<{ [key: string]: string }>;
 }) {
+  const searchParams = await props.searchParams;
   noStore();
 
   const data = await fetchChatData(searchParams);
diff --git a/web/src/app/search/page.tsx b/web/src/app/search/page.tsx
index 8bd743fe763..3572d7cbe40 100644
--- a/web/src/app/search/page.tsx
+++ b/web/src/app/search/page.tsx
@@ -30,23 +30,21 @@ import WrappedSearch from "./WrappedSearch";
 import { SearchProvider } from "@/components/context/SearchContext";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "../admin/configuration/llm/interfaces";
-import { AssistantsProvider } from "@/components/context/AssistantsContext";
 import { headers } from "next/headers";
 import {
   hasCompletedWelcomeFlowSS,
   WelcomeModal,
 } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
 
-export default async function Home({
-  searchParams,
-}: {
-  searchParams: { [key: string]: string | string[] | undefined };
+export default async function Home(props: {
+  searchParams: Promise<{ [key: string]: string | string[] | undefined }>;
 }) {
+  const searchParams = await props.searchParams;
   // Disable caching so we always get the up to date connector / document set / persona info
   // importantly, this prevents users from adding a connector, going back to the main page,
   // and then getting hit with a "No Connectors" popup
   noStore();
-
+  const requestCookies = await cookies();
   const tasks = [
     getAuthTypeMetadataSS(),
     getCurrentUserSS(),
@@ -88,7 +86,7 @@ export default async function Home({
   const authDisabled = authTypeMetadata?.authType === "disabled";
 
   if (!authDisabled && !user) {
-    const headersList = headers();
+    const headersList = await headers();
     const fullUrl = headersList.get("x-url") || "/search";
     const searchParamsString = new URLSearchParams(
       searchParams as unknown as Record<string, string>
@@ -146,7 +144,7 @@ export default async function Home({
   }
 
   // needs to be done in a non-client side component due to nextjs
-  const storedSearchType = cookies().get("searchType")?.value as
+  const storedSearchType = requestCookies.get("searchType")?.value as
     | string
     | undefined;
   const searchTypeDefault: SearchType =
@@ -159,7 +157,7 @@ export default async function Home({
 
   const shouldShowWelcomeModal =
     !llmProviders.length &&
-    !hasCompletedWelcomeFlowSS() &&
+    !hasCompletedWelcomeFlowSS(requestCookies) &&
     !hasAnyConnectors &&
     (!user || user.role === "admin");
 
@@ -168,8 +166,10 @@ export default async function Home({
     ccPairs.length === 0 &&
     !shouldShowWelcomeModal;
 
-  const sidebarToggled = cookies().get(SIDEBAR_TOGGLED_COOKIE_NAME);
-  const agenticSearchToggle = cookies().get(AGENTIC_SEARCH_TYPE_COOKIE_NAME);
+  const sidebarToggled = requestCookies.get(SIDEBAR_TOGGLED_COOKIE_NAME);
+  const agenticSearchToggle = requestCookies.get(
+    AGENTIC_SEARCH_TYPE_COOKIE_NAME
+  );
 
   const toggleSidebar = sidebarToggled
     ? sidebarToggled.value.toLocaleLowerCase() == "true" || false
@@ -183,7 +183,9 @@ export default async function Home({
     <>
       <HealthCheckBanner />
       <InstantSSRAutoRefresh />
-      {shouldShowWelcomeModal && <WelcomeModal user={user} />}
+      {shouldShowWelcomeModal && (
+        <WelcomeModal user={user} requestCookies={requestCookies} />
+      )}
       {/* ChatPopup is a custom popup that displays a admin-specified message on initial user visit. 
       Only used in the EE version of the app. */}
       <ChatPopup />
diff --git a/web/src/components/context/AppProvider.tsx b/web/src/components/context/AppProvider.tsx
index 66d8a0eff15..eda873214a2 100644
--- a/web/src/components/context/AppProvider.tsx
+++ b/web/src/components/context/AppProvider.tsx
@@ -1,4 +1,3 @@
-"use server";
 import { CombinedSettings } from "@/app/admin/settings/interfaces";
 import { UserProvider } from "../user/UserProvider";
 import { ProviderContextProvider } from "../chat_search/ProviderContext";
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 30f6f0bdccc..120d5807485 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -15,7 +15,6 @@ import {
   ArrowSquareOut,
 } from "@phosphor-icons/react";
 import {
-  FiCheck,
   FiChevronsDown,
   FiChevronsUp,
   FiEdit2,
@@ -30,17 +29,10 @@ import {
   FiChevronRight,
   FiChevronLeft,
   FiAlertTriangle,
-  FiZoomIn,
   FiCopy,
-  FiBookmark,
   FiCpu,
   FiInfo,
-  FiUploadCloud,
-  FiUser,
-  FiUsers,
   FiBarChart2,
-  FiDatabase,
-  FiSlack,
 } from "react-icons/fi";
 import { SiBookstack } from "react-icons/si";
 import Image, { StaticImageData } from "next/image";
diff --git a/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx b/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx
index 7355990f22e..3deaab06508 100644
--- a/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx
+++ b/web/src/components/initialSetup/welcome/WelcomeModalWrapper.tsx
@@ -5,17 +5,25 @@ import {
 } from "./WelcomeModal";
 import { COMPLETED_WELCOME_FLOW_COOKIE } from "./constants";
 import { User } from "@/lib/types";
+import { ReadonlyRequestCookies } from "next/dist/server/web/spec-extension/adapters/request-cookies";
 
-export function hasCompletedWelcomeFlowSS() {
-  const cookieStore = cookies();
+export function hasCompletedWelcomeFlowSS(
+  requestCookies: ReadonlyRequestCookies
+) {
   return (
-    cookieStore.get(COMPLETED_WELCOME_FLOW_COOKIE)?.value?.toLowerCase() ===
+    requestCookies.get(COMPLETED_WELCOME_FLOW_COOKIE)?.value?.toLowerCase() ===
     "true"
   );
 }
 
-export function WelcomeModal({ user }: { user: User | null }) {
-  const hasCompletedWelcomeFlow = hasCompletedWelcomeFlowSS();
+export function WelcomeModal({
+  user,
+  requestCookies,
+}: {
+  user: User | null;
+  requestCookies: ReadonlyRequestCookies;
+}) {
+  const hasCompletedWelcomeFlow = hasCompletedWelcomeFlowSS(requestCookies);
   if (hasCompletedWelcomeFlow) {
     return <_CompletedWelcomeFlowDummyComponent />;
   }
diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts
index 32838aeac6f..751aa4d005a 100644
--- a/web/src/lib/chat/fetchChatData.ts
+++ b/web/src/lib/chat/fetchChatData.ts
@@ -19,16 +19,13 @@ import { Settings } from "@/app/admin/settings/interfaces";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
-import { personaComparator } from "@/app/admin/assistants/lib";
 import { cookies, headers } from "next/headers";
 import {
   SIDEBAR_TOGGLED_COOKIE_NAME,
   DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME,
 } from "@/components/resizable/constants";
 import { hasCompletedWelcomeFlowSS } from "@/components/initialSetup/welcome/WelcomeModalWrapper";
-import { fetchAssistantsSS } from "../assistants/fetchAssistantsSS";
 import { NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN } from "../constants";
-import { checkLLMSupportsImageInput } from "../llm/utils";
 import { redirect } from "next/navigation";
 
 interface FetchChatDataResult {
@@ -51,6 +48,7 @@ interface FetchChatDataResult {
 export async function fetchChatData(searchParams: {
   [key: string]: string;
 }): Promise<FetchChatDataResult | { redirect: string }> {
+  const requestCookies = await cookies();
   const tasks = [
     getAuthTypeMetadataSS(),
     getCurrentUserSS(),
@@ -93,7 +91,7 @@ export async function fetchChatData(searchParams: {
 
   const authDisabled = authTypeMetadata?.authType === "disabled";
   if (!authDisabled && !user) {
-    const headersList = headers();
+    const headersList = await headers();
     const fullUrl = headersList.get("x-url") || "/chat";
     const searchParamsString = new URLSearchParams(
       searchParams as unknown as Record<string, string>
@@ -165,10 +163,10 @@ export async function fetchChatData(searchParams: {
     ? parseInt(defaultAssistantIdRaw)
     : undefined;
 
-  const documentSidebarCookieInitialWidth = cookies().get(
+  const documentSidebarCookieInitialWidth = requestCookies.get(
     DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME
   );
-  const sidebarToggled = cookies().get(SIDEBAR_TOGGLED_COOKIE_NAME);
+  const sidebarToggled = requestCookies.get(SIDEBAR_TOGGLED_COOKIE_NAME);
 
   const toggleSidebar = sidebarToggled
     ? sidebarToggled.value.toLocaleLowerCase() == "true" || false
@@ -181,7 +179,7 @@ export async function fetchChatData(searchParams: {
   const hasAnyConnectors = ccPairs.length > 0;
   const shouldShowWelcomeModal =
     !llmProviders.length &&
-    !hasCompletedWelcomeFlowSS() &&
+    !hasCompletedWelcomeFlowSS(requestCookies) &&
     !hasAnyConnectors &&
     (!user || user.role === "admin");
 
@@ -195,7 +193,7 @@ export async function fetchChatData(searchParams: {
     console.log(`Failed to fetch folders - ${foldersResponse?.status}`);
   }
 
-  const openedFoldersCookie = cookies().get("openedFolders");
+  const openedFoldersCookie = requestCookies.get("openedFolders");
   const openedFolders = openedFoldersCookie
     ? JSON.parse(openedFoldersCookie.value)
     : {};
diff --git a/web/src/lib/chat/fetchSomeChatData.ts b/web/src/lib/chat/fetchSomeChatData.ts
index fdfa55b5ea1..47c4c22465d 100644
--- a/web/src/lib/chat/fetchSomeChatData.ts
+++ b/web/src/lib/chat/fetchSomeChatData.ts
@@ -14,8 +14,6 @@ import {
 import { ChatSession } from "@/app/chat/interfaces";
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { InputPrompt } from "@/app/admin/prompt-library/interfaces";
-import { FullEmbeddingModelResponse } from "@/components/embedding/interfaces";
-import { Settings } from "@/app/admin/settings/interfaces";
 import { fetchLLMProvidersSS } from "@/lib/llm/fetchLLMs";
 import { LLMProviderDescriptor } from "@/app/admin/configuration/llm/interfaces";
 import { Folder } from "@/app/chat/folders/interfaces";
@@ -67,6 +65,7 @@ export async function fetchSomeChatData(
   searchParams: { [key: string]: string },
   fetchOptions: FetchOption[] = []
 ): Promise<FetchChatDataResult | { redirect: string }> {
+  const requestCookies = await cookies();
   const tasks: Promise<any>[] = [];
   const taskMap: Record<FetchOption, () => Promise<any>> = {
     user: getCurrentUserSS,
@@ -196,7 +195,7 @@ export async function fetchSomeChatData(
   }
 
   if (fetchOptions.includes("folders")) {
-    const openedFoldersCookie = cookies().get("openedFolders");
+    const openedFoldersCookie = requestCookies.get("openedFolders");
     result.openedFolders = openedFoldersCookie
       ? JSON.parse(openedFoldersCookie.value)
       : {};
@@ -207,10 +206,10 @@ export async function fetchSomeChatData(
     ? parseInt(defaultAssistantIdRaw)
     : undefined;
 
-  const documentSidebarCookieInitialWidth = cookies().get(
+  const documentSidebarCookieInitialWidth = requestCookies.get(
     DOCUMENT_SIDEBAR_WIDTH_COOKIE_NAME
   );
-  const sidebarToggled = cookies().get(SIDEBAR_TOGGLED_COOKIE_NAME);
+  const sidebarToggled = requestCookies.get(SIDEBAR_TOGGLED_COOKIE_NAME);
 
   result.toggleSidebar = sidebarToggled
     ? sidebarToggled.value.toLowerCase() === "true"
@@ -223,7 +222,7 @@ export async function fetchSomeChatData(
   if (fetchOptions.includes("ccPairs") && result.ccPairs) {
     const hasAnyConnectors = result.ccPairs.length > 0;
     result.shouldShowWelcomeModal =
-      !hasCompletedWelcomeFlowSS() &&
+      !hasCompletedWelcomeFlowSS(requestCookies) &&
       !hasAnyConnectors &&
       (!user || user.role === "admin");
 
diff --git a/web/src/lib/userSS.ts b/web/src/lib/userSS.ts
index 8ce25555b2f..55b916d34b3 100644
--- a/web/src/lib/userSS.ts
+++ b/web/src/lib/userSS.ts
@@ -143,7 +143,7 @@ export const getCurrentUserSS = async (): Promise<User | null> => {
       credentials: "include",
       next: { revalidate: 0 },
       headers: {
-        cookie: cookies()
+        cookie: (await cookies())
           .getAll()
           .map((cookie) => `${cookie.name}=${cookie.value}`)
           .join("; "),
diff --git a/web/src/lib/utilsSS.ts b/web/src/lib/utilsSS.ts
index 862804b9261..b54bb1a2be3 100644
--- a/web/src/lib/utilsSS.ts
+++ b/web/src/lib/utilsSS.ts
@@ -15,12 +15,12 @@ export function buildUrl(path: string) {
   return `${INTERNAL_URL}/${path}`;
 }
 
-export function fetchSS(url: string, options?: RequestInit) {
+export async function fetchSS(url: string, options?: RequestInit) {
   const init = options || {
     credentials: "include",
     cache: "no-store",
     headers: {
-      cookie: cookies()
+      cookie: (await cookies())
         .getAll()
         .map((cookie) => `${cookie.name}=${cookie.value}`)
         .join("; "),
diff --git a/web/src/middleware.ts b/web/src/middleware.ts
index 06c106fffc4..a98f24b5f52 100644
--- a/web/src/middleware.ts
+++ b/web/src/middleware.ts
@@ -4,24 +4,26 @@ import { SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED } from "./lib/consta
 
 // NOTE: have to have the "/:path*" here since NextJS doesn't allow any real JS to
 // be run before the config is defined e.g. if we try and do a .map it will complain
-const eePaths = [
-  "/admin/groups/:path*",
-  "/admin/api-key/:path*",
-  "/admin/performance/usage/:path*",
-  "/admin/performance/query-history/:path*",
-  "/admin/whitelabeling/:path*",
-  "/admin/performance/custom-analytics/:path*",
-  "/admin/standard-answer/:path*",
-  ...(process.env.NEXT_PUBLIC_CLOUD_ENABLED
-    ? ["/admin/cloud-settings/:path*"]
-    : []),
-];
+export const config = {
+  matcher: [
+    "/admin/groups/:path*",
+    "/admin/api-key/:path*",
+    "/admin/performance/usage/:path*",
+    "/admin/performance/query-history/:path*",
+    "/admin/whitelabeling/:path*",
+    "/admin/performance/custom-analytics/:path*",
+    "/admin/standard-answer/:path*",
+
+    // Cloud only
+    "/admin/cloud-settings/:path*",
+  ],
+};
 
 // removes the "/:path*" from the end
 const stripPath = (path: string) =>
   path.replace(/(.*):\path\*$/, "$1").replace(/\/$/, "");
 
-const strippedEEPaths = eePaths.map(stripPath);
+const strippedEEPaths = config.matcher.map(stripPath);
 
 export async function middleware(request: NextRequest) {
   if (SERVER_SIDE_ONLY__PAID_ENTERPRISE_FEATURES_ENABLED) {
@@ -43,8 +45,3 @@ export async function middleware(request: NextRequest) {
   // Continue with the response if no rewrite is needed
   return NextResponse.next();
 }
-
-// Specify the paths that the middleware should run for
-export const config = {
-  matcher: eePaths,
-};
diff --git a/web/tailwind.config.js b/web/tailwind.config.js
index 2cf0ca98aa4..4917e66c6ff 100644
--- a/web/tailwind.config.js
+++ b/web/tailwind.config.js
@@ -1,11 +1,18 @@
 var merge = require("lodash/merge");
 
+// Use relative paths for imports
 const baseThemes = require("./tailwind-themes/tailwind.config.js");
-const customThemes = process.env.NEXT_PUBLIC_THEME
-  ? require(
+
+let customThemes = null;
+try {
+  if (process.env.NEXT_PUBLIC_THEME) {
+    customThemes = require(
       `./tailwind-themes/custom/${process.env.NEXT_PUBLIC_THEME}/tailwind.config.js`
-    )
-  : null;
+    );
+  }
+} catch (error) {
+  console.warn(`Custom theme not found for: ${process.env.NEXT_PUBLIC_THEME}`);
+}
 
 /** @type {import('tailwindcss').Config} */
 module.exports = customThemes ? merge(baseThemes, customThemes) : baseThemes;

From 19d6b63fd3429567319d8e00c1fc4385e983ff4e Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 10:05:33 -0800
Subject: [PATCH 349/376] temporary update (#3037)

---
 .../daily/connectors/google_drive/test_google_drive_oauth.py     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
index f39b15600b4..3faee8474d7 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
@@ -134,6 +134,7 @@ def test_drive_one_only(
         + DRIVE_ID_MAPPING["FOLDER_1"]
         + DRIVE_ID_MAPPING["FOLDER_1_1"]
         + DRIVE_ID_MAPPING["FOLDER_1_2"]
+        + ["file_61.txt"]  # Temporary patch so merge queue works (TODO: update mapping)
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,

From d4f38bba8bad77b864a7aa18f8472b4679e434fb Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 10:27:06 -0800
Subject: [PATCH 350/376] Revert temporary modifications (#3038)

* Revert temporary modifications

* nit
---
 .../daily/connectors/google_drive/test_google_drive_oauth.py    | 1 -
 web/package-lock.json                                           | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
index 3faee8474d7..f39b15600b4 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
@@ -134,7 +134,6 @@ def test_drive_one_only(
         + DRIVE_ID_MAPPING["FOLDER_1"]
         + DRIVE_ID_MAPPING["FOLDER_1_1"]
         + DRIVE_ID_MAPPING["FOLDER_1_2"]
-        + ["file_61.txt"]  # Temporary patch so merge queue works (TODO: update mapping)
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
diff --git a/web/package-lock.json b/web/package-lock.json
index 2b02ef81659..c105c298991 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -14853,4 +14853,4 @@
       }
     }
   }
-}
+}
\ No newline at end of file

From 46e5ffa3ae98c6dd010e5f0578be999979acca28 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 10:27:25 -0800
Subject: [PATCH 351/376] add validated + reformatted dynamic beat acquisition
 (#3006)

* add validated + reformatted dynamic beat acquisition

* validate

* reorg

* nit

* address comments

* update

* typing

* ensure versioned apps capture

* Remove locks (#3017)

* add validated + reformatted dynamic beat acquisition

* initial removal of locks!

* minor

* remove unecessary locks

* update

* nit

* k

* K8s jobs (#3033)

* add k8s configs

* k

* update config

* k

* improved timeouts + worker configs

* improve workers
---
 .../background/celery/apps/app_base.py        |  72 +++----
 .../danswer/background/celery/apps/beat.py    | 190 +++++++++++-------
 .../danswer/background/celery/apps/light.py   |   1 +
 .../danswer/background/celery/apps/primary.py | 165 ++++++---------
 .../background/celery/apps/scheduler.py       |  96 +++++++++
 .../background/celery/tasks/beat_schedule.py  |  48 +++++
 .../background/celery/versioned_apps/beat.py  |   6 +-
 .../ee/danswer/background/celery/apps/beat.py |  52 -----
 .../background/celery/tasks/beat_schedule.py  |  33 +++
 .../cloud_kubernetes/hpa/workers_hpa.yaml     |  56 ++++++
 deployment/cloud_kubernetes/workers/beat.yaml |  44 ++++
 .../workers/heavy_worker.yaml                 |  60 ++++++
 .../workers/indexing_worker.yaml              |  60 ++++++
 .../workers/light_worker.yaml                 |  60 ++++++
 .../cloud_kubernetes/workers/primary.yaml     |  60 ++++++
 web/package-lock.json                         |  15 ++
 16 files changed, 748 insertions(+), 270 deletions(-)
 create mode 100644 backend/danswer/background/celery/apps/scheduler.py
 create mode 100644 backend/danswer/background/celery/tasks/beat_schedule.py
 delete mode 100644 backend/ee/danswer/background/celery/apps/beat.py
 create mode 100644 backend/ee/danswer/background/celery/tasks/beat_schedule.py
 create mode 100644 deployment/cloud_kubernetes/hpa/workers_hpa.yaml
 create mode 100644 deployment/cloud_kubernetes/workers/beat.yaml
 create mode 100644 deployment/cloud_kubernetes/workers/heavy_worker.yaml
 create mode 100644 deployment/cloud_kubernetes/workers/indexing_worker.yaml
 create mode 100644 deployment/cloud_kubernetes/workers/light_worker.yaml
 create mode 100644 deployment/cloud_kubernetes/workers/primary.yaml

diff --git a/backend/danswer/background/celery/apps/app_base.py b/backend/danswer/background/celery/apps/app_base.py
index 8649227ea36..117293f1c2f 100644
--- a/backend/danswer/background/celery/apps/app_base.py
+++ b/backend/danswer/background/celery/apps/app_base.py
@@ -16,7 +16,6 @@
 from danswer.background.celery.apps.task_formatters import CeleryTaskPlainFormatter
 from danswer.background.celery.celery_utils import celery_is_worker_primary
 from danswer.configs.constants import DanswerRedisLocks
-from danswer.db.engine import get_all_tenant_ids
 from danswer.redis.redis_connector import RedisConnector
 from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
 from danswer.redis.redis_connector_delete import RedisConnectorDelete
@@ -27,6 +26,7 @@
 from danswer.utils.logger import ColoredFormatter
 from danswer.utils.logger import PlainFormatter
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import SENTRY_DSN
 
 
@@ -172,44 +172,30 @@ def wait_for_redis(sender: Any, **kwargs: Any) -> None:
 
 
 def on_secondary_worker_init(sender: Any, **kwargs: Any) -> None:
+    logger.info("Running as a secondary celery worker.")
+
+    # Exit early if multi-tenant since primary worker check not needed
+    if MULTI_TENANT:
+        return
+
+    # Set up variables for waiting on primary worker
     WAIT_INTERVAL = 5
     WAIT_LIMIT = 60
-
-    logger.info("Running as a secondary celery worker.")
-    logger.info("Waiting for all tenant primary workers to be ready...")
+    r = get_redis_client(tenant_id=None)
     time_start = time.monotonic()
 
+    logger.info("Waiting for primary worker to be ready...")
     while True:
-        tenant_ids = get_all_tenant_ids()
-        # Check if we have a primary worker lock for each tenant
-        all_tenants_ready = all(
-            get_redis_client(tenant_id=tenant_id).exists(
-                DanswerRedisLocks.PRIMARY_WORKER
-            )
-            for tenant_id in tenant_ids
-        )
-
-        if all_tenants_ready:
+        if r.exists(DanswerRedisLocks.PRIMARY_WORKER):
             break
 
         time_elapsed = time.monotonic() - time_start
-        ready_tenants = sum(
-            1
-            for tenant_id in tenant_ids
-            if get_redis_client(tenant_id=tenant_id).exists(
-                DanswerRedisLocks.PRIMARY_WORKER
-            )
-        )
-
         logger.info(
-            f"Not all tenant primary workers are ready yet. "
-            f"Ready tenants: {ready_tenants}/{len(tenant_ids)} "
-            f"elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
+            f"Primary worker is not ready yet. elapsed={time_elapsed:.1f} timeout={WAIT_LIMIT:.1f}"
         )
-
         if time_elapsed > WAIT_LIMIT:
             msg = (
-                f"Not all tenant primary workers were ready within the timeout "
+                f"Primary worker was not ready within the timeout. "
                 f"({WAIT_LIMIT} seconds). Exiting..."
             )
             logger.error(msg)
@@ -217,7 +203,7 @@ def on_secondary_worker_init(sender: Any, **kwargs: Any) -> None:
 
         time.sleep(WAIT_INTERVAL)
 
-    logger.info("All tenant primary workers are ready. Continuing...")
+    logger.info("Wait for primary worker completed successfully. Continuing...")
     return
 
 
@@ -229,26 +215,20 @@ def on_worker_shutdown(sender: Any, **kwargs: Any) -> None:
     if not celery_is_worker_primary(sender):
         return
 
-    if not hasattr(sender, "primary_worker_locks"):
+    if not sender.primary_worker_lock:
         return
 
-    for tenant_id, lock in sender.primary_worker_locks.items():
-        try:
-            if lock and lock.owned():
-                logger.debug(f"Attempting to release lock for tenant {tenant_id}")
-                try:
-                    lock.release()
-                    logger.debug(f"Successfully released lock for tenant {tenant_id}")
-                except Exception as e:
-                    logger.error(
-                        f"Failed to release lock for tenant {tenant_id}. Error: {str(e)}"
-                    )
-                finally:
-                    sender.primary_worker_locks[tenant_id] = None
-        except Exception as e:
-            logger.error(
-                f"Error checking lock status for tenant {tenant_id}. Error: {str(e)}"
-            )
+    logger.info("Releasing primary worker lock.")
+    lock = sender.primary_worker_lock
+    try:
+        if lock.owned():
+            try:
+                lock.release()
+                sender.primary_worker_lock = None
+            except Exception as e:
+                logger.error(f"Failed to release primary worker lock: {e}")
+    except Exception as e:
+        logger.error(f"Failed to check if primary worker lock is owned: {e}")
 
 
 def on_setup_logging(
diff --git a/backend/danswer/background/celery/apps/beat.py b/backend/danswer/background/celery/apps/beat.py
index 097f8f5cfa3..5ef887121dc 100644
--- a/backend/danswer/background/celery/apps/beat.py
+++ b/backend/danswer/background/celery/apps/beat.py
@@ -3,26 +3,144 @@
 
 from celery import Celery
 from celery import signals
+from celery.beat import PersistentScheduler  # type: ignore
 from celery.signals import beat_init
 
 import danswer.background.celery.apps.app_base as app_base
-from danswer.configs.constants import DanswerCeleryPriority
 from danswer.configs.constants import POSTGRES_CELERY_BEAT_APP_NAME
 from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import SqlEngine
 from danswer.utils.logger import setup_logger
+from danswer.utils.variable_functionality import fetch_versioned_implementation
 
-logger = setup_logger()
+logger = setup_logger(__name__)
 
 celery_app = Celery(__name__)
 celery_app.config_from_object("danswer.background.celery.configs.beat")
 
 
+class DynamicTenantScheduler(PersistentScheduler):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        logger.info("Initializing DynamicTenantScheduler")
+        super().__init__(*args, **kwargs)
+        self._reload_interval = timedelta(minutes=2)
+        self._last_reload = self.app.now() - self._reload_interval
+        # Let the parent class handle store initialization
+        self.setup_schedule()
+        self._update_tenant_tasks()
+        logger.info(f"Set reload interval to {self._reload_interval}")
+
+    def setup_schedule(self) -> None:
+        logger.info("Setting up initial schedule")
+        super().setup_schedule()
+        logger.info("Initial schedule setup complete")
+
+    def tick(self) -> float:
+        retval = super().tick()
+        now = self.app.now()
+        if (
+            self._last_reload is None
+            or (now - self._last_reload) > self._reload_interval
+        ):
+            logger.info("Reload interval reached, initiating tenant task update")
+            self._update_tenant_tasks()
+            self._last_reload = now
+            logger.info("Tenant task update completed, reset reload timer")
+        return retval
+
+    def _update_tenant_tasks(self) -> None:
+        logger.info("Starting tenant task update process")
+        try:
+            logger.info("Fetching all tenant IDs")
+            tenant_ids = get_all_tenant_ids()
+            logger.info(f"Found {len(tenant_ids)} tenants")
+
+            logger.info("Fetching tasks to schedule")
+            tasks_to_schedule = fetch_versioned_implementation(
+                "danswer.background.celery.tasks.beat_schedule", "get_tasks_to_schedule"
+            )
+
+            new_beat_schedule: dict[str, dict[str, Any]] = {}
+
+            current_schedule = self.schedule.items()
+
+            existing_tenants = set()
+            for task_name, _ in current_schedule:
+                if "-" in task_name:
+                    existing_tenants.add(task_name.split("-")[-1])
+            logger.info(f"Found {len(existing_tenants)} existing tenants in schedule")
+
+            for tenant_id in tenant_ids:
+                if tenant_id not in existing_tenants:
+                    logger.info(f"Processing new tenant: {tenant_id}")
+
+                for task in tasks_to_schedule():
+                    task_name = f"{task['name']}-{tenant_id}"
+                    logger.debug(f"Creating task configuration for {task_name}")
+                    new_task = {
+                        "task": task["task"],
+                        "schedule": task["schedule"],
+                        "kwargs": {"tenant_id": tenant_id},
+                    }
+                    if options := task.get("options"):
+                        logger.debug(f"Adding options to task {task_name}: {options}")
+                        new_task["options"] = options
+                    new_beat_schedule[task_name] = new_task
+
+            if self._should_update_schedule(current_schedule, new_beat_schedule):
+                logger.info(
+                    "Schedule update required",
+                    extra={
+                        "new_tasks": len(new_beat_schedule),
+                        "current_tasks": len(current_schedule),
+                    },
+                )
+
+                # Create schedule entries
+                entries = {}
+                for name, entry in new_beat_schedule.items():
+                    entries[name] = self.Entry(
+                        name=name,
+                        app=self.app,
+                        task=entry["task"],
+                        schedule=entry["schedule"],
+                        options=entry.get("options", {}),
+                        kwargs=entry.get("kwargs", {}),
+                    )
+
+                # Update the schedule using the scheduler's methods
+                self.schedule.clear()
+                self.schedule.update(entries)
+
+                # Ensure changes are persisted
+                self.sync()
+
+                logger.info("Schedule update completed successfully")
+            else:
+                logger.info("Schedule is up to date, no changes needed")
+
+        except (AttributeError, KeyError) as e:
+            logger.exception(f"Failed to process task configuration: {str(e)}")
+        except Exception as e:
+            logger.exception(f"Unexpected error updating tenant tasks: {str(e)}")
+
+    def _should_update_schedule(
+        self, current_schedule: dict, new_schedule: dict
+    ) -> bool:
+        """Compare schedules to determine if an update is needed."""
+        logger.debug("Comparing current and new schedules")
+        current_tasks = set(name for name, _ in current_schedule)
+        new_tasks = set(new_schedule.keys())
+        needs_update = current_tasks != new_tasks
+        logger.debug(f"Schedule update needed: {needs_update}")
+        return needs_update
+
+
 @beat_init.connect
 def on_beat_init(sender: Any, **kwargs: Any) -> None:
     logger.info("beat_init signal received.")
 
-    # celery beat shouldn't touch the db at all. But just setting a low minimum here.
+    # Celery beat shouldn't touch the db at all. But just setting a low minimum here.
     SqlEngine.set_app_name(POSTGRES_CELERY_BEAT_APP_NAME)
     SqlEngine.init_engine(pool_size=2, max_overflow=0)
     app_base.wait_for_redis(sender, **kwargs)
@@ -35,68 +153,4 @@ def on_setup_logging(
     app_base.on_setup_logging(loglevel, logfile, format, colorize, **kwargs)
 
 
-#####
-# Celery Beat (Periodic Tasks) Settings
-#####
-
-tenant_ids = get_all_tenant_ids()
-
-tasks_to_schedule = [
-    {
-        "name": "check-for-vespa-sync",
-        "task": "check_for_vespa_sync_task",
-        "schedule": timedelta(seconds=5),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "check-for-connector-deletion",
-        "task": "check_for_connector_deletion_task",
-        "schedule": timedelta(seconds=60),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "check-for-indexing",
-        "task": "check_for_indexing",
-        "schedule": timedelta(seconds=10),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "check-for-prune",
-        "task": "check_for_pruning",
-        "schedule": timedelta(seconds=10),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-    {
-        "name": "kombu-message-cleanup",
-        "task": "kombu_message_cleanup_task",
-        "schedule": timedelta(seconds=3600),
-        "options": {"priority": DanswerCeleryPriority.LOWEST},
-    },
-    {
-        "name": "monitor-vespa-sync",
-        "task": "monitor_vespa_sync",
-        "schedule": timedelta(seconds=5),
-        "options": {"priority": DanswerCeleryPriority.HIGH},
-    },
-]
-
-
-# Build the celery beat schedule dynamically
-beat_schedule = {}
-
-for tenant_id in tenant_ids:
-    for task in tasks_to_schedule:
-        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
-        beat_schedule[task_name] = {
-            "task": task["task"],
-            "schedule": task["schedule"],
-            "options": task["options"],
-            "kwargs": {"tenant_id": tenant_id},  # Must pass tenant_id as an argument
-        }
-
-# Include any existing beat schedules
-existing_beat_schedule = celery_app.conf.beat_schedule or {}
-beat_schedule.update(existing_beat_schedule)
-
-# Update the Celery app configuration once
-celery_app.conf.beat_schedule = beat_schedule
+celery_app.conf.beat_scheduler = DynamicTenantScheduler
diff --git a/backend/danswer/background/celery/apps/light.py b/backend/danswer/background/celery/apps/light.py
index 6f39074b601..033ba084ee8 100644
--- a/backend/danswer/background/celery/apps/light.py
+++ b/backend/danswer/background/celery/apps/light.py
@@ -85,5 +85,6 @@ def on_setup_logging(
     [
         "danswer.background.celery.tasks.shared",
         "danswer.background.celery.tasks.vespa",
+        "danswer.background.celery.tasks.connector_deletion",
     ]
 )
diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 5de692bd842..7db4b998254 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -17,17 +17,11 @@
 from danswer.configs.constants import CELERY_PRIMARY_WORKER_LOCK_TIMEOUT
 from danswer.configs.constants import DanswerRedisLocks
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
-from danswer.db.engine import get_all_tenant_ids
 from danswer.db.engine import SqlEngine
 from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
-from danswer.redis.redis_connector_delete import RedisConnectorDelete
-from danswer.redis.redis_connector_index import RedisConnectorIndex
-from danswer.redis.redis_connector_prune import RedisConnectorPrune
-from danswer.redis.redis_connector_stop import RedisConnectorStop
-from danswer.redis.redis_document_set import RedisDocumentSet
 from danswer.redis.redis_pool import get_redis_client
-from danswer.redis.redis_usergroup import RedisUserGroup
 from danswer.utils.logger import setup_logger
+from shared_configs.configs import MULTI_TENANT
 
 
 logger = setup_logger()
@@ -79,58 +73,45 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
 
     logger.info("Running as the primary celery worker.")
 
-    sender.primary_worker_locks = {}
+    if MULTI_TENANT:
+        return
 
     # This is singleton work that should be done on startup exactly once
-    # by the primary worker
-    tenant_ids = get_all_tenant_ids()
-    for tenant_id in tenant_ids:
-        r = get_redis_client(tenant_id=tenant_id)
-
-        # For the moment, we're assuming that we are the only primary worker
-        # that should be running.
-        # TODO: maybe check for or clean up another zombie primary worker if we detect it
-        r.delete(DanswerRedisLocks.PRIMARY_WORKER)
-
-        # this process wide lock is taken to help other workers start up in order.
-        # it is planned to use this lock to enforce singleton behavior on the primary
-        # worker, since the primary worker does redis cleanup on startup, but this isn't
-        # implemented yet.
-        lock = r.lock(
-            DanswerRedisLocks.PRIMARY_WORKER,
-            timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
-        )
-
-        logger.info("Primary worker lock: Acquire starting.")
-        acquired = lock.acquire(blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2)
-        if acquired:
-            logger.info("Primary worker lock: Acquire succeeded.")
-        else:
-            logger.error("Primary worker lock: Acquire failed!")
-            raise WorkerShutdown("Primary worker lock could not be acquired!")
-
-        # tacking on our own user data to the sender
-        sender.primary_worker_locks[tenant_id] = lock
-
-        # As currently designed, when this worker starts as "primary", we reinitialize redis
-        # to a clean state (for our purposes, anyway)
-        r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
-        r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
-
-        r.delete(RedisConnectorCredentialPair.get_taskset_key())
-        r.delete(RedisConnectorCredentialPair.get_fence_key())
-
-        RedisDocumentSet.reset_all(r)
-
-        RedisUserGroup.reset_all(r)
-
-        RedisConnectorDelete.reset_all(r)
-
-        RedisConnectorPrune.reset_all(r)
-
-        RedisConnectorIndex.reset_all(r)
-
-        RedisConnectorStop.reset_all(r)
+    # by the primary worker. This is unnecessary in the multi tenant scenario
+    r = get_redis_client(tenant_id=None)
+
+    # For the moment, we're assuming that we are the only primary worker
+    # that should be running.
+    # TODO: maybe check for or clean up another zombie primary worker if we detect it
+    r.delete(DanswerRedisLocks.PRIMARY_WORKER)
+
+    # this process wide lock is taken to help other workers start up in order.
+    # it is planned to use this lock to enforce singleton behavior on the primary
+    # worker, since the primary worker does redis cleanup on startup, but this isn't
+    # implemented yet.
+    lock = r.lock(
+        DanswerRedisLocks.PRIMARY_WORKER,
+        timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
+    )
+
+    logger.info("Primary worker lock: Acquire starting.")
+    acquired = lock.acquire(blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2)
+    if acquired:
+        logger.info("Primary worker lock: Acquire succeeded.")
+    else:
+        logger.error("Primary worker lock: Acquire failed!")
+        raise WorkerShutdown("Primary worker lock could not be acquired!")
+
+    # tacking on our own user data to the sender
+    sender.primary_worker_lock = lock
+
+    # As currently designed, when this worker starts as "primary", we reinitialize redis
+    # to a clean state (for our purposes, anyway)
+    r.delete(DanswerRedisLocks.CHECK_VESPA_SYNC_BEAT_LOCK)
+    r.delete(DanswerRedisLocks.MONITOR_VESPA_SYNC_BEAT_LOCK)
+
+    r.delete(RedisConnectorCredentialPair.get_taskset_key())
+    r.delete(RedisConnectorCredentialPair.get_fence_key())
 
 
 @worker_ready.connect
@@ -183,52 +164,36 @@ def run_periodic_task(self, worker: Any) -> None:
             if not celery_is_worker_primary(worker):
                 return
 
-            if not hasattr(worker, "primary_worker_locks"):
+            if not hasattr(worker, "primary_worker_lock"):
                 return
 
-            # Retrieve all tenant IDs
-            tenant_ids = get_all_tenant_ids()
-
-            for tenant_id in tenant_ids:
-                lock = worker.primary_worker_locks.get(tenant_id)
-                if not lock:
-                    continue  # Skip if no lock for this tenant
-
-                r = get_redis_client(tenant_id=tenant_id)
-
-                if lock.owned():
-                    task_logger.debug(
-                        f"Reacquiring primary worker lock for tenant {tenant_id}."
-                    )
-                    lock.reacquire()
+            lock = worker.primary_worker_lock
+
+            r = get_redis_client(tenant_id=None)
+
+            if lock.owned():
+                task_logger.debug("Reacquiring primary worker lock.")
+                lock.reacquire()
+            else:
+                task_logger.warning(
+                    "Full acquisition of primary worker lock. "
+                    "Reasons could be worker restart or lock expiration."
+                )
+                lock = r.lock(
+                    DanswerRedisLocks.PRIMARY_WORKER,
+                    timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
+                )
+
+                task_logger.info("Primary worker lock: Acquire starting.")
+                acquired = lock.acquire(
+                    blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2
+                )
+                if acquired:
+                    task_logger.info("Primary worker lock: Acquire succeeded.")
+                    worker.primary_worker_lock = lock
                 else:
-                    task_logger.warning(
-                        f"Full acquisition of primary worker lock for tenant {tenant_id}. "
-                        "Reasons could be worker restart or lock expiration."
-                    )
-                    lock = r.lock(
-                        DanswerRedisLocks.PRIMARY_WORKER,
-                        timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT,
-                    )
-
-                    task_logger.info(
-                        f"Primary worker lock for tenant {tenant_id}: Acquire starting."
-                    )
-                    acquired = lock.acquire(
-                        blocking_timeout=CELERY_PRIMARY_WORKER_LOCK_TIMEOUT / 2
-                    )
-                    if acquired:
-                        task_logger.info(
-                            f"Primary worker lock for tenant {tenant_id}: Acquire succeeded."
-                        )
-                        worker.primary_worker_locks[tenant_id] = lock
-                    else:
-                        task_logger.error(
-                            f"Primary worker lock for tenant {tenant_id}: Acquire failed!"
-                        )
-                        raise TimeoutError(
-                            f"Primary worker lock for tenant {tenant_id} could not be acquired!"
-                        )
+                    task_logger.error("Primary worker lock: Acquire failed!")
+                    raise TimeoutError("Primary worker lock could not be acquired!")
 
         except Exception:
             task_logger.exception("Periodic task failed.")
diff --git a/backend/danswer/background/celery/apps/scheduler.py b/backend/danswer/background/celery/apps/scheduler.py
new file mode 100644
index 00000000000..3ddf1dc169c
--- /dev/null
+++ b/backend/danswer/background/celery/apps/scheduler.py
@@ -0,0 +1,96 @@
+from datetime import timedelta
+from typing import Any
+
+from celery.beat import PersistentScheduler  # type: ignore
+from celery.utils.log import get_task_logger
+
+from danswer.db.engine import get_all_tenant_ids
+from danswer.utils.variable_functionality import fetch_versioned_implementation
+
+logger = get_task_logger(__name__)
+
+
+class DynamicTenantScheduler(PersistentScheduler):
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self._reload_interval = timedelta(minutes=1)
+        self._last_reload = self.app.now() - self._reload_interval
+
+    def setup_schedule(self) -> None:
+        super().setup_schedule()
+
+    def tick(self) -> float:
+        retval = super().tick()
+        now = self.app.now()
+        if (
+            self._last_reload is None
+            or (now - self._last_reload) > self._reload_interval
+        ):
+            logger.info("Reloading schedule to check for new tenants...")
+            self._update_tenant_tasks()
+            self._last_reload = now
+        return retval
+
+    def _update_tenant_tasks(self) -> None:
+        logger.info("Checking for tenant task updates...")
+        try:
+            tenant_ids = get_all_tenant_ids()
+            tasks_to_schedule = fetch_versioned_implementation(
+                "danswer.background.celery.tasks.beat_schedule", "get_tasks_to_schedule"
+            )
+
+            new_beat_schedule: dict[str, dict[str, Any]] = {}
+
+            current_schedule = getattr(self, "_store", {"entries": {}}).get(
+                "entries", {}
+            )
+
+            existing_tenants = set()
+            for task_name in current_schedule.keys():
+                if "-" in task_name:
+                    existing_tenants.add(task_name.split("-")[-1])
+
+            for tenant_id in tenant_ids:
+                if tenant_id not in existing_tenants:
+                    logger.info(f"Found new tenant: {tenant_id}")
+
+                for task in tasks_to_schedule():
+                    task_name = f"{task['name']}-{tenant_id}"
+                    new_task = {
+                        "task": task["task"],
+                        "schedule": task["schedule"],
+                        "kwargs": {"tenant_id": tenant_id},
+                    }
+                    if options := task.get("options"):
+                        new_task["options"] = options
+                    new_beat_schedule[task_name] = new_task
+
+            if self._should_update_schedule(current_schedule, new_beat_schedule):
+                logger.info(
+                    "Updating schedule",
+                    extra={
+                        "new_tasks": len(new_beat_schedule),
+                        "current_tasks": len(current_schedule),
+                    },
+                )
+                if not hasattr(self, "_store"):
+                    self._store: dict[str, dict] = {"entries": {}}
+                self.update_from_dict(new_beat_schedule)
+                logger.info(f"New schedule: {new_beat_schedule}")
+
+                logger.info("Tenant tasks updated successfully")
+            else:
+                logger.debug("No schedule updates needed")
+
+        except (AttributeError, KeyError):
+            logger.exception("Failed to process task configuration")
+        except Exception:
+            logger.exception("Unexpected error updating tenant tasks")
+
+    def _should_update_schedule(
+        self, current_schedule: dict, new_schedule: dict
+    ) -> bool:
+        """Compare schedules to determine if an update is needed."""
+        current_tasks = set(current_schedule.keys())
+        new_tasks = set(new_schedule.keys())
+        return current_tasks != new_tasks
diff --git a/backend/danswer/background/celery/tasks/beat_schedule.py b/backend/danswer/background/celery/tasks/beat_schedule.py
new file mode 100644
index 00000000000..6a20c6ba5c1
--- /dev/null
+++ b/backend/danswer/background/celery/tasks/beat_schedule.py
@@ -0,0 +1,48 @@
+from datetime import timedelta
+from typing import Any
+
+from danswer.configs.constants import DanswerCeleryPriority
+
+
+tasks_to_schedule = [
+    {
+        "name": "check-for-vespa-sync",
+        "task": "check_for_vespa_sync_task",
+        "schedule": timedelta(seconds=5),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "check-for-connector-deletion",
+        "task": "check_for_connector_deletion_task",
+        "schedule": timedelta(seconds=20),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "check-for-indexing",
+        "task": "check_for_indexing",
+        "schedule": timedelta(seconds=10),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "check-for-prune",
+        "task": "check_for_pruning",
+        "schedule": timedelta(seconds=10),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+    {
+        "name": "kombu-message-cleanup",
+        "task": "kombu_message_cleanup_task",
+        "schedule": timedelta(seconds=3600),
+        "options": {"priority": DanswerCeleryPriority.LOWEST},
+    },
+    {
+        "name": "monitor-vespa-sync",
+        "task": "monitor_vespa_sync",
+        "schedule": timedelta(seconds=5),
+        "options": {"priority": DanswerCeleryPriority.HIGH},
+    },
+]
+
+
+def get_tasks_to_schedule() -> list[dict[str, Any]]:
+    return tasks_to_schedule
diff --git a/backend/danswer/background/celery/versioned_apps/beat.py b/backend/danswer/background/celery/versioned_apps/beat.py
index d1b7dc591d9..af407f93c64 100644
--- a/backend/danswer/background/celery/versioned_apps/beat.py
+++ b/backend/danswer/background/celery/versioned_apps/beat.py
@@ -1,8 +1,6 @@
 """Factory stub for running celery worker / celery beat."""
-from danswer.utils.variable_functionality import fetch_versioned_implementation
+from danswer.background.celery.apps.beat import celery_app
 from danswer.utils.variable_functionality import set_is_ee_based_on_env_variable
 
 set_is_ee_based_on_env_variable()
-app = fetch_versioned_implementation(
-    "danswer.background.celery.apps.beat", "celery_app"
-)
+app = celery_app
diff --git a/backend/ee/danswer/background/celery/apps/beat.py b/backend/ee/danswer/background/celery/apps/beat.py
deleted file mode 100644
index 980eb5e3214..00000000000
--- a/backend/ee/danswer/background/celery/apps/beat.py
+++ /dev/null
@@ -1,52 +0,0 @@
-#####
-# Celery Beat (Periodic Tasks) Settings
-#####
-from datetime import timedelta
-
-from danswer.background.celery.apps.beat import celery_app
-from danswer.db.engine import get_all_tenant_ids
-
-
-tenant_ids = get_all_tenant_ids()
-
-tasks_to_schedule = [
-    {
-        "name": "sync-external-doc-permissions",
-        "task": "check_sync_external_doc_permissions_task",
-        "schedule": timedelta(seconds=30),  # TODO: optimize this
-    },
-    {
-        "name": "sync-external-group-permissions",
-        "task": "check_sync_external_group_permissions_task",
-        "schedule": timedelta(seconds=60),  # TODO: optimize this
-    },
-    {
-        "name": "autogenerate_usage_report",
-        "task": "autogenerate_usage_report_task",
-        "schedule": timedelta(days=30),  # TODO: change this to config flag
-    },
-    {
-        "name": "check-ttl-management",
-        "task": "check_ttl_management_task",
-        "schedule": timedelta(hours=1),
-    },
-]
-
-# Build the celery beat schedule dynamically
-beat_schedule = {}
-
-for tenant_id in tenant_ids:
-    for task in tasks_to_schedule:
-        task_name = f"{task['name']}-{tenant_id}"  # Unique name for each scheduled task
-        beat_schedule[task_name] = {
-            "task": task["task"],
-            "schedule": task["schedule"],
-            "kwargs": {"tenant_id": tenant_id},  # Must pass tenant_id as an argument
-        }
-
-# Include any existing beat schedules
-existing_beat_schedule = celery_app.conf.beat_schedule or {}
-beat_schedule.update(existing_beat_schedule)
-
-# Update the Celery app configuration
-celery_app.conf.beat_schedule = beat_schedule
diff --git a/backend/ee/danswer/background/celery/tasks/beat_schedule.py b/backend/ee/danswer/background/celery/tasks/beat_schedule.py
new file mode 100644
index 00000000000..05e2b92bde1
--- /dev/null
+++ b/backend/ee/danswer/background/celery/tasks/beat_schedule.py
@@ -0,0 +1,33 @@
+from datetime import timedelta
+from typing import Any
+
+from danswer.background.celery.tasks.beat_schedule import (
+    tasks_to_schedule as base_tasks_to_schedule,
+)
+
+ee_tasks_to_schedule = [
+    {
+        "name": "sync-external-doc-permissions",
+        "task": "check_sync_external_doc_permissions_task",
+        "schedule": timedelta(seconds=30),  # TODO: optimize this
+    },
+    {
+        "name": "sync-external-group-permissions",
+        "task": "check_sync_external_group_permissions_task",
+        "schedule": timedelta(seconds=60),  # TODO: optimize this
+    },
+    {
+        "name": "autogenerate_usage_report",
+        "task": "autogenerate_usage_report_task",
+        "schedule": timedelta(days=30),  # TODO: change this to config flag
+    },
+    {
+        "name": "check-ttl-management",
+        "task": "check_ttl_management_task",
+        "schedule": timedelta(hours=1),
+    },
+]
+
+
+def get_tasks_to_schedule() -> list[dict[str, Any]]:
+    return ee_tasks_to_schedule + base_tasks_to_schedule
diff --git a/deployment/cloud_kubernetes/hpa/workers_hpa.yaml b/deployment/cloud_kubernetes/hpa/workers_hpa.yaml
new file mode 100644
index 00000000000..fd24b9eeac3
--- /dev/null
+++ b/deployment/cloud_kubernetes/hpa/workers_hpa.yaml
@@ -0,0 +1,56 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: celery-worker-heavy-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: celery-worker-heavy
+  minReplicas: 1
+  maxReplicas: 5
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 60
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: celery-worker-light-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: celery-worker-light
+  minReplicas: 1
+  maxReplicas: 10
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: celery-worker-indexing-hpa
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: celery-worker-indexing
+  minReplicas: 1
+  maxReplicas: 10
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: Utilization
+          averageUtilization: 70
diff --git a/deployment/cloud_kubernetes/workers/beat.yaml b/deployment/cloud_kubernetes/workers/beat.yaml
new file mode 100644
index 00000000000..a9d053f7295
--- /dev/null
+++ b/deployment/cloud_kubernetes/workers/beat.yaml
@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-beat
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: celery-beat
+  template:
+    metadata:
+      labels:
+        app: celery-beat
+    spec:
+      containers:
+        - name: celery-beat
+          image: danswer/danswer-backend:v0.11.0-cloud.beta.4
+          imagePullPolicy: Always
+          command:
+            [
+              "celery",
+              "-A",
+              "danswer.background.celery.versioned_apps.beat",
+              "beat",
+              "--loglevel=INFO",
+            ]
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+            - name: DANSWER_VERSION
+              value: "v0.11.0-cloud.beta.4"
+          envFrom:
+            - configMapRef:
+                name: env-configmap
+          resources:
+            requests:
+              cpu: "250m"
+              memory: "512Mi"
+            limits:
+              cpu: "500m"
+              memory: "1Gi"
diff --git a/deployment/cloud_kubernetes/workers/heavy_worker.yaml b/deployment/cloud_kubernetes/workers/heavy_worker.yaml
new file mode 100644
index 00000000000..682cadee647
--- /dev/null
+++ b/deployment/cloud_kubernetes/workers/heavy_worker.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-worker-heavy
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: celery-worker-heavy
+  template:
+    metadata:
+      labels:
+        app: celery-worker-heavy
+    spec:
+      containers:
+        - name: celery-worker-heavy
+          image: danswer/danswer-backend:v0.11.0-cloud.beta.4
+          imagePullPolicy: Always
+          command:
+            [
+              "celery",
+              "-A",
+              "danswer.background.celery.versioned_apps.heavy",
+              "worker",
+              "--loglevel=INFO",
+              "--hostname=heavy@%n",
+              "-Q",
+              "connector_pruning",
+            ]
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+            - name: DANSWER_VERSION
+              value: "v0.11.0-cloud.beta.4"
+          envFrom:
+            - configMapRef:
+                name: env-configmap
+          volumeMounts:
+            - name: vespa-certificates
+              mountPath: "/app/certs"
+              readOnly: true
+          resources:
+            requests:
+              cpu: "1000m"
+              memory: "2Gi"
+            limits:
+              cpu: "2000m"
+              memory: "4Gi"
+      volumes:
+        - name: vespa-certificates
+          secret:
+            secretName: vespa-certificates
+            items:
+              - key: cert.pem
+                path: cert.pem
+              - key: key.pem
+                path: key.pem
diff --git a/deployment/cloud_kubernetes/workers/indexing_worker.yaml b/deployment/cloud_kubernetes/workers/indexing_worker.yaml
new file mode 100644
index 00000000000..47d92999f2e
--- /dev/null
+++ b/deployment/cloud_kubernetes/workers/indexing_worker.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-worker-indexing
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: celery-worker-indexing
+  template:
+    metadata:
+      labels:
+        app: celery-worker-indexing
+    spec:
+      containers:
+        - name: celery-worker-indexing
+          image: danswer/danswer-backend:v0.11.0-cloud.beta.4
+          imagePullPolicy: Always
+          command:
+            [
+              "celery",
+              "-A",
+              "danswer.background.celery.versioned_apps.indexing",
+              "worker",
+              "--loglevel=INFO",
+              "--hostname=indexing@%n",
+              "-Q",
+              "connector_indexing",
+            ]
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+            - name: DANSWER_VERSION
+              value: "v0.11.0-cloud.beta.4"
+          envFrom:
+            - configMapRef:
+                name: env-configmap
+          volumeMounts:
+            - name: vespa-certificates
+              mountPath: "/app/certs"
+              readOnly: true
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"
+            limits:
+              cpu: "1000m"
+              memory: "2Gi"
+      volumes:
+        - name: vespa-certificates
+          secret:
+            secretName: vespa-certificates
+            items:
+              - key: cert.pem
+                path: cert.pem
+              - key: key.pem
+                path: key.pem
diff --git a/deployment/cloud_kubernetes/workers/light_worker.yaml b/deployment/cloud_kubernetes/workers/light_worker.yaml
new file mode 100644
index 00000000000..c3c59354514
--- /dev/null
+++ b/deployment/cloud_kubernetes/workers/light_worker.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-worker-light
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: celery-worker-light
+  template:
+    metadata:
+      labels:
+        app: celery-worker-light
+    spec:
+      containers:
+        - name: celery-worker-light
+          image: danswer/danswer-backend:v0.11.0-cloud.beta.4
+          imagePullPolicy: Always
+          command:
+            [
+              "celery",
+              "-A",
+              "danswer.background.celery.versioned_apps.light",
+              "worker",
+              "--loglevel=INFO",
+              "--hostname=light@%n",
+              "-Q",
+              "vespa_metadata_sync,connector_deletion",
+            ]
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+            - name: DANSWER_VERSION
+              value: "v0.11.0-cloud.beta.4"
+          envFrom:
+            - configMapRef:
+                name: env-configmap
+          volumeMounts:
+            - name: vespa-certificates
+              mountPath: "/app/certs"
+              readOnly: true
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"
+            limits:
+              cpu: "1000m"
+              memory: "2Gi"
+      volumes:
+        - name: vespa-certificates
+          secret:
+            secretName: vespa-certificates
+            items:
+              - key: cert.pem
+                path: cert.pem
+              - key: key.pem
+                path: key.pem
diff --git a/deployment/cloud_kubernetes/workers/primary.yaml b/deployment/cloud_kubernetes/workers/primary.yaml
new file mode 100644
index 00000000000..7408e3bfb42
--- /dev/null
+++ b/deployment/cloud_kubernetes/workers/primary.yaml
@@ -0,0 +1,60 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: celery-worker-primary
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: celery-worker-primary
+  template:
+    metadata:
+      labels:
+        app: celery-worker-primary
+    spec:
+      containers:
+        - name: celery-worker-primary
+          image: danswer/danswer-backend:v0.11.0-cloud.beta.4
+          imagePullPolicy: Always
+          command:
+            [
+              "celery",
+              "-A",
+              "danswer.background.celery.versioned_apps.primary",
+              "worker",
+              "--loglevel=INFO",
+              "--hostname=primary@%n",
+              "-Q",
+              "celery,periodic_tasks,vespa_metadata_sync",
+            ]
+          env:
+            - name: REDIS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: danswer-secrets
+                  key: redis_password
+            - name: DANSWER_VERSION
+              value: "v0.11.0-cloud.beta.4"
+          envFrom:
+            - configMapRef:
+                name: env-configmap
+          volumeMounts:
+            - name: vespa-certificates
+              mountPath: "/app/certs"
+              readOnly: true
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"
+            limits:
+              cpu: "1000m"
+              memory: "2Gi"
+      volumes:
+        - name: vespa-certificates
+          secret:
+            secretName: vespa-certificates
+            items:
+              - key: cert.pem
+                path: cert.pem
+              - key: key.pem
+                path: key.pem
diff --git a/web/package-lock.json b/web/package-lock.json
index c105c298991..2c72891b7cb 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -14851,6 +14851,21 @@
         "type": "github",
         "url": "https://github.com/sponsors/wooorm"
       }
+    },
+    "node_modules/@next/swc-win32-ia32-msvc": {
+      "version": "14.2.3",
+      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz",
+      "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==",
+      "cpu": [
+        "ia32"
+      ],
+      "optional": true,
+      "os": [
+        "win32"
+      ],
+      "engines": {
+        "node": ">= 10"
+      }
     }
   }
 }
\ No newline at end of file

From 93d0104d3ce9f4ecf3a3da153a806a9384f04308 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 10:42:52 -0800
Subject: [PATCH 352/376] slight upgrade to image generation prompts (#3036)

* slight upgrade to prompts

* k

* nit
---
 backend/danswer/chat/prompts.yaml | 40 +++++++++++++++----------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/backend/danswer/chat/prompts.yaml b/backend/danswer/chat/prompts.yaml
index d83d4ede4b5..85752b4d20d 100644
--- a/backend/danswer/chat/prompts.yaml
+++ b/backend/danswer/chat/prompts.yaml
@@ -9,19 +9,19 @@ prompts:
     system: >
       You are a question answering system that is constantly learning and improving.
       The current date is DANSWER_DATETIME_REPLACEMENT.
-      
+
       You can process and comprehend vast amounts of text and utilize this knowledge to provide
       grounded, accurate, and concise answers to diverse queries.
-      
+
       You always clearly communicate ANY UNCERTAINTY in your answer.
     # Task Prompt (as shown in UI)
     task: >
       Answer my query based on the documents provided.
       The documents may not all be relevant, ignore any documents that are not directly relevant
       to the most recent user query.
-      
+
       I have not read or seen any of the documents and do not want to read them.
-      
+
       If there are no relevant documents, refer to the chat history and your internal knowledge.
     # Inject a statement at the end of system prompt to inform the LLM of the current date/time
     # If the DANSWER_DATETIME_REPLACEMENT is set, the date/time is inserted there instead
@@ -30,21 +30,21 @@ prompts:
     # Prompts the LLM to include citations in the for [1], [2] etc.
     # which get parsed to match the passed in sources
     include_citations: true
-  
+
   - name: "ImageGeneration"
-    description: "Generates images based on user prompts!"
+    description: "Generates images from user descriptions!"
     system: >
-      You are an advanced image generation system capable of creating diverse and detailed images.
-      
-      You can interpret user prompts and generate high-quality, creative images that match their descriptions.
-      
-      You always strive to create safe and appropriate content, avoiding any harmful or offensive imagery.
+      You are an AI image generation assistant. Your role is to create high-quality images based on user descriptions.
+
+      For appropriate requests, you will generate an image that matches the user's requirements.
+      For inappropriate or unsafe requests, you will politely decline and explain why the request cannot be fulfilled.
+
+      You aim to be helpful while maintaining appropriate content standards.
     task: >
-      Generate an image based on the user's description.
-      
-      Provide a detailed description of the generated image, including key elements, colors, and composition.
-      
-      If the request is not possible or appropriate, explain why and suggest alternatives.
+      Based on the user's description, create a high-quality image that accurately reflects their request. 
+      Pay close attention to the specified details, styles, and desired elements.
+
+      If the request is not appropriate or cannot be fulfilled, explain why and suggest alternatives.
     datetime_aware: true
     include_citations: false
 
@@ -64,14 +64,13 @@ prompts:
     datetime_aware: true
     include_citations: true
 
-
   - name: "Summarize"
     description: "Summarize relevant information from retrieved context!"
     system: >
       You are a text summarizing assistant that highlights the most important knowledge from the
       context provided, prioritizing the information that relates to the user query.
       The current date is DANSWER_DATETIME_REPLACEMENT.
-      
+
       You ARE NOT creative and always stick to the provided documents.
       If there are no documents, refer to the conversation history.
 
@@ -84,7 +83,6 @@ prompts:
     datetime_aware: true
     include_citations: true
 
-
   - name: "Paraphrase"
     description: "Recites information from retrieved context! Least creative but most safe!"
     system: >
@@ -92,10 +90,10 @@ prompts:
       The current date is DANSWER_DATETIME_REPLACEMENT.
 
       You only provide quotes that are EXACT substrings from provided documents!
-      
+
       If there are no documents provided,
       simply tell the user that there are no documents to reference.
-      
+
       You NEVER generate new text or phrases outside of the citation.
       DO NOT explain your responses, only provide the quotes and NOTHING ELSE.
     task: >

From a7002dfa1d8d91655c1b304e8a93885500f6321a Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 10:43:05 -0800
Subject: [PATCH 353/376] add CSV display (#3028)

* add CSV display

* add downloading

* restructure

* create portal for modal

* update requirements

* nit
---
 backend/danswer/file_store/models.py          |   1 +
 backend/danswer/llm/utils.py                  |  28 +++-
 .../server/query_and_chat/chat_backend.py     |  16 +-
 backend/requirements/dev.txt                  |   4 +-
 web/src/app/chat/ChatPage.tsx                 |   1 +
 .../chat/files/documents/DocumentPreview.tsx  |  46 +++---
 web/src/app/chat/interfaces.ts                |   1 +
 web/src/app/chat/message/Messages.tsx         |  39 ++++-
 web/src/components/Modal.tsx                  |  22 ++-
 web/src/components/icons/icons.tsx            |  96 +++++++++++
 web/src/components/tools/CSVContent.tsx       | 156 ++++++++++++++++++
 .../tools/ExpandableContentWrapper.tsx        | 137 +++++++++++++++
 web/src/components/tools/ToolResult.tsx       |  27 +++
 web/src/components/tooltip/CustomTooltip.tsx  |   9 +-
 14 files changed, 549 insertions(+), 34 deletions(-)
 create mode 100644 web/src/components/tools/CSVContent.tsx
 create mode 100644 web/src/components/tools/ExpandableContentWrapper.tsx
 create mode 100644 web/src/components/tools/ToolResult.tsx

diff --git a/backend/danswer/file_store/models.py b/backend/danswer/file_store/models.py
index d944a2fd270..5bf964287e3 100644
--- a/backend/danswer/file_store/models.py
+++ b/backend/danswer/file_store/models.py
@@ -13,6 +13,7 @@ class ChatFileType(str, Enum):
     DOC = "document"
     # Plain text only contain the text
     PLAIN_TEXT = "plain_text"
+    CSV = "csv"
 
 
 class FileDescriptor(TypedDict):
diff --git a/backend/danswer/llm/utils.py b/backend/danswer/llm/utils.py
index af480f83955..77e13755241 100644
--- a/backend/danswer/llm/utils.py
+++ b/backend/danswer/llm/utils.py
@@ -1,3 +1,4 @@
+import io
 import json
 from collections.abc import Callable
 from collections.abc import Iterator
@@ -7,6 +8,7 @@
 from typing import Union
 
 import litellm  # type: ignore
+import pandas as pd
 import tiktoken
 from langchain.prompts.base import StringPromptValue
 from langchain.prompts.chat import ChatPromptValue
@@ -135,6 +137,18 @@ def translate_history_to_basemessages(
     return history_basemessages, history_token_counts
 
 
+def _process_csv_file(file: InMemoryChatFile) -> str:
+    df = pd.read_csv(io.StringIO(file.content.decode("utf-8")))
+    csv_preview = df.head().to_string()
+
+    file_name_section = (
+        f"CSV FILE NAME: {file.filename}\n"
+        if file.filename
+        else "CSV FILE (NO NAME PROVIDED):\n"
+    )
+    return f"{file_name_section}{CODE_BLOCK_PAT.format(csv_preview)}\n\n\n"
+
+
 def _build_content(
     message: str,
     files: list[InMemoryChatFile] | None = None,
@@ -145,16 +159,26 @@ def _build_content(
         if files
         else None
     )
-    if not text_files:
+
+    csv_files = (
+        [file for file in files if file.file_type == ChatFileType.CSV]
+        if files
+        else None
+    )
+
+    if not text_files and not csv_files:
         return message
 
     final_message_with_files = "FILES:\n\n"
-    for file in text_files:
+    for file in text_files or []:
         file_content = file.content.decode("utf-8")
         file_name_section = f"DOCUMENT: {file.filename}\n" if file.filename else ""
         final_message_with_files += (
             f"{file_name_section}{CODE_BLOCK_PAT.format(file_content.strip())}\n\n\n"
         )
+    for file in csv_files or []:
+        final_message_with_files += _process_csv_file(file)
+
     final_message_with_files += message
 
     return final_message_with_files
diff --git a/backend/danswer/server/query_and_chat/chat_backend.py b/backend/danswer/server/query_and_chat/chat_backend.py
index e14f6f25c5d..c1f4a7b3970 100644
--- a/backend/danswer/server/query_and_chat/chat_backend.py
+++ b/backend/danswer/server/query_and_chat/chat_backend.py
@@ -557,9 +557,9 @@ def upload_files_for_chat(
     _: User | None = Depends(current_user),
 ) -> dict[str, list[FileDescriptor]]:
     image_content_types = {"image/jpeg", "image/png", "image/webp"}
+    csv_content_types = {"text/csv"}
     text_content_types = {
         "text/plain",
-        "text/csv",
         "text/markdown",
         "text/x-markdown",
         "text/x-config",
@@ -578,8 +578,10 @@ def upload_files_for_chat(
         "application/epub+zip",
     }
 
-    allowed_content_types = image_content_types.union(text_content_types).union(
-        document_content_types
+    allowed_content_types = (
+        image_content_types.union(text_content_types)
+        .union(document_content_types)
+        .union(csv_content_types)
     )
 
     for file in files:
@@ -589,6 +591,10 @@ def upload_files_for_chat(
             elif file.content_type in text_content_types:
                 error_detail = "Unsupported text file type. Supported text types include .txt, .csv, .md, .mdx, .conf, "
                 ".log, .tsv."
+            elif file.content_type in csv_content_types:
+                error_detail = (
+                    "Unsupported CSV file type. Supported CSV types include .csv."
+                )
             else:
                 error_detail = (
                     "Unsupported document file type. Supported document types include .pdf, .docx, .pptx, .xlsx, "
@@ -614,6 +620,10 @@ def upload_files_for_chat(
             file_type = ChatFileType.IMAGE
             # Convert image to JPEG
             file_content, new_content_type = convert_to_jpeg(file)
+        elif file.content_type in csv_content_types:
+            file_type = ChatFileType.CSV
+            file_content = io.BytesIO(file.file.read())
+            new_content_type = file.content_type or ""
         elif file.content_type in document_content_types:
             file_type = ChatFileType.DOC
             file_content = io.BytesIO(file.file.read())
diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt
index fb589509d64..84c1c9ea4bf 100644
--- a/backend/requirements/dev.txt
+++ b/backend/requirements/dev.txt
@@ -24,4 +24,6 @@ types-urllib3==1.26.25.11
 trafilatura==1.12.2
 lxml==5.3.0
 lxml_html_clean==0.2.2
-boto3-stubs[s3]==1.34.133
\ No newline at end of file
+boto3-stubs[s3]==1.34.133
+pandas==2.2.3
+pandas-stubs==2.2.3.241009
\ No newline at end of file
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 57e915423ea..03b53fe0993 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -1490,6 +1490,7 @@ export function ChatPage({
     const imageFiles = acceptedFiles.filter((file) =>
       file.type.startsWith("image/")
     );
+
     if (imageFiles.length > 0 && !llmAcceptsImages) {
       setPopup({
         type: "error",
diff --git a/web/src/app/chat/files/documents/DocumentPreview.tsx b/web/src/app/chat/files/documents/DocumentPreview.tsx
index 07dddd53cfd..e4bc05c541f 100644
--- a/web/src/app/chat/files/documents/DocumentPreview.tsx
+++ b/web/src/app/chat/files/documents/DocumentPreview.tsx
@@ -1,70 +1,78 @@
 import { FiFileText } from "react-icons/fi";
 import { useState, useRef, useEffect } from "react";
 import { Tooltip } from "@/components/tooltip/Tooltip";
+import { ExpandTwoIcon } from "@/components/icons/icons";
 
 export function DocumentPreview({
   fileName,
   maxWidth,
   alignBubble,
+  open,
 }: {
   fileName: string;
+  open?: () => void;
   maxWidth?: string;
   alignBubble?: boolean;
 }) {
-  const [isOverflowing, setIsOverflowing] = useState(false);
   const fileNameRef = useRef<HTMLDivElement>(null);
 
-  useEffect(() => {
-    if (fileNameRef.current) {
-      setIsOverflowing(
-        fileNameRef.current.scrollWidth > fileNameRef.current.clientWidth
-      );
-    }
-  }, [fileName]);
-
   return (
     <div
       className={`
         ${alignBubble && "w-64"}
         flex
         items-center
-        p-2
+        p-3
         bg-hover
         border
         border-border
-        rounded-md
+        rounded-lg
         box-border
-        h-16
+        h-20
+        hover:shadow-sm
+        transition-all
       `}
     >
       <div className="flex-shrink-0">
         <div
           className="
-            w-12
-            h-12
+            w-14
+            h-14
             bg-document
             flex
             items-center
             justify-center
-            rounded-md
+            rounded-lg
+            transition-all
+            duration-200
+            hover:bg-document-dark
           "
         >
-          <FiFileText className="w-6 h-6 text-white" />
+          <FiFileText className="w-7 h-7 text-white" />
         </div>
       </div>
-      <div className="ml-4 relative">
+      <div className="ml-4 flex-grow">
         <Tooltip content={fileName} side="top" align="start">
           <div
             ref={fileNameRef}
-            className={`font-medium text-sm line-clamp-1 break-all ellipses ${
+            className={`font-medium text-sm line-clamp-1 break-all ellipsis ${
               maxWidth ? maxWidth : "max-w-48"
             }`}
           >
             {fileName}
           </div>
         </Tooltip>
-        <div className="text-subtle text-sm">Document</div>
+        <div className="text-subtle text-xs mt-1">Document</div>
       </div>
+      {open && (
+        <button
+          onClick={() => open()}
+          className="ml-2 p-2 rounded-full hover:bg-gray-200 transition-colors duration-200"
+          aria-label="Expand document"
+        >
+          <ExpandTwoIcon className="w-5 h-5 text-gray-600" />
+        </button>
+      )}
     </div>
   );
 }
diff --git a/web/src/app/chat/interfaces.ts b/web/src/app/chat/interfaces.ts
index 5e25566a7d5..f77bf50d6ad 100644
--- a/web/src/app/chat/interfaces.ts
+++ b/web/src/app/chat/interfaces.ts
@@ -32,6 +32,7 @@ export enum ChatFileType {
   IMAGE = "image",
   DOCUMENT = "document",
   PLAIN_TEXT = "plain_text",
+  CSV = "csv",
 }
 
 export interface FileDescriptor {
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 8526be1b922..101bcdbb67c 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -55,6 +55,8 @@ import { LlmOverride } from "@/lib/hooks";
 import { ContinueGenerating } from "./ContinueMessage";
 import { MemoizedLink, MemoizedParagraph } from "./MemoizedTextComponents";
 import { extractCodeText } from "./codeUtils";
+import ToolResult from "../../../components/tools/ToolResult";
+import CsvContent from "../../../components/tools/CSVContent";
 
 const TOOLS_WITH_CUSTOM_HANDLING = [
   SEARCH_TOOL_NAME,
@@ -69,8 +71,13 @@ function FileDisplay({
   files: FileDescriptor[];
   alignBubble?: boolean;
 }) {
+  const [close, setClose] = useState(true);
   const imageFiles = files.filter((file) => file.type === ChatFileType.IMAGE);
-  const nonImgFiles = files.filter((file) => file.type !== ChatFileType.IMAGE);
+  const nonImgFiles = files.filter(
+    (file) => file.type !== ChatFileType.IMAGE && file.type !== ChatFileType.CSV
+  );
+
+  const csvImgFiles = files.filter((file) => file.type == ChatFileType.CSV);
 
   return (
     <>
@@ -94,6 +101,7 @@ function FileDisplay({
           </div>
         </div>
       )}
+
       {imageFiles && imageFiles.length > 0 && (
         <div
           id="danswer-image"
@@ -106,6 +114,35 @@ function FileDisplay({
           </div>
         </div>
       )}
+
+      {csvImgFiles && csvImgFiles.length > 0 && (
+        <div className={` ${alignBubble && "ml-auto"} mt-2 auto mb-4`}>
+          <div className="flex flex-col gap-2">
+            {csvImgFiles.map((file) => {
+              return (
+                <div key={file.id} className="w-fit">
+                  {close ? (
+                    <>
+                      <ToolResult
+                        csvFileDescriptor={file}
+                        close={() => setClose(false)}
+                        contentComponent={CsvContent}
+                      />
+                    </>
+                  ) : (
+                    <DocumentPreview
+                      open={() => setClose(true)}
+                      fileName={file.name || file.id}
+                      maxWidth="max-w-64"
+                      alignBubble={alignBubble}
+                    />
+                  )}
+                </div>
+              );
+            })}
+          </div>
+        </div>
+      )}
     </>
   );
 }
diff --git a/web/src/components/Modal.tsx b/web/src/components/Modal.tsx
index 4abbfd8b9aa..4582ed8a558 100644
--- a/web/src/components/Modal.tsx
+++ b/web/src/components/Modal.tsx
@@ -4,6 +4,8 @@ import { FiX } from "react-icons/fi";
 import { IconProps, XIcon } from "./icons/icons";
 import { useRef } from "react";
 import { isEventWithinRef } from "@/lib/contains";
+import ReactDOM from "react-dom";
+import { useEffect, useState } from "react";
 
 interface ModalProps {
   icon?: ({ size, className }: IconProps) => JSX.Element;
@@ -14,6 +16,7 @@ interface ModalProps {
   width?: string;
   titleSize?: string;
   hideDividerForTitle?: boolean;
+  hideCloseButton?: boolean;
   noPadding?: boolean;
 }
 
@@ -27,8 +30,17 @@ export function Modal({
   hideDividerForTitle,
   noPadding,
   icon,
+  hideCloseButton,
 }: ModalProps) {
   const modalRef = useRef<HTMLDivElement>(null);
+  const [isMounted, setIsMounted] = useState(false);
+
+  useEffect(() => {
+    setIsMounted(true);
+    return () => {
+      setIsMounted(false);
+    };
+  }, []);
 
   const handleMouseDown = (e: React.MouseEvent<HTMLDivElement>) => {
     if (
@@ -41,11 +53,11 @@ export function Modal({
     }
   };
 
-  return (
+  const modalContent = (
     <div
       onMouseDown={handleMouseDown}
       className={`fixed inset-0 bg-black bg-opacity-25 backdrop-blur-sm h-full
-        flex items-center justify-center z-50 transition-opacity duration-300 ease-in-out`}
+        flex items-center justify-center z-[9999] transition-opacity duration-300 ease-in-out`}
     >
       <div
         ref={modalRef}
@@ -54,13 +66,13 @@ export function Modal({
             e.stopPropagation();
           }
         }}
-        className={`bg-background  text-emphasis rounded shadow-2xl 
+        className={`bg-background text-emphasis rounded shadow-2xl 
           transform transition-all duration-300 ease-in-out
           ${width ?? "w-11/12 max-w-4xl"}
           ${noPadding ? "" : "p-10"}
           ${className || ""}`}
       >
-        {onOutsideClick && (
+        {onOutsideClick && !hideCloseButton && (
           <div className="absolute top-2 right-2">
             <button
               onClick={onOutsideClick}
@@ -93,4 +105,6 @@ export function Modal({
       </div>
     </div>
   );
+
+  return isMounted ? ReactDOM.createPortal(modalContent, document.body) : null;
 }
diff --git a/web/src/components/icons/icons.tsx b/web/src/components/icons/icons.tsx
index 120d5807485..dc1156f5787 100644
--- a/web/src/components/icons/icons.tsx
+++ b/web/src/components/icons/icons.tsx
@@ -2588,3 +2588,99 @@ export const WindowsIcon = ({
     </svg>
   );
 };
+
+export const OpenIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <svg
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+      xmlns="http://www.w3.org/2000/svg"
+      width="200"
+      height="200"
+      viewBox="0 0 14 14"
+    >
+      <path
+        fill="none"
+        stroke="currentColor"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        d="M7 13.5a9.26 9.26 0 0 0-5.61-2.95a1 1 0 0 1-.89-1V1.5A1 1 0 0 1 1.64.51A9.3 9.3 0 0 1 7 3.43zm0 0a9.26 9.26 0 0 1 5.61-2.95a1 1 0 0 0 .89-1V1.5a1 1 0 0 0-1.14-.99A9.3 9.3 0 0 0 7 3.43z"
+      />
+    </svg>
+  );
+};
+
+export const DexpandTwoIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <svg
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+      xmlns="http://www.w3.org/2000/svg"
+      width="200"
+      height="200"
+      viewBox="0 0 14 14"
+    >
+      <path
+        fill="none"
+        stroke="currentColor"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        d="m.5 13.5l5-5m-4 0h4v4m8-12l-5 5m4 0h-4v-4"
+      />
+    </svg>
+  );
+};
+
+export const ExpandTwoIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <svg
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+      xmlns="http://www.w3.org/2000/svg"
+      width="200"
+      height="200"
+      viewBox="0 0 14 14"
+    >
+      <path
+        fill="none"
+        stroke="currentColor"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        d="m8.5 5.5l5-5m-4 0h4v4m-8 4l-5 5m4 0h-4v-4"
+      />
+    </svg>
+  );
+};
+
+export const DownloadCSVIcon = ({
+  size = 16,
+  className = defaultTailwindCSS,
+}: IconProps) => {
+  return (
+    <svg
+      style={{ width: `${size}px`, height: `${size}px` }}
+      className={`w-[${size}px] h-[${size}px] ` + className}
+      xmlns="http://www.w3.org/2000/svg"
+      width="200"
+      height="200"
+      viewBox="0 0 14 14"
+    >
+      <path
+        fill="none"
+        stroke="currentColor"
+        stroke-linecap="round"
+        stroke-linejoin="round"
+        d="M.5 10.5v1a2 2 0 0 0 2 2h9a2 2 0 0 0 2-2v-1M4 6l3 3.5L10 6M7 9.5v-9"
+      />
+    </svg>
+  );
+};
diff --git a/web/src/components/tools/CSVContent.tsx b/web/src/components/tools/CSVContent.tsx
new file mode 100644
index 00000000000..5dd6dcbf9ef
--- /dev/null
+++ b/web/src/components/tools/CSVContent.tsx
@@ -0,0 +1,156 @@
+// CsvContent
+import React, { useState, useEffect } from "react";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeader,
+  TableRow,
+} from "@/components/ui/table";
+import { ContentComponentProps } from "./ExpandableContentWrapper";
+import { WarningCircle } from "@phosphor-icons/react";
+
+const CsvContent: React.FC<ContentComponentProps> = ({
+  fileDescriptor,
+  isLoading,
+  fadeIn,
+  expanded = false,
+}) => {
+  const [data, setData] = useState<Record<string, string>[]>([]);
+  const [headers, setHeaders] = useState<string[]>([]);
+
+  useEffect(() => {
+    fetchCSV(fileDescriptor.id);
+  }, [fileDescriptor.id]);
+
+  const fetchCSV = async (id: string) => {
+    try {
+      const response = await fetch(`api/chat/file/${id}`);
+      if (!response.ok) {
+        throw new Error("Failed to fetch CSV file");
+      }
+
+      const contentLength = response.headers.get("Content-Length");
+      const fileSizeInMB = contentLength
+        ? parseInt(contentLength) / (1024 * 1024)
+        : 0;
+      const MAX_FILE_SIZE_MB = 5;
+
+      if (fileSizeInMB > MAX_FILE_SIZE_MB) {
+        throw new Error("File size exceeds the maximum limit of 5MB");
+      }
+
+      const csvData = await response.text();
+      const rows = csvData.trim().split("\n");
+      const parsedHeaders = rows[0].split(",");
+      setHeaders(parsedHeaders);
+
+      const parsedData: Record<string, string>[] = rows.slice(1).map((row) => {
+        const values = row.split(",");
+        return parsedHeaders.reduce<Record<string, string>>(
+          (obj, header, index) => {
+            obj[header] = values[index];
+            return obj;
+          },
+          {}
+        );
+      });
+      setData(parsedData);
+    } catch (error) {
+      console.error("Error fetching CSV file:", error);
+      setData([]);
+      setHeaders([]);
+    }
+  };
+
+  if (isLoading) {
+    return (
+      <div className="flex items-center justify-center h-[300px]">
+        <div className="animate-pulse flex space-x-4">
+          <div className="rounded-full bg-background-200 h-10 w-10"></div>
+          <div className="w-full flex-1 space-y-4 py-1">
+            <div className="h-2 w-full bg-background-200 rounded"></div>
+            <div className="w-full space-y-3">
+              <div className="grid grid-cols-3 gap-4">
+                <div className="h-2 bg-background-200 rounded col-span-2"></div>
+                <div className="h-2 bg-background-200 rounded col-span-1"></div>
+              </div>
+              <div className="h-2 bg-background-200 rounded"></div>
+            </div>
+          </div>
+        </div>
+      </div>
+    );
+  }
+
+  return (
+    <div
+      className={`transition-opacity transform relative duration-1000 ease-in-out ${
+        fadeIn ? "opacity-100" : "opacity-0"
+      }`}
+    >
+      <div
+        className={`overflow-y-hidden flex relative ${
+          expanded ? "max-h-2/3" : "max-h-[300px]"
+        }`}
+      >
+        <Table>
+          <TableHeader className="sticky z-[1000] top-0">
+            <TableRow className="hover:bg-background-125 bg-background-125">
+              {headers.map((header, index) => (
+                <TableHead key={index}>
+                  <p className="text-text-600 line-clamp-2 my-2 font-medium">
+                    {header}
+                  </p>
+                </TableHead>
+              ))}
+            </TableRow>
+          </TableHeader>
+
+          <TableBody className="h-[300px] overflow-y-scroll">
+            {data.length > 0 ? (
+              data.map((row, rowIndex) => (
+                <TableRow key={rowIndex}>
+                  {headers.map((header, cellIndex) => (
+                    <TableCell
+                      className={`${
+                        cellIndex === 0 ? "sticky left-0 bg-background-100" : ""
+                      }`}
+                      key={cellIndex}
+                    >
+                      {row[header]}
+                    </TableCell>
+                  ))}
+                </TableRow>
+              ))
+            ) : (
+              <TableRow>
+                <TableCell
+                  colSpan={headers.length}
+                  className="text-center py-8"
+                >
+                  <div className="flex flex-col items-center justify-center space-y-2">
+                    <WarningCircle className="w-8 h-8 text-error" />
+                    <p className="text-text-600 font-medium">
+                      {headers.length === 0
+                        ? "Error loading CSV"
+                        : "No data available"}
+                    </p>
+                    <p className="text-text-400 text-sm">
+                      {headers.length === 0
+                        ? "The CSV file may be too large or couldn't be loaded properly."
+                        : "The CSV file appears to be empty."}
+                    </p>
+                  </div>
+                </TableCell>
+              </TableRow>
+            )}
+          </TableBody>
+        </Table>
+      </div>
+    </div>
+  );
+};
+
+export default CsvContent;
diff --git a/web/src/components/tools/ExpandableContentWrapper.tsx b/web/src/components/tools/ExpandableContentWrapper.tsx
new file mode 100644
index 00000000000..4e7635e0841
--- /dev/null
+++ b/web/src/components/tools/ExpandableContentWrapper.tsx
@@ -0,0 +1,137 @@
+// ExpandableContentWrapper
+import React, { useState, useEffect } from "react";
+import {
+  CustomTooltip,
+  TooltipGroup,
+} from "@/components/tooltip/CustomTooltip";
+import {
+  DexpandTwoIcon,
+  DownloadCSVIcon,
+  ExpandTwoIcon,
+  OpenIcon,
+} from "@/components/icons/icons";
+import { Card, CardHeader, CardTitle, CardContent } from "@/components/ui/card";
+import { Modal } from "@/components/Modal";
+import { FileDescriptor } from "@/app/chat/interfaces";
+
+export interface ExpandableContentWrapperProps {
+  fileDescriptor: FileDescriptor;
+  close: () => void;
+  ContentComponent: React.ComponentType<ContentComponentProps>;
+}
+
+export interface ContentComponentProps {
+  fileDescriptor: FileDescriptor;
+  isLoading: boolean;
+  fadeIn: boolean;
+  expanded?: boolean;
+}
+
+const ExpandableContentWrapper: React.FC<ExpandableContentWrapperProps> = ({
+  fileDescriptor,
+  close,
+  ContentComponent,
+}) => {
+  const [expanded, setExpanded] = useState(false);
+  const [isLoading, setIsLoading] = useState(true);
+  const [fadeIn, setFadeIn] = useState(false);
+
+  const toggleExpand = () => setExpanded((prev) => !prev);
+
+  // Prevent a jarring fade in
+  useEffect(() => {
+    setTimeout(() => setIsLoading(false), 300);
+  }, []);
+
+  useEffect(() => {
+    if (!isLoading) {
+      setTimeout(() => setFadeIn(true), 50);
+    } else {
+      setFadeIn(false);
+    }
+  }, [isLoading]);
+
+  const downloadFile = () => {
+    const a = document.createElement("a");
+    a.href = `api/chat/file/${fileDescriptor.id}`;
+    a.download = fileDescriptor.name || "download.csv";
+    a.setAttribute("download", fileDescriptor.name || "download.csv");
+    document.body.appendChild(a);
+    a.click();
+    document.body.removeChild(a);
+  };
+
+  const Content = (
+    <div
+      className={`${
+        !expanded ? "w-message-sm" : "w-full"
+      } !rounded !rounded-lg overflow-y-hidden border h-full border-border`}
+    >
+      <CardHeader className="w-full py-4 border-b border-border bg-white z-[10] top-0">
+        <div className="flex justify-between items-center">
+          <CardTitle className="text-ellipsis line-clamp-1 text-xl font-semibold text-text-700 pr-4">
+            {fileDescriptor.name || "Untitled"}
+          </CardTitle>
+          <div className="flex items-center">
+            <TooltipGroup gap="gap-x-4">
+              <CustomTooltip showTick line content="Download file">
+                <button onClick={downloadFile}>
+                  <DownloadCSVIcon className="cursor-pointer hover:text-text-800 h-6 w-6 text-text-400" />
+                </button>
+              </CustomTooltip>
+              <CustomTooltip
+                line
+                showTick
+                content={expanded ? "Minimize" : "Full screen"}
+              >
+                <button onClick={toggleExpand}>
+                  {!expanded ? (
+                    <ExpandTwoIcon className="hover:text-text-800 h-6 w-6 cursor-pointer text-text-400" />
+                  ) : (
+                    <DexpandTwoIcon className="hover:text-text-800 h-6 w-6 cursor-pointer text-text-400" />
+                  )}
+                </button>
+              </CustomTooltip>
+              <CustomTooltip showTick line content="Hide">
+                <button onClick={close}>
+                  <OpenIcon className="hover:text-text-800 h-6 w-6 cursor-pointer text-text-400" />
+                </button>
+              </CustomTooltip>
+            </TooltipGroup>
+          </div>
+        </div>
+      </CardHeader>
+      <Card
+        className={`!rounded-none w-full ${
+          expanded ? "max-h-[600px]" : "max-h-[300px] h"
+        } p-0 relative overflow-x-scroll overflow-y-scroll mx-auto`}
+      >
+        <CardContent className="p-0">
+          <ContentComponent
+            fileDescriptor={fileDescriptor}
+            isLoading={isLoading}
+            fadeIn={fadeIn}
+            expanded={expanded}
+          />
+        </CardContent>
+      </Card>
+    </div>
+  );
+
+  return (
+    <>
+      {expanded && (
+        <Modal
+          hideCloseButton
+          onOutsideClick={() => setExpanded(false)}
+          className="!max-w-5xl overflow-hidden rounded-lg !p-0 !m-0"
+        >
+          {Content}
+        </Modal>
+      )}
+      {!expanded && Content}
+    </>
+  );
+};
+
+export default ExpandableContentWrapper;
diff --git a/web/src/components/tools/ToolResult.tsx b/web/src/components/tools/ToolResult.tsx
new file mode 100644
index 00000000000..64c6ec96f0c
--- /dev/null
+++ b/web/src/components/tools/ToolResult.tsx
@@ -0,0 +1,27 @@
+import React from "react";
+import ExpandableContentWrapper, {
+  ContentComponentProps,
+} from "./ExpandableContentWrapper";
+import { FileDescriptor } from "@/app/chat/interfaces";
+
+interface ToolResultProps {
+  csvFileDescriptor: FileDescriptor;
+  close: () => void;
+  contentComponent: React.ComponentType<ContentComponentProps>;
+}
+
+const ToolResult: React.FC<ToolResultProps> = ({
+  csvFileDescriptor,
+  close,
+  contentComponent,
+}) => {
+  return (
+    <ExpandableContentWrapper
+      fileDescriptor={csvFileDescriptor}
+      close={close}
+      ContentComponent={contentComponent}
+    />
+  );
+};
+
+export default ToolResult;
diff --git a/web/src/components/tooltip/CustomTooltip.tsx b/web/src/components/tooltip/CustomTooltip.tsx
index 42efdbe2e4a..3737d0c1ee7 100644
--- a/web/src/components/tooltip/CustomTooltip.tsx
+++ b/web/src/components/tooltip/CustomTooltip.tsx
@@ -19,9 +19,10 @@ const TooltipGroupContext = createContext<{
   hoverCountRef: { current: false },
 });
 
-export const TooltipGroup: React.FC<{ children: React.ReactNode }> = ({
-  children,
-}) => {
+export const TooltipGroup: React.FC<{
+  children: React.ReactNode;
+  gap?: string;
+}> = ({ children, gap }) => {
   const [groupHovered, setGroupHovered] = useState(false);
   const hoverCountRef = useRef(false);
 
@@ -29,7 +30,7 @@ export const TooltipGroup: React.FC<{ children: React.ReactNode }> = ({
     <TooltipGroupContext.Provider
       value={{ groupHovered, setGroupHovered, hoverCountRef }}
     >
-      <div className="inline-flex">{children}</div>
+      <div className={`inline-flex ${gap}`}>{children}</div>
     </TooltipGroupContext.Provider>
   );
 };

From 51b79f688a893fa16e5c02dff8fcf4a7591fd50b Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 10:51:51 -0800
Subject: [PATCH 354/376] Tool call per message (#3025)

* single tool call per message

* finalize migration

* minor image generation fix

* validate simplify

* k

* remove print

* validated
---
 ...cb72ea4d80_single_tool_call_per_message.py | 50 +++++++++++++++++++
 backend/danswer/chat/process_message.py       | 18 +++----
 backend/danswer/db/chat.py                    | 23 ++++-----
 backend/danswer/db/models.py                  | 16 ++++--
 backend/danswer/file_store/utils.py           | 12 +++--
 backend/danswer/llm/answering/models.py       | 17 +++----
 backend/danswer/llm/chat_llm.py               |  6 ++-
 .../danswer/server/query_and_chat/models.py   |  2 +-
 web/src/app/chat/ChatPage.tsx                 | 48 ++++++++----------
 web/src/app/chat/interfaces.ts                |  4 +-
 web/src/app/chat/lib.tsx                      |  2 +-
 web/src/app/chat/message/Messages.tsx         |  2 +-
 12 files changed, 126 insertions(+), 74 deletions(-)
 create mode 100644 backend/alembic/versions/33cb72ea4d80_single_tool_call_per_message.py

diff --git a/backend/alembic/versions/33cb72ea4d80_single_tool_call_per_message.py b/backend/alembic/versions/33cb72ea4d80_single_tool_call_per_message.py
new file mode 100644
index 00000000000..0cd3da444bc
--- /dev/null
+++ b/backend/alembic/versions/33cb72ea4d80_single_tool_call_per_message.py
@@ -0,0 +1,50 @@
+"""single tool call per message
+
+Revision ID: 33cb72ea4d80
+Revises: 5b29123cd710
+Create Date: 2024-11-01 12:51:01.535003
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "33cb72ea4d80"
+down_revision = "5b29123cd710"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Step 1: Delete extraneous ToolCall entries
+    # Keep only the ToolCall with the smallest 'id' for each 'message_id'
+    op.execute(
+        sa.text(
+            """
+            DELETE FROM tool_call
+            WHERE id NOT IN (
+                SELECT MIN(id)
+                FROM tool_call
+                WHERE message_id IS NOT NULL
+                GROUP BY message_id
+            );
+        """
+        )
+    )
+
+    # Step 2: Add a unique constraint on message_id
+    op.create_unique_constraint(
+        constraint_name="uq_tool_call_message_id",
+        table_name="tool_call",
+        columns=["message_id"],
+    )
+
+
+def downgrade() -> None:
+    # Step 1: Drop the unique constraint on message_id
+    op.drop_constraint(
+        constraint_name="uq_tool_call_message_id",
+        table_name="tool_call",
+        type_="unique",
+    )
diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index 0394f34b828..252f6df0f1e 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -864,17 +864,15 @@ def stream_chat_message_objects(
             if message_specific_citations
             else None,
             error=None,
-            tool_calls=(
-                [
-                    ToolCall(
-                        tool_id=tool_name_to_tool_id[tool_result.tool_name],
-                        tool_name=tool_result.tool_name,
-                        tool_arguments=tool_result.tool_args,
-                        tool_result=tool_result.tool_result,
-                    )
-                ]
+            tool_call=(
+                ToolCall(
+                    tool_id=tool_name_to_tool_id[tool_result.tool_name],
+                    tool_name=tool_result.tool_name,
+                    tool_arguments=tool_result.tool_args,
+                    tool_result=tool_result.tool_result,
+                )
                 if tool_result
-                else []
+                else None
             ),
         )
 
diff --git a/backend/danswer/db/chat.py b/backend/danswer/db/chat.py
index d885e1efd63..4aaee092972 100644
--- a/backend/danswer/db/chat.py
+++ b/backend/danswer/db/chat.py
@@ -388,7 +388,7 @@ def get_chat_messages_by_session(
     )
 
     if prefetch_tool_calls:
-        stmt = stmt.options(joinedload(ChatMessage.tool_calls))
+        stmt = stmt.options(joinedload(ChatMessage.tool_call))
         result = db_session.scalars(stmt).unique().all()
     else:
         result = db_session.scalars(stmt).all()
@@ -474,7 +474,7 @@ def create_new_chat_message(
     alternate_assistant_id: int | None = None,
     # Maps the citation number [n] to the DB SearchDoc
     citations: dict[int, int] | None = None,
-    tool_calls: list[ToolCall] | None = None,
+    tool_call: ToolCall | None = None,
     commit: bool = True,
     reserved_message_id: int | None = None,
     overridden_model: str | None = None,
@@ -494,7 +494,7 @@ def create_new_chat_message(
         existing_message.message_type = message_type
         existing_message.citations = citations
         existing_message.files = files
-        existing_message.tool_calls = tool_calls if tool_calls else []
+        existing_message.tool_call = tool_call
         existing_message.error = error
         existing_message.alternate_assistant_id = alternate_assistant_id
         existing_message.overridden_model = overridden_model
@@ -513,7 +513,7 @@ def create_new_chat_message(
             message_type=message_type,
             citations=citations,
             files=files,
-            tool_calls=tool_calls if tool_calls else [],
+            tool_call=tool_call,
             error=error,
             alternate_assistant_id=alternate_assistant_id,
             overridden_model=overridden_model,
@@ -749,14 +749,13 @@ def translate_db_message_to_chat_message_detail(
         time_sent=chat_message.time_sent,
         citations=chat_message.citations,
         files=chat_message.files or [],
-        tool_calls=[
-            ToolCallFinalResult(
-                tool_name=tool_call.tool_name,
-                tool_args=tool_call.tool_arguments,
-                tool_result=tool_call.tool_result,
-            )
-            for tool_call in chat_message.tool_calls
-        ],
+        tool_call=ToolCallFinalResult(
+            tool_name=chat_message.tool_call.tool_name,
+            tool_args=chat_message.tool_call.tool_arguments,
+            tool_result=chat_message.tool_call.tool_result,
+        )
+        if chat_message.tool_call
+        else None,
         alternate_assistant_id=chat_message.alternate_assistant_id,
         overridden_model=chat_message.overridden_model,
     )
diff --git a/backend/danswer/db/models.py b/backend/danswer/db/models.py
index 3f63d1d0dba..3ff2133155b 100644
--- a/backend/danswer/db/models.py
+++ b/backend/danswer/db/models.py
@@ -918,10 +918,15 @@ class ToolCall(Base):
     tool_arguments: Mapped[dict[str, JSON_ro]] = mapped_column(postgresql.JSONB())
     tool_result: Mapped[JSON_ro] = mapped_column(postgresql.JSONB())
 
-    message_id: Mapped[int] = mapped_column(ForeignKey("chat_message.id"))
+    message_id: Mapped[int | None] = mapped_column(
+        ForeignKey("chat_message.id"), nullable=False
+    )
 
+    # Update the relationship
     message: Mapped["ChatMessage"] = relationship(
-        "ChatMessage", back_populates="tool_calls"
+        "ChatMessage",
+        back_populates="tool_call",
+        uselist=False,
     )
 
 
@@ -1052,12 +1057,13 @@ class ChatMessage(Base):
         secondary=ChatMessage__SearchDoc.__table__,
         back_populates="chat_messages",
     )
-    # NOTE: Should always be attached to the `assistant` message.
-    # represents the tool calls used to generate this message
-    tool_calls: Mapped[list["ToolCall"]] = relationship(
+
+    tool_call: Mapped["ToolCall"] = relationship(
         "ToolCall",
         back_populates="message",
+        uselist=False,
     )
+
     standard_answers: Mapped[list["StandardAnswer"]] = relationship(
         "StandardAnswer",
         secondary=ChatMessage__StandardAnswer.__table__,
diff --git a/backend/danswer/file_store/utils.py b/backend/danswer/file_store/utils.py
index b71d20bbbb4..e9eea2c262d 100644
--- a/backend/danswer/file_store/utils.py
+++ b/backend/danswer/file_store/utils.py
@@ -8,12 +8,13 @@
 from sqlalchemy.orm import Session
 
 from danswer.configs.constants import FileOrigin
-from danswer.db.engine import get_session_context_manager
+from danswer.db.engine import get_session_with_tenant
 from danswer.db.models import ChatMessage
 from danswer.file_store.file_store import get_default_file_store
 from danswer.file_store.models import FileDescriptor
 from danswer.file_store.models import InMemoryChatFile
 from danswer.utils.threadpool_concurrency import run_functions_tuples_in_parallel
+from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 
 def load_chat_file(
@@ -52,11 +53,11 @@ def load_all_chat_files(
     return files
 
 
-def save_file_from_url(url: str) -> str:
+def save_file_from_url(url: str, tenant_id: str) -> str:
     """NOTE: using multiple sessions here, since this is often called
     using multithreading. In practice, sharing a session has resulted in
     weird errors."""
-    with get_session_context_manager() as db_session:
+    with get_session_with_tenant(tenant_id) as db_session:
         response = requests.get(url)
         response.raise_for_status()
 
@@ -75,7 +76,10 @@ def save_file_from_url(url: str) -> str:
 
 
 def save_files_from_urls(urls: list[str]) -> list[str]:
+    tenant_id = CURRENT_TENANT_ID_CONTEXTVAR.get()
+
     funcs: list[tuple[Callable[..., Any], tuple[Any, ...]]] = [
-        (save_file_from_url, (url,)) for url in urls
+        (save_file_from_url, (url, tenant_id)) for url in urls
     ]
+    # Must pass in tenant_id here, since this is called by multithreading
     return run_functions_tuples_in_parallel(funcs)
diff --git a/backend/danswer/llm/answering/models.py b/backend/danswer/llm/answering/models.py
index 87c1297fe92..03f72a0968c 100644
--- a/backend/danswer/llm/answering/models.py
+++ b/backend/danswer/llm/answering/models.py
@@ -33,7 +33,7 @@ class PreviousMessage(BaseModel):
     token_count: int
     message_type: MessageType
     files: list[InMemoryChatFile]
-    tool_calls: list[ToolCallFinalResult]
+    tool_call: ToolCallFinalResult | None
 
     @classmethod
     def from_chat_message(
@@ -51,14 +51,13 @@ def from_chat_message(
                 for file in available_files
                 if str(file.file_id) in message_file_ids
             ],
-            tool_calls=[
-                ToolCallFinalResult(
-                    tool_name=tool_call.tool_name,
-                    tool_args=tool_call.tool_arguments,
-                    tool_result=tool_call.tool_result,
-                )
-                for tool_call in chat_message.tool_calls
-            ],
+            tool_call=ToolCallFinalResult(
+                tool_name=chat_message.tool_call.tool_name,
+                tool_args=chat_message.tool_call.tool_arguments,
+                tool_result=chat_message.tool_call.tool_result,
+            )
+            if chat_message.tool_call
+            else None,
         )
 
     def to_langchain_msg(self) -> BaseMessage:
diff --git a/backend/danswer/llm/chat_llm.py b/backend/danswer/llm/chat_llm.py
index d450fff0a63..6bd4b51d40b 100644
--- a/backend/danswer/llm/chat_llm.py
+++ b/backend/danswer/llm/chat_llm.py
@@ -83,8 +83,10 @@ def _convert_litellm_message_to_langchain_message(
                     "args": json.loads(tool_call.function.arguments),
                     "id": tool_call.id,
                 }
-                for tool_call in (tool_calls if tool_calls else [])
-            ],
+                for tool_call in tool_calls
+            ]
+            if tool_calls
+            else [],
         )
     elif role == "system":
         return SystemMessage(content=content)
diff --git a/backend/danswer/server/query_and_chat/models.py b/backend/danswer/server/query_and_chat/models.py
index 1ca14f9283f..13b3b1ec0a8 100644
--- a/backend/danswer/server/query_and_chat/models.py
+++ b/backend/danswer/server/query_and_chat/models.py
@@ -188,7 +188,7 @@ class ChatMessageDetail(BaseModel):
     chat_session_id: UUID | None = None
     citations: dict[int, int] | None = None
     files: list[FileDescriptor]
-    tool_calls: list[ToolCallFinalResult]
+    tool_call: ToolCallFinalResult | None
 
     def model_dump(self, *args: list, **kwargs: dict[str, Any]) -> dict[str, Any]:  # type: ignore
         initial_dict = super().model_dump(mode="json", *args, **kwargs)  # type: ignore
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 03b53fe0993..dcf1baf14df 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -277,13 +277,13 @@ export function ChatPage({
     if (
       lastMessage &&
       lastMessage.type === "assistant" &&
-      lastMessage.toolCalls[0] &&
-      lastMessage.toolCalls[0].tool_result === undefined
+      lastMessage.toolCall &&
+      lastMessage.toolCall.tool_result === undefined
     ) {
       const newCompleteMessageMap = new Map(
         currentMessageMap(completeMessageDetail)
       );
-      const updatedMessage = { ...lastMessage, toolCalls: [] };
+      const updatedMessage = { ...lastMessage, toolCall: null };
       newCompleteMessageMap.set(lastMessage.messageId, updatedMessage);
       updateCompleteMessageDetail(currentSession, newCompleteMessageMap);
     }
@@ -513,7 +513,7 @@ export function ChatPage({
         message: "",
         type: "system",
         files: [],
-        toolCalls: [],
+        toolCall: null,
         parentMessageId: null,
         childrenMessageIds: [firstMessageId],
         latestChildMessageId: firstMessageId,
@@ -1104,7 +1104,7 @@ export function ChatPage({
     let stackTrace: string | null = null;
 
     let finalMessage: BackendMessage | null = null;
-    let toolCalls: ToolCallMetadata[] = [];
+    let toolCall: ToolCallMetadata | null = null;
 
     let initialFetchDetails: null | {
       user_message_id: number;
@@ -1209,7 +1209,7 @@ export function ChatPage({
                 message: currMessage,
                 type: "user",
                 files: currentMessageFiles,
-                toolCalls: [],
+                toolCall: null,
                 parentMessageId: parentMessage?.messageId || SYSTEM_MESSAGE_ID,
               },
             ];
@@ -1262,17 +1262,14 @@ export function ChatPage({
                 setSelectedMessageForDocDisplay(user_message_id);
               }
             } else if (Object.hasOwn(packet, "tool_name")) {
-              toolCalls = [
-                {
-                  tool_name: (packet as ToolCallMetadata).tool_name,
-                  tool_args: (packet as ToolCallMetadata).tool_args,
-                  tool_result: (packet as ToolCallMetadata).tool_result,
-                },
-              ];
-              if (
-                !toolCalls[0].tool_result ||
-                toolCalls[0].tool_result == undefined
-              ) {
+              // Will only ever be one tool call per message
+              toolCall = {
+                tool_name: (packet as ToolCallMetadata).tool_name,
+                tool_args: (packet as ToolCallMetadata).tool_args,
+                tool_result: (packet as ToolCallMetadata).tool_result,
+              };
+
+              if (!toolCall.tool_result || toolCall.tool_result == undefined) {
                 updateChatState("toolBuilding", frozenSessionId);
               } else {
                 updateChatState("streaming", frozenSessionId);
@@ -1280,8 +1277,8 @@ export function ChatPage({
 
               // This will be consolidated in upcoming tool calls udpate,
               // but for now, we need to set query as early as possible
-              if (toolCalls[0].tool_name == SEARCH_TOOL_NAME) {
-                query = toolCalls[0].tool_args["query"];
+              if (toolCall.tool_name == SEARCH_TOOL_NAME) {
+                query = toolCall.tool_args["query"];
               }
             } else if (Object.hasOwn(packet, "file_ids")) {
               aiMessageImages = (packet as ImageGenerationDisplay).file_ids.map(
@@ -1339,7 +1336,7 @@ export function ChatPage({
                 message: currMessage,
                 type: "user",
                 files: currentMessageFiles,
-                toolCalls: [],
+                toolCall: null,
                 parentMessageId: error ? null : lastSuccessfulMessageId,
                 childrenMessageIds: [
                   ...(regenerationRequest?.parentMessage?.childrenMessageIds ||
@@ -1358,7 +1355,7 @@ export function ChatPage({
                   finalMessage?.context_docs?.top_documents || documents,
                 citations: finalMessage?.citations || {},
                 files: finalMessage?.files || aiMessageImages || [],
-                toolCalls: finalMessage?.tool_calls || toolCalls,
+                toolCall: finalMessage?.tool_call || null,
                 parentMessageId: regenerationRequest
                   ? regenerationRequest?.parentMessage?.messageId!
                   : initialFetchDetails.user_message_id,
@@ -1381,7 +1378,7 @@ export function ChatPage({
             message: currMessage,
             type: "user",
             files: currentMessageFiles,
-            toolCalls: [],
+            toolCall: null,
             parentMessageId: parentMessage?.messageId || SYSTEM_MESSAGE_ID,
           },
           {
@@ -1391,7 +1388,7 @@ export function ChatPage({
             message: errorMsg,
             type: "error",
             files: aiMessageImages || [],
-            toolCalls: [],
+            toolCall: null,
             parentMessageId:
               initialFetchDetails?.user_message_id || TEMP_USER_MESSAGE_ID,
           },
@@ -2238,10 +2235,7 @@ export function ChatPage({
                                       citedDocuments={getCitedDocumentsFromMessage(
                                         message
                                       )}
-                                      toolCall={
-                                        message.toolCalls &&
-                                        message.toolCalls[0]
-                                      }
+                                      toolCall={message.toolCall}
                                       isComplete={
                                         i !== messageHistory.length - 1 ||
                                         (currentSessionChatState !=
diff --git a/web/src/app/chat/interfaces.ts b/web/src/app/chat/interfaces.ts
index f77bf50d6ad..20ea4e7d289 100644
--- a/web/src/app/chat/interfaces.ts
+++ b/web/src/app/chat/interfaces.ts
@@ -86,7 +86,7 @@ export interface Message {
   documents?: DanswerDocument[] | null;
   citations?: CitationMap;
   files: FileDescriptor[];
-  toolCalls: ToolCallMetadata[];
+  toolCall: ToolCallMetadata | null;
   // for rebuilding the message tree
   parentMessageId: number | null;
   childrenMessageIds?: number[];
@@ -121,7 +121,7 @@ export interface BackendMessage {
   time_sent: string;
   citations: CitationMap;
   files: FileDescriptor[];
-  tool_calls: ToolCallFinalResult[];
+  tool_call: ToolCallFinalResult | null;
   alternate_assistant_id?: number | null;
   overridden_model?: string;
 }
diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx
index 38fdac037a6..cda037a08b4 100644
--- a/web/src/app/chat/lib.tsx
+++ b/web/src/app/chat/lib.tsx
@@ -428,7 +428,7 @@ export function processRawChatHistory(
             citations: messageInfo?.citations || {},
           }
         : {}),
-      toolCalls: messageInfo.tool_calls,
+      toolCall: messageInfo.tool_call,
       parentMessageId: messageInfo.parent_message,
       childrenMessageIds: [],
       latestChildMessageId: messageInfo.latest_child_message,
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 101bcdbb67c..2939c74d3f4 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -189,7 +189,7 @@ export const AIMessage = ({
   files?: FileDescriptor[];
   query?: string;
   citedDocuments?: [string, DanswerDocument][] | null;
-  toolCall?: ToolCallMetadata;
+  toolCall?: ToolCallMetadata | null;
   isComplete?: boolean;
   hasDocs?: boolean;
   handleFeedback?: (feedbackType: FeedbackType) => void;

From fac2b100a10890cbfcdcdcdbb18f07fd22944ed4 Mon Sep 17 00:00:00 2001
From: Yuhong Sun <yuhongsun96@gmail.com>
Date: Sun, 3 Nov 2024 11:24:04 -0800
Subject: [PATCH 355/376] Last Message Too Large Logging (#3039)

---
 backend/danswer/prompts/prompt_utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/backend/danswer/prompts/prompt_utils.py b/backend/danswer/prompts/prompt_utils.py
index cd59e97061f..5e1a3e298fb 100644
--- a/backend/danswer/prompts/prompt_utils.py
+++ b/backend/danswer/prompts/prompt_utils.py
@@ -13,6 +13,10 @@
 from danswer.prompts.chat_prompts import CITATION_REMINDER
 from danswer.prompts.constants import CODE_BLOCK_PAT
 from danswer.search.models import InferenceChunk
+from danswer.utils.logger import setup_logger
+
+
+logger = setup_logger()
 
 
 MOST_BASIC_PROMPT = "You are a helpful AI assistant."
@@ -136,14 +140,23 @@ def find_last_index(lst: list[int], max_prompt_tokens: int) -> int:
     before the list exceeds the maximum"""
     running_sum = 0
 
+    if not lst:
+        logger.warning("Empty message history passed to find_last_index")
+        return 0
+
     last_ind = 0
     for i in range(len(lst) - 1, -1, -1):
         running_sum += lst[i] + _PER_MESSAGE_TOKEN_BUFFER
         if running_sum > max_prompt_tokens:
             last_ind = i + 1
             break
+
     if last_ind >= len(lst):
+        logger.error(
+            f"Last message alone is too large! max_prompt_tokens: {max_prompt_tokens}, message_token_counts: {lst}"
+        )
         raise ValueError("Last message alone is too large!")
+
     return last_ind
 
 

From 56c3a5ff5bee8faf9ac104c15f2c4bb972668c92 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Sun, 3 Nov 2024 13:58:12 -0800
Subject: [PATCH 356/376] add POSTGRES_IDLE_SESSIONS_TIMEOUT (#3019)

Co-authored-by: Richard Kuo <rkuo@rkuo.com>
---
 backend/danswer/configs/app_configs.py | 11 +++++++++++
 backend/danswer/db/engine.py           |  5 +++++
 2 files changed, 16 insertions(+)

diff --git a/backend/danswer/configs/app_configs.py b/backend/danswer/configs/app_configs.py
index b76ad6042d1..3e2695072a2 100644
--- a/backend/danswer/configs/app_configs.py
+++ b/backend/danswer/configs/app_configs.py
@@ -163,6 +163,17 @@
 except ValueError:
     POSTGRES_POOL_RECYCLE = POSTGRES_POOL_RECYCLE_DEFAULT
 
+# Experimental setting to control idle transactions
+POSTGRES_IDLE_SESSIONS_TIMEOUT_DEFAULT = 0  # milliseconds
+try:
+    POSTGRES_IDLE_SESSIONS_TIMEOUT = int(
+        os.environ.get(
+            "POSTGRES_IDLE_SESSIONS_TIMEOUT", POSTGRES_IDLE_SESSIONS_TIMEOUT_DEFAULT
+        )
+    )
+except ValueError:
+    POSTGRES_IDLE_SESSIONS_TIMEOUT = POSTGRES_IDLE_SESSIONS_TIMEOUT_DEFAULT
+
 REDIS_SSL = os.getenv("REDIS_SSL", "").lower() == "true"
 REDIS_HOST = os.environ.get("REDIS_HOST") or "localhost"
 REDIS_PORT = int(os.environ.get("REDIS_PORT", 6379))
diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 9dc6024ab8c..6b45f390940 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -29,6 +29,7 @@
 from danswer.configs.app_configs import POSTGRES_API_SERVER_POOL_SIZE
 from danswer.configs.app_configs import POSTGRES_DB
 from danswer.configs.app_configs import POSTGRES_HOST
+from danswer.configs.app_configs import POSTGRES_IDLE_SESSIONS_TIMEOUT
 from danswer.configs.app_configs import POSTGRES_PASSWORD
 from danswer.configs.app_configs import POSTGRES_POOL_PRE_PING
 from danswer.configs.app_configs import POSTGRES_POOL_RECYCLE
@@ -352,6 +353,10 @@ def get_session_with_tenant(
             cursor = dbapi_connection.cursor()
             try:
                 cursor.execute(f'SET search_path = "{tenant_id}"')
+                if POSTGRES_IDLE_SESSIONS_TIMEOUT:
+                    cursor.execute(
+                        f"SET SESSION idle_in_transaction_session_timeout = {POSTGRES_IDLE_SESSIONS_TIMEOUT}"
+                    )
             finally:
                 cursor.close()
 

From c2d04f591d70cfde21c6fd2f44f7c62c1a2aa56e Mon Sep 17 00:00:00 2001
From: Chris Weaver <25087905+Weves@users.noreply.github.com>
Date: Sun, 3 Nov 2024 14:10:45 -0800
Subject: [PATCH 357/376] Add drive sections (#3040)

* ADd header support for drive

* Fix mypy

* Comment change

* Improve

* Cleanup

* Add comment
---
 .../connectors/google_drive/connector.py      |  72 +++++---
 .../connectors/google_drive/connector_auth.py |   2 +
 .../connectors/google_drive/doc_conversion.py | 161 ++++++++++++------
 .../connectors/google_drive/google_utils.py   |   2 +-
 .../connectors/google_drive/resources.py      |  52 ++++++
 .../google_drive/section_extraction.py        | 105 ++++++++++++
 .../google_drive/doc_sync.py                  |   6 +-
 .../google_drive/group_sync.py                |   6 +-
 .../daily/connectors/google_drive/conftest.py |  28 ++-
 .../daily/connectors/google_drive/helpers.py  |  36 +++-
 .../google_drive/test_google_drive_oauth.py   |   2 +
 .../test_google_drive_sections.py             |  71 ++++++++
 .../test_google_drive_service_acct.py         |   2 +
 .../test_google_drive_slim_docs.py            |   7 +-
 14 files changed, 459 insertions(+), 93 deletions(-)
 create mode 100644 backend/danswer/connectors/google_drive/resources.py
 create mode 100644 backend/danswer/connectors/google_drive/section_extraction.py
 create mode 100644 backend/tests/daily/connectors/google_drive/test_google_drive_sections.py

diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index 4ddd51f749f..8b2433d31c8 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -3,8 +3,6 @@
 
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
-from googleapiclient.discovery import build  # type: ignore
-from googleapiclient.discovery import Resource  # type: ignore
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.connectors.google_drive.connector_auth import (
@@ -24,6 +22,9 @@
 from danswer.connectors.google_drive.file_retrieval import get_files_in_shared_drive
 from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
 from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.connectors.google_drive.resources import get_admin_service
+from danswer.connectors.google_drive.resources import get_drive_service
+from danswer.connectors.google_drive.resources import get_google_docs_service
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
@@ -103,42 +104,49 @@ def __init__(
         shared_folder_url_list = _extract_str_list_from_comma_str(shared_folder_urls)
         self.shared_folder_ids = _extract_ids_from_urls(shared_folder_url_list)
 
-        self.primary_admin_email: str | None = None
+        self._primary_admin_email: str | None = None
         self.google_domain: str | None = None
 
-        self.creds: OAuthCredentials | ServiceAccountCredentials | None = None
+        self._creds: OAuthCredentials | ServiceAccountCredentials | None = None
 
         self._TRAVERSED_PARENT_IDS: set[str] = set()
 
+    @property
+    def primary_admin_email(self) -> str:
+        if self._primary_admin_email is None:
+            raise RuntimeError(
+                "Primary admin email missing, "
+                "should not call this property "
+                "before calling load_credentials"
+            )
+        return self._primary_admin_email
+
+    @property
+    def creds(self) -> OAuthCredentials | ServiceAccountCredentials:
+        if self._creds is None:
+            raise RuntimeError(
+                "Creds missing, "
+                "should not call this property "
+                "before calling load_credentials"
+            )
+        return self._creds
+
     def _update_traversed_parent_ids(self, folder_id: str) -> None:
         self._TRAVERSED_PARENT_IDS.add(folder_id)
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:
         primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]
         self.google_domain = primary_admin_email.split("@")[1]
-        self.primary_admin_email = primary_admin_email
+        self._primary_admin_email = primary_admin_email
 
-        self.creds, new_creds_dict = get_google_drive_creds(credentials)
+        self._creds, new_creds_dict = get_google_drive_creds(credentials)
         return new_creds_dict
 
-    def get_google_resource(
-        self,
-        service_name: str = "drive",
-        service_version: str = "v3",
-        user_email: str | None = None,
-    ) -> Resource:
-        if isinstance(self.creds, ServiceAccountCredentials):
-            creds = self.creds.with_subject(user_email or self.primary_admin_email)
-            service = build(service_name, service_version, credentials=creds)
-        elif isinstance(self.creds, OAuthCredentials):
-            service = build(service_name, service_version, credentials=self.creds)
-        else:
-            raise PermissionError("No credentials found")
-
-        return service
-
     def _get_all_user_emails(self) -> list[str]:
-        admin_service = self.get_google_resource("admin", "directory_v1")
+        admin_service = get_admin_service(
+            creds=self.creds,
+            user_email=self.primary_admin_email,
+        )
         emails = []
         for user in execute_paginated_retrieval(
             retrieval_function=admin_service.users().list,
@@ -156,7 +164,10 @@ def _fetch_drive_items(
         start: SecondsSinceUnixEpoch | None = None,
         end: SecondsSinceUnixEpoch | None = None,
     ) -> Iterator[GoogleDriveFileType]:
-        primary_drive_service = self.get_google_resource()
+        primary_drive_service = get_drive_service(
+            creds=self.creds,
+            user_email=self.primary_admin_email,
+        )
 
         if self.include_shared_drives:
             shared_drive_urls = self.shared_drive_ids
@@ -212,7 +223,7 @@ def _fetch_drive_items(
 
             for email in all_user_emails:
                 logger.info(f"Fetching personal files for user: {email}")
-                user_drive_service = self.get_google_resource(user_email=email)
+                user_drive_service = get_drive_service(self.creds, user_email=email)
 
                 yield from get_files_in_my_drive(
                     service=user_drive_service,
@@ -233,11 +244,16 @@ def _extract_docs_from_google_drive(
             start=start,
             end=end,
         ):
-            user_email = file.get("owners", [{}])[0].get("emailAddress")
-            service = self.get_google_resource(user_email=user_email)
+            user_email = (
+                file.get("owners", [{}])[0].get("emailAddress")
+                or self.primary_admin_email
+            )
+            user_drive_service = get_drive_service(self.creds, user_email=user_email)
+            docs_service = get_google_docs_service(self.creds, user_email=user_email)
             if doc := convert_drive_item_to_document(
                 file=file,
-                service=service,
+                drive_service=user_drive_service,
+                docs_service=docs_service,
             ):
                 doc_batch.append(doc)
             if len(doc_batch) >= self.batch_size:
diff --git a/backend/danswer/connectors/google_drive/connector_auth.py b/backend/danswer/connectors/google_drive/connector_auth.py
index 80cbda6772a..6ea2784662b 100644
--- a/backend/danswer/connectors/google_drive/connector_auth.py
+++ b/backend/danswer/connectors/google_drive/connector_auth.py
@@ -28,6 +28,8 @@
 
 logger = setup_logger()
 
+# NOTE: do not need https://www.googleapis.com/auth/documents.readonly
+# this is counted under `/auth/drive.readonly`
 GOOGLE_DRIVE_SCOPES = [
     "https://www.googleapis.com/auth/drive.readonly",
     "https://www.googleapis.com/auth/drive.metadata.readonly",
diff --git a/backend/danswer/connectors/google_drive/doc_conversion.py b/backend/danswer/connectors/google_drive/doc_conversion.py
index 688190c2267..81e709f53f4 100644
--- a/backend/danswer/connectors/google_drive/doc_conversion.py
+++ b/backend/danswer/connectors/google_drive/doc_conversion.py
@@ -2,7 +2,6 @@
 from datetime import datetime
 from datetime import timezone
 
-from googleapiclient.discovery import Resource  # type: ignore
 from googleapiclient.errors import HttpError  # type: ignore
 
 from danswer.configs.app_configs import CONTINUE_ON_CONNECTOR_FAILURE
@@ -13,6 +12,9 @@
 from danswer.connectors.google_drive.constants import UNSUPPORTED_FILE_TYPE_CONTENT
 from danswer.connectors.google_drive.models import GDriveMimeType
 from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.connectors.google_drive.resources import GoogleDocsService
+from danswer.connectors.google_drive.resources import GoogleDriveService
+from danswer.connectors.google_drive.section_extraction import get_document_sections
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
 from danswer.file_processing.extract_file_text import docx_to_text
@@ -25,86 +27,137 @@
 logger = setup_logger()
 
 
-def _extract_text(file: dict[str, str], service: Resource) -> str:
+def _extract_sections_basic(
+    file: dict[str, str], service: GoogleDriveService
+) -> list[Section]:
     mime_type = file["mimeType"]
+    link = file["webViewLink"]
 
     if mime_type not in set(item.value for item in GDriveMimeType):
         # Unsupported file types can still have a title, finding this way is still useful
-        return UNSUPPORTED_FILE_TYPE_CONTENT
-
-    if mime_type in [
-        GDriveMimeType.DOC.value,
-        GDriveMimeType.PPT.value,
-        GDriveMimeType.SPREADSHEET.value,
-    ]:
-        export_mime_type = (
-            "text/plain"
-            if mime_type != GDriveMimeType.SPREADSHEET.value
-            else "text/csv"
-        )
-        return (
-            service.files()
-            .export(fileId=file["id"], mimeType=export_mime_type)
-            .execute()
-            .decode("utf-8")
-        )
-    elif mime_type in [
-        GDriveMimeType.PLAIN_TEXT.value,
-        GDriveMimeType.MARKDOWN.value,
-    ]:
-        return service.files().get_media(fileId=file["id"]).execute().decode("utf-8")
-    if mime_type in [
-        GDriveMimeType.WORD_DOC.value,
-        GDriveMimeType.POWERPOINT.value,
-        GDriveMimeType.PDF.value,
-    ]:
-        response = service.files().get_media(fileId=file["id"]).execute()
-        if get_unstructured_api_key():
-            return unstructured_to_text(
-                file=io.BytesIO(response), file_name=file.get("name", file["id"])
+        return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
+
+    try:
+        if mime_type in [
+            GDriveMimeType.DOC.value,
+            GDriveMimeType.PPT.value,
+            GDriveMimeType.SPREADSHEET.value,
+        ]:
+            export_mime_type = (
+                "text/plain"
+                if mime_type != GDriveMimeType.SPREADSHEET.value
+                else "text/csv"
             )
+            text = (
+                service.files()
+                .export(fileId=file["id"], mimeType=export_mime_type)
+                .execute()
+                .decode("utf-8")
+            )
+            return [Section(link=link, text=text)]
+        elif mime_type in [
+            GDriveMimeType.PLAIN_TEXT.value,
+            GDriveMimeType.MARKDOWN.value,
+        ]:
+            return [
+                Section(
+                    link=link,
+                    text=service.files()
+                    .get_media(fileId=file["id"])
+                    .execute()
+                    .decode("utf-8"),
+                )
+            ]
+        if mime_type in [
+            GDriveMimeType.WORD_DOC.value,
+            GDriveMimeType.POWERPOINT.value,
+            GDriveMimeType.PDF.value,
+        ]:
+            response = service.files().get_media(fileId=file["id"]).execute()
+            if get_unstructured_api_key():
+                return [
+                    Section(
+                        link=link,
+                        text=unstructured_to_text(
+                            file=io.BytesIO(response),
+                            file_name=file.get("name", file["id"]),
+                        ),
+                    )
+                ]
+
+            if mime_type == GDriveMimeType.WORD_DOC.value:
+                return [
+                    Section(link=link, text=docx_to_text(file=io.BytesIO(response)))
+                ]
+            elif mime_type == GDriveMimeType.PDF.value:
+                text, _ = read_pdf_file(file=io.BytesIO(response))
+                return [Section(link=link, text=text)]
+            elif mime_type == GDriveMimeType.POWERPOINT.value:
+                return [
+                    Section(link=link, text=pptx_to_text(file=io.BytesIO(response)))
+                ]
 
-        if mime_type == GDriveMimeType.WORD_DOC.value:
-            return docx_to_text(file=io.BytesIO(response))
-        elif mime_type == GDriveMimeType.PDF.value:
-            text, _ = read_pdf_file(file=io.BytesIO(response))
-            return text
-        elif mime_type == GDriveMimeType.POWERPOINT.value:
-            return pptx_to_text(file=io.BytesIO(response))
+        return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
 
-    return UNSUPPORTED_FILE_TYPE_CONTENT
+    except Exception:
+        return [Section(link=link, text=UNSUPPORTED_FILE_TYPE_CONTENT)]
 
 
 def convert_drive_item_to_document(
-    file: GoogleDriveFileType, service: Resource
+    file: GoogleDriveFileType,
+    drive_service: GoogleDriveService,
+    docs_service: GoogleDocsService,
 ) -> Document | None:
     try:
         # Skip files that are shortcuts
         if file.get("mimeType") == DRIVE_SHORTCUT_TYPE:
             logger.info("Ignoring Drive Shortcut Filetype")
             return None
-        try:
-            text_contents = _extract_text(file, service) or ""
-        except HttpError as e:
-            reason = e.error_details[0]["reason"] if e.error_details else e.reason
-            message = e.error_details[0]["message"] if e.error_details else e.reason
-            if e.status_code == 403 and reason in ERRORS_TO_CONTINUE_ON:
+
+        sections: list[Section] = []
+
+        # Special handling for Google Docs to preserve structure, link
+        # to headers
+        if file.get("mimeType") == GDriveMimeType.DOC.value:
+            try:
+                sections = get_document_sections(docs_service, file["id"])
+            except Exception as e:
                 logger.warning(
-                    f"Could not export file '{file['name']}' due to '{message}', skipping..."
+                    f"Ran into exception '{e}' when pulling sections from Google Doc '{file['name']}'."
+                    " Falling back to basic extraction."
                 )
-                return None
 
-            raise
+        # NOTE: this will run for either (1) the above failed or (2) the file is not a Google Doc
+        if not sections:
+            try:
+                # For all other file types just extract the text
+                sections = _extract_sections_basic(file, drive_service)
+
+            except HttpError as e:
+                reason = e.error_details[0]["reason"] if e.error_details else e.reason
+                message = e.error_details[0]["message"] if e.error_details else e.reason
+                if e.status_code == 403 and reason in ERRORS_TO_CONTINUE_ON:
+                    logger.warning(
+                        f"Could not export file '{file['name']}' due to '{message}', skipping..."
+                    )
+                    return None
+
+                raise
+
+        if not sections:
+            return None
 
         return Document(
             id=file["webViewLink"],
-            sections=[Section(link=file["webViewLink"], text=text_contents)],
+            sections=sections,
             source=DocumentSource.GOOGLE_DRIVE,
             semantic_identifier=file["name"],
             doc_updated_at=datetime.fromisoformat(file["modifiedTime"]).astimezone(
                 timezone.utc
             ),
-            metadata={} if text_contents else {IGNORE_FOR_QA: "True"},
+            metadata={}
+            if any(section.text for section in sections)
+            else {IGNORE_FOR_QA: "True"},
             additional_info=file.get("id"),
         )
     except Exception as e:
diff --git a/backend/danswer/connectors/google_drive/google_utils.py b/backend/danswer/connectors/google_drive/google_utils.py
index 5f772e5ad63..a2e029a41df 100644
--- a/backend/danswer/connectors/google_drive/google_utils.py
+++ b/backend/danswer/connectors/google_drive/google_utils.py
@@ -28,7 +28,7 @@ def execute_paginated_retrieval(
         if next_page_token:
             request_kwargs["pageToken"] = next_page_token
 
-        results = add_retries(lambda: retrieval_function(**request_kwargs).execute())()
+        results = (lambda: retrieval_function(**request_kwargs).execute())()
 
         next_page_token = results.get("nextPageToken")
         for item in results.get(list_key, []):
diff --git a/backend/danswer/connectors/google_drive/resources.py b/backend/danswer/connectors/google_drive/resources.py
new file mode 100644
index 00000000000..df5b36376b5
--- /dev/null
+++ b/backend/danswer/connectors/google_drive/resources.py
@@ -0,0 +1,52 @@
+from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
+from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
+from googleapiclient.discovery import build  # type: ignore
+from googleapiclient.discovery import Resource  # type: ignore
+
+
+class GoogleDriveService(Resource):
+    pass
+
+
+class GoogleDocsService(Resource):
+    pass
+
+
+class AdminService(Resource):
+    pass
+
+
+def _get_google_service(
+    service_name: str,
+    service_version: str,
+    creds: ServiceAccountCredentials | OAuthCredentials,
+    user_email: str | None = None,
+) -> GoogleDriveService:
+    if isinstance(creds, ServiceAccountCredentials):
+        creds = creds.with_subject(user_email)
+        service = build(service_name, service_version, credentials=creds)
+    elif isinstance(creds, OAuthCredentials):
+        service = build(service_name, service_version, credentials=creds)
+
+    return service
+
+
+def get_google_docs_service(
+    creds: ServiceAccountCredentials | OAuthCredentials,
+    user_email: str | None = None,
+) -> GoogleDocsService:
+    return _get_google_service("docs", "v1", creds, user_email)
+
+
+def get_drive_service(
+    creds: ServiceAccountCredentials | OAuthCredentials,
+    user_email: str | None = None,
+) -> GoogleDriveService:
+    return _get_google_service("drive", "v3", creds, user_email)
+
+
+def get_admin_service(
+    creds: ServiceAccountCredentials | OAuthCredentials,
+    user_email: str,
+) -> AdminService:
+    return _get_google_service("admin", "directory_v1", creds, user_email)
diff --git a/backend/danswer/connectors/google_drive/section_extraction.py b/backend/danswer/connectors/google_drive/section_extraction.py
new file mode 100644
index 00000000000..bcd162b1c79
--- /dev/null
+++ b/backend/danswer/connectors/google_drive/section_extraction.py
@@ -0,0 +1,105 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from danswer.connectors.google_drive.resources import GoogleDocsService
+from danswer.connectors.models import Section
+
+
+class CurrentHeading(BaseModel):
+    id: str
+    text: str
+
+
+def _build_gdoc_section_link(doc_id: str, heading_id: str) -> str:
+    """Builds a Google Doc link that jumps to a specific heading"""
+    # NOTE: doesn't support docs with multiple tabs atm, if we need that ask
+    # @Chris
+    return (
+        f"https://docs.google.com/document/d/{doc_id}/edit?tab=t.0#heading={heading_id}"
+    )
+
+
+def _extract_id_from_heading(paragraph: dict[str, Any]) -> str:
+    """Extracts the id from a heading paragraph element"""
+    return paragraph["paragraphStyle"]["headingId"]
+
+
+def _extract_text_from_paragraph(paragraph: dict[str, Any]) -> str:
+    """Extracts the text content from a paragraph element"""
+    text_elements = []
+    for element in paragraph.get("elements", []):
+        if "textRun" in element:
+            text_elements.append(element["textRun"].get("content", ""))
+    return "".join(text_elements)
+
+
+def get_document_sections(
+    docs_service: GoogleDocsService,
+    doc_id: str,
+) -> list[Section]:
+    """Extracts sections from a Google Doc, including their headings and content"""
+    # Fetch the document structure
+    doc = docs_service.documents().get(documentId=doc_id).execute()
+
+    # Get the content
+    content = doc.get("body", {}).get("content", [])
+
+    sections: list[Section] = []
+    current_section: list[str] = []
+    current_heading: CurrentHeading | None = None
+
+    for element in content:
+        if "paragraph" not in element:
+            continue
+
+        paragraph = element["paragraph"]
+
+        # Check if this is a heading
+        if (
+            "paragraphStyle" in paragraph
+            and "namedStyleType" in paragraph["paragraphStyle"]
+        ):
+            style = paragraph["paragraphStyle"]["namedStyleType"]
+            is_heading = style.startswith("HEADING_")
+            is_title = style.startswith("TITLE")
+
+            if is_heading or is_title:
+                # If we were building a previous section, add it to sections list
+                if current_heading is not None and current_section:
+                    heading_text = current_heading.text
+                    section_text = f"{heading_text}\n" + "\n".join(current_section)
+                    sections.append(
+                        Section(
+                            text=section_text.strip(),
+                            link=_build_gdoc_section_link(doc_id, current_heading.id),
+                        )
+                    )
+                    current_section = []
+
+                # Start new heading
+                heading_id = _extract_id_from_heading(paragraph)
+                heading_text = _extract_text_from_paragraph(paragraph)
+                current_heading = CurrentHeading(
+                    id=heading_id,
+                    text=heading_text,
+                )
+                continue
+
+        # Add content to current section
+        if current_heading is not None:
+            text = _extract_text_from_paragraph(paragraph)
+            if text.strip():
+                current_section.append(text)
+
+    # Don't forget to add the last section
+    if current_heading is not None and current_section:
+        section_text = f"{current_heading.text}\n" + "\n".join(current_section)
+        sections.append(
+            Section(
+                text=section_text.strip(),
+                link=_build_gdoc_section_link(doc_id, current_heading.id),
+            )
+        )
+
+    return sections
diff --git a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
index d1df0cb0846..9f736b22a00 100644
--- a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
@@ -7,6 +7,7 @@
 from danswer.access.models import ExternalAccess
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
 from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_drive.resources import get_drive_service
 from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.models import SlimDocument
 from danswer.db.models import ConnectorCredentialPair
@@ -56,7 +57,10 @@ def _fetch_permissions_for_permission_ids(
         return permissions
 
     owner_email = permission_info.get("owner_email")
-    drive_service = google_drive_connector.get_google_resource(user_email=owner_email)
+    drive_service = get_drive_service(
+        creds=google_drive_connector.creds,
+        user_email=(owner_email or google_drive_connector.primary_admin_email),
+    )
 
     # Otherwise, fetch all permissions and update cache
     fetched_permissions = execute_paginated_retrieval(
diff --git a/backend/ee/danswer/external_permissions/google_drive/group_sync.py b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
index c3afa962392..919866749ff 100644
--- a/backend/ee/danswer/external_permissions/google_drive/group_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
@@ -2,6 +2,7 @@
 
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
 from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_drive.resources import get_admin_service
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
@@ -19,8 +20,9 @@ def gdrive_group_sync(
         **cc_pair.connector.connector_specific_config
     )
     google_drive_connector.load_credentials(cc_pair.credential.credential_json)
-
-    admin_service = google_drive_connector.get_google_resource("admin", "directory_v1")
+    admin_service = get_admin_service(
+        google_drive_connector.creds, google_drive_connector.primary_admin_email
+    )
 
     danswer_groups: list[ExternalUserGroup] = []
     for group in execute_paginated_retrieval(
diff --git a/backend/tests/daily/connectors/google_drive/conftest.py b/backend/tests/daily/connectors/google_drive/conftest.py
index 0b516d0359c..c330dcf2d60 100644
--- a/backend/tests/daily/connectors/google_drive/conftest.py
+++ b/backend/tests/daily/connectors/google_drive/conftest.py
@@ -31,6 +31,30 @@ def load_env_vars(env_file: str = ".env") -> None:
 load_env_vars()
 
 
+def parse_credentials(env_str: str) -> dict:
+    """
+    Parse a double-escaped JSON string from environment variables into a Python dictionary.
+
+    Args:
+        env_str (str): The double-escaped JSON string from environment variables
+
+    Returns:
+        dict: Parsed OAuth credentials
+    """
+    # first try normally
+    try:
+        return json.loads(env_str)
+    except Exception:
+        # First, try remove extra escaping backslashes
+        unescaped = env_str.replace('\\"', '"')
+
+        # remove leading / trailing quotes
+        unescaped = unescaped.strip('"')
+
+        # Now parse the JSON
+        return json.loads(unescaped)
+
+
 @pytest.fixture
 def google_drive_oauth_connector_factory() -> Callable[..., GoogleDriveConnector]:
     def _connector_factory(
@@ -50,7 +74,7 @@ def _connector_factory(
         )
 
         json_string = os.environ["GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR"]
-        refried_json_string = json.loads(json_string)
+        refried_json_string = json.dumps(parse_credentials(json_string))
 
         credentials_json = {
             DB_CREDENTIALS_DICT_TOKEN_KEY: refried_json_string,
@@ -84,7 +108,7 @@ def _connector_factory(
         )
 
         json_string = os.environ["GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR"]
-        refried_json_string = json.loads(json_string)
+        refried_json_string = json.dumps(parse_credentials(json_string))
 
         # Load Service Account Credentials
         connector.load_credentials(
diff --git a/backend/tests/daily/connectors/google_drive/helpers.py b/backend/tests/daily/connectors/google_drive/helpers.py
index a1bc8feec38..7a120412e9d 100644
--- a/backend/tests/daily/connectors/google_drive/helpers.py
+++ b/backend/tests/daily/connectors/google_drive/helpers.py
@@ -18,6 +18,7 @@
 _FOLDER_2_FILE_IDS = list(range(45, 50))
 _FOLDER_2_1_FILE_IDS = list(range(50, 55))
 _FOLDER_2_2_FILE_IDS = list(range(55, 60))
+_SECTIONS_FILE_IDS = [61]
 
 _PUBLIC_FOLDER_RANGE = _FOLDER_1_2_FILE_IDS
 _PUBLIC_FILE_IDS = list(range(55, 57))
@@ -64,6 +65,7 @@
     "FOLDER_2": _FOLDER_2_FILE_IDS,
     "FOLDER_2_1": _FOLDER_2_1_FILE_IDS,
     "FOLDER_2_2": _FOLDER_2_2_FILE_IDS,
+    "SECTIONS": _SECTIONS_FILE_IDS,
 }
 
 # Dictionary for emails
@@ -100,6 +102,7 @@
         + _FOLDER_2_FILE_IDS
         + _FOLDER_2_1_FILE_IDS
         + _FOLDER_2_2_FILE_IDS
+        + _SECTIONS_FILE_IDS
     ),
     # This user has access to drive 1
     # This user has redundant access to folder 1 because of group access
@@ -127,6 +130,21 @@
     "TEST_USER_3": _TEST_USER_3_FILE_IDS,
 }
 
+SPECIAL_FILE_ID_TO_CONTENT_MAP: dict[int, str] = {
+    61: (
+        "Title\n\n"
+        "This is a Google Doc with sections - "
+        "Section 1\n\n"
+        "Section 1 content - "
+        "Sub-Section 1-1\n\n"
+        "Sub-Section 1-1 content - "
+        "Sub-Section 1-2\n\n"
+        "Sub-Section 1-2 content - "
+        "Section 2\n\n"
+        "Section 2 content"
+    ),
+}
+
 
 file_name_template = "file_{}.txt"
 file_text_template = "This is file {}"
@@ -142,18 +160,28 @@ def print_discrepencies(expected: set[str], retrieved: set[str]) -> None:
         print(expected - retrieved)
 
 
+def get_file_content(file_id: int) -> str:
+    if file_id in SPECIAL_FILE_ID_TO_CONTENT_MAP:
+        return SPECIAL_FILE_ID_TO_CONTENT_MAP[file_id]
+
+    return file_text_template.format(file_id)
+
+
 def assert_retrieved_docs_match_expected(
     retrieved_docs: list[Document], expected_file_ids: Sequence[int]
 ) -> None:
     expected_file_names = {
         file_name_template.format(file_id) for file_id in expected_file_ids
     }
-    expected_file_texts = {
-        file_text_template.format(file_id) for file_id in expected_file_ids
-    }
+    expected_file_texts = {get_file_content(file_id) for file_id in expected_file_ids}
 
     retrieved_file_names = set([doc.semantic_identifier for doc in retrieved_docs])
-    retrieved_texts = set([doc.sections[0].text for doc in retrieved_docs])
+    retrieved_texts = set(
+        [
+            " - ".join([section.text for section in doc.sections])
+            for doc in retrieved_docs
+        ]
+    )
 
     # Check file names
     print_discrepencies(expected_file_names, retrieved_file_names)
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
index f39b15600b4..a4adaa4673a 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
@@ -41,6 +41,7 @@ def test_include_all(
         + DRIVE_ID_MAPPING["FOLDER_2"]
         + DRIVE_ID_MAPPING["FOLDER_2_1"]
         + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        + DRIVE_ID_MAPPING["SECTIONS"]
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
@@ -75,6 +76,7 @@ def test_include_shared_drives_only(
         + DRIVE_ID_MAPPING["FOLDER_2"]
         + DRIVE_ID_MAPPING["FOLDER_2_1"]
         + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        + DRIVE_ID_MAPPING["SECTIONS"]
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_sections.py b/backend/tests/daily/connectors/google_drive/test_google_drive_sections.py
new file mode 100644
index 00000000000..4f4556a06d6
--- /dev/null
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_sections.py
@@ -0,0 +1,71 @@
+import time
+from collections.abc import Callable
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from danswer.connectors.google_drive.connector import GoogleDriveConnector
+from danswer.connectors.models import Document
+
+
+SECTIONS_FOLDER_URL = (
+    "https://drive.google.com/drive/u/5/folders/1loe6XJ-pJxu9YYPv7cF3Hmz296VNzA33"
+)
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_google_drive_sections(
+    mock_get_api_key: MagicMock,
+    google_drive_oauth_connector_factory: Callable[..., GoogleDriveConnector],
+    google_drive_service_acct_connector_factory: Callable[..., GoogleDriveConnector],
+) -> None:
+    oauth_connector = google_drive_oauth_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=False,
+        shared_folder_urls=SECTIONS_FOLDER_URL,
+    )
+    service_acct_connector = google_drive_service_acct_connector_factory(
+        include_shared_drives=False,
+        include_my_drives=False,
+        shared_folder_urls=SECTIONS_FOLDER_URL,
+    )
+    for connector in [oauth_connector, service_acct_connector]:
+        retrieved_docs: list[Document] = []
+        for doc_batch in connector.poll_source(0, time.time()):
+            retrieved_docs.extend(doc_batch)
+
+        # Verify we got the 1 doc with sections
+        assert len(retrieved_docs) == 1
+
+        # Verify each section has the expected structure
+        doc = retrieved_docs[0]
+        assert len(doc.sections) == 5
+
+        header_section = doc.sections[0]
+        assert header_section.text == "Title\n\nThis is a Google Doc with sections"
+        assert header_section.link is not None
+        assert header_section.link.endswith(
+            "?tab=t.0#heading=h.hfjc17k6qwzt"
+        ) or header_section.link.endswith("?tab=t.0#heading=h.hfjc17k6qwzt")
+
+        section_1 = doc.sections[1]
+        assert section_1.text == "Section 1\n\nSection 1 content"
+        assert section_1.link is not None
+        assert section_1.link.endswith("?tab=t.0#heading=h.8slfx752a3g5")
+
+        section_2 = doc.sections[2]
+        assert section_2.text == "Sub-Section 1-1\n\nSub-Section 1-1 content"
+        assert section_2.link is not None
+        assert section_2.link.endswith("?tab=t.0#heading=h.4kj3ayade1bp")
+
+        section_3 = doc.sections[3]
+        assert section_3.text == "Sub-Section 1-2\n\nSub-Section 1-2 content"
+        assert section_3.link is not None
+        assert section_3.link.endswith("?tab=t.0#heading=h.pm6wrpzgk69l")
+
+        section_4 = doc.sections[4]
+        assert section_4.text == "Section 2\n\nSection 2 content"
+        assert section_4.link is not None
+        assert section_4.link.endswith("?tab=t.0#heading=h.2m0s9youe2k9")
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py b/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
index b36a53b30f6..a7f081fd323 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
@@ -44,6 +44,7 @@ def test_include_all(
         + DRIVE_ID_MAPPING["FOLDER_2"]
         + DRIVE_ID_MAPPING["FOLDER_2_1"]
         + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        + DRIVE_ID_MAPPING["SECTIONS"]
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
@@ -78,6 +79,7 @@ def test_include_shared_drives_only(
         + DRIVE_ID_MAPPING["FOLDER_2"]
         + DRIVE_ID_MAPPING["FOLDER_2_1"]
         + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        + DRIVE_ID_MAPPING["SECTIONS"]
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py b/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
index e731c8b27ce..d7e79da09d5 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
@@ -6,6 +6,7 @@
 from danswer.access.models import ExternalAccess
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
 from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_drive.resources import get_admin_service
 from ee.danswer.external_permissions.google_drive.doc_sync import (
     _get_permissions_from_slim_doc,
 )
@@ -72,7 +73,10 @@ def assert_correct_access_for_user(
 # This function is supposed to map to the group_sync.py file for the google drive connector
 # TODO: Call it directly
 def get_group_map(google_drive_connector: GoogleDriveConnector) -> dict[str, list[str]]:
-    admin_service = google_drive_connector.get_google_resource("admin", "directory_v1")
+    admin_service = get_admin_service(
+        creds=google_drive_connector.creds,
+        user_email=google_drive_connector.primary_admin_email,
+    )
 
     group_map: dict[str, list[str]] = {}
     for group in execute_paginated_retrieval(
@@ -138,6 +142,7 @@ def test_all_permissions(
         + DRIVE_ID_MAPPING["FOLDER_2"]
         + DRIVE_ID_MAPPING["FOLDER_2_1"]
         + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        + DRIVE_ID_MAPPING["SECTIONS"]
     )
 
     # Should get everything

From c6e8bf2d286b025cb485f84890223018d11fd55e Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 15:54:19 -0800
Subject: [PATCH 358/376] add multiple formats to tools (#3041)

---
 backend/danswer/chat/models.py                |   4 +-
 backend/danswer/chat/process_message.py       |  35 +++-
 backend/danswer/llm/utils.py                  |  18 ++-
 .../one_shot_answer/answer_question.py        |   2 +-
 .../custom/custom_tool.py                     | 151 +++++++++++++++++-
 .../tool_implementations/custom/prompt.py     |  25 +++
 .../danswer/tools/custom/test_custom_tools.py |   1 +
 web/src/app/chat/ChatPage.tsx                 |   4 +-
 web/src/app/chat/interfaces.ts                |   2 +-
 web/src/app/chat/lib.tsx                      |   4 +-
 10 files changed, 220 insertions(+), 26 deletions(-)
 create mode 100644 backend/danswer/tools/tool_implementations/custom/prompt.py

diff --git a/backend/danswer/chat/models.py b/backend/danswer/chat/models.py
index d5925fc2ed9..159506c0734 100644
--- a/backend/danswer/chat/models.py
+++ b/backend/danswer/chat/models.py
@@ -156,7 +156,7 @@ class QAResponse(SearchResponse, DanswerAnswer):
     error_msg: str | None = None
 
 
-class ImageGenerationDisplay(BaseModel):
+class FileChatDisplay(BaseModel):
     file_ids: list[str]
 
 
@@ -170,7 +170,7 @@ class CustomToolResponse(BaseModel):
     | DanswerQuotes
     | CitationInfo
     | DanswerContexts
-    | ImageGenerationDisplay
+    | FileChatDisplay
     | CustomToolResponse
     | StreamingError
     | StreamStopInfo
diff --git a/backend/danswer/chat/process_message.py b/backend/danswer/chat/process_message.py
index 252f6df0f1e..314e432b86a 100644
--- a/backend/danswer/chat/process_message.py
+++ b/backend/danswer/chat/process_message.py
@@ -11,8 +11,8 @@
 from danswer.chat.models import CitationInfo
 from danswer.chat.models import CustomToolResponse
 from danswer.chat.models import DanswerAnswerPiece
+from danswer.chat.models import FileChatDisplay
 from danswer.chat.models import FinalUsedContextDocsResponse
-from danswer.chat.models import ImageGenerationDisplay
 from danswer.chat.models import LLMRelevanceFilterResponse
 from danswer.chat.models import MessageResponseIDInfo
 from danswer.chat.models import MessageSpecificCitations
@@ -275,7 +275,7 @@ def _get_force_search_settings(
     | DanswerAnswerPiece
     | AllCitations
     | CitationInfo
-    | ImageGenerationDisplay
+    | FileChatDisplay
     | CustomToolResponse
     | MessageSpecificCitations
     | MessageResponseIDInfo
@@ -769,7 +769,6 @@ def stream_chat_message_objects(
                         yield LLMRelevanceFilterResponse(
                             llm_selected_doc_indices=llm_indices
                         )
-
                 elif packet.id == FINAL_CONTEXT_DOCUMENTS_ID:
                     yield FinalUsedContextDocsResponse(
                         final_context_docs=packet.response
@@ -787,7 +786,7 @@ def stream_chat_message_objects(
                         FileDescriptor(id=str(file_id), type=ChatFileType.IMAGE)
                         for file_id in file_ids
                     ]
-                    yield ImageGenerationDisplay(
+                    yield FileChatDisplay(
                         file_ids=[str(file_id) for file_id in file_ids]
                     )
                 elif packet.id == INTERNET_SEARCH_RESPONSE_ID:
@@ -801,10 +800,30 @@ def stream_chat_message_objects(
                     yield qa_docs_response
                 elif packet.id == CUSTOM_TOOL_RESPONSE_ID:
                     custom_tool_response = cast(CustomToolCallSummary, packet.response)
-                    yield CustomToolResponse(
-                        response=custom_tool_response.tool_result,
-                        tool_name=custom_tool_response.tool_name,
-                    )
+
+                    if (
+                        custom_tool_response.response_type == "image"
+                        or custom_tool_response.response_type == "csv"
+                    ):
+                        file_ids = custom_tool_response.tool_result.file_ids
+                        ai_message_files = [
+                            FileDescriptor(
+                                id=str(file_id),
+                                type=ChatFileType.IMAGE
+                                if custom_tool_response.response_type == "image"
+                                else ChatFileType.CSV,
+                            )
+                            for file_id in file_ids
+                        ]
+                        yield FileChatDisplay(
+                            file_ids=[str(file_id) for file_id in file_ids]
+                        )
+                    else:
+                        yield CustomToolResponse(
+                            response=custom_tool_response.tool_result,
+                            tool_name=custom_tool_response.tool_name,
+                        )
+
             elif isinstance(packet, StreamStopInfo):
                 pass
             else:
diff --git a/backend/danswer/llm/utils.py b/backend/danswer/llm/utils.py
index 77e13755241..24017146912 100644
--- a/backend/danswer/llm/utils.py
+++ b/backend/danswer/llm/utils.py
@@ -109,11 +109,10 @@ def translate_danswer_msg_to_langchain(
     files: list[InMemoryChatFile] = []
 
     # If the message is a `ChatMessage`, it doesn't have the downloaded files
-    # attached. Just ignore them for now. Also, OpenAI doesn't allow files to
-    # be attached to AI messages, so we must remove them
-    if not isinstance(msg, ChatMessage) and msg.message_type != MessageType.ASSISTANT:
+    # attached. Just ignore them for now.
+    if not isinstance(msg, ChatMessage):
         files = msg.files
-    content = build_content_with_imgs(msg.message, files)
+    content = build_content_with_imgs(msg.message, files, message_type=msg.message_type)
 
     if msg.message_type == MessageType.SYSTEM:
         raise ValueError("System messages are not currently part of history")
@@ -188,10 +187,19 @@ def build_content_with_imgs(
     message: str,
     files: list[InMemoryChatFile] | None = None,
     img_urls: list[str] | None = None,
+    message_type: MessageType = MessageType.USER,
 ) -> str | list[str | dict[str, Any]]:  # matching Langchain's BaseMessage content type
     files = files or []
-    img_files = [file for file in files if file.file_type == ChatFileType.IMAGE]
+
+    # Only include image files for user messages
+    img_files = (
+        [file for file in files if file.file_type == ChatFileType.IMAGE]
+        if message_type == MessageType.USER
+        else []
+    )
+
     img_urls = img_urls or []
+
     message_main_content = _build_content(message, files)
 
     if not img_files and not img_urls:
diff --git a/backend/danswer/one_shot_answer/answer_question.py b/backend/danswer/one_shot_answer/answer_question.py
index 9ece5f4bba2..f3cbe2b60af 100644
--- a/backend/danswer/one_shot_answer/answer_question.py
+++ b/backend/danswer/one_shot_answer/answer_question.py
@@ -253,7 +253,7 @@ def stream_answer_objects(
         return_contexts=query_req.return_contexts,
         skip_gen_ai_answer_generation=query_req.skip_gen_ai_answer_generation,
     )
-    # won't be any ImageGenerationDisplay responses since that tool is never passed in
+    # won't be any FileChatDisplay responses since that tool is never passed in
     for packet in cast(AnswerObjectIterator, answer.processed_streamed_output):
         # for one-shot flow, don't currently do anything with these
         if isinstance(packet, ToolResponse):
diff --git a/backend/danswer/tools/tool_implementations/custom/custom_tool.py b/backend/danswer/tools/tool_implementations/custom/custom_tool.py
index a1fb4bb699e..eace6d53a3b 100644
--- a/backend/danswer/tools/tool_implementations/custom/custom_tool.py
+++ b/backend/danswer/tools/tool_implementations/custom/custom_tool.py
@@ -1,22 +1,34 @@
+import csv
 import json
+import uuid
 from collections.abc import Generator
+from io import BytesIO
+from io import StringIO
 from typing import Any
 from typing import cast
+from typing import Dict
+from typing import List
 
 import requests
 from langchain_core.messages import HumanMessage
 from langchain_core.messages import SystemMessage
 from pydantic import BaseModel
 
+from danswer.configs.constants import FileOrigin
+from danswer.db.engine import get_session_with_tenant
+from danswer.file_store.file_store import get_default_file_store
+from danswer.file_store.models import ChatFileType
+from danswer.file_store.models import InMemoryChatFile
 from danswer.key_value_store.interface import JSON_ro
 from danswer.llm.answering.models import PreviousMessage
+from danswer.llm.answering.prompts.build import AnswerPromptBuilder
 from danswer.llm.interfaces import LLM
 from danswer.tools.base_tool import BaseTool
+from danswer.tools.message import ToolCallSummary
 from danswer.tools.models import CHAT_SESSION_ID_PLACEHOLDER
 from danswer.tools.models import DynamicSchemaInfo
 from danswer.tools.models import MESSAGE_ID_PLACEHOLDER
 from danswer.tools.models import ToolResponse
-from danswer.tools.tool_implementations.custom.base_tool_types import ToolResultType
 from danswer.tools.tool_implementations.custom.custom_tool_prompts import (
     SHOULD_USE_CUSTOM_TOOL_SYSTEM_PROMPT,
 )
@@ -39,6 +51,9 @@
 from danswer.tools.tool_implementations.custom.openapi_parsing import (
     validate_openapi_schema,
 )
+from danswer.tools.tool_implementations.custom.prompt import (
+    build_custom_image_generation_user_prompt,
+)
 from danswer.utils.headers import header_list_to_header_dict
 from danswer.utils.headers import HeaderItemDict
 from danswer.utils.logger import setup_logger
@@ -48,9 +63,14 @@
 CUSTOM_TOOL_RESPONSE_ID = "custom_tool_response"
 
 
+class CustomToolFileResponse(BaseModel):
+    file_ids: List[str]  # References to saved images or CSVs
+
+
 class CustomToolCallSummary(BaseModel):
     tool_name: str
-    tool_result: ToolResultType
+    response_type: str  # e.g., 'json', 'image', 'csv', 'graph'
+    tool_result: Any  # The response data
 
 
 class CustomTool(BaseTool):
@@ -91,6 +111,12 @@ def build_tool_message_content(
         self, *args: ToolResponse
     ) -> str | list[str | dict[str, Any]]:
         response = cast(CustomToolCallSummary, args[0].response)
+
+        if response.response_type == "image" or response.response_type == "csv":
+            image_response = cast(CustomToolFileResponse, response.tool_result)
+            return json.dumps({"file_ids": image_response.file_ids})
+
+        # For JSON or other responses, return as-is
         return json.dumps(response.tool_result)
 
     """For LLMs which do NOT support explicit tool calling"""
@@ -158,6 +184,38 @@ def get_args_for_non_tool_calling_llm(
         )
         return None
 
+    def _save_and_get_file_references(
+        self, file_content: bytes | str, content_type: str
+    ) -> List[str]:
+        with get_session_with_tenant() as db_session:
+            file_store = get_default_file_store(db_session)
+
+            file_id = str(uuid.uuid4())
+
+            # Handle both binary and text content
+            if isinstance(file_content, str):
+                content = BytesIO(file_content.encode())
+            else:
+                content = BytesIO(file_content)
+
+            file_store.save_file(
+                file_name=file_id,
+                content=content,
+                display_name=file_id,
+                file_origin=FileOrigin.CHAT_UPLOAD,
+                file_type=content_type,
+                file_metadata={
+                    "content_type": content_type,
+                },
+            )
+
+        return [file_id]
+
+    def _parse_csv(self, csv_text: str) -> List[Dict[str, Any]]:
+        csv_file = StringIO(csv_text)
+        reader = csv.DictReader(csv_file)
+        return [row for row in reader]
+
     """Actual execution of the tool"""
 
     def run(self, **kwargs: Any) -> Generator[ToolResponse, None, None]:
@@ -177,20 +235,103 @@ def run(self, **kwargs: Any) -> Generator[ToolResponse, None, None]:
 
         url = self._method_spec.build_url(self._base_url, path_params, query_params)
         method = self._method_spec.method
-        # Log request details
+
         response = requests.request(
             method, url, json=request_body, headers=self.headers
         )
+        content_type = response.headers.get("Content-Type", "")
+
+        if "text/csv" in content_type:
+            file_ids = self._save_and_get_file_references(
+                response.content, content_type
+            )
+            tool_result = CustomToolFileResponse(file_ids=file_ids)
+            response_type = "csv"
+
+        elif "image/" in content_type:
+            file_ids = self._save_and_get_file_references(
+                response.content, content_type
+            )
+            tool_result = CustomToolFileResponse(file_ids=file_ids)
+            response_type = "image"
+
+        else:
+            tool_result = response.json()
+            response_type = "json"
+
+        logger.info(
+            f"Returning tool response for {self._name} with type {response_type}"
+        )
 
         yield ToolResponse(
             id=CUSTOM_TOOL_RESPONSE_ID,
             response=CustomToolCallSummary(
-                tool_name=self._name, tool_result=response.json()
+                tool_name=self._name,
+                response_type=response_type,
+                tool_result=tool_result,
             ),
         )
 
+    def build_next_prompt(
+        self,
+        prompt_builder: AnswerPromptBuilder,
+        tool_call_summary: ToolCallSummary,
+        tool_responses: list[ToolResponse],
+        using_tool_calling_llm: bool,
+    ) -> AnswerPromptBuilder:
+        response = cast(CustomToolCallSummary, tool_responses[0].response)
+
+        # Handle non-file responses using parent class behavior
+        if response.response_type not in ["image", "csv"]:
+            return super().build_next_prompt(
+                prompt_builder,
+                tool_call_summary,
+                tool_responses,
+                using_tool_calling_llm,
+            )
+
+        # Handle image and CSV file responses
+        file_type = (
+            ChatFileType.IMAGE
+            if response.response_type == "image"
+            else ChatFileType.CSV
+        )
+
+        # Load files from storage
+        files = []
+        with get_session_with_tenant() as db_session:
+            file_store = get_default_file_store(db_session)
+
+            for file_id in response.tool_result.file_ids:
+                try:
+                    file_io = file_store.read_file(file_id, mode="b")
+                    files.append(
+                        InMemoryChatFile(
+                            file_id=file_id,
+                            filename=file_id,
+                            content=file_io.read(),
+                            file_type=file_type,
+                        )
+                    )
+                except Exception:
+                    logger.exception(f"Failed to read file {file_id}")
+
+            # Update prompt with file content
+            prompt_builder.update_user_prompt(
+                build_custom_image_generation_user_prompt(
+                    query=prompt_builder.get_user_message_content(),
+                    files=files,
+                    file_type=file_type,
+                )
+            )
+
+        return prompt_builder
+
     def final_result(self, *args: ToolResponse) -> JSON_ro:
-        return cast(CustomToolCallSummary, args[0].response).tool_result
+        response = cast(CustomToolCallSummary, args[0].response)
+        if isinstance(response.tool_result, CustomToolFileResponse):
+            return response.tool_result.model_dump()
+        return response.tool_result
 
 
 def build_custom_tools_from_openapi_schema_and_headers(
diff --git a/backend/danswer/tools/tool_implementations/custom/prompt.py b/backend/danswer/tools/tool_implementations/custom/prompt.py
new file mode 100644
index 00000000000..9911594a917
--- /dev/null
+++ b/backend/danswer/tools/tool_implementations/custom/prompt.py
@@ -0,0 +1,25 @@
+from langchain_core.messages import HumanMessage
+
+from danswer.file_store.models import ChatFileType
+from danswer.file_store.models import InMemoryChatFile
+from danswer.llm.utils import build_content_with_imgs
+
+
+CUSTOM_IMG_GENERATION_SUMMARY_PROMPT = """
+You have just created the attached {file_type} file in response to the following query: "{query}".
+
+Can you please summarize it in a sentence or two? Do NOT include image urls or bulleted lists.
+"""
+
+
+def build_custom_image_generation_user_prompt(
+    query: str, file_type: ChatFileType, files: list[InMemoryChatFile] | None = None
+) -> HumanMessage:
+    return HumanMessage(
+        content=build_content_with_imgs(
+            message=CUSTOM_IMG_GENERATION_SUMMARY_PROMPT.format(
+                query=query, file_type=file_type.value
+            ).strip(),
+            files=files,
+        )
+    )
diff --git a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
index f56336809e4..4d47a8761ff 100644
--- a/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
+++ b/backend/tests/unit/danswer/tools/custom/test_custom_tools.py
@@ -215,6 +215,7 @@ def test_custom_tool_final_result(self) -> None:
         mock_response = ToolResponse(
             id=CUSTOM_TOOL_RESPONSE_ID,
             response=CustomToolCallSummary(
+                response_type="json",
                 tool_name="getAssistant",
                 tool_result={"id": "789", "name": "Final Assistant"},
             ),
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index dcf1baf14df..43ce519a13b 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -10,7 +10,7 @@ import {
   ChatSessionSharedStatus,
   DocumentsResponse,
   FileDescriptor,
-  ImageGenerationDisplay,
+  FileChatDisplay,
   Message,
   MessageResponseIDInfo,
   RetrievalType,
@@ -1281,7 +1281,7 @@ export function ChatPage({
                 query = toolCall.tool_args["query"];
               }
             } else if (Object.hasOwn(packet, "file_ids")) {
-              aiMessageImages = (packet as ImageGenerationDisplay).file_ids.map(
+              aiMessageImages = (packet as FileChatDisplay).file_ids.map(
                 (fileId) => {
                   return {
                     id: fileId,
diff --git a/web/src/app/chat/interfaces.ts b/web/src/app/chat/interfaces.ts
index 20ea4e7d289..dd736837402 100644
--- a/web/src/app/chat/interfaces.ts
+++ b/web/src/app/chat/interfaces.ts
@@ -136,7 +136,7 @@ export interface DocumentsResponse {
   rephrased_query: string | null;
 }
 
-export interface ImageGenerationDisplay {
+export interface FileChatDisplay {
   file_ids: string[];
 }
 
diff --git a/web/src/app/chat/lib.tsx b/web/src/app/chat/lib.tsx
index cda037a08b4..41a83eee1b8 100644
--- a/web/src/app/chat/lib.tsx
+++ b/web/src/app/chat/lib.tsx
@@ -12,7 +12,7 @@ import {
   ChatSession,
   DocumentsResponse,
   FileDescriptor,
-  ImageGenerationDisplay,
+  FileChatDisplay,
   Message,
   MessageResponseIDInfo,
   RetrievalType,
@@ -103,7 +103,7 @@ export type PacketType =
   | BackendMessage
   | AnswerPiecePacket
   | DocumentsResponse
-  | ImageGenerationDisplay
+  | FileChatDisplay
   | StreamingError
   | MessageResponseIDInfo
   | StreamStopInfo;

From bafb95d920eb8d7210e4b4e1be56a2b9ec8d2b74 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Sun, 3 Nov 2024 15:57:11 -0800
Subject: [PATCH 359/376] Misc color clean up (#3026)

* misc color clean up

* additional nits

* nit

* nit

* additional minor nits

* ensure tailwind config evaluates properly + update textarea -> input

* ensure tool call renders

* formatting
---
 web/components.json                           |  2 +-
 web/package-lock.json                         | 17 +--------------
 .../admin/bot/SlackBotConfigCreationForm.tsx  |  1 +
 .../llm/ConfiguredLLMProviderDisplay.tsx      |  2 +-
 .../[ccPairId]/IndexingAttemptsTable.tsx      |  1 -
 .../pages/DynamicConnectorCreationForm.tsx    |  1 -
 web/src/app/admin/documents/sets/page.tsx     |  8 +++++--
 .../status/CCPairIndexingStatusTable.tsx      | 14 ++++++-------
 .../assistants/gallery/AssistantsGallery.tsx  |  6 ++++--
 .../app/assistants/mine/AssistantsList.tsx    |  6 ++++--
 web/src/app/chat/ChatPage.tsx                 |  2 +-
 web/src/app/chat/RegenerateOption.tsx         |  2 +-
 web/src/components/admin/connectors/Field.tsx | 21 ++++++++++++-------
 web/src/components/ui/button.tsx              |  2 +-
 web/src/components/ui/select.tsx              | 10 +++++++--
 web/src/lib/connectors/connectors.tsx         |  2 --
 web/tailwind.config.js                        | 17 +++++++--------
 17 files changed, 56 insertions(+), 58 deletions(-)

diff --git a/web/components.json b/web/components.json
index bfcba8e4a85..cbde1f415b0 100644
--- a/web/components.json
+++ b/web/components.json
@@ -17,4 +17,4 @@
     "lib": "@/lib",
     "hooks": "@/hooks"
   }
-}
\ No newline at end of file
+}
diff --git a/web/package-lock.json b/web/package-lock.json
index 2c72891b7cb..2b02ef81659 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -14851,21 +14851,6 @@
         "type": "github",
         "url": "https://github.com/sponsors/wooorm"
       }
-    },
-    "node_modules/@next/swc-win32-ia32-msvc": {
-      "version": "14.2.3",
-      "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz",
-      "integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==",
-      "cpu": [
-        "ia32"
-      ],
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">= 10"
-      }
     }
   }
-}
\ No newline at end of file
+}
diff --git a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
index cf4e018dc66..e7113717949 100644
--- a/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
+++ b/web/src/app/admin/bot/SlackBotConfigCreationForm.tsx
@@ -370,6 +370,7 @@ export const SlackBotCreationForm = ({
                 <div className="flex">
                   <Button
                     type="submit"
+                    variant="submit"
                     disabled={isSubmitting}
                     className="mx-auto w-64"
                   >
diff --git a/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx b/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
index ea6149e08ad..5921a9a2ada 100644
--- a/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
+++ b/web/src/app/admin/configuration/llm/ConfiguredLLMProviderDisplay.tsx
@@ -122,7 +122,7 @@ function LLMProviderDisplay({
 
         <div className="ml-auto">
           <Button
-            variant={existingLlmProvider ? "outline" : "next"}
+            variant={existingLlmProvider ? "success-reverse" : "navigate"}
             onClick={() => setFormIsVisible(true)}
           >
             {existingLlmProvider ? "Edit" : "Set up"}
diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index 4494e782ded..ac39f3c93d4 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -206,7 +206,6 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
       <Callout
         className="mt-4"
         title="No indexing attempts scheduled yet"
-        icon={"info"}
         type="notice"
       >
         Index attempts are scheduled in the background, and may take some time
diff --git a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
index a6ac93441e6..dc7f75e0632 100644
--- a/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/DynamicConnectorCreationForm.tsx
@@ -104,7 +104,6 @@ const RenderField: FC<RenderFieldProps> = ({
           type={field.type}
           label={label}
           name={field.name}
-          isTextArea={true}
         />
       )}
     </>
diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx
index 9db9c15ee7b..2d5296d33f2 100644
--- a/web/src/app/admin/documents/sets/page.tsx
+++ b/web/src/app/admin/documents/sets/page.tsx
@@ -207,13 +207,17 @@ const DocumentSetTable = ({
                   </TableCell>
                   <TableCell>
                     {documentSet.is_public ? (
-                      <Badge size="md" variant="success" icon={FiUnlock}>
+                      <Badge
+                        size="md"
+                        variant={isEditable ? "success" : "default"}
+                        icon={FiUnlock}
+                      >
                         Public
                       </Badge>
                     ) : (
                       <Badge
                         size="md"
-                        variant={isEditable ? "purple" : "outline"}
+                        variant={isEditable ? "in_progress" : "outline"}
                         icon={FiLock}
                       >
                         Private
diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
index 7ade9b0cfd7..833f1de674e 100644
--- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
+++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
@@ -168,7 +168,7 @@ function ConnectorRow({
         );
       case "not_started":
         return (
-          <Badge circle variant="secondary">
+          <Badge circle variant="purple">
             Scheduled
           </Badge>
         );
@@ -183,7 +183,7 @@ function ConnectorRow({
 
   return (
     <TableRow
-      className={`border hover:bg-hover-light ${
+      className={`hover:bg-hover-light ${
         invisible ? "invisible !h-0 !-mb-10" : "!border !border-border"
       }  w-full cursor-pointer relative `}
       onClick={() => {
@@ -202,21 +202,19 @@ function ConnectorRow({
       {isPaidEnterpriseFeaturesEnabled && (
         <TableCell>
           {ccPairsIndexingStatus.access_type === "public" ? (
-            <Badge variant="success" icon={FiUnlock}>
+            <Badge variant={isEditable ? "success" : "default"} icon={FiUnlock}>
               Public
             </Badge>
           ) : ccPairsIndexingStatus.access_type === "sync" ? (
             <Badge
-              variant="outline"
-              color={isEditable ? "orange" : "gray"}
+              variant={isEditable ? "orange" : "default"}
               icon={FiRefreshCw}
             >
               Sync
             </Badge>
           ) : (
             <Badge
-              size="md"
-              variant={isEditable ? "outline" : "default"}
+              variant={isEditable ? "in_progress" : "default"}
               icon={FiLock}
             >
               Private
@@ -419,7 +417,7 @@ export function CCPairIndexingStatusTable({
           placeholder="Search connectors..."
           value={searchTerm}
           onChange={(e) => setSearchTerm(e.target.value)}
-          className="ml-1 w-96 h-9 flex-none rounded-md border border-border bg-background-50 px-3 py-1 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
+          className="ml-1 w-96 h-9 flex-none rounded-md bg-background-50 px-3 py-1 text-sm shadow-sm transition-colors placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring"
         />
 
         <Button className="h-9" onClick={() => toggleSources()}>
diff --git a/web/src/app/assistants/gallery/AssistantsGallery.tsx b/web/src/app/assistants/gallery/AssistantsGallery.tsx
index e2d2e17004d..cf5b81a9a28 100644
--- a/web/src/app/assistants/gallery/AssistantsGallery.tsx
+++ b/web/src/app/assistants/gallery/AssistantsGallery.tsx
@@ -179,7 +179,8 @@ export function AssistantsGallery() {
         <div className="grid grid-cols-2 gap-4 mt-4 mb-6">
           <Button
             onClick={() => router.push("/assistants/new")}
-            className="w-full py-3 text-lg rounded-full bg-background-800 text-white hover:bg-background-800 transition duration-300 ease-in-out"
+            variant="default"
+            className="p-6 text-base"
             icon={FiPlus}
           >
             Create New Assistant
@@ -187,7 +188,8 @@ export function AssistantsGallery() {
 
           <Button
             onClick={() => router.push("/assistants/mine")}
-            className="w-full hover:border-border-strong py-3 text-lg rounded-full bg-white border border-border shadow text-text-700 hover:bg-background-50 transition duration-300 ease-in-out"
+            variant="outline"
+            className="text-base py-6"
             icon={FiList}
           >
             Your Assistants
diff --git a/web/src/app/assistants/mine/AssistantsList.tsx b/web/src/app/assistants/mine/AssistantsList.tsx
index 0af7a98fd20..0290973067e 100644
--- a/web/src/app/assistants/mine/AssistantsList.tsx
+++ b/web/src/app/assistants/mine/AssistantsList.tsx
@@ -394,8 +394,9 @@ export function AssistantsList() {
 
         <div className="grid grid-cols-2 gap-4 mt-4 mb-8">
           <Button
+            variant="default"
+            className="p-6 text-base"
             onClick={() => router.push("/assistants/new")}
-            className="w-full py-3 text-lg rounded-full bg-background-800 text-white hover:bg-background-800 transition duration-300 ease-in-out"
             icon={FiPlus}
           >
             Create New Assistant
@@ -403,7 +404,8 @@ export function AssistantsList() {
 
           <Button
             onClick={() => router.push("/assistants/gallery")}
-            className="w-full hover:border-border-strong py-3 text-lg rounded-full bg-white border !border-border shadow text-text-700 hover:bg-background-50 transition duration-300 ease-in-out"
+            variant="outline"
+            className="text-base py-6"
             icon={FiList}
           >
             Assistant Gallery
diff --git a/web/src/app/chat/ChatPage.tsx b/web/src/app/chat/ChatPage.tsx
index 43ce519a13b..53df687f2ff 100644
--- a/web/src/app/chat/ChatPage.tsx
+++ b/web/src/app/chat/ChatPage.tsx
@@ -1355,7 +1355,7 @@ export function ChatPage({
                   finalMessage?.context_docs?.top_documents || documents,
                 citations: finalMessage?.citations || {},
                 files: finalMessage?.files || aiMessageImages || [],
-                toolCall: finalMessage?.tool_call || null,
+                toolCall: finalMessage?.tool_call || toolCall,
                 parentMessageId: regenerationRequest
                   ? regenerationRequest?.parentMessage?.messageId!
                   : initialFetchDetails.user_message_id,
diff --git a/web/src/app/chat/RegenerateOption.tsx b/web/src/app/chat/RegenerateOption.tsx
index 606f5ed589d..dd92bee0dbc 100644
--- a/web/src/app/chat/RegenerateOption.tsx
+++ b/web/src/app/chat/RegenerateOption.tsx
@@ -97,7 +97,7 @@ export function RegenerateDropdown({
           ) : (
             <Hoverable
               size={16}
-              icon={StarFeedback as IconType}
+              icon={FiRefreshCw as IconType}
               hoverText={getDisplayNameForModel(alternate)}
             />
           )}
diff --git a/web/src/components/admin/connectors/Field.tsx b/web/src/components/admin/connectors/Field.tsx
index 601d4d5ffdd..56431954552 100644
--- a/web/src/components/admin/connectors/Field.tsx
+++ b/web/src/components/admin/connectors/Field.tsx
@@ -594,7 +594,6 @@ export function SelectorFormField({
   label,
   options,
   subtext,
-  includeDefault = false,
   side = "bottom",
   maxHeight,
   onSelect,
@@ -615,26 +614,34 @@ export function SelectorFormField({
       {subtext && <SubLabel>{subtext}</SubLabel>}
       <div className="mt-2">
         <Select
-          value={field.value}
+          value={field.value || defaultValue}
           onValueChange={
             onSelect || ((selected) => setFieldValue(name, selected))
           }
           defaultValue={defaultValue}
         >
           <SelectTrigger>
-            <SelectValue
-              placeholder={includeDefault ? "Select..." : undefined}
-            />
+            <SelectValue placeholder="Select...">
+              {field.value || defaultValue || ""}
+            </SelectValue>
           </SelectTrigger>
           <SelectContent
             side={side}
             className={maxHeight ? `max-h-[${maxHeight}]` : undefined}
           >
-            {includeDefault && (
+            {options.length == 0 && (
               <SelectItem value="default">Select...</SelectItem>
             )}
+            {defaultValue && (
+              <SelectItem value={defaultValue}>{defaultValue}</SelectItem>
+            )}
             {options.map((option) => (
-              <SelectItem key={option.value} value={String(option.value)}>
+              <SelectItem
+                icon={option.icon}
+                key={option.value}
+                value={String(option.value)}
+                selected={field.value === option.value}
+              >
                 {option.name}
               </SelectItem>
             ))}
diff --git a/web/src/components/ui/button.tsx b/web/src/components/ui/button.tsx
index 91cffa3a926..bf0b3c448b3 100644
--- a/web/src/components/ui/button.tsx
+++ b/web/src/components/ui/button.tsx
@@ -12,7 +12,7 @@ const buttonVariants = cva(
         success:
           "bg-green-100 text-green-600 hover:bg-green-500/90 dark:bg-blue-500 dark:text-neutral-50 dark:hover:bg-green-900/90",
         "success-reverse":
-          "bg-green-500 text-green-100 hover:bg-green-600/90 dark:bg-neutral-50 dark:text-blue-500 dark:hover:bg-green-100/90",
+          "bg-green-500 text-white hover:bg-green-600/90 dark:bg-neutral-50 dark:text-blue-500 dark:hover:bg-green-100/90",
 
         default:
           "bg-neutral-900 border-border text-neutral-50 hover:bg-neutral-900/90 dark:bg-neutral-50 dark:text-neutral-900 dark:hover:bg-neutral-50/90",
diff --git a/web/src/components/ui/select.tsx b/web/src/components/ui/select.tsx
index 8bbcc971748..7f6941201c9 100644
--- a/web/src/components/ui/select.tsx
+++ b/web/src/components/ui/select.tsx
@@ -115,8 +115,9 @@ const SelectItem = React.forwardRef<
   React.ElementRef<typeof SelectPrimitive.Item>,
   React.ComponentPropsWithoutRef<typeof SelectPrimitive.Item> & {
     icon?: React.ElementType;
+    selected?: boolean;
   }
->(({ className, children, icon: Icon, ...props }, ref) => (
+>(({ className, children, icon: Icon, selected, ...props }, ref) => (
   <SelectPrimitive.Item
     ref={ref}
     className={cn(
@@ -127,9 +128,14 @@ const SelectItem = React.forwardRef<
   >
     <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
       <SelectPrimitive.ItemIndicator>
-        {Icon ? <Icon className="h-4 w-4" /> : <Check className="h-4 w-4" />}
+        <Check className="h-4 w-4" />
       </SelectPrimitive.ItemIndicator>
     </span>
+    {!selected && Icon && (
+      <span className="absolute left-2 flex h-3.5 w-3.5 items-center justify-center">
+        <Icon className="h-4 w-4" />
+      </span>
+    )}
 
     <SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
   </SelectPrimitive.Item>
diff --git a/web/src/lib/connectors/connectors.tsx b/web/src/lib/connectors/connectors.tsx
index 1ec265049d0..3c4b015906a 100644
--- a/web/src/lib/connectors/connectors.tsx
+++ b/web/src/lib/connectors/connectors.tsx
@@ -950,7 +950,6 @@ For example, specifying .*-support.* as a "channel" will cause the connector to
     values: [],
     advanced_values: [],
   },
-  
 };
 export function createConnectorInitialValues(
   connector: ConfigurableSources
@@ -1210,7 +1209,6 @@ export interface AsanaConfig {
 
 export interface FreshdeskConfig {}
 
-
 export interface MediaWikiConfig extends MediaWikiBaseConfig {
   hostname: string;
 }
diff --git a/web/tailwind.config.js b/web/tailwind.config.js
index 4917e66c6ff..8d5c2f72704 100644
--- a/web/tailwind.config.js
+++ b/web/tailwind.config.js
@@ -3,16 +3,13 @@ var merge = require("lodash/merge");
 // Use relative paths for imports
 const baseThemes = require("./tailwind-themes/tailwind.config.js");
 
-let customThemes = null;
-try {
-  if (process.env.NEXT_PUBLIC_THEME) {
-    customThemes = require(
-      `./tailwind-themes/custom/${process.env.NEXT_PUBLIC_THEME}/tailwind.config.js`
-    );
-  }
-} catch (error) {
-  console.warn(`Custom theme not found for: ${process.env.NEXT_PUBLIC_THEME}`);
-}
+const customThemes = process.env.NEXT_PUBLIC_THEME
+  ? require(
+      process.env.NEXT_PUBLIC_THEME
+        ? `./tailwind-themes/custom/${process.env.NEXT_PUBLIC_THEME}/tailwind.config.js`
+        : "./tailwind-themes/custom/default/tailwind.config.js"
+    )
+  : null;
 
 /** @type {import('tailwindcss').Config} */
 module.exports = customThemes ? merge(baseThemes, customThemes) : baseThemes;

From 8e55566f6678ae1dc004a07fac449ce19e7dfbb1 Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Sun, 3 Nov 2024 17:37:20 -0800
Subject: [PATCH 360/376] Fix slack bot form + LLM provider form

---
 web/src/components/admin/connectors/Field.tsx | 57 +++++++++++--------
 web/src/components/ui/select.tsx              |  8 ++-
 2 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/web/src/components/admin/connectors/Field.tsx b/web/src/components/admin/connectors/Field.tsx
index 56431954552..271a7447f3d 100644
--- a/web/src/components/admin/connectors/Field.tsx
+++ b/web/src/components/admin/connectors/Field.tsx
@@ -25,7 +25,7 @@ import {
 } from "@radix-ui/react-tooltip";
 import ReactMarkdown from "react-markdown";
 import { FaMarkdown } from "react-icons/fa";
-import { useState } from "react";
+import { useRef, useState } from "react";
 import remarkGfm from "remark-gfm";
 import { EditIcon } from "@/components/icons/icons";
 import { Button } from "@/components/ui/button";
@@ -602,6 +602,11 @@ export function SelectorFormField({
 }: SelectorFormFieldProps) {
   const [field] = useField<string>(name);
   const { setFieldValue } = useFormikContext();
+  const [container, setContainer] = useState<HTMLDivElement | null>(null);
+
+  const currentlySelected = options.find(
+    (option) => option.value?.toString() === field.value?.toString()
+  );
 
   return (
     <div>
@@ -612,7 +617,7 @@ export function SelectorFormField({
         </div>
       )}
       {subtext && <SubLabel>{subtext}</SubLabel>}
-      <div className="mt-2">
+      <div className="mt-2" ref={setContainer}>
         <Select
           value={field.value || defaultValue}
           onValueChange={
@@ -622,30 +627,34 @@ export function SelectorFormField({
         >
           <SelectTrigger>
             <SelectValue placeholder="Select...">
-              {field.value || defaultValue || ""}
+              {currentlySelected?.name || defaultValue || ""}
             </SelectValue>
           </SelectTrigger>
-          <SelectContent
-            side={side}
-            className={maxHeight ? `max-h-[${maxHeight}]` : undefined}
-          >
-            {options.length == 0 && (
-              <SelectItem value="default">Select...</SelectItem>
-            )}
-            {defaultValue && (
-              <SelectItem value={defaultValue}>{defaultValue}</SelectItem>
-            )}
-            {options.map((option) => (
-              <SelectItem
-                icon={option.icon}
-                key={option.value}
-                value={String(option.value)}
-                selected={field.value === option.value}
-              >
-                {option.name}
-              </SelectItem>
-            ))}
-          </SelectContent>
+
+          {container && (
+            <SelectContent
+              side={side}
+              className={maxHeight ? `max-h-[${maxHeight}]` : undefined}
+              container={container}
+            >
+              {options.length == 0 && (
+                <SelectItem value="default">Select...</SelectItem>
+              )}
+              {defaultValue && (
+                <SelectItem value={defaultValue}>{defaultValue}</SelectItem>
+              )}
+              {options.map((option) => (
+                <SelectItem
+                  icon={option.icon}
+                  key={option.value}
+                  value={String(option.value)}
+                  selected={field.value === option.value}
+                >
+                  {option.name}
+                </SelectItem>
+              ))}
+            </SelectContent>
+          )}
         </Select>
       </div>
 
diff --git a/web/src/components/ui/select.tsx b/web/src/components/ui/select.tsx
index 7f6941201c9..0efc6be15f9 100644
--- a/web/src/components/ui/select.tsx
+++ b/web/src/components/ui/select.tsx
@@ -69,9 +69,11 @@ SelectScrollDownButton.displayName =
 
 const SelectContent = React.forwardRef<
   React.ElementRef<typeof SelectPrimitive.Content>,
-  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content>
->(({ className, children, position = "popper", ...props }, ref) => (
-  <SelectPrimitive.Portal>
+  React.ComponentPropsWithoutRef<typeof SelectPrimitive.Content> & {
+    container?: HTMLElement | null;
+  }
+>(({ className, children, position = "popper", container, ...props }, ref) => (
+  <SelectPrimitive.Portal container={container}>
     <SelectPrimitive.Content
       ref={ref}
       className={cn(

From 2cd1e6be00fcb896f8c810766a42e2cfa07e6a0f Mon Sep 17 00:00:00 2001
From: hagen-danswer <hagen@danswer.ai>
Date: Mon, 4 Nov 2024 10:06:23 -0800
Subject: [PATCH 361/376] gmail refactor + permission syncing (#3021)

* initial frontend changes and shared google refactoring

* gmail connector is reworked

* added permission syncing for gmail

* tested!

* Added tests for gmail connector

* fixed tests and mypy

* temp fix

* testing done!

* rename

* test fixes maybe?

* removed irrelevant tests

* anotha one

* refactoring changes

* refactor finished

* maybe these fixes work

* dumps

* final fixes
---
 .../workflows/pr-python-connector-tests.yml   |   2 +
 backend/danswer/connectors/gmail/connector.py | 533 ++++++++++--------
 .../connectors/gmail/connector_auth.py        | 197 -------
 backend/danswer/connectors/gmail/constants.py |   4 -
 .../connectors/google_drive/connector.py      |  44 +-
 .../connectors/google_drive/connector_auth.py | 238 --------
 .../connectors/google_drive/constants.py      |  32 --
 .../connectors/google_drive/doc_conversion.py |  13 +-
 .../connectors/google_drive/file_retrieval.py |  15 +-
 .../connectors/google_drive/google_utils.py   |  35 --
 .../google_drive/section_extraction.py        |   2 +-
 .../connectors/google_utils/__init__.py       |   0
 .../connectors/google_utils/google_auth.py    | 107 ++++
 .../connectors/google_utils/google_kv.py      | 207 +++++++
 .../connectors/google_utils/google_utils.py   | 102 ++++
 .../resources.py                              |  13 +-
 .../google_utils/shared_constants.py          |  40 ++
 backend/danswer/db/credentials.py             |   8 +-
 backend/danswer/server/documents/connector.py |  99 ++--
 backend/danswer/server/documents/models.py    |   6 +-
 backend/danswer/server/manage/users.py        |   4 +-
 .../external_permissions/gmail/doc_sync.py    |  68 +++
 .../google_drive/doc_sync.py                  |   4 +-
 .../google_drive/group_sync.py                |   4 +-
 .../external_permissions/sync_params.py       |   2 +
 .../tests/daily/connectors/gmail/conftest.py  |  89 +++
 .../connectors/gmail/test_gmail_connector.py  | 125 ++++
 .../daily/connectors/google_drive/conftest.py |  29 +-
 .../google_drive/test_google_drive_oauth.py   |  29 +-
 .../test_google_drive_service_acct.py         |  43 +-
 .../test_google_drive_slim_docs.py            |  47 +-
 backend/tests/load_env_vars.py                |  16 +
 .../connectors/gmail/test_connector.py        | 201 +------
 .../unit/danswer/connectors/gmail/thread.json | 349 ++++++++++++
 .../[connector]/AddConnectorPage.tsx          |   4 +-
 .../[connector]/pages/gdrive/Credential.tsx   |   2 +-
 .../[connector]/pages/gmail/Credential.tsx    |  29 +-
 .../[connector]/pages/gmail/GmailPage.tsx     |   7 +-
 .../[connector]/pages/utils/hooks.ts          |  16 +-
 .../lib/connectors/AutoSyncOptionFields.tsx   |   1 +
 web/src/lib/connectors/credentials.ts         |   5 +-
 web/src/lib/types.ts                          |   1 +
 42 files changed, 1653 insertions(+), 1119 deletions(-)
 delete mode 100644 backend/danswer/connectors/gmail/connector_auth.py
 delete mode 100644 backend/danswer/connectors/gmail/constants.py
 delete mode 100644 backend/danswer/connectors/google_drive/connector_auth.py
 delete mode 100644 backend/danswer/connectors/google_drive/google_utils.py
 create mode 100644 backend/danswer/connectors/google_utils/__init__.py
 create mode 100644 backend/danswer/connectors/google_utils/google_auth.py
 create mode 100644 backend/danswer/connectors/google_utils/google_kv.py
 create mode 100644 backend/danswer/connectors/google_utils/google_utils.py
 rename backend/danswer/connectors/{google_drive => google_utils}/resources.py (83%)
 create mode 100644 backend/danswer/connectors/google_utils/shared_constants.py
 create mode 100644 backend/ee/danswer/external_permissions/gmail/doc_sync.py
 create mode 100644 backend/tests/daily/connectors/gmail/conftest.py
 create mode 100644 backend/tests/daily/connectors/gmail/test_gmail_connector.py
 create mode 100644 backend/tests/load_env_vars.py
 create mode 100644 backend/tests/unit/danswer/connectors/gmail/thread.json

diff --git a/.github/workflows/pr-python-connector-tests.yml b/.github/workflows/pr-python-connector-tests.yml
index fa7df201b5e..88053427726 100644
--- a/.github/workflows/pr-python-connector-tests.yml
+++ b/.github/workflows/pr-python-connector-tests.yml
@@ -21,6 +21,8 @@ env:
   # Google
   GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR }}
   GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
+  GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }}
+  GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }}
 
 jobs:
   connectors-check:
diff --git a/backend/danswer/connectors/gmail/connector.py b/backend/danswer/connectors/gmail/connector.py
index 376f03e71d0..268ad871f7c 100644
--- a/backend/danswer/connectors/gmail/connector.py
+++ b/backend/danswer/connectors/gmail/connector.py
@@ -1,283 +1,360 @@
-import re
-import time
 from base64 import urlsafe_b64decode
-from datetime import datetime
-from datetime import timezone
 from typing import Any
-from typing import cast
 from typing import Dict
 
 from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
-from googleapiclient import discovery  # type: ignore
-from googleapiclient.errors import HttpError  # type: ignore
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from danswer.connectors.gmail.connector_auth import (
-    get_gmail_creds_for_authorized_user,
-)
-from danswer.connectors.gmail.connector_auth import (
-    get_gmail_creds_for_service_account,
-)
-from danswer.connectors.gmail.constants import (
-    DB_CREDENTIALS_DICT_DELEGATED_USER_KEY,
-)
-from danswer.connectors.gmail.constants import DB_CREDENTIALS_DICT_TOKEN_KEY
-from danswer.connectors.gmail.constants import (
-    GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+from danswer.connectors.google_utils.google_auth import get_google_creds
+from danswer.connectors.google_utils.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_utils.resources import get_admin_service
+from danswer.connectors.google_utils.resources import get_gmail_service
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
 )
+from danswer.connectors.google_utils.shared_constants import MISSING_SCOPES_ERROR_STR
+from danswer.connectors.google_utils.shared_constants import ONYX_SCOPE_INSTRUCTIONS
+from danswer.connectors.google_utils.shared_constants import SLIM_BATCH_SIZE
+from danswer.connectors.google_utils.shared_constants import USER_FIELDS
 from danswer.connectors.interfaces import GenerateDocumentsOutput
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
 from danswer.connectors.interfaces import PollConnector
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
+from danswer.connectors.interfaces import SlimConnector
+from danswer.connectors.models import BasicExpertInfo
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
+from danswer.connectors.models import SlimDocument
 from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
 
-logger = setup_logger()
 
+logger = setup_logger()
 
-def _execute_with_retry(request: Any) -> Any:
-    max_attempts = 10
-    attempt = 0
+# This is for the initial list call to get the thread ids
+THREAD_LIST_FIELDS = "nextPageToken, threads(id)"
+
+# These are the fields to retrieve using the ID from the initial list call
+PARTS_FIELDS = "parts(body(data), mimeType)"
+PAYLOAD_FIELDS = f"payload(headers, {PARTS_FIELDS})"
+MESSAGES_FIELDS = f"messages(id, {PAYLOAD_FIELDS})"
+THREADS_FIELDS = f"threads(id, {MESSAGES_FIELDS})"
+THREAD_FIELDS = f"id, {MESSAGES_FIELDS}"
+
+EMAIL_FIELDS = [
+    "cc",
+    "bcc",
+    "from",
+    "to",
+]
+
+add_retries = retry_builder(tries=50, max_delay=30)
+
+
+def _build_time_range_query(
+    time_range_start: SecondsSinceUnixEpoch | None = None,
+    time_range_end: SecondsSinceUnixEpoch | None = None,
+) -> str | None:
+    query = ""
+    if time_range_start is not None and time_range_start != 0:
+        query += f"after:{int(time_range_start)}"
+    if time_range_end is not None and time_range_end != 0:
+        query += f" before:{int(time_range_end)}"
+    query = query.strip()
+
+    if len(query) == 0:
+        return None
+
+    return query
+
+
+def _clean_email_and_extract_name(email: str) -> tuple[str, str | None]:
+    email = email.strip()
+    if "<" in email and ">" in email:
+        # Handle format: "Display Name <email@domain.com>"
+        display_name = email[: email.find("<")].strip()
+        email_address = email[email.find("<") + 1 : email.find(">")].strip()
+        return email_address, display_name if display_name else None
+    else:
+        # Handle plain email address
+        return email.strip(), None
+
+
+def _get_owners_from_emails(emails: dict[str, str | None]) -> list[BasicExpertInfo]:
+    owners = []
+    for email, names in emails.items():
+        if names:
+            name_parts = names.split(" ")
+            first_name = " ".join(name_parts[:-1])
+            last_name = name_parts[-1]
+        else:
+            first_name = None
+            last_name = None
+        owners.append(
+            BasicExpertInfo(email=email, first_name=first_name, last_name=last_name)
+        )
+    return owners
+
+
+def _get_message_body(payload: dict[str, Any]) -> str:
+    parts = payload.get("parts", [])
+    message_body = ""
+    for part in parts:
+        mime_type = part.get("mimeType")
+        body = part.get("body")
+        if mime_type == "text/plain" and body:
+            data = body.get("data", "")
+            text = urlsafe_b64decode(data).decode()
+            message_body += text
+    return message_body
+
+
+def message_to_section(message: Dict[str, Any]) -> tuple[Section, dict[str, str]]:
+    link = f"https://mail.google.com/mail/u/0/#inbox/{message['id']}"
+
+    payload = message.get("payload", {})
+    headers = payload.get("headers", [])
+    metadata: dict[str, Any] = {}
+    for header in headers:
+        name = header.get("name").lower()
+        value = header.get("value")
+        if name in EMAIL_FIELDS:
+            metadata[name] = value
+        if name == "subject":
+            metadata["subject"] = value
+        if name == "date":
+            metadata["updated_at"] = value
+
+    if labels := message.get("labelIds"):
+        metadata["labels"] = labels
 
-    while attempt < max_attempts:
-        # Note for reasons unknown, the Google API will sometimes return a 429
-        # and even after waiting the retry period, it will return another 429.
-        # It could be due to a few possibilities:
-        # 1. Other things are also requesting from the Gmail API with the same key
-        # 2. It's a rolling rate limit so the moment we get some amount of requests cleared, we hit it again very quickly
-        # 3. The retry-after has a maximum and we've already hit the limit for the day
-        # or it's something else...
-        try:
-            return request.execute()
-        except HttpError as error:
-            attempt += 1
-
-            if error.resp.status == 429:
-                # Attempt to get 'Retry-After' from headers
-                retry_after = error.resp.get("Retry-After")
-                if retry_after:
-                    sleep_time = int(retry_after)
+    message_data = ""
+    for name, value in metadata.items():
+        # updated at isnt super useful for the llm
+        if name != "updated_at":
+            message_data += f"{name}: {value}\n"
+
+    message_body_text: str = _get_message_body(payload)
+
+    return Section(link=link, text=message_body_text + message_data), metadata
+
+
+def thread_to_document(full_thread: Dict[str, Any]) -> Document | None:
+    all_messages = full_thread.get("messages", [])
+    if not all_messages:
+        return None
+
+    sections = []
+    semantic_identifier = ""
+    updated_at = None
+    from_emails: dict[str, str | None] = {}
+    other_emails: dict[str, str | None] = {}
+    for message in all_messages:
+        section, message_metadata = message_to_section(message)
+        sections.append(section)
+
+        for name, value in message_metadata.items():
+            if name in EMAIL_FIELDS:
+                email, display_name = _clean_email_and_extract_name(value)
+                if name == "from":
+                    from_emails[email] = (
+                        display_name if not from_emails.get(email) else None
+                    )
                 else:
-                    # Extract 'Retry after' timestamp from error message
-                    match = re.search(
-                        r"Retry after (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)",
-                        str(error),
+                    other_emails[email] = (
+                        display_name if not other_emails.get(email) else None
                     )
-                    if match:
-                        retry_after_timestamp = match.group(1)
-                        retry_after_dt = datetime.strptime(
-                            retry_after_timestamp, "%Y-%m-%dT%H:%M:%S.%fZ"
-                        ).replace(tzinfo=timezone.utc)
-                        current_time = datetime.now(timezone.utc)
-                        sleep_time = max(
-                            int((retry_after_dt - current_time).total_seconds()),
-                            0,
-                        )
-                    else:
-                        logger.error(
-                            f"No Retry-After header or timestamp found in error message: {error}"
-                        )
-                        sleep_time = 60
-
-                sleep_time += 3  # Add a buffer to be safe
-
-                logger.info(
-                    f"Rate limit exceeded. Attempt {attempt}/{max_attempts}. Sleeping for {sleep_time} seconds."
-                )
-                time.sleep(sleep_time)
-
-            else:
-                raise
 
-    # If we've exhausted all attempts
-    raise Exception(f"Failed to execute request after {max_attempts} attempts")
+        # If we haven't set the semantic identifier yet, set it to the subject of the first message
+        if not semantic_identifier:
+            semantic_identifier = message_metadata.get("subject", "")
+
+        if message_metadata.get("updated_at"):
+            updated_at = message_metadata.get("updated_at")
+
+    updated_at_datetime = None
+    if updated_at:
+        updated_at_datetime = time_str_to_utc(updated_at)
+
+    id = full_thread.get("id")
+    if not id:
+        raise ValueError("Thread ID is required")
+
+    primary_owners = _get_owners_from_emails(from_emails)
+    secondary_owners = _get_owners_from_emails(other_emails)
+
+    return Document(
+        id=id,
+        semantic_identifier=semantic_identifier,
+        sections=sections,
+        source=DocumentSource.GMAIL,
+        # This is used to perform permission sync
+        primary_owners=primary_owners,
+        secondary_owners=secondary_owners,
+        doc_updated_at=updated_at_datetime,
+        # Not adding emails to metadata because it's already in the sections
+        metadata={},
+    )
 
 
-class GmailConnector(LoadConnector, PollConnector):
+class GmailConnector(LoadConnector, PollConnector, SlimConnector):
     def __init__(self, batch_size: int = INDEX_BATCH_SIZE) -> None:
         self.batch_size = batch_size
-        self.creds: OAuthCredentials | ServiceAccountCredentials | None = None
 
-    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:
-        """Checks for two different types of credentials.
-        (1) A credential which holds a token acquired via a user going thorugh
-        the Google OAuth flow.
-        (2) A credential which holds a service account key JSON file, which
-        can then be used to impersonate any user in the workspace.
-        """
-        creds: OAuthCredentials | ServiceAccountCredentials | None = None
-        new_creds_dict = None
-        if DB_CREDENTIALS_DICT_TOKEN_KEY in credentials:
-            access_token_json_str = cast(
-                str, credentials[DB_CREDENTIALS_DICT_TOKEN_KEY]
-            )
-            creds = get_gmail_creds_for_authorized_user(
-                token_json_str=access_token_json_str
-            )
+        self._creds: OAuthCredentials | ServiceAccountCredentials | None = None
+        self._primary_admin_email: str | None = None
 
-            # tell caller to update token stored in DB if it has changed
-            # (e.g. the token has been refreshed)
-            new_creds_json_str = creds.to_json() if creds else ""
-            if new_creds_json_str != access_token_json_str:
-                new_creds_dict = {DB_CREDENTIALS_DICT_TOKEN_KEY: new_creds_json_str}
-
-        if GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY in credentials:
-            service_account_key_json_str = credentials[
-                GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY
-            ]
-            creds = get_gmail_creds_for_service_account(
-                service_account_key_json_str=service_account_key_json_str
+    @property
+    def primary_admin_email(self) -> str:
+        if self._primary_admin_email is None:
+            raise RuntimeError(
+                "Primary admin email missing, "
+                "should not call this property "
+                "before calling load_credentials"
             )
-
-            # "Impersonate" a user if one is specified
-            delegated_user_email = cast(
-                str | None, credentials.get(DB_CREDENTIALS_DICT_DELEGATED_USER_KEY)
+        return self._primary_admin_email
+
+    @property
+    def google_domain(self) -> str:
+        if self._primary_admin_email is None:
+            raise RuntimeError(
+                "Primary admin email missing, "
+                "should not call this property "
+                "before calling load_credentials"
             )
-            if delegated_user_email:
-                creds = creds.with_subject(delegated_user_email) if creds else None  # type: ignore
-
-        if creds is None:
-            raise PermissionError(
-                "Unable to access Gmail - unknown credential structure."
+        return self._primary_admin_email.split("@")[-1]
+
+    @property
+    def creds(self) -> OAuthCredentials | ServiceAccountCredentials:
+        if self._creds is None:
+            raise RuntimeError(
+                "Creds missing, "
+                "should not call this property "
+                "before calling load_credentials"
             )
+        return self._creds
 
-        self.creds = creds
-        return new_creds_dict
+    def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:
+        primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]
+        self._primary_admin_email = primary_admin_email
 
-    def _get_email_body(self, payload: dict[str, Any]) -> str:
-        parts = payload.get("parts", [])
-        email_body = ""
-        for part in parts:
-            mime_type = part.get("mimeType")
-            body = part.get("body")
-            if mime_type == "text/plain":
-                data = body.get("data", "")
-                text = urlsafe_b64decode(data).decode()
-                email_body += text
-        return email_body
-
-    def _email_to_document(self, full_email: Dict[str, Any]) -> Document:
-        email_id = full_email["id"]
-        payload = full_email["payload"]
-        headers = payload.get("headers")
-        labels = full_email.get("labelIds", [])
-        metadata = {}
-        if headers:
-            for header in headers:
-                name = header.get("name").lower()
-                value = header.get("value")
-                if name in ["from", "to", "subject", "date", "cc", "bcc"]:
-                    metadata[name] = value
-        email_data = ""
-        for name, value in metadata.items():
-            email_data += f"{name}: {value}\n"
-        metadata["labels"] = labels
-        logger.debug(f"{email_data}")
-        email_body_text: str = self._get_email_body(payload)
-        date_str = metadata.get("date")
-        email_updated_at = time_str_to_utc(date_str) if date_str else None
-        link = f"https://mail.google.com/mail/u/0/#inbox/{email_id}"
-        return Document(
-            id=email_id,
-            sections=[Section(link=link, text=email_data + email_body_text)],
+        self._creds, new_creds_dict = get_google_creds(
+            credentials=credentials,
             source=DocumentSource.GMAIL,
-            title=metadata.get("subject"),
-            semantic_identifier=metadata.get("subject", "Untitled Email"),
-            doc_updated_at=email_updated_at,
-            metadata=metadata,
         )
+        return new_creds_dict
 
-    @staticmethod
-    def _build_time_range_query(
+    def _get_all_user_emails(self) -> list[str]:
+        admin_service = get_admin_service(self.creds, self.primary_admin_email)
+        emails = []
+        for user in execute_paginated_retrieval(
+            retrieval_function=admin_service.users().list,
+            list_key="users",
+            fields=USER_FIELDS,
+            domain=self.google_domain,
+        ):
+            if email := user.get("primaryEmail"):
+                emails.append(email)
+        return emails
+
+    def _fetch_threads(
+        self,
         time_range_start: SecondsSinceUnixEpoch | None = None,
         time_range_end: SecondsSinceUnixEpoch | None = None,
-    ) -> str | None:
-        query = ""
-        if time_range_start is not None and time_range_start != 0:
-            query += f"after:{int(time_range_start)}"
-        if time_range_end is not None and time_range_end != 0:
-            query += f" before:{int(time_range_end)}"
-        query = query.strip()
-
-        if len(query) == 0:
-            return None
-
-        return query
+    ) -> GenerateDocumentsOutput:
+        query = _build_time_range_query(time_range_start, time_range_end)
+        doc_batch = []
+        for user_email in self._get_all_user_emails():
+            gmail_service = get_gmail_service(self.creds, user_email)
+            for thread in execute_paginated_retrieval(
+                retrieval_function=gmail_service.users().threads().list,
+                list_key="threads",
+                userId=user_email,
+                fields=THREAD_LIST_FIELDS,
+                q=query,
+            ):
+                full_thread = add_retries(
+                    lambda: gmail_service.users()
+                    .threads()
+                    .get(
+                        userId=user_email,
+                        id=thread["id"],
+                        fields=THREAD_FIELDS,
+                    )
+                    .execute()
+                )()
+                doc = thread_to_document(full_thread)
+                if doc is None:
+                    continue
+                doc_batch.append(doc)
+                if len(doc_batch) > self.batch_size:
+                    yield doc_batch
+                    doc_batch = []
+        if doc_batch:
+            yield doc_batch
 
-    def _fetch_mails_from_gmail(
+    def _fetch_slim_threads(
         self,
         time_range_start: SecondsSinceUnixEpoch | None = None,
         time_range_end: SecondsSinceUnixEpoch | None = None,
-    ) -> GenerateDocumentsOutput:
-        if self.creds is None:
-            raise PermissionError("Not logged into Gmail")
-        page_token = ""
-        query = GmailConnector._build_time_range_query(time_range_start, time_range_end)
-        service = discovery.build("gmail", "v1", credentials=self.creds)
-        while page_token is not None:
-            result = _execute_with_retry(
-                service.users()
-                .messages()
-                .list(
-                    userId="me",
-                    pageToken=page_token,
-                    q=query,
-                    maxResults=self.batch_size,
-                )
-            )
-
-            page_token = result.get("nextPageToken")
-            messages = result.get("messages", [])
-            doc_batch = []
-            for message in messages:
-                message_id = message["id"]
-                msg = _execute_with_retry(
-                    service.users()
-                    .messages()
-                    .get(userId="me", id=message_id, format="full")
+    ) -> GenerateSlimDocumentOutput:
+        query = _build_time_range_query(time_range_start, time_range_end)
+        doc_batch = []
+        for user_email in self._get_all_user_emails():
+            gmail_service = get_gmail_service(self.creds, user_email)
+            for thread in execute_paginated_retrieval(
+                retrieval_function=gmail_service.users().threads().list,
+                list_key="threads",
+                userId=user_email,
+                fields=THREAD_LIST_FIELDS,
+                q=query,
+            ):
+                doc_batch.append(
+                    SlimDocument(
+                        id=thread["id"],
+                        perm_sync_data={"user_email": user_email},
+                    )
                 )
-                doc = self._email_to_document(msg)
-                doc_batch.append(doc)
-            if len(doc_batch) > 0:
-                yield doc_batch
+                if len(doc_batch) > SLIM_BATCH_SIZE:
+                    yield doc_batch
+                    doc_batch = []
+        if doc_batch:
+            yield doc_batch
 
     def load_from_state(self) -> GenerateDocumentsOutput:
-        yield from self._fetch_mails_from_gmail()
+        try:
+            yield from self._fetch_threads()
+        except Exception as e:
+            if MISSING_SCOPES_ERROR_STR in str(e):
+                raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+            raise e
 
     def poll_source(
         self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch
     ) -> GenerateDocumentsOutput:
-        yield from self._fetch_mails_from_gmail(start, end)
+        try:
+            yield from self._fetch_threads(start, end)
+        except Exception as e:
+            if MISSING_SCOPES_ERROR_STR in str(e):
+                raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+            raise e
+
+    def retrieve_all_slim_documents(
+        self,
+        start: SecondsSinceUnixEpoch | None = None,
+        end: SecondsSinceUnixEpoch | None = None,
+    ) -> GenerateSlimDocumentOutput:
+        try:
+            yield from self._fetch_slim_threads(start, end)
+        except Exception as e:
+            if MISSING_SCOPES_ERROR_STR in str(e):
+                raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+            raise e
 
 
 if __name__ == "__main__":
-    import json
-    import os
-
-    service_account_json_path = os.environ.get("GOOGLE_SERVICE_ACCOUNT_KEY_JSON_PATH")
-    if not service_account_json_path:
-        raise ValueError(
-            "Please set GOOGLE_SERVICE_ACCOUNT_KEY_JSON_PATH environment variable"
-        )
-    with open(service_account_json_path) as f:
-        creds = json.load(f)
-
-    credentials_dict = {
-        DB_CREDENTIALS_DICT_TOKEN_KEY: json.dumps(creds),
-    }
-    delegated_user = os.environ.get("GMAIL_DELEGATED_USER")
-    if delegated_user:
-        credentials_dict[DB_CREDENTIALS_DICT_DELEGATED_USER_KEY] = delegated_user
-
-    connector = GmailConnector()
-    connector.load_credentials(
-        json.loads(credentials_dict[DB_CREDENTIALS_DICT_TOKEN_KEY])
-    )
-    document_batch_generator = connector.load_from_state()
-    for document_batch in document_batch_generator:
-        print(document_batch)
-        break
+    pass
diff --git a/backend/danswer/connectors/gmail/connector_auth.py b/backend/danswer/connectors/gmail/connector_auth.py
deleted file mode 100644
index 7d996c5e687..00000000000
--- a/backend/danswer/connectors/gmail/connector_auth.py
+++ /dev/null
@@ -1,197 +0,0 @@
-import json
-from typing import cast
-from urllib.parse import parse_qs
-from urllib.parse import ParseResult
-from urllib.parse import urlparse
-
-from google.auth.transport.requests import Request  # type: ignore
-from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
-from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
-from google_auth_oauthlib.flow import InstalledAppFlow  # type: ignore
-from sqlalchemy.orm import Session
-
-from danswer.configs.app_configs import WEB_DOMAIN
-from danswer.configs.constants import DocumentSource
-from danswer.configs.constants import KV_CRED_KEY
-from danswer.configs.constants import KV_GMAIL_CRED_KEY
-from danswer.configs.constants import KV_GMAIL_SERVICE_ACCOUNT_KEY
-from danswer.connectors.gmail.constants import (
-    DB_CREDENTIALS_DICT_DELEGATED_USER_KEY,
-)
-from danswer.connectors.gmail.constants import DB_CREDENTIALS_DICT_TOKEN_KEY
-from danswer.connectors.gmail.constants import (
-    GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
-)
-from danswer.connectors.gmail.constants import SCOPES
-from danswer.db.credentials import update_credential_json
-from danswer.db.models import User
-from danswer.key_value_store.factory import get_kv_store
-from danswer.server.documents.models import CredentialBase
-from danswer.server.documents.models import GoogleAppCredentials
-from danswer.server.documents.models import GoogleServiceAccountKey
-from danswer.utils.logger import setup_logger
-
-logger = setup_logger()
-
-
-def _build_frontend_gmail_redirect() -> str:
-    return f"{WEB_DOMAIN}/admin/connectors/gmail/auth/callback"
-
-
-def get_gmail_creds_for_authorized_user(
-    token_json_str: str,
-) -> OAuthCredentials | None:
-    creds_json = json.loads(token_json_str)
-    creds = OAuthCredentials.from_authorized_user_info(creds_json, SCOPES)
-    if creds.valid:
-        return creds
-
-    if creds.expired and creds.refresh_token:
-        try:
-            creds.refresh(Request())
-            if creds.valid:
-                logger.notice("Refreshed Gmail tokens.")
-                return creds
-        except Exception as e:
-            logger.exception(f"Failed to refresh gmail access token due to: {e}")
-            return None
-
-    return None
-
-
-def get_gmail_creds_for_service_account(
-    service_account_key_json_str: str,
-) -> ServiceAccountCredentials | None:
-    service_account_key = json.loads(service_account_key_json_str)
-    creds = ServiceAccountCredentials.from_service_account_info(
-        service_account_key, scopes=SCOPES
-    )
-    if not creds.valid or not creds.expired:
-        creds.refresh(Request())
-    return creds if creds.valid else None
-
-
-def verify_csrf(credential_id: int, state: str) -> None:
-    csrf = get_kv_store().load(KV_CRED_KEY.format(str(credential_id)))
-    if csrf != state:
-        raise PermissionError(
-            "State from Gmail Connector callback does not match expected"
-        )
-
-
-def get_gmail_auth_url(credential_id: int) -> str:
-    creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
-    credential_json = json.loads(creds_str)
-    flow = InstalledAppFlow.from_client_config(
-        credential_json,
-        scopes=SCOPES,
-        redirect_uri=_build_frontend_gmail_redirect(),
-    )
-    auth_url, _ = flow.authorization_url(prompt="consent")
-
-    parsed_url = cast(ParseResult, urlparse(auth_url))
-    params = parse_qs(parsed_url.query)
-
-    get_kv_store().store(
-        KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
-    )  # type: ignore
-    return str(auth_url)
-
-
-def get_auth_url(credential_id: int) -> str:
-    creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
-    credential_json = json.loads(creds_str)
-    flow = InstalledAppFlow.from_client_config(
-        credential_json,
-        scopes=SCOPES,
-        redirect_uri=_build_frontend_gmail_redirect(),
-    )
-    auth_url, _ = flow.authorization_url(prompt="consent")
-
-    parsed_url = cast(ParseResult, urlparse(auth_url))
-    params = parse_qs(parsed_url.query)
-
-    get_kv_store().store(
-        KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
-    )  # type: ignore
-    return str(auth_url)
-
-
-def update_gmail_credential_access_tokens(
-    auth_code: str,
-    credential_id: int,
-    user: User,
-    db_session: Session,
-) -> OAuthCredentials | None:
-    app_credentials = get_google_app_gmail_cred()
-    flow = InstalledAppFlow.from_client_config(
-        app_credentials.model_dump(),
-        scopes=SCOPES,
-        redirect_uri=_build_frontend_gmail_redirect(),
-    )
-    flow.fetch_token(code=auth_code)
-    creds = flow.credentials
-    token_json_str = creds.to_json()
-    new_creds_dict = {DB_CREDENTIALS_DICT_TOKEN_KEY: token_json_str}
-
-    if not update_credential_json(credential_id, new_creds_dict, user, db_session):
-        return None
-    return creds
-
-
-def build_service_account_creds(
-    delegated_user_email: str | None = None,
-) -> CredentialBase:
-    service_account_key = get_gmail_service_account_key()
-
-    credential_dict = {
-        GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY: service_account_key.json(),
-    }
-    if delegated_user_email:
-        credential_dict[DB_CREDENTIALS_DICT_DELEGATED_USER_KEY] = delegated_user_email
-
-    return CredentialBase(
-        source=DocumentSource.GMAIL,
-        credential_json=credential_dict,
-        admin_public=True,
-    )
-
-
-def get_google_app_gmail_cred() -> GoogleAppCredentials:
-    creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
-    return GoogleAppCredentials(**json.loads(creds_str))
-
-
-def upsert_google_app_gmail_cred(app_credentials: GoogleAppCredentials) -> None:
-    get_kv_store().store(KV_GMAIL_CRED_KEY, app_credentials.json(), encrypt=True)
-
-
-def delete_google_app_gmail_cred() -> None:
-    get_kv_store().delete(KV_GMAIL_CRED_KEY)
-
-
-def get_gmail_service_account_key() -> GoogleServiceAccountKey:
-    creds_str = str(get_kv_store().load(KV_GMAIL_SERVICE_ACCOUNT_KEY))
-    return GoogleServiceAccountKey(**json.loads(creds_str))
-
-
-def upsert_gmail_service_account_key(
-    service_account_key: GoogleServiceAccountKey,
-) -> None:
-    get_kv_store().store(
-        KV_GMAIL_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
-    )
-
-
-def upsert_service_account_key(service_account_key: GoogleServiceAccountKey) -> None:
-    get_kv_store().store(
-        KV_GMAIL_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
-    )
-
-
-def delete_gmail_service_account_key() -> None:
-    get_kv_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
-
-
-def delete_service_account_key() -> None:
-    get_kv_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
diff --git a/backend/danswer/connectors/gmail/constants.py b/backend/danswer/connectors/gmail/constants.py
deleted file mode 100644
index 36eff081818..00000000000
--- a/backend/danswer/connectors/gmail/constants.py
+++ /dev/null
@@ -1,4 +0,0 @@
-DB_CREDENTIALS_DICT_TOKEN_KEY = "gmail_tokens"
-GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY = "gmail_service_account_key"
-DB_CREDENTIALS_DICT_DELEGATED_USER_KEY = "gmail_delegated_user"
-SCOPES = ["https://www.googleapis.com/auth/gmail.readonly"]
diff --git a/backend/danswer/connectors/google_drive/connector.py b/backend/danswer/connectors/google_drive/connector.py
index 8b2433d31c8..6d23606260d 100644
--- a/backend/danswer/connectors/google_drive/connector.py
+++ b/backend/danswer/connectors/google_drive/connector.py
@@ -5,26 +5,27 @@
 from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
 
 from danswer.configs.app_configs import INDEX_BATCH_SIZE
-from danswer.connectors.google_drive.connector_auth import (
-    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
-)
-from danswer.connectors.google_drive.connector_auth import get_google_drive_creds
-from danswer.connectors.google_drive.constants import MISSING_SCOPES_ERROR_STR
-from danswer.connectors.google_drive.constants import ONYX_SCOPE_INSTRUCTIONS
-from danswer.connectors.google_drive.constants import SCOPE_DOC_URL
-from danswer.connectors.google_drive.constants import SLIM_BATCH_SIZE
-from danswer.connectors.google_drive.constants import USER_FIELDS
+from danswer.configs.constants import DocumentSource
 from danswer.connectors.google_drive.doc_conversion import (
     convert_drive_item_to_document,
 )
 from danswer.connectors.google_drive.file_retrieval import crawl_folders_for_files
 from danswer.connectors.google_drive.file_retrieval import get_files_in_my_drive
 from danswer.connectors.google_drive.file_retrieval import get_files_in_shared_drive
-from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
 from danswer.connectors.google_drive.models import GoogleDriveFileType
-from danswer.connectors.google_drive.resources import get_admin_service
-from danswer.connectors.google_drive.resources import get_drive_service
-from danswer.connectors.google_drive.resources import get_google_docs_service
+from danswer.connectors.google_utils.google_auth import get_google_creds
+from danswer.connectors.google_utils.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_utils.resources import get_admin_service
+from danswer.connectors.google_utils.resources import get_drive_service
+from danswer.connectors.google_utils.resources import get_google_docs_service
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import MISSING_SCOPES_ERROR_STR
+from danswer.connectors.google_utils.shared_constants import ONYX_SCOPE_INSTRUCTIONS
+from danswer.connectors.google_utils.shared_constants import SCOPE_DOC_URL
+from danswer.connectors.google_utils.shared_constants import SLIM_BATCH_SIZE
+from danswer.connectors.google_utils.shared_constants import USER_FIELDS
 from danswer.connectors.interfaces import GenerateDocumentsOutput
 from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.interfaces import LoadConnector
@@ -105,7 +106,6 @@ def __init__(
         self.shared_folder_ids = _extract_ids_from_urls(shared_folder_url_list)
 
         self._primary_admin_email: str | None = None
-        self.google_domain: str | None = None
 
         self._creds: OAuthCredentials | ServiceAccountCredentials | None = None
 
@@ -121,6 +121,16 @@ def primary_admin_email(self) -> str:
             )
         return self._primary_admin_email
 
+    @property
+    def google_domain(self) -> str:
+        if self._primary_admin_email is None:
+            raise RuntimeError(
+                "Primary admin email missing, "
+                "should not call this property "
+                "before calling load_credentials"
+            )
+        return self._primary_admin_email.split("@")[-1]
+
     @property
     def creds(self) -> OAuthCredentials | ServiceAccountCredentials:
         if self._creds is None:
@@ -136,10 +146,12 @@ def _update_traversed_parent_ids(self, folder_id: str) -> None:
 
     def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:
         primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]
-        self.google_domain = primary_admin_email.split("@")[1]
         self._primary_admin_email = primary_admin_email
 
-        self._creds, new_creds_dict = get_google_drive_creds(credentials)
+        self._creds, new_creds_dict = get_google_creds(
+            credentials=credentials,
+            source=DocumentSource.GOOGLE_DRIVE,
+        )
         return new_creds_dict
 
     def _get_all_user_emails(self) -> list[str]:
diff --git a/backend/danswer/connectors/google_drive/connector_auth.py b/backend/danswer/connectors/google_drive/connector_auth.py
deleted file mode 100644
index 6ea2784662b..00000000000
--- a/backend/danswer/connectors/google_drive/connector_auth.py
+++ /dev/null
@@ -1,238 +0,0 @@
-import json
-from typing import cast
-from urllib.parse import parse_qs
-from urllib.parse import ParseResult
-from urllib.parse import urlparse
-
-from google.auth.transport.requests import Request  # type: ignore
-from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
-from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
-from google_auth_oauthlib.flow import InstalledAppFlow  # type: ignore
-from googleapiclient.discovery import build  # type: ignore
-from sqlalchemy.orm import Session
-
-from danswer.configs.app_configs import WEB_DOMAIN
-from danswer.configs.constants import DocumentSource
-from danswer.configs.constants import KV_CRED_KEY
-from danswer.configs.constants import KV_GOOGLE_DRIVE_CRED_KEY
-from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
-from danswer.connectors.google_drive.constants import MISSING_SCOPES_ERROR_STR
-from danswer.connectors.google_drive.constants import ONYX_SCOPE_INSTRUCTIONS
-from danswer.db.credentials import update_credential_json
-from danswer.db.models import User
-from danswer.key_value_store.factory import get_kv_store
-from danswer.server.documents.models import CredentialBase
-from danswer.server.documents.models import GoogleAppCredentials
-from danswer.server.documents.models import GoogleServiceAccountKey
-from danswer.utils.logger import setup_logger
-
-logger = setup_logger()
-
-# NOTE: do not need https://www.googleapis.com/auth/documents.readonly
-# this is counted under `/auth/drive.readonly`
-GOOGLE_DRIVE_SCOPES = [
-    "https://www.googleapis.com/auth/drive.readonly",
-    "https://www.googleapis.com/auth/drive.metadata.readonly",
-    "https://www.googleapis.com/auth/admin.directory.group.readonly",
-    "https://www.googleapis.com/auth/admin.directory.user.readonly",
-]
-DB_CREDENTIALS_DICT_TOKEN_KEY = "google_drive_tokens"
-DB_CREDENTIALS_PRIMARY_ADMIN_KEY = "google_drive_primary_admin"
-
-
-def _build_frontend_google_drive_redirect() -> str:
-    return f"{WEB_DOMAIN}/admin/connectors/google-drive/auth/callback"
-
-
-def get_google_drive_creds_for_authorized_user(
-    token_json_str: str, scopes: list[str]
-) -> OAuthCredentials | None:
-    creds_json = json.loads(token_json_str)
-    creds = OAuthCredentials.from_authorized_user_info(creds_json, scopes)
-    if creds.valid:
-        return creds
-
-    if creds.expired and creds.refresh_token:
-        try:
-            creds.refresh(Request())
-            if creds.valid:
-                logger.notice("Refreshed Google Drive tokens.")
-                return creds
-        except Exception as e:
-            logger.exception(f"Failed to refresh google drive access token due to: {e}")
-            return None
-
-    return None
-
-
-def get_google_drive_creds(
-    credentials: dict[str, str], scopes: list[str] = GOOGLE_DRIVE_SCOPES
-) -> tuple[ServiceAccountCredentials | OAuthCredentials, dict[str, str] | None]:
-    """Checks for two different types of credentials.
-    (1) A credential which holds a token acquired via a user going thorough
-    the Google OAuth flow.
-    (2) A credential which holds a service account key JSON file, which
-    can then be used to impersonate any user in the workspace.
-    """
-    oauth_creds = None
-    service_creds = None
-    new_creds_dict = None
-    if DB_CREDENTIALS_DICT_TOKEN_KEY in credentials:
-        access_token_json_str = cast(str, credentials[DB_CREDENTIALS_DICT_TOKEN_KEY])
-        oauth_creds = get_google_drive_creds_for_authorized_user(
-            token_json_str=access_token_json_str, scopes=scopes
-        )
-
-        # tell caller to update token stored in DB if it has changed
-        # (e.g. the token has been refreshed)
-        new_creds_json_str = oauth_creds.to_json() if oauth_creds else ""
-        if new_creds_json_str != access_token_json_str:
-            new_creds_dict = {
-                DB_CREDENTIALS_DICT_TOKEN_KEY: new_creds_json_str,
-                DB_CREDENTIALS_PRIMARY_ADMIN_KEY: credentials[
-                    DB_CREDENTIALS_PRIMARY_ADMIN_KEY
-                ],
-            }
-
-    elif KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY in credentials:
-        service_account_key_json_str = credentials[KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY]
-        service_account_key = json.loads(service_account_key_json_str)
-
-        service_creds = ServiceAccountCredentials.from_service_account_info(
-            service_account_key, scopes=scopes
-        )
-
-        if not service_creds.valid or not service_creds.expired:
-            service_creds.refresh(Request())
-
-        if not service_creds.valid:
-            raise PermissionError(
-                "Unable to access Google Drive - service account credentials are invalid."
-            )
-
-    creds: ServiceAccountCredentials | OAuthCredentials | None = (
-        oauth_creds or service_creds
-    )
-    if creds is None:
-        raise PermissionError(
-            "Unable to access Google Drive - unknown credential structure."
-        )
-
-    return creds, new_creds_dict
-
-
-def verify_csrf(credential_id: int, state: str) -> None:
-    csrf = get_kv_store().load(KV_CRED_KEY.format(str(credential_id)))
-    if csrf != state:
-        raise PermissionError(
-            "State from Google Drive Connector callback does not match expected"
-        )
-
-
-def get_auth_url(credential_id: int) -> str:
-    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
-    credential_json = json.loads(creds_str)
-    flow = InstalledAppFlow.from_client_config(
-        credential_json,
-        scopes=GOOGLE_DRIVE_SCOPES,
-        redirect_uri=_build_frontend_google_drive_redirect(),
-    )
-    auth_url, _ = flow.authorization_url(prompt="consent")
-
-    parsed_url = cast(ParseResult, urlparse(auth_url))
-    params = parse_qs(parsed_url.query)
-
-    get_kv_store().store(
-        KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
-    )  # type: ignore
-    return str(auth_url)
-
-
-def update_credential_access_tokens(
-    auth_code: str,
-    credential_id: int,
-    user: User,
-    db_session: Session,
-) -> OAuthCredentials | None:
-    app_credentials = get_google_app_cred()
-    flow = InstalledAppFlow.from_client_config(
-        app_credentials.model_dump(),
-        scopes=GOOGLE_DRIVE_SCOPES,
-        redirect_uri=_build_frontend_google_drive_redirect(),
-    )
-    flow.fetch_token(code=auth_code)
-    creds = flow.credentials
-    token_json_str = creds.to_json()
-
-    # Get user email from Google API so we know who
-    # the primary admin is for this connector
-    try:
-        admin_service = build("drive", "v3", credentials=creds)
-        user_info = (
-            admin_service.about()
-            .get(
-                fields="user(emailAddress)",
-            )
-            .execute()
-        )
-        email = user_info.get("user", {}).get("emailAddress")
-    except Exception as e:
-        if MISSING_SCOPES_ERROR_STR in str(e):
-            raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
-        raise e
-
-    new_creds_dict = {
-        DB_CREDENTIALS_DICT_TOKEN_KEY: token_json_str,
-        DB_CREDENTIALS_PRIMARY_ADMIN_KEY: email,
-    }
-
-    if not update_credential_json(credential_id, new_creds_dict, user, db_session):
-        return None
-    return creds
-
-
-def build_service_account_creds(
-    source: DocumentSource,
-    primary_admin_email: str | None = None,
-) -> CredentialBase:
-    service_account_key = get_service_account_key()
-
-    credential_dict = {
-        KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY: service_account_key.json(),
-    }
-    if primary_admin_email:
-        credential_dict[DB_CREDENTIALS_PRIMARY_ADMIN_KEY] = primary_admin_email
-
-    return CredentialBase(
-        credential_json=credential_dict,
-        admin_public=True,
-        source=DocumentSource.GOOGLE_DRIVE,
-    )
-
-
-def get_google_app_cred() -> GoogleAppCredentials:
-    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
-    return GoogleAppCredentials(**json.loads(creds_str))
-
-
-def upsert_google_app_cred(app_credentials: GoogleAppCredentials) -> None:
-    get_kv_store().store(KV_GOOGLE_DRIVE_CRED_KEY, app_credentials.json(), encrypt=True)
-
-
-def delete_google_app_cred() -> None:
-    get_kv_store().delete(KV_GOOGLE_DRIVE_CRED_KEY)
-
-
-def get_service_account_key() -> GoogleServiceAccountKey:
-    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY))
-    return GoogleServiceAccountKey(**json.loads(creds_str))
-
-
-def upsert_service_account_key(service_account_key: GoogleServiceAccountKey) -> None:
-    get_kv_store().store(
-        KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
-    )
-
-
-def delete_service_account_key() -> None:
-    get_kv_store().delete(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY)
diff --git a/backend/danswer/connectors/google_drive/constants.py b/backend/danswer/connectors/google_drive/constants.py
index 848a21fffe6..4fdfb23d57b 100644
--- a/backend/danswer/connectors/google_drive/constants.py
+++ b/backend/danswer/connectors/google_drive/constants.py
@@ -2,35 +2,3 @@
 DRIVE_FOLDER_TYPE = "application/vnd.google-apps.folder"
 DRIVE_SHORTCUT_TYPE = "application/vnd.google-apps.shortcut"
 DRIVE_FILE_TYPE = "application/vnd.google-apps.file"
-
-FILE_FIELDS = (
-    "nextPageToken, files(mimeType, id, name, permissions, modifiedTime, webViewLink, "
-    "shortcutDetails, owners(emailAddress))"
-)
-SLIM_FILE_FIELDS = (
-    "nextPageToken, files(mimeType, id, name, permissions(emailAddress, type), "
-    "permissionIds, webViewLink, owners(emailAddress))"
-)
-FOLDER_FIELDS = "nextPageToken, files(id, name, permissions, modifiedTime, webViewLink, shortcutDetails)"
-USER_FIELDS = "nextPageToken, users(primaryEmail)"
-
-# these errors don't represent a failure in the connector, but simply files
-# that can't / shouldn't be indexed
-ERRORS_TO_CONTINUE_ON = [
-    "cannotExportFile",
-    "exportSizeLimitExceeded",
-    "cannotDownloadFile",
-]
-
-# Error message substrings
-MISSING_SCOPES_ERROR_STR = "client not authorized for any of the scopes requested"
-
-# Documentation and error messages
-SCOPE_DOC_URL = "https://docs.danswer.dev/connectors/google_drive/overview"
-ONYX_SCOPE_INSTRUCTIONS = (
-    "You have upgraded Danswer without updating the Google Drive scopes. "
-    f"Please refer to the documentation to learn how to update the scopes: {SCOPE_DOC_URL}"
-)
-
-# Batch sizes
-SLIM_BATCH_SIZE = 500
diff --git a/backend/danswer/connectors/google_drive/doc_conversion.py b/backend/danswer/connectors/google_drive/doc_conversion.py
index 81e709f53f4..44a03818088 100644
--- a/backend/danswer/connectors/google_drive/doc_conversion.py
+++ b/backend/danswer/connectors/google_drive/doc_conversion.py
@@ -8,13 +8,12 @@
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import IGNORE_FOR_QA
 from danswer.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
-from danswer.connectors.google_drive.constants import ERRORS_TO_CONTINUE_ON
 from danswer.connectors.google_drive.constants import UNSUPPORTED_FILE_TYPE_CONTENT
 from danswer.connectors.google_drive.models import GDriveMimeType
 from danswer.connectors.google_drive.models import GoogleDriveFileType
-from danswer.connectors.google_drive.resources import GoogleDocsService
-from danswer.connectors.google_drive.resources import GoogleDriveService
 from danswer.connectors.google_drive.section_extraction import get_document_sections
+from danswer.connectors.google_utils.resources import GoogleDocsService
+from danswer.connectors.google_utils.resources import GoogleDriveService
 from danswer.connectors.models import Document
 from danswer.connectors.models import Section
 from danswer.file_processing.extract_file_text import docx_to_text
@@ -26,6 +25,14 @@
 
 logger = setup_logger()
 
+# these errors don't represent a failure in the connector, but simply files
+# that can't / shouldn't be indexed
+ERRORS_TO_CONTINUE_ON = [
+    "cannotExportFile",
+    "exportSizeLimitExceeded",
+    "cannotDownloadFile",
+]
+
 
 def _extract_sections_basic(
     file: dict[str, str], service: GoogleDriveService
diff --git a/backend/danswer/connectors/google_drive/file_retrieval.py b/backend/danswer/connectors/google_drive/file_retrieval.py
index ea4e7d49466..4f8cddab6df 100644
--- a/backend/danswer/connectors/google_drive/file_retrieval.py
+++ b/backend/danswer/connectors/google_drive/file_retrieval.py
@@ -6,16 +6,23 @@
 
 from danswer.connectors.google_drive.constants import DRIVE_FOLDER_TYPE
 from danswer.connectors.google_drive.constants import DRIVE_SHORTCUT_TYPE
-from danswer.connectors.google_drive.constants import FILE_FIELDS
-from danswer.connectors.google_drive.constants import FOLDER_FIELDS
-from danswer.connectors.google_drive.constants import SLIM_FILE_FIELDS
-from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
 from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.connectors.google_utils.google_utils import execute_paginated_retrieval
 from danswer.connectors.interfaces import SecondsSinceUnixEpoch
 from danswer.utils.logger import setup_logger
 
 logger = setup_logger()
 
+FILE_FIELDS = (
+    "nextPageToken, files(mimeType, id, name, permissions, modifiedTime, webViewLink, "
+    "shortcutDetails, owners(emailAddress))"
+)
+SLIM_FILE_FIELDS = (
+    "nextPageToken, files(mimeType, id, name, permissions(emailAddress, type), "
+    "permissionIds, webViewLink, owners(emailAddress))"
+)
+FOLDER_FIELDS = "nextPageToken, files(id, name, permissions, modifiedTime, webViewLink, shortcutDetails)"
+
 
 def _generate_time_range_filter(
     start: SecondsSinceUnixEpoch | None = None,
diff --git a/backend/danswer/connectors/google_drive/google_utils.py b/backend/danswer/connectors/google_drive/google_utils.py
deleted file mode 100644
index a2e029a41df..00000000000
--- a/backend/danswer/connectors/google_drive/google_utils.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from collections.abc import Callable
-from collections.abc import Iterator
-from typing import Any
-
-from danswer.connectors.google_drive.models import GoogleDriveFileType
-from danswer.utils.retry_wrapper import retry_builder
-
-
-# Google Drive APIs are quite flakey and may 500 for an
-# extended period of time. Trying to combat here by adding a very
-# long retry period (~20 minutes of trying every minute)
-add_retries = retry_builder(tries=50, max_delay=30)
-
-
-def execute_paginated_retrieval(
-    retrieval_function: Callable,
-    list_key: str,
-    **kwargs: Any,
-) -> Iterator[GoogleDriveFileType]:
-    """Execute a paginated retrieval from Google Drive API
-    Args:
-        retrieval_function: The specific list function to call (e.g., service.files().list)
-        **kwargs: Arguments to pass to the list function
-    """
-    next_page_token = ""
-    while next_page_token is not None:
-        request_kwargs = kwargs.copy()
-        if next_page_token:
-            request_kwargs["pageToken"] = next_page_token
-
-        results = (lambda: retrieval_function(**request_kwargs).execute())()
-
-        next_page_token = results.get("nextPageToken")
-        for item in results.get(list_key, []):
-            yield item
diff --git a/backend/danswer/connectors/google_drive/section_extraction.py b/backend/danswer/connectors/google_drive/section_extraction.py
index bcd162b1c79..b5809204893 100644
--- a/backend/danswer/connectors/google_drive/section_extraction.py
+++ b/backend/danswer/connectors/google_drive/section_extraction.py
@@ -2,7 +2,7 @@
 
 from pydantic import BaseModel
 
-from danswer.connectors.google_drive.resources import GoogleDocsService
+from danswer.connectors.google_utils.resources import GoogleDocsService
 from danswer.connectors.models import Section
 
 
diff --git a/backend/danswer/connectors/google_utils/__init__.py b/backend/danswer/connectors/google_utils/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/backend/danswer/connectors/google_utils/google_auth.py b/backend/danswer/connectors/google_utils/google_auth.py
new file mode 100644
index 00000000000..8a8c59d6af3
--- /dev/null
+++ b/backend/danswer/connectors/google_utils/google_auth.py
@@ -0,0 +1,107 @@
+import json
+from typing import cast
+
+from google.auth.transport.requests import Request  # type: ignore
+from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
+from google.oauth2.service_account import Credentials as ServiceAccountCredentials  # type: ignore
+
+from danswer.configs.constants import DocumentSource
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_TOKEN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    GOOGLE_SCOPES,
+)
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def get_google_oauth_creds(
+    token_json_str: str, source: DocumentSource
+) -> OAuthCredentials | None:
+    creds_json = json.loads(token_json_str)
+    creds = OAuthCredentials.from_authorized_user_info(
+        info=creds_json,
+        scopes=GOOGLE_SCOPES[source],
+    )
+    if creds.valid:
+        return creds
+
+    if creds.expired and creds.refresh_token:
+        try:
+            creds.refresh(Request())
+            if creds.valid:
+                logger.notice("Refreshed Google Drive tokens.")
+                return creds
+        except Exception:
+            logger.exception("Failed to refresh google drive access token due to:")
+            return None
+
+    return None
+
+
+def get_google_creds(
+    credentials: dict[str, str],
+    source: DocumentSource,
+) -> tuple[ServiceAccountCredentials | OAuthCredentials, dict[str, str] | None]:
+    """Checks for two different types of credentials.
+    (1) A credential which holds a token acquired via a user going thorough
+    the Google OAuth flow.
+    (2) A credential which holds a service account key JSON file, which
+    can then be used to impersonate any user in the workspace.
+    """
+    oauth_creds = None
+    service_creds = None
+    new_creds_dict = None
+    if DB_CREDENTIALS_DICT_TOKEN_KEY in credentials:
+        # OAUTH
+        access_token_json_str = cast(str, credentials[DB_CREDENTIALS_DICT_TOKEN_KEY])
+        oauth_creds = get_google_oauth_creds(
+            token_json_str=access_token_json_str, source=source
+        )
+
+        # tell caller to update token stored in DB if it has changed
+        # (e.g. the token has been refreshed)
+        new_creds_json_str = oauth_creds.to_json() if oauth_creds else ""
+        if new_creds_json_str != access_token_json_str:
+            new_creds_dict = {
+                DB_CREDENTIALS_DICT_TOKEN_KEY: new_creds_json_str,
+                DB_CREDENTIALS_PRIMARY_ADMIN_KEY: credentials[
+                    DB_CREDENTIALS_PRIMARY_ADMIN_KEY
+                ],
+            }
+    elif DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY in credentials:
+        # SERVICE ACCOUNT
+        service_account_key_json_str = credentials[
+            DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY
+        ]
+        service_account_key = json.loads(service_account_key_json_str)
+
+        service_creds = ServiceAccountCredentials.from_service_account_info(
+            service_account_key, scopes=GOOGLE_SCOPES[source]
+        )
+
+        if not service_creds.valid or not service_creds.expired:
+            service_creds.refresh(Request())
+
+        if not service_creds.valid:
+            raise PermissionError(
+                f"Unable to access {source} - service account credentials are invalid."
+            )
+
+    creds: ServiceAccountCredentials | OAuthCredentials | None = (
+        oauth_creds or service_creds
+    )
+    if creds is None:
+        raise PermissionError(
+            f"Unable to access {source} - unknown credential structure."
+        )
+
+    return creds, new_creds_dict
diff --git a/backend/danswer/connectors/google_utils/google_kv.py b/backend/danswer/connectors/google_utils/google_kv.py
new file mode 100644
index 00000000000..1231293be69
--- /dev/null
+++ b/backend/danswer/connectors/google_utils/google_kv.py
@@ -0,0 +1,207 @@
+import json
+from typing import cast
+from urllib.parse import parse_qs
+from urllib.parse import ParseResult
+from urllib.parse import urlparse
+
+from google.oauth2.credentials import Credentials as OAuthCredentials  # type: ignore
+from google_auth_oauthlib.flow import InstalledAppFlow  # type: ignore
+from googleapiclient.discovery import build  # type: ignore
+from sqlalchemy.orm import Session
+
+from danswer.configs.app_configs import WEB_DOMAIN
+from danswer.configs.constants import DocumentSource
+from danswer.configs.constants import KV_CRED_KEY
+from danswer.configs.constants import KV_GMAIL_CRED_KEY
+from danswer.configs.constants import KV_GMAIL_SERVICE_ACCOUNT_KEY
+from danswer.configs.constants import KV_GOOGLE_DRIVE_CRED_KEY
+from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_TOKEN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    GOOGLE_SCOPES,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    MISSING_SCOPES_ERROR_STR,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    ONYX_SCOPE_INSTRUCTIONS,
+)
+from danswer.db.credentials import update_credential_json
+from danswer.db.models import User
+from danswer.key_value_store.factory import get_kv_store
+from danswer.server.documents.models import CredentialBase
+from danswer.server.documents.models import GoogleAppCredentials
+from danswer.server.documents.models import GoogleServiceAccountKey
+from danswer.utils.logger import setup_logger
+
+logger = setup_logger()
+
+
+def _build_frontend_google_drive_redirect() -> str:
+    return f"{WEB_DOMAIN}/admin/connectors/google-drive/auth/callback"
+
+
+def verify_csrf(credential_id: int, state: str) -> None:
+    csrf = get_kv_store().load(KV_CRED_KEY.format(str(credential_id)))
+    if csrf != state:
+        raise PermissionError(
+            "State from Google Drive Connector callback does not match expected"
+        )
+
+
+def update_credential_access_tokens(
+    auth_code: str,
+    credential_id: int,
+    user: User,
+    db_session: Session,
+    source: DocumentSource,
+) -> OAuthCredentials | None:
+    app_credentials = get_google_app_cred(source)
+    flow = InstalledAppFlow.from_client_config(
+        app_credentials.model_dump(),
+        scopes=GOOGLE_SCOPES,
+        redirect_uri=_build_frontend_google_drive_redirect(),
+    )
+    flow.fetch_token(code=auth_code)
+    creds = flow.credentials
+    token_json_str = creds.to_json()
+
+    # Get user email from Google API so we know who
+    # the primary admin is for this connector
+    try:
+        admin_service = build("drive", "v3", credentials=creds)
+        user_info = (
+            admin_service.about()
+            .get(
+                fields="user(emailAddress)",
+            )
+            .execute()
+        )
+        email = user_info.get("user", {}).get("emailAddress")
+    except Exception as e:
+        if MISSING_SCOPES_ERROR_STR in str(e):
+            raise PermissionError(ONYX_SCOPE_INSTRUCTIONS) from e
+        raise e
+
+    new_creds_dict = {
+        DB_CREDENTIALS_DICT_TOKEN_KEY: token_json_str,
+        DB_CREDENTIALS_PRIMARY_ADMIN_KEY: email,
+    }
+
+    if not update_credential_json(credential_id, new_creds_dict, user, db_session):
+        return None
+    return creds
+
+
+def build_service_account_creds(
+    source: DocumentSource,
+    primary_admin_email: str | None = None,
+) -> CredentialBase:
+    service_account_key = get_service_account_key(source=source)
+
+    credential_dict = {
+        DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY: service_account_key.json(),
+    }
+    if primary_admin_email:
+        credential_dict[DB_CREDENTIALS_PRIMARY_ADMIN_KEY] = primary_admin_email
+
+    return CredentialBase(
+        credential_json=credential_dict,
+        admin_public=True,
+        source=source,
+    )
+
+
+def get_auth_url(credential_id: int) -> str:
+    creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
+    credential_json = json.loads(creds_str)
+    flow = InstalledAppFlow.from_client_config(
+        credential_json,
+        scopes=GOOGLE_SCOPES,
+        redirect_uri=_build_frontend_google_drive_redirect(),
+    )
+    auth_url, _ = flow.authorization_url(prompt="consent")
+
+    parsed_url = cast(ParseResult, urlparse(auth_url))
+    params = parse_qs(parsed_url.query)
+
+    get_kv_store().store(
+        KV_CRED_KEY.format(credential_id), params.get("state", [None])[0], encrypt=True
+    )  # type: ignore
+    return str(auth_url)
+
+
+def get_google_app_cred(source: DocumentSource) -> GoogleAppCredentials:
+    if source == DocumentSource.GOOGLE_DRIVE:
+        creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_CRED_KEY))
+    elif source == DocumentSource.GMAIL:
+        creds_str = str(get_kv_store().load(KV_GMAIL_CRED_KEY))
+    else:
+        raise ValueError(f"Unsupported source: {source}")
+    return GoogleAppCredentials(**json.loads(creds_str))
+
+
+def upsert_google_app_cred(
+    app_credentials: GoogleAppCredentials, source: DocumentSource
+) -> None:
+    if source == DocumentSource.GOOGLE_DRIVE:
+        get_kv_store().store(
+            KV_GOOGLE_DRIVE_CRED_KEY, app_credentials.json(), encrypt=True
+        )
+    elif source == DocumentSource.GMAIL:
+        get_kv_store().store(KV_GMAIL_CRED_KEY, app_credentials.json(), encrypt=True)
+    else:
+        raise ValueError(f"Unsupported source: {source}")
+
+
+def delete_google_app_cred(source: DocumentSource) -> None:
+    if source == DocumentSource.GOOGLE_DRIVE:
+        get_kv_store().delete(KV_GOOGLE_DRIVE_CRED_KEY)
+    elif source == DocumentSource.GMAIL:
+        get_kv_store().delete(KV_GMAIL_CRED_KEY)
+    else:
+        raise ValueError(f"Unsupported source: {source}")
+
+
+def get_service_account_key(source: DocumentSource) -> GoogleServiceAccountKey:
+    if source == DocumentSource.GOOGLE_DRIVE:
+        creds_str = str(get_kv_store().load(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY))
+    elif source == DocumentSource.GMAIL:
+        creds_str = str(get_kv_store().load(KV_GMAIL_SERVICE_ACCOUNT_KEY))
+    else:
+        raise ValueError(f"Unsupported source: {source}")
+    return GoogleServiceAccountKey(**json.loads(creds_str))
+
+
+def upsert_service_account_key(
+    service_account_key: GoogleServiceAccountKey, source: DocumentSource
+) -> None:
+    if source == DocumentSource.GOOGLE_DRIVE:
+        get_kv_store().store(
+            KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY,
+            service_account_key.json(),
+            encrypt=True,
+        )
+    elif source == DocumentSource.GMAIL:
+        get_kv_store().store(
+            KV_GMAIL_SERVICE_ACCOUNT_KEY, service_account_key.json(), encrypt=True
+        )
+    else:
+        raise ValueError(f"Unsupported source: {source}")
+
+
+def delete_service_account_key(source: DocumentSource) -> None:
+    if source == DocumentSource.GOOGLE_DRIVE:
+        get_kv_store().delete(KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY)
+    elif source == DocumentSource.GMAIL:
+        get_kv_store().delete(KV_GMAIL_SERVICE_ACCOUNT_KEY)
+    else:
+        raise ValueError(f"Unsupported source: {source}")
diff --git a/backend/danswer/connectors/google_utils/google_utils.py b/backend/danswer/connectors/google_utils/google_utils.py
new file mode 100644
index 00000000000..037ec3731ac
--- /dev/null
+++ b/backend/danswer/connectors/google_utils/google_utils.py
@@ -0,0 +1,102 @@
+import re
+import time
+from collections.abc import Callable
+from collections.abc import Iterator
+from datetime import datetime
+from datetime import timezone
+from typing import Any
+
+from googleapiclient.errors import HttpError  # type: ignore
+
+from danswer.connectors.google_drive.models import GoogleDriveFileType
+from danswer.utils.logger import setup_logger
+from danswer.utils.retry_wrapper import retry_builder
+
+logger = setup_logger()
+
+
+# Google Drive APIs are quite flakey and may 500 for an
+# extended period of time. Trying to combat here by adding a very
+# long retry period (~20 minutes of trying every minute)
+add_retries = retry_builder(tries=50, max_delay=30)
+
+
+def _execute_with_retry(request: Any) -> Any:
+    max_attempts = 10
+    attempt = 0
+
+    while attempt < max_attempts:
+        # Note for reasons unknown, the Google API will sometimes return a 429
+        # and even after waiting the retry period, it will return another 429.
+        # It could be due to a few possibilities:
+        # 1. Other things are also requesting from the Gmail API with the same key
+        # 2. It's a rolling rate limit so the moment we get some amount of requests cleared, we hit it again very quickly
+        # 3. The retry-after has a maximum and we've already hit the limit for the day
+        # or it's something else...
+        try:
+            return request.execute()
+        except HttpError as error:
+            attempt += 1
+
+            if error.resp.status == 429:
+                # Attempt to get 'Retry-After' from headers
+                retry_after = error.resp.get("Retry-After")
+                if retry_after:
+                    sleep_time = int(retry_after)
+                else:
+                    # Extract 'Retry after' timestamp from error message
+                    match = re.search(
+                        r"Retry after (\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z)",
+                        str(error),
+                    )
+                    if match:
+                        retry_after_timestamp = match.group(1)
+                        retry_after_dt = datetime.strptime(
+                            retry_after_timestamp, "%Y-%m-%dT%H:%M:%S.%fZ"
+                        ).replace(tzinfo=timezone.utc)
+                        current_time = datetime.now(timezone.utc)
+                        sleep_time = max(
+                            int((retry_after_dt - current_time).total_seconds()),
+                            0,
+                        )
+                    else:
+                        logger.error(
+                            f"No Retry-After header or timestamp found in error message: {error}"
+                        )
+                        sleep_time = 60
+
+                sleep_time += 3  # Add a buffer to be safe
+
+                logger.info(
+                    f"Rate limit exceeded. Attempt {attempt}/{max_attempts}. Sleeping for {sleep_time} seconds."
+                )
+                time.sleep(sleep_time)
+
+            else:
+                raise
+
+    # If we've exhausted all attempts
+    raise Exception(f"Failed to execute request after {max_attempts} attempts")
+
+
+def execute_paginated_retrieval(
+    retrieval_function: Callable,
+    list_key: str,
+    **kwargs: Any,
+) -> Iterator[GoogleDriveFileType]:
+    """Execute a paginated retrieval from Google Drive API
+    Args:
+        retrieval_function: The specific list function to call (e.g., service.files().list)
+        **kwargs: Arguments to pass to the list function
+    """
+    next_page_token = ""
+    while next_page_token is not None:
+        request_kwargs = kwargs.copy()
+        if next_page_token:
+            request_kwargs["pageToken"] = next_page_token
+
+        results = add_retries(lambda: retrieval_function(**request_kwargs).execute())()
+
+        next_page_token = results.get("nextPageToken")
+        for item in results.get(list_key, []):
+            yield item
diff --git a/backend/danswer/connectors/google_drive/resources.py b/backend/danswer/connectors/google_utils/resources.py
similarity index 83%
rename from backend/danswer/connectors/google_drive/resources.py
rename to backend/danswer/connectors/google_utils/resources.py
index df5b36376b5..341c99df155 100644
--- a/backend/danswer/connectors/google_drive/resources.py
+++ b/backend/danswer/connectors/google_utils/resources.py
@@ -16,12 +16,16 @@ class AdminService(Resource):
     pass
 
 
+class GmailService(Resource):
+    pass
+
+
 def _get_google_service(
     service_name: str,
     service_version: str,
     creds: ServiceAccountCredentials | OAuthCredentials,
     user_email: str | None = None,
-) -> GoogleDriveService:
+) -> GoogleDriveService | GoogleDocsService | AdminService | GmailService:
     if isinstance(creds, ServiceAccountCredentials):
         creds = creds.with_subject(user_email)
         service = build(service_name, service_version, credentials=creds)
@@ -50,3 +54,10 @@ def get_admin_service(
     user_email: str,
 ) -> AdminService:
     return _get_google_service("admin", "directory_v1", creds, user_email)
+
+
+def get_gmail_service(
+    creds: ServiceAccountCredentials | OAuthCredentials,
+    user_email: str,
+) -> GmailService:
+    return _get_google_service("gmail", "v1", creds, user_email)
diff --git a/backend/danswer/connectors/google_utils/shared_constants.py b/backend/danswer/connectors/google_utils/shared_constants.py
new file mode 100644
index 00000000000..3aa3d1048b6
--- /dev/null
+++ b/backend/danswer/connectors/google_utils/shared_constants.py
@@ -0,0 +1,40 @@
+from danswer.configs.constants import DocumentSource
+
+# NOTE: do not need https://www.googleapis.com/auth/documents.readonly
+# this is counted under `/auth/drive.readonly`
+GOOGLE_SCOPES = {
+    DocumentSource.GOOGLE_DRIVE: [
+        "https://www.googleapis.com/auth/drive.readonly",
+        "https://www.googleapis.com/auth/drive.metadata.readonly",
+        "https://www.googleapis.com/auth/admin.directory.group.readonly",
+        "https://www.googleapis.com/auth/admin.directory.user.readonly",
+    ],
+    DocumentSource.GMAIL: [
+        "https://www.googleapis.com/auth/gmail.readonly",
+        "https://www.googleapis.com/auth/admin.directory.user.readonly",
+        "https://www.googleapis.com/auth/admin.directory.group.readonly",
+    ],
+}
+
+# This is the Oauth token
+DB_CREDENTIALS_DICT_TOKEN_KEY = "google_tokens"
+# This is the service account key
+DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY = "google_service_account_key"
+# The email saved for both auth types
+DB_CREDENTIALS_PRIMARY_ADMIN_KEY = "google_primary_admin"
+
+USER_FIELDS = "nextPageToken, users(primaryEmail)"
+
+# Error message substrings
+MISSING_SCOPES_ERROR_STR = "client not authorized for any of the scopes requested"
+
+# Documentation and error messages
+SCOPE_DOC_URL = "https://docs.danswer.dev/connectors/google_drive/overview"
+ONYX_SCOPE_INSTRUCTIONS = (
+    "You have upgraded Danswer without updating the Google Drive scopes. "
+    f"Please refer to the documentation to learn how to update the scopes: {SCOPE_DOC_URL}"
+)
+
+
+# This is the maximum number of threads that can be retrieved at once
+SLIM_BATCH_SIZE = 500
diff --git a/backend/danswer/db/credentials.py b/backend/danswer/db/credentials.py
index 58be604a724..86532dfc542 100644
--- a/backend/danswer/db/credentials.py
+++ b/backend/danswer/db/credentials.py
@@ -11,8 +11,8 @@
 from danswer.auth.schemas import UserRole
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
-from danswer.connectors.gmail.constants import (
-    GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
 )
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.models import Credential
@@ -427,9 +427,7 @@ def delete_gmail_service_account_credentials(
 ) -> None:
     credentials = fetch_credentials(db_session=db_session, user=user)
     for credential in credentials:
-        if credential.credential_json.get(
-            GMAIL_DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY
-        ):
+        if credential.credential_json.get(DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY):
             db_session.delete(credential)
 
     db_session.commit()
diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py
index 1933503ea5b..0e2c00d67bc 100644
--- a/backend/danswer/server/documents/connector.py
+++ b/backend/danswer/server/documents/connector.py
@@ -22,35 +22,38 @@
 from danswer.configs.app_configs import ENABLED_CONNECTOR_TYPES
 from danswer.configs.constants import DocumentSource
 from danswer.configs.constants import FileOrigin
-from danswer.connectors.gmail.connector_auth import delete_gmail_service_account_key
-from danswer.connectors.gmail.connector_auth import delete_google_app_gmail_cred
-from danswer.connectors.gmail.connector_auth import get_gmail_auth_url
-from danswer.connectors.gmail.connector_auth import get_gmail_service_account_key
-from danswer.connectors.gmail.connector_auth import get_google_app_gmail_cred
-from danswer.connectors.gmail.connector_auth import (
-    update_gmail_credential_access_tokens,
+from danswer.connectors.google_utils.google_auth import (
+    get_google_oauth_creds,
 )
-from danswer.connectors.gmail.connector_auth import (
-    upsert_gmail_service_account_key,
+from danswer.connectors.google_utils.google_kv import (
+    build_service_account_creds,
 )
-from danswer.connectors.gmail.connector_auth import upsert_google_app_gmail_cred
-from danswer.connectors.google_drive.connector_auth import build_service_account_creds
-from danswer.connectors.google_drive.connector_auth import DB_CREDENTIALS_DICT_TOKEN_KEY
-from danswer.connectors.google_drive.connector_auth import delete_google_app_cred
-from danswer.connectors.google_drive.connector_auth import delete_service_account_key
-from danswer.connectors.google_drive.connector_auth import get_auth_url
-from danswer.connectors.google_drive.connector_auth import get_google_app_cred
-from danswer.connectors.google_drive.connector_auth import (
-    get_google_drive_creds_for_authorized_user,
+from danswer.connectors.google_utils.google_kv import (
+    delete_google_app_cred,
 )
-from danswer.connectors.google_drive.connector_auth import get_service_account_key
-from danswer.connectors.google_drive.connector_auth import GOOGLE_DRIVE_SCOPES
-from danswer.connectors.google_drive.connector_auth import (
+from danswer.connectors.google_utils.google_kv import (
+    delete_service_account_key,
+)
+from danswer.connectors.google_utils.google_kv import get_auth_url
+from danswer.connectors.google_utils.google_kv import (
+    get_google_app_cred,
+)
+from danswer.connectors.google_utils.google_kv import (
+    get_service_account_key,
+)
+from danswer.connectors.google_utils.google_kv import (
     update_credential_access_tokens,
 )
-from danswer.connectors.google_drive.connector_auth import upsert_google_app_cred
-from danswer.connectors.google_drive.connector_auth import upsert_service_account_key
-from danswer.connectors.google_drive.connector_auth import verify_csrf
+from danswer.connectors.google_utils.google_kv import (
+    upsert_google_app_cred,
+)
+from danswer.connectors.google_utils.google_kv import (
+    upsert_service_account_key,
+)
+from danswer.connectors.google_utils.google_kv import verify_csrf
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_TOKEN_KEY,
+)
 from danswer.db.connector import create_connector
 from danswer.db.connector import delete_connector
 from danswer.db.connector import fetch_connector_by_id
@@ -125,7 +128,7 @@ def check_google_app_gmail_credentials_exist(
     _: User = Depends(current_curator_or_admin_user),
 ) -> dict[str, str]:
     try:
-        return {"client_id": get_google_app_gmail_cred().web.client_id}
+        return {"client_id": get_google_app_cred(DocumentSource.GMAIL).web.client_id}
     except KvKeyNotFoundError:
         raise HTTPException(status_code=404, detail="Google App Credentials not found")
 
@@ -135,7 +138,7 @@ def upsert_google_app_gmail_credentials(
     app_credentials: GoogleAppCredentials, _: User = Depends(current_admin_user)
 ) -> StatusResponse:
     try:
-        upsert_google_app_gmail_cred(app_credentials)
+        upsert_google_app_cred(app_credentials, DocumentSource.GMAIL)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -150,7 +153,7 @@ def delete_google_app_gmail_credentials(
     db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
-        delete_google_app_gmail_cred()
+        delete_google_app_cred(DocumentSource.GMAIL)
         cleanup_gmail_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -165,7 +168,9 @@ def check_google_app_credentials_exist(
     _: User = Depends(current_curator_or_admin_user),
 ) -> dict[str, str]:
     try:
-        return {"client_id": get_google_app_cred().web.client_id}
+        return {
+            "client_id": get_google_app_cred(DocumentSource.GOOGLE_DRIVE).web.client_id
+        }
     except KvKeyNotFoundError:
         raise HTTPException(status_code=404, detail="Google App Credentials not found")
 
@@ -175,7 +180,7 @@ def upsert_google_app_credentials(
     app_credentials: GoogleAppCredentials, _: User = Depends(current_admin_user)
 ) -> StatusResponse:
     try:
-        upsert_google_app_cred(app_credentials)
+        upsert_google_app_cred(app_credentials, DocumentSource.GOOGLE_DRIVE)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -190,7 +195,7 @@ def delete_google_app_credentials(
     db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
-        delete_google_app_cred()
+        delete_google_app_cred(DocumentSource.GOOGLE_DRIVE)
         cleanup_google_drive_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -205,7 +210,11 @@ def check_google_service_gmail_account_key_exist(
     _: User = Depends(current_curator_or_admin_user),
 ) -> dict[str, str]:
     try:
-        return {"service_account_email": get_gmail_service_account_key().client_email}
+        return {
+            "service_account_email": get_service_account_key(
+                DocumentSource.GMAIL
+            ).client_email
+        }
     except KvKeyNotFoundError:
         raise HTTPException(
             status_code=404, detail="Google Service Account Key not found"
@@ -217,7 +226,7 @@ def upsert_google_service_gmail_account_key(
     service_account_key: GoogleServiceAccountKey, _: User = Depends(current_admin_user)
 ) -> StatusResponse:
     try:
-        upsert_gmail_service_account_key(service_account_key)
+        upsert_service_account_key(service_account_key, DocumentSource.GMAIL)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -232,7 +241,7 @@ def delete_google_service_gmail_account_key(
     db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
-        delete_gmail_service_account_key()
+        delete_service_account_key(DocumentSource.GMAIL)
         cleanup_gmail_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -247,7 +256,11 @@ def check_google_service_account_key_exist(
     _: User = Depends(current_curator_or_admin_user),
 ) -> dict[str, str]:
     try:
-        return {"service_account_email": get_service_account_key().client_email}
+        return {
+            "service_account_email": get_service_account_key(
+                DocumentSource.GOOGLE_DRIVE
+            ).client_email
+        }
     except KvKeyNotFoundError:
         raise HTTPException(
             status_code=404, detail="Google Service Account Key not found"
@@ -259,7 +272,7 @@ def upsert_google_service_account_key(
     service_account_key: GoogleServiceAccountKey, _: User = Depends(current_admin_user)
 ) -> StatusResponse:
     try:
-        upsert_service_account_key(service_account_key)
+        upsert_service_account_key(service_account_key, DocumentSource.GOOGLE_DRIVE)
     except ValueError as e:
         raise HTTPException(status_code=400, detail=str(e))
 
@@ -274,7 +287,7 @@ def delete_google_service_account_key(
     db_session: Session = Depends(get_session),
 ) -> StatusResponse:
     try:
-        delete_service_account_key()
+        delete_service_account_key(DocumentSource.GOOGLE_DRIVE)
         cleanup_google_drive_credentials(db_session=db_session)
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -322,7 +335,7 @@ def upsert_gmail_service_account_credential(
     try:
         credential_base = build_service_account_creds(
             DocumentSource.GMAIL,
-            primary_admin_email=service_account_credential_request.gmail_delegated_user,
+            primary_admin_email=service_account_credential_request.gmail_primary_admin,
         )
     except KvKeyNotFoundError as e:
         raise HTTPException(status_code=400, detail=str(e))
@@ -349,9 +362,9 @@ def check_drive_tokens(
     ):
         return AuthStatus(authenticated=False)
     token_json_str = str(db_credentials.credential_json[DB_CREDENTIALS_DICT_TOKEN_KEY])
-    google_drive_creds = get_google_drive_creds_for_authorized_user(
+    google_drive_creds = get_google_oauth_creds(
         token_json_str=token_json_str,
-        scopes=GOOGLE_DRIVE_SCOPES,
+        source=DocumentSource.GOOGLE_DRIVE,
     )
     if google_drive_creds is None:
         return AuthStatus(authenticated=False)
@@ -881,7 +894,7 @@ def gmail_auth(
         httponly=True,
         max_age=600,
     )
-    return AuthUrl(auth_url=get_gmail_auth_url(int(credential_id)))
+    return AuthUrl(auth_url=get_auth_url(int(credential_id)))
 
 
 @router.get("/connector/google-drive/authorize/{credential_id}")
@@ -913,8 +926,8 @@ def gmail_callback(
     credential_id = int(credential_id_cookie)
     verify_csrf(credential_id, callback.state)
     if (
-        update_gmail_credential_access_tokens(
-            callback.code, credential_id, user, db_session
+        update_credential_access_tokens(
+            callback.code, credential_id, user, db_session, DocumentSource.GMAIL
         )
         is None
     ):
@@ -941,7 +954,7 @@ def google_drive_callback(
     verify_csrf(credential_id, callback.state)
 
     credentials: Credentials | None = update_credential_access_tokens(
-        callback.code, credential_id, user, db_session
+        callback.code, credential_id, user, db_session, DocumentSource.GOOGLE_DRIVE
     )
     if credentials is None:
         raise HTTPException(
diff --git a/backend/danswer/server/documents/models.py b/backend/danswer/server/documents/models.py
index e45d6eabff0..d885641e398 100644
--- a/backend/danswer/server/documents/models.py
+++ b/backend/danswer/server/documents/models.py
@@ -378,15 +378,15 @@ class GoogleServiceAccountKey(BaseModel):
 
 class GoogleServiceAccountCredentialRequest(BaseModel):
     google_drive_primary_admin: str | None = None  # email of user to impersonate
-    gmail_delegated_user: str | None = None  # email of user to impersonate
+    gmail_primary_admin: str | None = None  # email of user to impersonate
 
     @model_validator(mode="after")
     def check_user_delegation(self) -> "GoogleServiceAccountCredentialRequest":
         if (self.google_drive_primary_admin is None) == (
-            self.gmail_delegated_user is None
+            self.gmail_primary_admin is None
         ):
             raise ValueError(
-                "Exactly one of google_drive_primary_admin or gmail_delegated_user must be set"
+                "Exactly one of google_drive_primary_admin or gmail_primary_admin must be set"
             )
         return self
 
diff --git a/backend/danswer/server/manage/users.py b/backend/danswer/server/manage/users.py
index 9e385908fa4..59c4de89a71 100644
--- a/backend/danswer/server/manage/users.py
+++ b/backend/danswer/server/manage/users.py
@@ -197,10 +197,10 @@ def bulk_invite_users(
             email_info = validate_email(email)
             normalized_emails.append(email_info.normalized)  # type: ignore
 
-    except (EmailUndeliverableError, EmailNotValidError):
+    except (EmailUndeliverableError, EmailNotValidError) as e:
         raise HTTPException(
             status_code=400,
-            detail="One or more emails in the list are invalid",
+            detail=f"Invalid email address: {email} - {str(e)}",
         )
 
     if MULTI_TENANT:
diff --git a/backend/ee/danswer/external_permissions/gmail/doc_sync.py b/backend/ee/danswer/external_permissions/gmail/doc_sync.py
new file mode 100644
index 00000000000..2748443f022
--- /dev/null
+++ b/backend/ee/danswer/external_permissions/gmail/doc_sync.py
@@ -0,0 +1,68 @@
+from datetime import datetime
+from datetime import timezone
+
+from sqlalchemy.orm import Session
+
+from danswer.access.models import ExternalAccess
+from danswer.connectors.gmail.connector import GmailConnector
+from danswer.connectors.interfaces import GenerateSlimDocumentOutput
+from danswer.db.models import ConnectorCredentialPair
+from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
+from danswer.utils.logger import setup_logger
+from ee.danswer.db.document import upsert_document_external_perms__no_commit
+
+logger = setup_logger()
+
+
+def _get_slim_doc_generator(
+    cc_pair: ConnectorCredentialPair,
+    gmail_connector: GmailConnector,
+) -> GenerateSlimDocumentOutput:
+    current_time = datetime.now(timezone.utc)
+    start_time = (
+        cc_pair.last_time_perm_sync.replace(tzinfo=timezone.utc).timestamp()
+        if cc_pair.last_time_perm_sync
+        else 0.0
+    )
+
+    return gmail_connector.retrieve_all_slim_documents(
+        start=start_time, end=current_time.timestamp()
+    )
+
+
+def gmail_doc_sync(
+    db_session: Session,
+    cc_pair: ConnectorCredentialPair,
+) -> None:
+    """
+    Adds the external permissions to the documents in postgres
+    if the document doesn't already exists in postgres, we create
+    it in postgres so that when it gets created later, the permissions are
+    already populated
+    """
+    gmail_connector = GmailConnector(**cc_pair.connector.connector_specific_config)
+    gmail_connector.load_credentials(cc_pair.credential.credential_json)
+
+    slim_doc_generator = _get_slim_doc_generator(cc_pair, gmail_connector)
+
+    for slim_doc_batch in slim_doc_generator:
+        for slim_doc in slim_doc_batch:
+            if slim_doc.perm_sync_data is None:
+                logger.warning(f"No permissions found for document {slim_doc.id}")
+                continue
+            if user_email := slim_doc.perm_sync_data.get("user_email"):
+                ext_access = ExternalAccess(
+                    external_user_emails=set([user_email]),
+                    external_user_group_ids=set(),
+                    is_public=False,
+                )
+                batch_add_non_web_user_if_not_exists__no_commit(
+                    db_session=db_session,
+                    emails=list(ext_access.external_user_emails),
+                )
+                upsert_document_external_perms__no_commit(
+                    db_session=db_session,
+                    doc_id=slim_doc.id,
+                    external_access=ext_access,
+                    source_type=cc_pair.connector.source,
+                )
diff --git a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
index 9f736b22a00..fddb0e72171 100644
--- a/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/doc_sync.py
@@ -6,8 +6,8 @@
 
 from danswer.access.models import ExternalAccess
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
-from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
-from danswer.connectors.google_drive.resources import get_drive_service
+from danswer.connectors.google_utils.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_utils.resources import get_drive_service
 from danswer.connectors.interfaces import GenerateSlimDocumentOutput
 from danswer.connectors.models import SlimDocument
 from danswer.db.models import ConnectorCredentialPair
diff --git a/backend/ee/danswer/external_permissions/google_drive/group_sync.py b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
index 919866749ff..e9ca40b3dcb 100644
--- a/backend/ee/danswer/external_permissions/google_drive/group_sync.py
+++ b/backend/ee/danswer/external_permissions/google_drive/group_sync.py
@@ -1,8 +1,8 @@
 from sqlalchemy.orm import Session
 
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
-from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
-from danswer.connectors.google_drive.resources import get_admin_service
+from danswer.connectors.google_utils.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_utils.resources import get_admin_service
 from danswer.db.models import ConnectorCredentialPair
 from danswer.db.users import batch_add_non_web_user_if_not_exists__no_commit
 from danswer.utils.logger import setup_logger
diff --git a/backend/ee/danswer/external_permissions/sync_params.py b/backend/ee/danswer/external_permissions/sync_params.py
index 10b080cd7f4..1fd09ca1509 100644
--- a/backend/ee/danswer/external_permissions/sync_params.py
+++ b/backend/ee/danswer/external_permissions/sync_params.py
@@ -6,6 +6,7 @@
 from danswer.db.models import ConnectorCredentialPair
 from ee.danswer.external_permissions.confluence.doc_sync import confluence_doc_sync
 from ee.danswer.external_permissions.confluence.group_sync import confluence_group_sync
+from ee.danswer.external_permissions.gmail.doc_sync import gmail_doc_sync
 from ee.danswer.external_permissions.google_drive.doc_sync import gdrive_doc_sync
 from ee.danswer.external_permissions.google_drive.group_sync import gdrive_group_sync
 from ee.danswer.external_permissions.slack.doc_sync import slack_doc_sync
@@ -28,6 +29,7 @@
     DocumentSource.GOOGLE_DRIVE: gdrive_doc_sync,
     DocumentSource.CONFLUENCE: confluence_doc_sync,
     DocumentSource.SLACK: slack_doc_sync,
+    DocumentSource.GMAIL: gmail_doc_sync,
 }
 
 # These functions update:
diff --git a/backend/tests/daily/connectors/gmail/conftest.py b/backend/tests/daily/connectors/gmail/conftest.py
new file mode 100644
index 00000000000..5010d0b5133
--- /dev/null
+++ b/backend/tests/daily/connectors/gmail/conftest.py
@@ -0,0 +1,89 @@
+import json
+import os
+from collections.abc import Callable
+
+import pytest
+
+from danswer.connectors.gmail.connector import GmailConnector
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_TOKEN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
+)
+from tests.load_env_vars import load_env_vars
+
+
+# Load environment variables at the module level
+load_env_vars()
+
+
+def parse_credentials(env_str: str) -> dict:
+    """
+    Parse a double-escaped JSON string from environment variables into a Python dictionary.
+
+    Args:
+        env_str (str): The double-escaped JSON string from environment variables
+
+    Returns:
+        dict: Parsed OAuth credentials
+    """
+    # first try normally
+    try:
+        return json.loads(env_str)
+    except Exception:
+        # First, try remove extra escaping backslashes
+        unescaped = env_str.replace('\\"', '"')
+
+        # remove leading / trailing quotes
+        unescaped = unescaped.strip('"')
+
+        # Now parse the JSON
+        return json.loads(unescaped)
+
+
+@pytest.fixture
+def google_gmail_oauth_connector_factory() -> Callable[..., GmailConnector]:
+    def _connector_factory(
+        primary_admin_email: str = "admin@onyx-test.com",
+    ) -> GmailConnector:
+        print("Creating GmailConnector with OAuth credentials")
+        connector = GmailConnector()
+
+        json_string = os.environ["GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR"]
+        refried_json_string = json.dumps(parse_credentials(json_string))
+
+        credentials_json = {
+            DB_CREDENTIALS_DICT_TOKEN_KEY: refried_json_string,
+            DB_CREDENTIALS_PRIMARY_ADMIN_KEY: primary_admin_email,
+        }
+        connector.load_credentials(credentials_json)
+        return connector
+
+    return _connector_factory
+
+
+@pytest.fixture
+def google_gmail_service_acct_connector_factory() -> Callable[..., GmailConnector]:
+    def _connector_factory(
+        primary_admin_email: str = "admin@onyx-test.com",
+    ) -> GmailConnector:
+        print("Creating GmailConnector with service account credentials")
+        connector = GmailConnector()
+
+        json_string = os.environ["GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR"]
+        refried_json_string = json.dumps(parse_credentials(json_string))
+
+        # Load Service Account Credentials
+        connector.load_credentials(
+            {
+                DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY: refried_json_string,
+                DB_CREDENTIALS_PRIMARY_ADMIN_KEY: primary_admin_email,
+            }
+        )
+        return connector
+
+    return _connector_factory
diff --git a/backend/tests/daily/connectors/gmail/test_gmail_connector.py b/backend/tests/daily/connectors/gmail/test_gmail_connector.py
new file mode 100644
index 00000000000..73cab1136c7
--- /dev/null
+++ b/backend/tests/daily/connectors/gmail/test_gmail_connector.py
@@ -0,0 +1,125 @@
+from collections.abc import Callable
+from typing import Any
+from unittest.mock import MagicMock
+from unittest.mock import patch
+
+from danswer.connectors.gmail.connector import GmailConnector
+from danswer.connectors.models import Document
+from danswer.connectors.models import SlimDocument
+
+
+_THREAD_1_START_TIME = 1730568700
+_THREAD_1_END_TIME = 1730569000
+
+"""
+This thread was 4 emails long:
+    admin@onyx-test.com -> test-group-1@onyx-test.com (conaining test_user_1 and test_user_2)
+    test_user_1@onyx-test.com -> admin@onyx-test.com
+    admin@onyx-test.com -> test_user_2@onyx-test.com + BCC: test_user_3@onyx-test.com
+    test_user_3@onyx-test.com -> admin@onyx-test.com
+"""
+_THREAD_1_BY_ID: dict[str, dict[str, Any]] = {
+    "192edefb315737c3": {
+        "email": "admin@onyx-test.com",
+        "sections_count": 4,
+        "primary_owners": set(
+            [
+                "admin@onyx-test.com",
+                "test_user_1@onyx-test.com",
+                "test_user_3@onyx-test.com",
+            ]
+        ),
+        "secondary_owners": set(
+            [
+                "test-group-1@onyx-test.com",
+                "admin@onyx-test.com",
+                "test_user_2@onyx-test.com",
+                "test_user_3@onyx-test.com",
+            ]
+        ),
+    },
+    "192edf020d2f5def": {
+        "email": "test_user_1@onyx-test.com",
+        "sections_count": 2,
+        "primary_owners": set(["admin@onyx-test.com", "test_user_1@onyx-test.com"]),
+        "secondary_owners": set(["test-group-1@onyx-test.com", "admin@onyx-test.com"]),
+    },
+    "192edf020ae90aab": {
+        "email": "test_user_2@onyx-test.com",
+        "sections_count": 2,
+        "primary_owners": set(["admin@onyx-test.com"]),
+        "secondary_owners": set(
+            ["test-group-1@onyx-test.com", "test_user_2@onyx-test.com"]
+        ),
+    },
+    "192edf18316015fa": {
+        "email": "test_user_3@onyx-test.com",
+        "sections_count": 2,
+        "primary_owners": set(["admin@onyx-test.com", "test_user_3@onyx-test.com"]),
+        "secondary_owners": set(
+            [
+                "admin@onyx-test.com",
+                "test_user_2@onyx-test.com",
+                "test_user_3@onyx-test.com",
+            ]
+        ),
+    },
+}
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_slim_docs_retrieval(
+    mock_get_api_key: MagicMock,
+    google_gmail_service_acct_connector_factory: Callable[..., GmailConnector],
+) -> None:
+    print("\n\nRunning test_slim_docs_retrieval")
+    connector = google_gmail_service_acct_connector_factory()
+    retrieved_slim_docs: list[SlimDocument] = []
+    for doc_batch in connector.retrieve_all_slim_documents(
+        _THREAD_1_START_TIME, _THREAD_1_END_TIME
+    ):
+        retrieved_slim_docs.extend(doc_batch)
+
+    assert len(retrieved_slim_docs) == 4
+
+    for doc in retrieved_slim_docs:
+        permission_info = doc.perm_sync_data
+        assert isinstance(permission_info, dict)
+        user_email = permission_info["user_email"]
+        assert _THREAD_1_BY_ID[doc.id]["email"] == user_email
+
+
+@patch(
+    "danswer.file_processing.extract_file_text.get_unstructured_api_key",
+    return_value=None,
+)
+def test_docs_retrieval(
+    mock_get_api_key: MagicMock,
+    google_gmail_service_acct_connector_factory: Callable[..., GmailConnector],
+) -> None:
+    print("\n\nRunning test_docs_retrieval")
+    connector = google_gmail_service_acct_connector_factory()
+    retrieved_docs: list[Document] = []
+    for doc_batch in connector.poll_source(_THREAD_1_START_TIME, _THREAD_1_END_TIME):
+        retrieved_docs.extend(doc_batch)
+
+    assert len(retrieved_docs) == 4
+
+    for doc in retrieved_docs:
+        id = doc.id
+        if doc.primary_owners:
+            retrieved_primary_owner_emails = set(
+                [owner.email for owner in doc.primary_owners]
+            )
+        if doc.secondary_owners:
+            retrieved_secondary_owner_emails = set(
+                [owner.email for owner in doc.secondary_owners]
+            )
+        assert _THREAD_1_BY_ID[id]["sections_count"] == len(doc.sections)
+        assert _THREAD_1_BY_ID[id]["primary_owners"] == retrieved_primary_owner_emails
+        assert (
+            _THREAD_1_BY_ID[id]["secondary_owners"] == retrieved_secondary_owner_emails
+        )
diff --git a/backend/tests/daily/connectors/google_drive/conftest.py b/backend/tests/daily/connectors/google_drive/conftest.py
index c330dcf2d60..4b618b28e1d 100644
--- a/backend/tests/daily/connectors/google_drive/conftest.py
+++ b/backend/tests/daily/connectors/google_drive/conftest.py
@@ -4,27 +4,17 @@
 
 import pytest
 
-from danswer.configs.constants import KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
-from danswer.connectors.google_drive.connector_auth import DB_CREDENTIALS_DICT_TOKEN_KEY
-from danswer.connectors.google_drive.connector_auth import (
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
+    DB_CREDENTIALS_DICT_TOKEN_KEY,
+)
+from danswer.connectors.google_utils.shared_constants import (
     DB_CREDENTIALS_PRIMARY_ADMIN_KEY,
 )
-
-
-def load_env_vars(env_file: str = ".env") -> None:
-    current_dir = os.path.dirname(os.path.abspath(__file__))
-    env_path = os.path.join(current_dir, env_file)
-    try:
-        with open(env_path, "r") as f:
-            for line in f:
-                line = line.strip()
-                if line and not line.startswith("#"):
-                    key, value = line.split("=", 1)
-                    os.environ[key] = value.strip()
-        print("Successfully loaded environment variables")
-    except FileNotFoundError:
-        print(f"File {env_file} not found")
+from tests.load_env_vars import load_env_vars
 
 
 # Load environment variables at the module level
@@ -65,6 +55,7 @@ def _connector_factory(
         my_drive_emails: str | None = None,
         shared_folder_urls: str | None = None,
     ) -> GoogleDriveConnector:
+        print("Creating GoogleDriveConnector with OAuth credentials")
         connector = GoogleDriveConnector(
             include_shared_drives=include_shared_drives,
             shared_drive_urls=shared_drive_urls,
@@ -113,7 +104,7 @@ def _connector_factory(
         # Load Service Account Credentials
         connector.load_credentials(
             {
-                KV_GOOGLE_DRIVE_SERVICE_ACCOUNT_KEY: refried_json_string,
+                DB_CREDENTIALS_DICT_SERVICE_ACCOUNT_KEY: refried_json_string,
                 DB_CREDENTIALS_PRIMARY_ADMIN_KEY: primary_admin_email,
             }
         )
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
index a4adaa4673a..9e788a7ffa2 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_oauth.py
@@ -102,7 +102,7 @@ def test_include_my_drives_only(
         retrieved_docs.extend(doc_batch)
 
     # Should only get everyone's My Drives
-    expected_file_ids = DRIVE_ID_MAPPING["ADMIN"]
+    expected_file_ids = list(range(0, 5))  # Admin's My Drive only
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
@@ -131,12 +131,7 @@ def test_drive_one_only(
         retrieved_docs.extend(doc_batch)
 
     # We ignore shared_drive_urls if include_shared_drives is False
-    expected_file_ids = (
-        DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-    )
+    expected_file_ids = list(range(20, 40))  # Shared Drive 1 and its folders
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
@@ -166,14 +161,9 @@ def test_folder_and_shared_drive(
 
     # Should
     expected_file_ids = (
-        DRIVE_ID_MAPPING["ADMIN"]
-        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2_1"]
-        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        list(range(0, 5))  # Admin's My Drive
+        + list(range(20, 40))  # Shared Drive 1 and its folders
+        + list(range(45, 60))  # Folder 2 and its subfolders
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
@@ -205,12 +195,9 @@ def test_folders_only(
     for doc_batch in connector.poll_source(0, time.time()):
         retrieved_docs.extend(doc_batch)
 
-    expected_file_ids = (
-        DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2_1"]
-        + DRIVE_ID_MAPPING["FOLDER_2_2"]
-    )
+    expected_file_ids = list(range(30, 40)) + list(  # Folders 1_1 and 1_2
+        range(50, 60)
+    )  # Folders 2_1 and 2_2
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py b/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
index a7f081fd323..3ea2c6ddf49 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_service_acct.py
@@ -105,12 +105,7 @@ def test_include_my_drives_only(
         retrieved_docs.extend(doc_batch)
 
     # Should only get everyone's My Drives
-    expected_file_ids = (
-        DRIVE_ID_MAPPING["ADMIN"]
-        + DRIVE_ID_MAPPING["TEST_USER_1"]
-        + DRIVE_ID_MAPPING["TEST_USER_2"]
-        + DRIVE_ID_MAPPING["TEST_USER_3"]
-    )
+    expected_file_ids = list(range(0, 20))  # All My Drives
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
@@ -137,12 +132,7 @@ def test_drive_one_only(
         retrieved_docs.extend(doc_batch)
 
     # We ignore shared_drive_urls if include_shared_drives is False
-    expected_file_ids = (
-        DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-    )
+    expected_file_ids = list(range(20, 40))  # Shared Drive 1 and its folders
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
@@ -174,17 +164,9 @@ def test_folder_and_shared_drive(
 
     # Should
     expected_file_ids = (
-        DRIVE_ID_MAPPING["ADMIN"]
-        + DRIVE_ID_MAPPING["TEST_USER_1"]
-        + DRIVE_ID_MAPPING["TEST_USER_2"]
-        + DRIVE_ID_MAPPING["TEST_USER_3"]
-        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2_1"]
-        + DRIVE_ID_MAPPING["FOLDER_2_2"]
+        list(range(0, 20))  # All My Drives
+        + list(range(20, 40))  # Shared Drive 1 and its folders
+        + list(range(45, 60))  # Folder 2 and its subfolders
     )
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
@@ -216,12 +198,9 @@ def test_folders_only(
     for doc_batch in connector.poll_source(0, time.time()):
         retrieved_docs.extend(doc_batch)
 
-    expected_file_ids = (
-        DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2_1"]
-        + DRIVE_ID_MAPPING["FOLDER_2_2"]
-    )
+    expected_file_ids = list(range(30, 40)) + list(  # Folders 1_1 and 1_2
+        range(50, 60)
+    )  # Folders 2_1 and 2_2
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
@@ -250,9 +229,9 @@ def test_specific_emails(
     for doc_batch in connector.poll_source(0, time.time()):
         retrieved_docs.extend(doc_batch)
 
-    expected_file_ids = (
-        DRIVE_ID_MAPPING["TEST_USER_1"] + DRIVE_ID_MAPPING["TEST_USER_3"]
-    )
+    expected_file_ids = list(range(5, 10)) + list(
+        range(15, 20)
+    )  # TEST_USER_1 and TEST_USER_3 My Drives
     assert_retrieved_docs_match_expected(
         retrieved_docs=retrieved_docs,
         expected_file_ids=expected_file_ids,
diff --git a/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py b/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
index d7e79da09d5..1361c198849 100644
--- a/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
+++ b/backend/tests/daily/connectors/google_drive/test_google_drive_slim_docs.py
@@ -5,13 +5,11 @@
 
 from danswer.access.models import ExternalAccess
 from danswer.connectors.google_drive.connector import GoogleDriveConnector
-from danswer.connectors.google_drive.google_utils import execute_paginated_retrieval
-from danswer.connectors.google_drive.resources import get_admin_service
+from danswer.connectors.google_utils.google_utils import execute_paginated_retrieval
+from danswer.connectors.google_utils.resources import get_admin_service
 from ee.danswer.external_permissions.google_drive.doc_sync import (
     _get_permissions_from_slim_doc,
 )
-from tests.daily.connectors.google_drive.helpers import ACCESS_MAPPING
-from tests.daily.connectors.google_drive.helpers import DRIVE_ID_MAPPING
 from tests.daily.connectors.google_drive.helpers import EMAIL_MAPPING
 from tests.daily.connectors.google_drive.helpers import file_name_template
 from tests.daily.connectors.google_drive.helpers import print_discrepencies
@@ -130,19 +128,19 @@ def test_all_permissions(
         print(file_name, external_access)
 
     expected_file_range = (
-        DRIVE_ID_MAPPING["ADMIN"]
-        + DRIVE_ID_MAPPING["TEST_USER_1"]
-        + DRIVE_ID_MAPPING["TEST_USER_2"]
-        + DRIVE_ID_MAPPING["TEST_USER_3"]
-        + DRIVE_ID_MAPPING["SHARED_DRIVE_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_1"]
-        + DRIVE_ID_MAPPING["FOLDER_1_2"]
-        + DRIVE_ID_MAPPING["SHARED_DRIVE_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2"]
-        + DRIVE_ID_MAPPING["FOLDER_2_1"]
-        + DRIVE_ID_MAPPING["FOLDER_2_2"]
-        + DRIVE_ID_MAPPING["SECTIONS"]
+        list(range(0, 5))  # Admin's My Drive
+        + list(range(5, 10))  # TEST_USER_1's My Drive
+        + list(range(10, 15))  # TEST_USER_2's My Drive
+        + list(range(15, 20))  # TEST_USER_3's My Drive
+        + list(range(20, 25))  # Shared Drive 1
+        + list(range(25, 30))  # Folder 1
+        + list(range(30, 35))  # Folder 1_1
+        + list(range(35, 40))  # Folder 1_2
+        + list(range(40, 45))  # Shared Drive 2
+        + list(range(45, 50))  # Folder 2
+        + list(range(50, 55))  # Folder 2_1
+        + list(range(55, 60))  # Folder 2_2
+        + [61]  # Sections
     )
 
     # Should get everything
@@ -154,26 +152,33 @@ def test_all_permissions(
 
     assert_correct_access_for_user(
         user_email=EMAIL_MAPPING["ADMIN"],
-        expected_access_ids=ACCESS_MAPPING["ADMIN"],
+        expected_access_ids=list(range(0, 5))  # Admin's My Drive
+        + list(range(20, 60))  # All shared drive content
+        + [61],  # Sections
         group_map=group_map,
         retrieved_access_map=access_map,
     )
     assert_correct_access_for_user(
         user_email=EMAIL_MAPPING["TEST_USER_1"],
-        expected_access_ids=ACCESS_MAPPING["TEST_USER_1"],
+        expected_access_ids=list(range(5, 10))  # TEST_USER_1's My Drive
+        + list(range(20, 40))  # Shared Drive 1 and its folders
+        + list(range(0, 2)),  # Access to some of Admin's files
         group_map=group_map,
         retrieved_access_map=access_map,
     )
 
     assert_correct_access_for_user(
         user_email=EMAIL_MAPPING["TEST_USER_2"],
-        expected_access_ids=ACCESS_MAPPING["TEST_USER_2"],
+        expected_access_ids=list(range(10, 15))  # TEST_USER_2's My Drive
+        + list(range(25, 40))  # Folder 1 and its subfolders
+        + list(range(50, 55))  # Folder 2_1
+        + list(range(45, 47)),  # Some files in Folder 2
         group_map=group_map,
         retrieved_access_map=access_map,
     )
     assert_correct_access_for_user(
         user_email=EMAIL_MAPPING["TEST_USER_3"],
-        expected_access_ids=ACCESS_MAPPING["TEST_USER_3"],
+        expected_access_ids=list(range(15, 20)),  # TEST_USER_3's My Drive only
         group_map=group_map,
         retrieved_access_map=access_map,
     )
diff --git a/backend/tests/load_env_vars.py b/backend/tests/load_env_vars.py
new file mode 100644
index 00000000000..2911ad4c598
--- /dev/null
+++ b/backend/tests/load_env_vars.py
@@ -0,0 +1,16 @@
+import os
+
+
+def load_env_vars(env_file: str = ".env") -> None:
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    env_path = os.path.join(current_dir, env_file)
+    try:
+        with open(env_path, "r") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    key, value = line.split("=", 1)
+                    os.environ[key] = value.strip()
+        print("Successfully loaded environment variables")
+    except FileNotFoundError:
+        print(f"File {env_file} not found")
diff --git a/backend/tests/unit/danswer/connectors/gmail/test_connector.py b/backend/tests/unit/danswer/connectors/gmail/test_connector.py
index 2689e2a2751..31661cbcc06 100644
--- a/backend/tests/unit/danswer/connectors/gmail/test_connector.py
+++ b/backend/tests/unit/danswer/connectors/gmail/test_connector.py
@@ -1,205 +1,42 @@
 import datetime
-
-import pytest
-from pytest_mock import MockFixture
+import json
+import os
 
 from danswer.configs.constants import DocumentSource
 from danswer.connectors.cross_connector_utils.miscellaneous_utils import time_str_to_utc
-from danswer.connectors.gmail.connector import GmailConnector
+from danswer.connectors.gmail.connector import _build_time_range_query
+from danswer.connectors.gmail.connector import thread_to_document
 from danswer.connectors.models import Document
 
 
-def test_email_to_document() -> None:
-    connector = GmailConnector()
-    email_id = "18cabedb1ea46b03"
-    email_subject = "Danswer Test Subject"
-    email_sender = "Google <no-reply@accounts.google.com>"
-    email_recipient = "test.mail@gmail.com"
-    email_date = "Wed, 27 Dec 2023 15:38:49 GMT"
-    email_labels = ["UNREAD", "IMPORTANT", "CATEGORY_UPDATES", "STARRED", "INBOX"]
-    full_email = {
-        "id": email_id,
-        "threadId": email_id,
-        "labelIds": email_labels,
-        "snippet": "A new sign-in. We noticed a new sign-in to your Google Account. If this was you, you don&#39;t need to do",
-        "payload": {
-            "partId": "",
-            "mimeType": "multipart/alternative",
-            "filename": "",
-            "headers": [
-                {"name": "Delivered-To", "value": email_recipient},
-                {"name": "Date", "value": email_date},
-                {
-                    "name": "Message-ID",
-                    "value": "<OhMtIhHwNS1NoOQRSQEWqw@notifications.google.com>",
-                },
-                {"name": "Subject", "value": email_subject},
-                {"name": "From", "value": email_sender},
-                {"name": "To", "value": email_recipient},
-            ],
-            "body": {"size": 0},
-            "parts": [
-                {
-                    "partId": "0",
-                    "mimeType": "text/plain",
-                    "filename": "",
-                    "headers": [
-                        {
-                            "name": "Content-Type",
-                            "value": 'text/plain; charset="UTF-8"; format=flowed; delsp=yes',
-                        },
-                        {"name": "Content-Transfer-Encoding", "value": "base64"},
-                    ],
-                    "body": {
-                        "size": 9,
-                        "data": "dGVzdCBkYXRh",
-                    },
-                },
-                {
-                    "partId": "1",
-                    "mimeType": "text/html",
-                    "filename": "",
-                    "headers": [
-                        {"name": "Content-Type", "value": 'text/html; charset="UTF-8"'},
-                        {
-                            "name": "Content-Transfer-Encoding",
-                            "value": "quoted-printable",
-                        },
-                    ],
-                    "body": {
-                        "size": 9,
-                        "data": "dGVzdCBkYXRh",
-                    },
-                },
-            ],
-        },
-        "sizeEstimate": 12048,
-        "historyId": "697762",
-        "internalDate": "1703691529000",
-    }
-    doc = connector._email_to_document(full_email)
+def test_thread_to_document() -> None:
+    json_path = os.path.join(os.path.dirname(__file__), "thread.json")
+    with open(json_path, "r") as f:
+        full_email_thread = json.load(f)
+
+    doc = thread_to_document(full_email_thread)
     assert type(doc) == Document
     assert doc.source == DocumentSource.GMAIL
-    assert doc.title == "Danswer Test Subject"
+    assert doc.semantic_identifier == "Email Chain 1"
     assert doc.doc_updated_at == datetime.datetime(
-        2023, 12, 27, 15, 38, 49, tzinfo=datetime.timezone.utc
+        2024, 11, 2, 17, 34, 55, tzinfo=datetime.timezone.utc
     )
-    assert doc.metadata == {
-        "labels": email_labels,
-        "from": email_sender,
-        "to": email_recipient,
-        "date": email_date,
-        "subject": email_subject,
-    }
-
-
-def test_fetch_mails_from_gmail_empty(mocker: MockFixture) -> None:
-    mock_discovery = mocker.patch("danswer.connectors.gmail.connector.discovery")
-    mock_discovery.build.return_value.users.return_value.messages.return_value.list.return_value.execute.return_value = {
-        "messages": []
-    }
-    connector = GmailConnector()
-    connector.creds = mocker.Mock()
-    with pytest.raises(StopIteration):
-        next(connector.load_from_state())
-
-
-def test_fetch_mails_from_gmail(mocker: MockFixture) -> None:
-    mock_discovery = mocker.patch("danswer.connectors.gmail.connector.discovery")
-    email_id = "18cabedb1ea46b03"
-    email_subject = "Danswer Test Subject"
-    email_sender = "Google <no-reply@accounts.google.com>"
-    email_recipient = "test.mail@gmail.com"
-    mock_discovery.build.return_value.users.return_value.messages.return_value.list.return_value.execute.return_value = {
-        "messages": [{"id": email_id, "threadId": email_id}],
-        "nextPageToken": "14473313008248105741",
-        "resultSizeEstimate": 201,
-    }
-    mock_discovery.build.return_value.users.return_value.messages.return_value.get.return_value.execute.return_value = {
-        "id": email_id,
-        "threadId": email_id,
-        "labelIds": ["UNREAD", "IMPORTANT", "CATEGORY_UPDATES", "STARRED", "INBOX"],
-        "snippet": "A new sign-in. We noticed a new sign-in to your Google Account. If this was you, you don&#39;t need to do",
-        "payload": {
-            "partId": "",
-            "mimeType": "multipart/alternative",
-            "filename": "",
-            "headers": [
-                {"name": "Delivered-To", "value": email_recipient},
-                {"name": "Date", "value": "Wed, 27 Dec 2023 15:38:49 GMT"},
-                {
-                    "name": "Message-ID",
-                    "value": "<OhMtIhHwNS1NoOQRSQEWqw@notifications.google.com>",
-                },
-                {"name": "Subject", "value": email_subject},
-                {"name": "From", "value": email_sender},
-                {"name": "To", "value": email_recipient},
-            ],
-            "body": {"size": 0},
-            "parts": [
-                {
-                    "partId": "0",
-                    "mimeType": "text/plain",
-                    "filename": "",
-                    "headers": [
-                        {
-                            "name": "Content-Type",
-                            "value": 'text/plain; charset="UTF-8"; format=flowed; delsp=yes',
-                        },
-                        {"name": "Content-Transfer-Encoding", "value": "base64"},
-                    ],
-                    "body": {
-                        "size": 9,
-                        "data": "dGVzdCBkYXRh",
-                    },
-                },
-                {
-                    "partId": "1",
-                    "mimeType": "text/html",
-                    "filename": "",
-                    "headers": [
-                        {"name": "Content-Type", "value": 'text/html; charset="UTF-8"'},
-                        {
-                            "name": "Content-Transfer-Encoding",
-                            "value": "quoted-printable",
-                        },
-                    ],
-                    "body": {
-                        "size": 9,
-                        "data": "dGVzdCBkYXRh",
-                    },
-                },
-            ],
-        },
-        "sizeEstimate": 12048,
-        "historyId": "697762",
-        "internalDate": "1703691529000",
-    }
-
-    connector = GmailConnector()
-    connector.creds = mocker.Mock()
-    docs = next(connector.load_from_state())
-    assert len(docs) == 1
-    doc: Document = docs[0]
-    assert type(doc) == Document
-    assert doc.id == email_id
-    assert doc.title == email_subject
-    assert email_recipient in doc.sections[0].text
-    assert email_sender in doc.sections[0].text
+    assert len(doc.sections) == 4
+    assert doc.metadata == {}
 
 
 def test_build_time_range_query() -> None:
     time_range_start = 1703066296.159339
     time_range_end = 1704984791.657404
-    query = GmailConnector._build_time_range_query(time_range_start, time_range_end)
+    query = _build_time_range_query(time_range_start, time_range_end)
     assert query == "after:1703066296 before:1704984791"
-    query = GmailConnector._build_time_range_query(time_range_start, None)
+    query = _build_time_range_query(time_range_start, None)
     assert query == "after:1703066296"
-    query = GmailConnector._build_time_range_query(None, time_range_end)
+    query = _build_time_range_query(None, time_range_end)
     assert query == "before:1704984791"
-    query = GmailConnector._build_time_range_query(0.0, time_range_end)
+    query = _build_time_range_query(0.0, time_range_end)
     assert query == "before:1704984791"
-    query = GmailConnector._build_time_range_query(None, None)
+    query = _build_time_range_query(None, None)
     assert query is None
 
 
diff --git a/backend/tests/unit/danswer/connectors/gmail/thread.json b/backend/tests/unit/danswer/connectors/gmail/thread.json
new file mode 100644
index 00000000000..53f0b83ce3a
--- /dev/null
+++ b/backend/tests/unit/danswer/connectors/gmail/thread.json
@@ -0,0 +1,349 @@
+{
+    "id": "192edefb315737c3",
+    "messages": [
+        {
+            "id": "192edeff0dc743cf",
+            "payload": {
+                "headers": [
+                    {
+                        "name": "MIME-Version",
+                        "value": "1.0"
+                    },
+                    {
+                        "name": "Date",
+                        "value": "Sat, 2 Nov 2024 10:32:57 -0700"
+                    },
+                    {
+                        "name": "Message-ID",
+                        "value": "<CABnEGTWbSYxvRDsxnXy1b2iQF=peGsHuOmrOcixpQFCJ9EBHHg@mail.gmail.com>"
+                    },
+                    {
+                        "name": "Subject",
+                        "value": "Email Chain 1"
+                    },
+                    {
+                        "name": "From",
+                        "value": "Test Admin Admin <admin@onyx-test.com>"
+                    },
+                    {
+                        "name": "To",
+                        "value": "test-group-1@onyx-test.com"
+                    },
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/alternative; boundary=\"0000000000004480480625f17117\""
+                    }
+                ],
+                "parts": [
+                    {
+                        "mimeType": "text/plain",
+                        "body": {
+                            "data": "VGhpcyBpcyBlbWFpbCAxIGluIGNoYWluIDENCg=="
+                        }
+                    },
+                    {
+                        "mimeType": "text/html",
+                        "body": {
+                            "data": "PGRpdiBkaXI9Imx0ciI-VGhpcyBpcyBlbWFpbCAxIGluIGNoYWluIDE8L2Rpdj4NCg=="
+                        }
+                    }
+                ]
+            }
+        },
+        {
+            "id": "192edf07fbcc8b2c",
+            "payload": {
+                "headers": [
+                    {
+                        "name": "Delivered-To",
+                        "value": "admin@onyx-test.com"
+                    },
+                    {
+                        "name": "Received",
+                        "value": "by 2002:a59:b3cc:0:b0:491:1bbc:5e54 with SMTP id g12csp1873533vqt;        Sat, 2 Nov 2024 10:33:34 -0700 (PDT)"
+                    },
+                    {
+                        "name": "X-Received",
+                        "value": "by 2002:a05:6102:1284:b0:4a9:555b:fb50 with SMTP id ada2fe7eead31-4a9555bfd21mr8428882137.20.1730568814436;        Sat, 02 Nov 2024 10:33:34 -0700 (PDT)"
+                    },
+                    {
+                        "name": "ARC-Seal",
+                        "value": "i=1; a=rsa-sha256; t=1730568814; cv=none;        d=google.com; s=arc-20240605;        b=A75GBczY/LN8OhNdpZ1VM3opx5VWU3HWYnwCIL9TLBqEpNz2X74TXNkCevJkImB3VF         BkFY7gHg7d8oGdsQvUp2EEdRBXKoYT8P4PTc3ZSD2W8LYU2XCudIbA5xtGObELmI0h0f         bCXT8dE7m6hGJPTg0WPSlkvGs2bY52bmSbCbrnrA/Mx/oyxYPzwv5cMw3CLMXo/8nOLO         FAzrnMTKRqYtn/QvYjUne7PpVSYPk0Edg5261/jn9qatyyL8VePU4FriQTffjAC85Ayc         jikVA5QnsYO79aXJE0SIw4xBHwtOgmyWhU9TPw2NfuQHZWrm39JudUYlmZb8MV4VpX6p         otxw=="
+                    },
+                    {
+                        "name": "ARC-Message-Signature",
+                        "value": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605;        h=to:subject:message-id:date:from:in-reply-to:references:mime-version         :dkim-signature;        bh=9Eo5wYdnqXP3axXBCAwTODK4DvptOqG5RNct/xfBak4=;        fh=/JhVJcrFVXWWzpGRY8HXA/cCDTQzCntn8VCeyDmjzic=;        b=bkhR3iHOUD64TOG3Mqfd9BMT/2IF9gHEjHZWR/tet5J05UKFhk2d4k69wuSLNJcxlF         dB6zzgt1vvEnCbSV+XBCEG1zW76T/sN6Ldn7+5xomsGFYvTZsW4E7OJqxkedfdpFeWwc         eBlgX765wnBs4ztktDhK6gO8igWx3CaYH5wbX72DV4wqcQpDNpMqNHK7sHrlOG2YJGzV         7i3tli4dJqu1zgQK+lo1or1QQyadFzhbwX2iFdSLTNSNR3s70kqqBOT69lDMv84dfKCp         +hXE0uwjOY/9lGG9rO1/e5WWEDC2BSZ7wzjvvyBRjDG+lavBqTggUizd8W+MlRYXONAX         t7Kg==;        dara=google.com"
+                    },
+                    {
+                        "name": "ARC-Authentication-Results",
+                        "value": "i=1; mx.google.com;       dkim=pass header.i=@onyx-test-com.20230601.gappssmtp.com header.s=20230601 header.b=Z57TqzI7;       spf=none (google.com: test_user_1@onyx-test.com does not designate permitted sender hosts) smtp.mailfrom=test_user_1@onyx-test.com;       dara=pass header.i=@onyx-test.com"
+                    },
+                    {
+                        "name": "Return-Path",
+                        "value": "<test_user_1@onyx-test.com>"
+                    },
+                    {
+                        "name": "Received",
+                        "value": "from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])        by mx.google.com with SMTPS id a1e0cc1a2514c-855dae589a1sor1192309241.6.2024.11.02.10.33.34        for <admin@onyx-test.com>        (Google Transport Security);        Sat, 02 Nov 2024 10:33:34 -0700 (PDT)"
+                    },
+                    {
+                        "name": "Received-SPF",
+                        "value": "none (google.com: test_user_1@onyx-test.com does not designate permitted sender hosts) client-ip=209.85.220.41;"
+                    },
+                    {
+                        "name": "Authentication-Results",
+                        "value": "mx.google.com;       dkim=pass header.i=@onyx-test-com.20230601.gappssmtp.com header.s=20230601 header.b=Z57TqzI7;       spf=none (google.com: test_user_1@onyx-test.com does not designate permitted sender hosts) smtp.mailfrom=test_user_1@onyx-test.com;       dara=pass header.i=@onyx-test.com"
+                    },
+                    {
+                        "name": "DKIM-Signature",
+                        "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=onyx-test-com.20230601.gappssmtp.com; s=20230601; t=1730568814; x=1731173614; darn=onyx-test.com;        h=to:subject:message-id:date:from:in-reply-to:references:mime-version         :from:to:cc:subject:date:message-id:reply-to;        bh=9Eo5wYdnqXP3axXBCAwTODK4DvptOqG5RNct/xfBak4=;        b=Z57TqzI7sEwwOumQx0z6YhibC1x2CHlNmBjwyQT1mNOUScZbzo6nmH8Ydo7slsTfgZ         rgwKEEYkf/CYlFWGUEzGzc22jVUCSMjNMFB0nEtfj+GPJaNjDR9FxjFLTUfSq64H/RCI         eO9+oEAJHaa5QmceX2yiSJFXNqmVEMJNT+K6CnlbN5gW6CUD2tBt46vW83PVJgxKMc76         A7/eaDxdZDLUvpjHes4SvM7x0eBM9t7w9wb/jEjGqA54HI2YHVcxM4HJxrbCChYn8UoG         7+UOpfOmHTZLdLYgMtSqYanJ3BTENEdyVp2LIOZOhlUT7Hbr9esyeVyy765XTuRAWxmo         DGPQ=="
+                    },
+                    {
+                        "name": "X-Google-DKIM-Signature",
+                        "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=1e100.net; s=20230601; t=1730568814; x=1731173614;        h=to:subject:message-id:date:from:in-reply-to:references:mime-version         :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to;        bh=9Eo5wYdnqXP3axXBCAwTODK4DvptOqG5RNct/xfBak4=;        b=fxuobWT2rW8kvQ14LUHbJEJOdCM4uBP+Obo7jL4w0BvwLrBNNbMPqMUc8d8u17dnS7         gczFCprOr5PZnVNmOZMQvmRTJ6poTkWOGQhsOyDOSLNI0IzuaN2wh9qjmFez6Z9nTx3f         Lo0I0uahwzNkExywHC9x0H3NOZlS4074qkyLJObgnOHa5vml8SEcChMuzOQuCSU9wNjO         t26urEoct8LArf0K/xztjxpEpDCgnf4Cr/KmZfi4/2Sjv4jwQzkLVuiwADraHIJbLv1m         UMNs92dakWYK0cBbuwOx/sYpUWWyhVmv6Q0LqXzJjtpY4Z0zsnpI2UCrkAdAOSh7geEJ         LCnw=="
+                    },
+                    {
+                        "name": "X-Gm-Message-State",
+                        "value": "AOJu0YyCYZOHIzoRHgMd7foUCpX2JYDwPS2XsTjWiMkkR364/mhFKFsQ vixTj7QM6pDecoDxn8pS0btM7b8z+cwo/8hFiYNgp26wK5L0aGymu+M8OuEk/73fuEthWVV0eko B9LvS5+qixa/oNO/HkRJpVTQmAH7OTT25KeZJj0Dd3x1JqsrfiNE="
+                    },
+                    {
+                        "name": "X-Google-Smtp-Source",
+                        "value": "AGHT+IHCMrQhOT9sgPUOQJL1oVfxMruiLg3BZ5DXqKMdQ7PYF2puka6Ovabv3BPg08CeyS1ovKydIdwHT2uleZkkAaU="
+                    },
+                    {
+                        "name": "X-Received",
+                        "value": "by 2002:a05:6102:5092:b0:4a3:e05e:f6a3 with SMTP id ada2fe7eead31-4a900e11589mr14462681137.3.1730568813787; Sat, 02 Nov 2024 10:33:33 -0700 (PDT)"
+                    },
+                    {
+                        "name": "MIME-Version",
+                        "value": "1.0"
+                    },
+                    {
+                        "name": "References",
+                        "value": "<CABnEGTWbSYxvRDsxnXy1b2iQF=peGsHuOmrOcixpQFCJ9EBHHg@mail.gmail.com>"
+                    },
+                    {
+                        "name": "In-Reply-To",
+                        "value": "<CABnEGTWbSYxvRDsxnXy1b2iQF=peGsHuOmrOcixpQFCJ9EBHHg@mail.gmail.com>"
+                    },
+                    {
+                        "name": "From",
+                        "value": "test_user_1 1 <test_user_1@onyx-test.com>"
+                    },
+                    {
+                        "name": "Date",
+                        "value": "Sat, 2 Nov 2024 10:33:22 -0700"
+                    },
+                    {
+                        "name": "Message-ID",
+                        "value": "<CANSSAx8n6=Kr4sQaGVYaKj63Hdb4=NCffD6OhAADYm+2fe7_dw@mail.gmail.com>"
+                    },
+                    {
+                        "name": "Subject",
+                        "value": "Re: Email Chain 1"
+                    },
+                    {
+                        "name": "To",
+                        "value": "Test Admin Admin <admin@onyx-test.com>"
+                    },
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/alternative; boundary=\"00000000000067dbf70625f1730f\""
+                    }
+                ],
+                "parts": [
+                    {
+                        "mimeType": "text/plain",
+                        "body": {
+                            "data": "VGhpcyBpcyBlbWFpbCAyIGluIGNoYWluIDENCg0KT24gU2F0LCBOb3YgMiwgMjAyNCBhdCAxMDozM-KAr0FNIFRlc3QgQWRtaW4gQWRtaW4gPGFkbWluQG9ueXgtdGVzdC5jb20-DQp3cm90ZToNCg0KPiBUaGlzIGlzIGVtYWlsIDEgaW4gY2hhaW4gMQ0KPg0K"
+                        }
+                    },
+                    {
+                        "mimeType": "text/html",
+                        "body": {
+                            "data": "PGRpdiBkaXI9Imx0ciI-VGhpcyBpcyBlbWFpbCAyIGluIGNoYWluIDE8L2Rpdj48YnI-PGRpdiBjbGFzcz0iZ21haWxfcXVvdGUiPjxkaXYgZGlyPSJsdHIiIGNsYXNzPSJnbWFpbF9hdHRyIj5PbiBTYXQsIE5vdiAyLCAyMDI0IGF0IDEwOjMz4oCvQU0gVGVzdCBBZG1pbiBBZG1pbiAmbHQ7PGEgaHJlZj0ibWFpbHRvOmFkbWluQG9ueXgtdGVzdC5jb20iPmFkbWluQG9ueXgtdGVzdC5jb208L2E-Jmd0OyB3cm90ZTo8YnI-PC9kaXY-PGJsb2NrcXVvdGUgY2xhc3M9ImdtYWlsX3F1b3RlIiBzdHlsZT0ibWFyZ2luOjBweCAwcHggMHB4IDAuOGV4O2JvcmRlci1sZWZ0OjFweCBzb2xpZCByZ2IoMjA0LDIwNCwyMDQpO3BhZGRpbmctbGVmdDoxZXgiPjxkaXYgZGlyPSJsdHIiPlRoaXMgaXMgZW1haWwgMSBpbiBjaGFpbiAxPC9kaXY-DQo8L2Jsb2NrcXVvdGU-PC9kaXY-DQo="
+                        }
+                    }
+                ]
+            }
+        },
+        {
+            "id": "192edf157175fcec",
+            "payload": {
+                "headers": [
+                    {
+                        "name": "MIME-Version",
+                        "value": "1.0"
+                    },
+                    {
+                        "name": "Date",
+                        "value": "Sat, 2 Nov 2024 10:34:29 -0700"
+                    },
+                    {
+                        "name": "References",
+                        "value": "<CABnEGTWbSYxvRDsxnXy1b2iQF=peGsHuOmrOcixpQFCJ9EBHHg@mail.gmail.com> <CANSSAx8n6=Kr4sQaGVYaKj63Hdb4=NCffD6OhAADYm+2fe7_dw@mail.gmail.com>"
+                    },
+                    {
+                        "name": "In-Reply-To",
+                        "value": "<CANSSAx8n6=Kr4sQaGVYaKj63Hdb4=NCffD6OhAADYm+2fe7_dw@mail.gmail.com>"
+                    },
+                    {
+                        "name": "Bcc",
+                        "value": "test_user_3@onyx-test.com"
+                    },
+                    {
+                        "name": "Message-ID",
+                        "value": "<CABnEGTUEDvhfyOWTCauhTCn5mVXGp6p1=yw65RUsGu8E=c2k4g@mail.gmail.com>"
+                    },
+                    {
+                        "name": "Subject",
+                        "value": "Fwd: Email Chain 1"
+                    },
+                    {
+                        "name": "From",
+                        "value": "Test Admin Admin <admin@onyx-test.com>"
+                    },
+                    {
+                        "name": "To",
+                        "value": "test_user_2 2 <test_user_2@onyx-test.com>"
+                    },
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/alternative; boundary=\"000000000000bf7afd0625f1764f\""
+                    }
+                ],
+                "parts": [
+                    {
+                        "mimeType": "text/plain",
+                        "body": {
+                            "data": "VGhpcyBpcyBlbWFpbCAzIGluIGNoYWluIDENCg0KLS0tLS0tLS0tLSBGb3J3YXJkZWQgbWVzc2FnZSAtLS0tLS0tLS0NCkZyb206IHRlc3RfdXNlcl8xIDEgPHRlc3RfdXNlcl8xQG9ueXgtdGVzdC5jb20-DQpEYXRlOiBTYXQsIE5vdiAyLCAyMDI0IGF0IDEwOjMz4oCvQU0NClN1YmplY3Q6IFJlOiBFbWFpbCBDaGFpbiAxDQpUbzogVGVzdCBBZG1pbiBBZG1pbiA8YWRtaW5Ab255eC10ZXN0LmNvbT4NCg0KDQpUaGlzIGlzIGVtYWlsIDIgaW4gY2hhaW4gMQ0KDQpPbiBTYXQsIE5vdiAyLCAyMDI0IGF0IDEwOjMz4oCvQU0gVGVzdCBBZG1pbiBBZG1pbiA8YWRtaW5Ab255eC10ZXN0LmNvbT4NCndyb3RlOg0KDQo-IFRoaXMgaXMgZW1haWwgMSBpbiBjaGFpbiAxDQo-DQo="
+                        }
+                    },
+                    {
+                        "mimeType": "text/html",
+                        "body": {
+                            "data": "PGRpdiBkaXI9Imx0ciI-PGRpdiBkaXI9Imx0ciI-VGhpcyBpcyBlbWFpbCAzIGluIGNoYWluIDE8L2Rpdj48YnI-PGRpdiBjbGFzcz0iZ21haWxfcXVvdGUiPjxkaXYgZGlyPSJsdHIiIGNsYXNzPSJnbWFpbF9hdHRyIj4tLS0tLS0tLS0tIEZvcndhcmRlZCBtZXNzYWdlIC0tLS0tLS0tLTxicj5Gcm9tOiA8c3Ryb25nIGNsYXNzPSJnbWFpbF9zZW5kZXJuYW1lIiBkaXI9ImF1dG8iPnRlc3RfdXNlcl8xIDE8L3N0cm9uZz4gPHNwYW4gZGlyPSJhdXRvIj4mbHQ7PGEgaHJlZj0ibWFpbHRvOnRlc3RfdXNlcl8xQG9ueXgtdGVzdC5jb20iPnRlc3RfdXNlcl8xQG9ueXgtdGVzdC5jb208L2E-Jmd0Ozwvc3Bhbj48YnI-RGF0ZTogU2F0LCBOb3YgMiwgMjAyNCBhdCAxMDozM-KAr0FNPGJyPlN1YmplY3Q6IFJlOiBFbWFpbCBDaGFpbiAxPGJyPlRvOiBUZXN0IEFkbWluIEFkbWluICZsdDs8YSBocmVmPSJtYWlsdG86YWRtaW5Ab255eC10ZXN0LmNvbSI-YWRtaW5Ab255eC10ZXN0LmNvbTwvYT4mZ3Q7PGJyPjwvZGl2Pjxicj48YnI-PGRpdiBkaXI9Imx0ciI-VGhpcyBpcyBlbWFpbCAyIGluIGNoYWluIDE8L2Rpdj48YnI-PGRpdiBjbGFzcz0iZ21haWxfcXVvdGUiPjxkaXYgZGlyPSJsdHIiIGNsYXNzPSJnbWFpbF9hdHRyIj5PbiBTYXQsIE5vdiAyLCAyMDI0IGF0IDEwOjMz4oCvQU0gVGVzdCBBZG1pbiBBZG1pbiAmbHQ7PGEgaHJlZj0ibWFpbHRvOmFkbWluQG9ueXgtdGVzdC5jb20iIHRhcmdldD0iX2JsYW5rIj5hZG1pbkBvbnl4LXRlc3QuY29tPC9hPiZndDsgd3JvdGU6PGJyPjwvZGl2PjxibG9ja3F1b3RlIGNsYXNzPSJnbWFpbF9xdW90ZSIgc3R5bGU9Im1hcmdpbjowcHggMHB4IDBweCAwLjhleDtib3JkZXItbGVmdDoxcHggc29saWQgcmdiKDIwNCwyMDQsMjA0KTtwYWRkaW5nLWxlZnQ6MWV4Ij48ZGl2IGRpcj0ibHRyIj5UaGlzIGlzIGVtYWlsIDEgaW4gY2hhaW4gMTwvZGl2Pg0KPC9ibG9ja3F1b3RlPjwvZGl2Pg0KPC9kaXY-PC9kaXY-DQo="
+                        }
+                    }
+                ]
+            }
+        },
+        {
+            "id": "192edf1e8f7ecbb4",
+            "payload": {
+                "headers": [
+                    {
+                        "name": "Delivered-To",
+                        "value": "admin@onyx-test.com"
+                    },
+                    {
+                        "name": "Received",
+                        "value": "by 2002:a59:b3cc:0:b0:491:1bbc:5e54 with SMTP id g12csp1874156vqt;        Sat, 2 Nov 2024 10:35:07 -0700 (PDT)"
+                    },
+                    {
+                        "name": "X-Received",
+                        "value": "by 2002:a05:6122:319c:b0:50d:81f9:5210 with SMTP id 71dfb90a1353d-5105d128958mr15853812e0c.13.1730568906834;        Sat, 02 Nov 2024 10:35:06 -0700 (PDT)"
+                    },
+                    {
+                        "name": "ARC-Seal",
+                        "value": "i=1; a=rsa-sha256; t=1730568906; cv=none;        d=google.com; s=arc-20240605;        b=JUd7S6ql1poKM5ox92op2g2Z67AS8sEkp5f/S+Mr5+7KSichsjAwixWg/YhhRhvaY/         UcykrbdaAeWfCuGtJgSq1nr1z5hB3iAltv/D2XCdJdOXzVDpVvaV9lT/YU6266VKtsnq         gFVKfjyMe/MnNKvDITQL67A2gRvhiR3XWxwEVvrMArMpUb9bbudlF/5L3MQY4BCIvWLL         9uBv1ZnclghscsxspoG3CkULkGqHGUTKq6bPoUn/hOljiVdsVVagoOwhbDEcyMRKUDnm         2t3H7iiujhlBIDbRoLJR/6C+A6AMyNKPAFA3axM6EXrTOADMZ8a0JqFj8O4rktYpRV+d         zHxQ=="
+                    },
+                    {
+                        "name": "ARC-Message-Signature",
+                        "value": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20240605;        h=to:subject:message-id:date:from:in-reply-to:references:mime-version         :dkim-signature;        bh=K0g0X/4URFSC1nuXjI7ZESJA66WnWcqwgfHOUDQ/kQo=;        fh=/JhVJcrFVXWWzpGRY8HXA/cCDTQzCntn8VCeyDmjzic=;        b=IarHhl5g5tjBhlMRRXo6WwTzaFOI4Q3w4ebNunftDUHwzV7Qu1hY0y7r3SRNaBb+qD         ZncYUI6PF/Oo7eMG65IloXfu+kHUI8NJMaoERUWgEk21Tj6cOSRO4x/W6V5PSX7a4lWZ         K1cNdAlaiWI09Esv07Vel975Bgrd+XiCwoVgJAAslHOJ2bZwSYWzvwLqdkCRVrAGJQ9/         I80kvOnNVesIFdIR6SGrhdz8xNIIoe60k8PjJRzkmzy/tEeKCYBz6W+NW4xoIaAVmKUw         RvjI8JozUVkGzh+LLyx64MakPCZPWM+ft+D35JodarYh+KesF+HV/Oe7rjaw7JXZ1WoE         OdJQ==;        dara=google.com"
+                    },
+                    {
+                        "name": "ARC-Authentication-Results",
+                        "value": "i=1; mx.google.com;       dkim=pass header.i=@onyx-test-com.20230601.gappssmtp.com header.s=20230601 header.b=1U8JkCbL;       spf=none (google.com: test_user_3@onyx-test.com does not designate permitted sender hosts) smtp.mailfrom=test_user_3@onyx-test.com;       dara=pass header.i=@onyx-test.com"
+                    },
+                    {
+                        "name": "Return-Path",
+                        "value": "<test_user_3@onyx-test.com>"
+                    },
+                    {
+                        "name": "Received",
+                        "value": "from mail-sor-f41.google.com (mail-sor-f41.google.com. [209.85.220.41])        by mx.google.com with SMTPS id 71dfb90a1353d-5106f3f9037sor1051490e0c.7.2024.11.02.10.35.06        for <admin@onyx-test.com>        (Google Transport Security);        Sat, 02 Nov 2024 10:35:06 -0700 (PDT)"
+                    },
+                    {
+                        "name": "Received-SPF",
+                        "value": "none (google.com: test_user_3@onyx-test.com does not designate permitted sender hosts) client-ip=209.85.220.41;"
+                    },
+                    {
+                        "name": "Authentication-Results",
+                        "value": "mx.google.com;       dkim=pass header.i=@onyx-test-com.20230601.gappssmtp.com header.s=20230601 header.b=1U8JkCbL;       spf=none (google.com: test_user_3@onyx-test.com does not designate permitted sender hosts) smtp.mailfrom=test_user_3@onyx-test.com;       dara=pass header.i=@onyx-test.com"
+                    },
+                    {
+                        "name": "DKIM-Signature",
+                        "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=onyx-test-com.20230601.gappssmtp.com; s=20230601; t=1730568906; x=1731173706; darn=onyx-test.com;        h=to:subject:message-id:date:from:in-reply-to:references:mime-version         :from:to:cc:subject:date:message-id:reply-to;        bh=K0g0X/4URFSC1nuXjI7ZESJA66WnWcqwgfHOUDQ/kQo=;        b=1U8JkCbLjicGtH7otVX3QjKv/XK5fGnmOIVMTD/b9cO1w8ai2GwCuJbBo+z1IuGqto         aRuNCcEqUIaFvVFiezvhL9xg7scIwHHvLOrSpmc0h0JMSx8q4kKaUGKEJpewsYvkStmr         DYv/cUIeaPTIChSuUDV7FVMhf7jIyIaYry3i9/EIlw+on18nD30C9kXwds5yWW8XGvtR         /OUuSdgJzuoNmypUt8v9Ebqd+LP23YTs+78/G1Ag+JjugxxF+C9cm7SxmooWueukRkm8         o8nQO5QVx/y/xsCZdM2XXcKCLcZIntuY48amlfFyIqrhG1/DEM6htD64meMGctNTptQf         jHrw=="
+                    },
+                    {
+                        "name": "X-Google-DKIM-Signature",
+                        "value": "v=1; a=rsa-sha256; c=relaxed/relaxed;        d=1e100.net; s=20230601; t=1730568906; x=1731173706;        h=to:subject:message-id:date:from:in-reply-to:references:mime-version         :x-gm-message-state:from:to:cc:subject:date:message-id:reply-to;        bh=K0g0X/4URFSC1nuXjI7ZESJA66WnWcqwgfHOUDQ/kQo=;        b=J4+ozlusGGM1Hn95EZkDeYbExgkyOlAdcY6LcV4Wx1zeI78HtEXGgvqcZ5sP7HzS1X         /A3i7WkgmjpC9bU2/zKLrfXDvYQ7udQwTJtKsKaUo4O65Al7Wtgz8e8rBDYikhqEEAZQ         GbEwqp+qa+v0T4rPhkQKd4zpIE3AUd3eh5u5iF/UEYc1NcyV35uMGWRP4jOK6F67MwS7         73MgObcGqmBH48I4K+ITYAkNEMGOBpY6fheGxCxyDpcG5gbf8swlWX2Dd0EM9H72o+Xb         jvAslOq1lZzPZUgyyZJ2wVEASxF8S7depiOLcTPKwsw+pgXIMAUBExBvu0u4PhO0qG+z         pftQ=="
+                    },
+                    {
+                        "name": "X-Gm-Message-State",
+                        "value": "AOJu0Yy2r0aT3w7HBU7t0JGla+x3AddG9WdnQT06r6T/HGZwZ9Wp9TUs Orb/HMtgvXivtYFkG14NJkMTBO4EqSynmzaxAvEheDXB1uYE2LS21XoqrvycvYQh3GUHBwUdS8L lE6BUjm4TJfXlZWAqKRxg4C0j1UFSuVdkXf6P1GCsdyKKTeS6A9eohw=="
+                    },
+                    {
+                        "name": "X-Google-Smtp-Source",
+                        "value": "AGHT+IHXTB7Ar9w/Q3G3gCT19SVELYvWl30pNGuNiTmkYZgMWFS7YUWTkG/DS4/mrjMRXpYuclOLHv8BeOmw9Jovkr4="
+                    },
+                    {
+                        "name": "X-Received",
+                        "value": "by 2002:a05:6102:3a10:b0:4a9:49:26d2 with SMTP id ada2fe7eead31-4a90109fb68mr15589362137.29.1730568906301; Sat, 02 Nov 2024 10:35:06 -0700 (PDT)"
+                    },
+                    {
+                        "name": "MIME-Version",
+                        "value": "1.0"
+                    },
+                    {
+                        "name": "References",
+                        "value": "<CABnEGTWbSYxvRDsxnXy1b2iQF=peGsHuOmrOcixpQFCJ9EBHHg@mail.gmail.com> <CANSSAx8n6=Kr4sQaGVYaKj63Hdb4=NCffD6OhAADYm+2fe7_dw@mail.gmail.com> <CABnEGTUEDvhfyOWTCauhTCn5mVXGp6p1=yw65RUsGu8E=c2k4g@mail.gmail.com>"
+                    },
+                    {
+                        "name": "In-Reply-To",
+                        "value": "<CABnEGTUEDvhfyOWTCauhTCn5mVXGp6p1=yw65RUsGu8E=c2k4g@mail.gmail.com>"
+                    },
+                    {
+                        "name": "From",
+                        "value": "test_user_3 3 <test_user_3@onyx-test.com>"
+                    },
+                    {
+                        "name": "Date",
+                        "value": "Sat, 2 Nov 2024 10:34:55 -0700"
+                    },
+                    {
+                        "name": "Message-ID",
+                        "value": "<CACcF+8GU1V2_CcYsUFNOh0+oSkMG=oN-ioyPPXRsD+0Ghr-u-Q@mail.gmail.com>"
+                    },
+                    {
+                        "name": "Subject",
+                        "value": "Re: Email Chain 1"
+                    },
+                    {
+                        "name": "To",
+                        "value": "Test Admin Admin <admin@onyx-test.com>"
+                    },
+                    {
+                        "name": "Content-Type",
+                        "value": "multipart/alternative; boundary=\"000000000000eb82a70625f178cf\""
+                    }
+                ],
+                "parts": [
+                    {
+                        "mimeType": "text/plain",
+                        "body": {
+                            "data": "VGhpcyBpcyBlbWFpbCA0IGluIGNoYWluIDENCg0KT24gU2F0LCBOb3YgMiwgMjAyNCBhdCAxMDozNOKAr0FNIFRlc3QgQWRtaW4gQWRtaW4gPGFkbWluQG9ueXgtdGVzdC5jb20-DQp3cm90ZToNCg0KPiBUaGlzIGlzIGVtYWlsIDMgaW4gY2hhaW4gMQ0KPg0KPiAtLS0tLS0tLS0tIEZvcndhcmRlZCBtZXNzYWdlIC0tLS0tLS0tLQ0KPiBGcm9tOiB0ZXN0X3VzZXJfMSAxIDx0ZXN0X3VzZXJfMUBvbnl4LXRlc3QuY29tPg0KPiBEYXRlOiBTYXQsIE5vdiAyLCAyMDI0IGF0IDEwOjMz4oCvQU0NCj4gU3ViamVjdDogUmU6IEVtYWlsIENoYWluIDENCj4gVG86IFRlc3QgQWRtaW4gQWRtaW4gPGFkbWluQG9ueXgtdGVzdC5jb20-DQo-DQo-DQo-IFRoaXMgaXMgZW1haWwgMiBpbiBjaGFpbiAxDQo-DQo-IE9uIFNhdCwgTm92IDIsIDIwMjQgYXQgMTA6MzPigK9BTSBUZXN0IEFkbWluIEFkbWluIDxhZG1pbkBvbnl4LXRlc3QuY29tPg0KPiB3cm90ZToNCj4NCj4-IFRoaXMgaXMgZW1haWwgMSBpbiBjaGFpbiAxDQo-Pg0KPg0K"
+                        }
+                    },
+                    {
+                        "mimeType": "text/html",
+                        "body": {
+                            "data": "PGRpdiBkaXI9Imx0ciI-VGhpcyBpcyBlbWFpbCA0IGluIGNoYWluIDE8YnIgY2xhc3M9ImdtYWlsLUFwcGxlLWludGVyY2hhbmdlLW5ld2xpbmUiPjwvZGl2Pjxicj48ZGl2IGNsYXNzPSJnbWFpbF9xdW90ZSI-PGRpdiBkaXI9Imx0ciIgY2xhc3M9ImdtYWlsX2F0dHIiPk9uIFNhdCwgTm92IDIsIDIwMjQgYXQgMTA6MzTigK9BTSBUZXN0IEFkbWluIEFkbWluICZsdDs8YSBocmVmPSJtYWlsdG86YWRtaW5Ab255eC10ZXN0LmNvbSI-YWRtaW5Ab255eC10ZXN0LmNvbTwvYT4mZ3Q7IHdyb3RlOjxicj48L2Rpdj48YmxvY2txdW90ZSBjbGFzcz0iZ21haWxfcXVvdGUiIHN0eWxlPSJtYXJnaW46MHB4IDBweCAwcHggMC44ZXg7Ym9yZGVyLWxlZnQ6MXB4IHNvbGlkIHJnYigyMDQsMjA0LDIwNCk7cGFkZGluZy1sZWZ0OjFleCI-PGRpdiBkaXI9Imx0ciI-PGRpdiBkaXI9Imx0ciI-VGhpcyBpcyBlbWFpbCAzIGluIGNoYWluIDE8L2Rpdj48YnI-PGRpdiBjbGFzcz0iZ21haWxfcXVvdGUiPjxkaXYgZGlyPSJsdHIiIGNsYXNzPSJnbWFpbF9hdHRyIj4tLS0tLS0tLS0tIEZvcndhcmRlZCBtZXNzYWdlIC0tLS0tLS0tLTxicj5Gcm9tOiA8c3Ryb25nIGNsYXNzPSJnbWFpbF9zZW5kZXJuYW1lIiBkaXI9ImF1dG8iPnRlc3RfdXNlcl8xIDE8L3N0cm9uZz4gPHNwYW4gZGlyPSJhdXRvIj4mbHQ7PGEgaHJlZj0ibWFpbHRvOnRlc3RfdXNlcl8xQG9ueXgtdGVzdC5jb20iIHRhcmdldD0iX2JsYW5rIj50ZXN0X3VzZXJfMUBvbnl4LXRlc3QuY29tPC9hPiZndDs8L3NwYW4-PGJyPkRhdGU6IFNhdCwgTm92IDIsIDIwMjQgYXQgMTA6MzPigK9BTTxicj5TdWJqZWN0OiBSZTogRW1haWwgQ2hhaW4gMTxicj5UbzogVGVzdCBBZG1pbiBBZG1pbiAmbHQ7PGEgaHJlZj0ibWFpbHRvOmFkbWluQG9ueXgtdGVzdC5jb20iIHRhcmdldD0iX2JsYW5rIj5hZG1pbkBvbnl4LXRlc3QuY29tPC9hPiZndDs8YnI-PC9kaXY-PGJyPjxicj48ZGl2IGRpcj0ibHRyIj5UaGlzIGlzIGVtYWlsIDIgaW4gY2hhaW4gMTwvZGl2Pjxicj48ZGl2IGNsYXNzPSJnbWFpbF9xdW90ZSI-PGRpdiBkaXI9Imx0ciIgY2xhc3M9ImdtYWlsX2F0dHIiPk9uIFNhdCwgTm92IDIsIDIwMjQgYXQgMTA6MzPigK9BTSBUZXN0IEFkbWluIEFkbWluICZsdDs8YSBocmVmPSJtYWlsdG86YWRtaW5Ab255eC10ZXN0LmNvbSIgdGFyZ2V0PSJfYmxhbmsiPmFkbWluQG9ueXgtdGVzdC5jb208L2E-Jmd0OyB3cm90ZTo8YnI-PC9kaXY-PGJsb2NrcXVvdGUgY2xhc3M9ImdtYWlsX3F1b3RlIiBzdHlsZT0ibWFyZ2luOjBweCAwcHggMHB4IDAuOGV4O2JvcmRlci1sZWZ0OjFweCBzb2xpZCByZ2IoMjA0LDIwNCwyMDQpO3BhZGRpbmctbGVmdDoxZXgiPjxkaXYgZGlyPSJsdHIiPlRoaXMgaXMgZW1haWwgMSBpbiBjaGFpbiAxPC9kaXY-DQo8L2Jsb2NrcXVvdGU-PC9kaXY-DQo8L2Rpdj48L2Rpdj4NCjwvYmxvY2txdW90ZT48L2Rpdj4NCg=="
+                        }
+                    }
+                ]
+            }
+        }
+    ]
+}
diff --git a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
index 95474914678..9ea868c9389 100644
--- a/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/AddConnectorPage.tsx
@@ -142,8 +142,8 @@ export default function AddConnector({
   const { popup, setPopup } = usePopup();
 
   // Hooks for Google Drive and Gmail credentials
-  const { liveGDriveCredential } = useGoogleDriveCredentials();
-  const { liveGmailCredential } = useGmailCredentials();
+  const { liveGDriveCredential } = useGoogleDriveCredentials(connector);
+  const { liveGmailCredential } = useGmailCredentials(connector);
 
   // Check if credential is activated
   const credentialActivated =
diff --git a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
index 8cf94d1b656..d0f5d58e858 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gdrive/Credential.tsx
@@ -413,7 +413,7 @@ export const DriveAuthSection = ({
             <Form>
               <TextFormField
                 name="google_drive_primary_admin"
-                label="User email to impersonate:"
+                label="Primary Admin Email:"
                 subtext="Enter the email of the user whose Google Drive access you want to delegate to the service account."
               />
               <div className="flex">
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
index 061ddddcde4..5b6787b301c 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/Credential.tsx
@@ -10,6 +10,7 @@ import { GMAIL_AUTH_IS_ADMIN_COOKIE_NAME } from "@/lib/constants";
 import Cookies from "js-cookie";
 import { TextFormField } from "@/components/admin/connectors/Field";
 import { Form, Formik } from "formik";
+import { User } from "@/lib/types";
 import CardSection from "@/components/admin/CardSection";
 import {
   Credential,
@@ -293,9 +294,10 @@ interface DriveCredentialSectionProps {
   setPopup: (popupSpec: PopupSpec | null) => void;
   refreshCredentials: () => void;
   connectorExists: boolean;
+  user: User | null;
 }
 
-export const GmailOAuthSection = ({
+export const GmailAuthSection = ({
   gmailPublicCredential,
   gmailServiceAccountCredential,
   serviceAccountKeyData,
@@ -303,6 +305,7 @@ export const GmailOAuthSection = ({
   setPopup,
   refreshCredentials,
   connectorExists,
+  user,
 }: DriveCredentialSectionProps) => {
   const router = useRouter();
 
@@ -342,24 +345,22 @@ export const GmailOAuthSection = ({
     return (
       <div>
         <p className="text-sm mb-2">
-          When using a Gmail Service Account, you can either have Danswer act as
-          the service account itself OR you can specify an account for the
-          service account to impersonate.
+          When using a Gmail Service Account, you must specify the email of the
+          primary admin that you would like the service account to impersonate.
           <br />
           <br />
-          If you want to use the service account itself, leave the{" "}
-          <b>&apos;User email to impersonate&apos;</b> field blank when
-          submitting. If you do choose this option, make sure you have shared
-          the documents you want to index with the service account.
+          For this connector to index all users Gmail, the primary admin email
+          should be an owner/admin of the Google Organization that being
+          indexed.
         </p>
 
         <CardSection>
           <Formik
             initialValues={{
-              gmail_delegated_user: "",
+              gmail_primary_admin: user?.email || "",
             }}
             validationSchema={Yup.object().shape({
-              gmail_delegated_user: Yup.string().optional(),
+              gmail_primary_admin: Yup.string().required(),
             })}
             onSubmit={async (values, formikHelpers) => {
               formikHelpers.setSubmitting(true);
@@ -372,7 +373,7 @@ export const GmailOAuthSection = ({
                     "Content-Type": "application/json",
                   },
                   body: JSON.stringify({
-                    gmail_delegated_user: values.gmail_delegated_user,
+                    gmail_primary_admin: values.gmail_primary_admin,
                   }),
                 }
               );
@@ -395,9 +396,9 @@ export const GmailOAuthSection = ({
             {({ isSubmitting }) => (
               <Form>
                 <TextFormField
-                  name="gmail_delegated_user"
-                  label="[Optional] User email to impersonate:"
-                  subtext="If left blank, Danswer will use the service account itself."
+                  name="gmail_primary_admin"
+                  label="Primary Admin Email:"
+                  subtext="You must provide an admin/owner account to retrieve all org emails."
                 />
                 <div className="flex">
                   <button
diff --git a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
index cf646916099..0ee95e0037c 100644
--- a/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
+++ b/web/src/app/admin/connectors/[connector]/pages/gmail/GmailPage.tsx
@@ -10,7 +10,7 @@ import {
   GmailCredentialJson,
   GmailServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
-import { GmailOAuthSection, GmailJsonUploadSection } from "./Credential";
+import { GmailAuthSection, GmailJsonUploadSection } from "./Credential";
 import {
   usePublicCredentials,
   useConnectorCredentialIndexingStatus,
@@ -20,7 +20,7 @@ import { GmailConfig } from "@/lib/connectors/connectors";
 import { useUser } from "@/components/user/UserProvider";
 
 export const GmailMain = () => {
-  const { isLoadingUser, isAdmin } = useUser();
+  const { isLoadingUser, isAdmin, user } = useUser();
 
   const {
     data: appCredentialData,
@@ -142,7 +142,7 @@ export const GmailMain = () => {
           <Title className="mb-2 mt-6 ml-auto mr-auto">
             Step 2: Authenticate with Danswer
           </Title>
-          <GmailOAuthSection
+          <GmailAuthSection
             setPopup={setPopup}
             refreshCredentials={refreshCredentials}
             gmailPublicCredential={gmailPublicCredential}
@@ -150,6 +150,7 @@ export const GmailMain = () => {
             appCredentialData={appCredentialData}
             serviceAccountKeyData={serviceAccountKeyData}
             connectorExists={gmailConnectorIndexingStatuses.length > 0}
+            user={user}
           />
         </>
       )}
diff --git a/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts b/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts
index d3a48e3a26b..5d5f61dd820 100644
--- a/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts
+++ b/web/src/app/admin/connectors/[connector]/pages/utils/hooks.ts
@@ -12,7 +12,7 @@ import {
   GoogleDriveServiceAccountCredentialJson,
 } from "@/lib/connectors/credentials";
 
-export const useGmailCredentials = () => {
+export const useGmailCredentials = (connector: string) => {
   const {
     data: credentialsData,
     isLoading: isCredentialsLoading,
@@ -23,7 +23,9 @@ export const useGmailCredentials = () => {
   const gmailPublicCredential: Credential<GmailCredentialJson> | undefined =
     credentialsData?.find(
       (credential) =>
-        credential.credential_json?.gmail_tokens && credential.admin_public
+        credential.credential_json?.google_service_account_key &&
+        credential.admin_public &&
+        credential.source === connector
     );
 
   const gmailServiceAccountCredential:
@@ -36,18 +38,20 @@ export const useGmailCredentials = () => {
     gmailPublicCredential || gmailServiceAccountCredential;
 
   return {
-    liveGmailCredential,
+    liveGmailCredential: liveGmailCredential,
   };
 };
 
-export const useGoogleDriveCredentials = () => {
+export const useGoogleDriveCredentials = (connector: string) => {
   const { data: credentialsData } = usePublicCredentials();
 
   const googleDrivePublicCredential:
     | Credential<GoogleDriveCredentialJson>
     | undefined = credentialsData?.find(
     (credential) =>
-      credential.credential_json?.google_drive_tokens && credential.admin_public
+      credential.credential_json?.google_service_account_key &&
+      credential.admin_public &&
+      credential.source === connector
   );
 
   const googleDriveServiceAccountCredential:
@@ -60,6 +64,6 @@ export const useGoogleDriveCredentials = () => {
     googleDrivePublicCredential || googleDriveServiceAccountCredential;
 
   return {
-    liveGDriveCredential,
+    liveGDriveCredential: liveGDriveCredential,
   };
 };
diff --git a/web/src/lib/connectors/AutoSyncOptionFields.tsx b/web/src/lib/connectors/AutoSyncOptionFields.tsx
index 4a8b44868e6..c39475bd89e 100644
--- a/web/src/lib/connectors/AutoSyncOptionFields.tsx
+++ b/web/src/lib/connectors/AutoSyncOptionFields.tsx
@@ -13,5 +13,6 @@ export const autoSyncConfigBySource: Record<
 > = {
   confluence: {},
   google_drive: {},
+  gmail: {},
   slack: {},
 };
diff --git a/web/src/lib/connectors/credentials.ts b/web/src/lib/connectors/credentials.ts
index c6da2c5f62c..86a922c74f2 100644
--- a/web/src/lib/connectors/credentials.ts
+++ b/web/src/lib/connectors/credentials.ts
@@ -54,6 +54,7 @@ export interface SlackCredentialJson {
 
 export interface GmailCredentialJson {
   gmail_tokens: string;
+  gmail_primary_admin: string;
 }
 
 export interface GoogleDriveCredentialJson {
@@ -63,7 +64,7 @@ export interface GoogleDriveCredentialJson {
 
 export interface GmailServiceAccountCredentialJson {
   gmail_service_account_key: string;
-  gmail_delegated_user: string;
+  gmail_primary_admin: string;
 }
 
 export interface GoogleDriveServiceAccountCredentialJson {
@@ -339,7 +340,7 @@ export const credentialDisplayNames: Record<string, string> = {
 
   // Gmail Service Account
   gmail_service_account_key: "Gmail Service Account Key",
-  gmail_delegated_user: "Gmail Delegated User",
+  gmail_primary_admin: "Gmail Primary Admin",
 
   // Google Drive Service Account
   google_drive_service_account_key: "Google Drive Service Account Key",
diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts
index f532a09ee31..6c5370bacfd 100644
--- a/web/src/lib/types.ts
+++ b/web/src/lib/types.ts
@@ -278,6 +278,7 @@ export type ConfigurableSources = Exclude<
 export const validAutoSyncSources = [
   "confluence",
   "google_drive",
+  "gmail",
   "slack",
 ] as const;
 export type ValidAutoSyncSources = (typeof validAutoSyncSources)[number];

From 2cb33b1fb4d01b00b0c17a92b4791996001ca729 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 4 Nov 2024 11:11:12 -0800
Subject: [PATCH 362/376] add default api keys for cloud users (#3044)

* add default api keys for cloud users

* add cohere as well

* naming
---
 backend/ee/danswer/configs/app_configs.py     |  4 +++
 backend/ee/danswer/server/tenants/api.py      |  4 ++-
 .../ee/danswer/server/tenants/provisioning.py | 31 +++++++++++++++++++
 .../search/filtering/FilterDropdown.tsx       |  2 +-
 4 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/backend/ee/danswer/configs/app_configs.py b/backend/ee/danswer/configs/app_configs.py
index 2c782241076..d36cef3daf8 100644
--- a/backend/ee/danswer/configs/app_configs.py
+++ b/backend/ee/danswer/configs/app_configs.py
@@ -25,3 +25,7 @@
 
 STRIPE_SECRET_KEY = os.environ.get("STRIPE_SECRET_KEY")
 STRIPE_PRICE_ID = os.environ.get("STRIPE_PRICE")
+
+OPENAI_DEFAULT_API_KEY = os.environ.get("OPENAI_DEFAULT_API_KEY")
+ANTHROPIC_DEFAULT_API_KEY = os.environ.get("ANTHROPIC_DEFAULT_API_KEY")
+COHERE_DEFAULT_API_KEY = os.environ.get("COHERE_DEFAULT_API_KEY")
diff --git a/backend/ee/danswer/server/tenants/api.py b/backend/ee/danswer/server/tenants/api.py
index 05d66b4c582..8e79c0b37b1 100644
--- a/backend/ee/danswer/server/tenants/api.py
+++ b/backend/ee/danswer/server/tenants/api.py
@@ -27,13 +27,13 @@
 from ee.danswer.server.tenants.models import ImpersonateRequest
 from ee.danswer.server.tenants.models import ProductGatingRequest
 from ee.danswer.server.tenants.provisioning import add_users_to_tenant
+from ee.danswer.server.tenants.provisioning import configure_default_api_keys
 from ee.danswer.server.tenants.provisioning import ensure_schema_exists
 from ee.danswer.server.tenants.provisioning import run_alembic_migrations
 from ee.danswer.server.tenants.provisioning import user_owns_a_tenant
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
-
 stripe.api_key = STRIPE_SECRET_KEY
 
 logger = setup_logger()
@@ -68,6 +68,8 @@ def create_tenant(
         with get_session_with_tenant(tenant_id) as db_session:
             setup_danswer(db_session, tenant_id)
 
+            configure_default_api_keys(db_session)
+
         add_users_to_tenant([email], tenant_id)
 
         return {
diff --git a/backend/ee/danswer/server/tenants/provisioning.py b/backend/ee/danswer/server/tenants/provisioning.py
index 311698e6a3d..9106821b5a5 100644
--- a/backend/ee/danswer/server/tenants/provisioning.py
+++ b/backend/ee/danswer/server/tenants/provisioning.py
@@ -10,9 +10,17 @@
 from danswer.db.engine import build_connection_string
 from danswer.db.engine import get_session_with_tenant
 from danswer.db.engine import get_sqlalchemy_engine
+from danswer.db.llm import upsert_cloud_embedding_provider
+from danswer.db.llm import upsert_llm_provider
 from danswer.db.models import UserTenantMapping
+from danswer.server.manage.embedding.models import CloudEmbeddingProviderCreationRequest
+from danswer.server.manage.llm.models import LLMProviderUpsertRequest
 from danswer.utils.logger import setup_logger
+from ee.danswer.configs.app_configs import ANTHROPIC_DEFAULT_API_KEY
+from ee.danswer.configs.app_configs import COHERE_DEFAULT_API_KEY
+from ee.danswer.configs.app_configs import OPENAI_DEFAULT_API_KEY
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
+from shared_configs.enums import EmbeddingProvider
 
 logger = setup_logger()
 
@@ -52,6 +60,29 @@ def run_alembic_migrations(schema_name: str) -> None:
         raise
 
 
+def configure_default_api_keys(db_session: Session) -> None:
+    open_provider = LLMProviderUpsertRequest(
+        name="OpenAI",
+        provider="OpenAI",
+        api_key=OPENAI_DEFAULT_API_KEY,
+        default_model_name="gpt-4o",
+    )
+    anthropic_provider = LLMProviderUpsertRequest(
+        name="Anthropic",
+        provider="Anthropic",
+        api_key=ANTHROPIC_DEFAULT_API_KEY,
+        default_model_name="claude-3-5-sonnet-20240620",
+    )
+    upsert_llm_provider(open_provider, db_session)
+    upsert_llm_provider(anthropic_provider, db_session)
+
+    cloud_embedding_provider = CloudEmbeddingProviderCreationRequest(
+        provider_type=EmbeddingProvider.COHERE,
+        api_key=COHERE_DEFAULT_API_KEY,
+    )
+    upsert_cloud_embedding_provider(db_session, cloud_embedding_provider)
+
+
 def ensure_schema_exists(tenant_id: str) -> bool:
     with Session(get_sqlalchemy_engine()) as db_session:
         with db_session.begin():
diff --git a/web/src/components/search/filtering/FilterDropdown.tsx b/web/src/components/search/filtering/FilterDropdown.tsx
index 444382fc73c..57c60f836c7 100644
--- a/web/src/components/search/filtering/FilterDropdown.tsx
+++ b/web/src/components/search/filtering/FilterDropdown.tsx
@@ -1,4 +1,4 @@
-import { FiCheck, FiChevronDown, FiX, FiXCircle } from "react-icons/fi";
+import { FiCheck, FiChevronDown, FiXCircle } from "react-icons/fi";
 import { CustomDropdown } from "../../Dropdown";
 
 interface Option {

From 1d8e662b79d05a677f08b50d27911da48adfd9c6 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Mon, 4 Nov 2024 11:48:15 -0800
Subject: [PATCH 363/376] ensure we reset all (#3048)

---
 .../danswer/background/celery/apps/primary.py  | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/backend/danswer/background/celery/apps/primary.py b/backend/danswer/background/celery/apps/primary.py
index 7db4b998254..b0950fc8f03 100644
--- a/backend/danswer/background/celery/apps/primary.py
+++ b/backend/danswer/background/celery/apps/primary.py
@@ -19,7 +19,13 @@
 from danswer.configs.constants import POSTGRES_CELERY_WORKER_PRIMARY_APP_NAME
 from danswer.db.engine import SqlEngine
 from danswer.redis.redis_connector_credential_pair import RedisConnectorCredentialPair
+from danswer.redis.redis_connector_delete import RedisConnectorDelete
+from danswer.redis.redis_connector_index import RedisConnectorIndex
+from danswer.redis.redis_connector_prune import RedisConnectorPrune
+from danswer.redis.redis_connector_stop import RedisConnectorStop
+from danswer.redis.redis_document_set import RedisDocumentSet
 from danswer.redis.redis_pool import get_redis_client
+from danswer.redis.redis_usergroup import RedisUserGroup
 from danswer.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 
@@ -113,6 +119,18 @@ def on_worker_init(sender: Any, **kwargs: Any) -> None:
     r.delete(RedisConnectorCredentialPair.get_taskset_key())
     r.delete(RedisConnectorCredentialPair.get_fence_key())
 
+    RedisDocumentSet.reset_all(r)
+
+    RedisUserGroup.reset_all(r)
+
+    RedisConnectorDelete.reset_all(r)
+
+    RedisConnectorPrune.reset_all(r)
+
+    RedisConnectorIndex.reset_all(r)
+
+    RedisConnectorStop.reset_all(r)
+
 
 @worker_ready.connect
 def on_worker_ready(sender: Any, **kwargs: Any) -> None:

From b6ecbbcf453ae0a1153ed93bb27cd85ff9d8b538 Mon Sep 17 00:00:00 2001
From: rkuo-danswer <rkuo@danswer.ai>
Date: Mon, 4 Nov 2024 12:47:56 -0800
Subject: [PATCH 364/376] add to async get session as well (#3046)

---
 backend/danswer/db/engine.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/backend/danswer/db/engine.py b/backend/danswer/db/engine.py
index 6b45f390940..511424e214e 100644
--- a/backend/danswer/db/engine.py
+++ b/backend/danswer/db/engine.py
@@ -310,8 +310,12 @@ async def get_async_session_with_tenant(
         try:
             # Set the search_path to the tenant's schema
             await session.execute(text(f'SET search_path = "{tenant_id}"'))
-        except Exception as e:
-            logger.error(f"Error setting search_path: {str(e)}")
+            if POSTGRES_IDLE_SESSIONS_TIMEOUT:
+                await session.execute(
+                    f"SET SESSION idle_in_transaction_session_timeout = {POSTGRES_IDLE_SESSIONS_TIMEOUT}"
+                )
+        except Exception:
+            logger.exception("Error setting search_path.")
             # You can choose to re-raise the exception or handle it
             # Here, we'll re-raise to prevent proceeding with an incorrect session
             raise

From 9032fb44676b069ba3783d600fc1accfba0c2e56 Mon Sep 17 00:00:00 2001
From: Weves <chrisweaver101@gmail.com>
Date: Mon, 4 Nov 2024 14:47:21 -0800
Subject: [PATCH 365/376] Improve background token refresh

---
 web/src/components/health/healthcheck.tsx | 92 +++++++++++++++--------
 web/src/components/health/refreshUtils.ts | 11 ++-
 web/src/lib/constants.ts                  |  3 +-
 3 files changed, 73 insertions(+), 33 deletions(-)

diff --git a/web/src/components/health/healthcheck.tsx b/web/src/components/health/healthcheck.tsx
index d537f0577bc..d571572dc51 100644
--- a/web/src/components/health/healthcheck.tsx
+++ b/web/src/components/health/healthcheck.tsx
@@ -7,7 +7,7 @@ import { useCallback, useEffect, useState } from "react";
 import { getSecondsUntilExpiration } from "@/lib/time";
 import { User } from "@/lib/types";
 import { mockedRefreshToken, refreshToken } from "./refreshUtils";
-import { CUSTOM_REFRESH_URL } from "@/lib/constants";
+import { NEXT_PUBLIC_CUSTOM_REFRESH_URL } from "@/lib/constants";
 
 export const HealthCheckBanner = () => {
   const { error } = useSWR("/api/health", errorHandlingFetcher);
@@ -35,57 +35,89 @@ export const HealthCheckBanner = () => {
   }, [user, updateExpirationTime]);
 
   useEffect(() => {
-    if (CUSTOM_REFRESH_URL) {
-      const refreshUrl = CUSTOM_REFRESH_URL;
-      let refreshTimeoutId: NodeJS.Timeout;
+    if (NEXT_PUBLIC_CUSTOM_REFRESH_URL) {
+      const refreshUrl = NEXT_PUBLIC_CUSTOM_REFRESH_URL;
+      let refreshIntervalId: NodeJS.Timer;
       let expireTimeoutId: NodeJS.Timeout;
 
       const attemptTokenRefresh = async () => {
-        try {
-          // NOTE: This is a mocked refresh token for testing purposes.
-          // const refreshTokenData = mockedRefreshToken();
-
-          const refreshTokenData = await refreshToken(refreshUrl);
-
-          const response = await fetch(
-            "/api/enterprise-settings/refresh-token",
-            {
-              method: "POST",
-              headers: {
-                "Content-Type": "application/json",
-              },
-              body: JSON.stringify(refreshTokenData),
+        let retryCount = 0;
+        const maxRetries = 3;
+
+        while (retryCount < maxRetries) {
+          try {
+            // NOTE: This is a mocked refresh token for testing purposes.
+            // const refreshTokenData = mockedRefreshToken();
+
+            const refreshTokenData = await refreshToken(refreshUrl);
+            if (!refreshTokenData) {
+              throw new Error("Failed to refresh token");
             }
-          );
-          if (!response.ok) {
-            throw new Error(`HTTP error! status: ${response.status}`);
-          }
-          await new Promise((resolve) => setTimeout(resolve, 4000));
 
-          await mutateUser(undefined, { revalidate: true });
-          updateExpirationTime();
-        } catch (error) {
-          console.error("Error refreshing token:", error);
+            const response = await fetch(
+              "/api/enterprise-settings/refresh-token",
+              {
+                method: "POST",
+                headers: {
+                  "Content-Type": "application/json",
+                },
+                body: JSON.stringify(refreshTokenData),
+              }
+            );
+            if (!response.ok) {
+              throw new Error(`HTTP error! status: ${response.status}`);
+            }
+            await new Promise((resolve) => setTimeout(resolve, 4000));
+
+            await mutateUser(undefined, { revalidate: true });
+            updateExpirationTime();
+            break; // Success - exit the retry loop
+          } catch (error) {
+            console.error(
+              `Error refreshing token (attempt ${
+                retryCount + 1
+              }/${maxRetries}):`,
+              error
+            );
+            retryCount++;
+
+            if (retryCount === maxRetries) {
+              console.error("Max retry attempts reached");
+            } else {
+              // Wait before retrying (exponential backoff)
+              await new Promise((resolve) =>
+                setTimeout(resolve, Math.pow(2, retryCount) * 1000)
+              );
+            }
+          }
         }
       };
 
       const scheduleRefreshAndExpire = () => {
         if (secondsUntilExpiration !== null) {
-          const timeUntilRefresh = (secondsUntilExpiration + 0.5) * 1000;
-          refreshTimeoutId = setTimeout(attemptTokenRefresh, timeUntilRefresh);
+          const refreshInterval = 60 * 15; // 15 mins
+          refreshIntervalId = setInterval(
+            attemptTokenRefresh,
+            refreshInterval * 1000
+          );
 
           const timeUntilExpire = (secondsUntilExpiration + 10) * 1000;
           expireTimeoutId = setTimeout(() => {
             console.debug("Session expired. Setting expired state to true.");
             setExpired(true);
           }, timeUntilExpire);
+
+          // if we're going to timeout before the next refresh, kick off a refresh now!
+          if (secondsUntilExpiration < refreshInterval) {
+            attemptTokenRefresh();
+          }
         }
       };
 
       scheduleRefreshAndExpire();
 
       return () => {
-        clearTimeout(refreshTimeoutId);
+        clearInterval(refreshIntervalId);
         clearTimeout(expireTimeoutId);
       };
     }
diff --git a/web/src/components/health/refreshUtils.ts b/web/src/components/health/refreshUtils.ts
index f478b5e4a49..fea9c76b210 100644
--- a/web/src/components/health/refreshUtils.ts
+++ b/web/src/components/health/refreshUtils.ts
@@ -42,14 +42,21 @@ export function mockedRefreshToken(): CustomRefreshTokenResponse {
 
 export async function refreshToken(
   customRefreshUrl: string
-): Promise<CustomRefreshTokenResponse> {
+): Promise<CustomRefreshTokenResponse | null> {
   try {
     console.debug("Sending request to custom refresh URL");
-    const url = new URL(customRefreshUrl);
+    // support both absolute and relative
+    const url = customRefreshUrl.startsWith("http")
+      ? new URL(customRefreshUrl)
+      : new URL(customRefreshUrl, window.location.origin);
     url.searchParams.append("info", "json");
     url.searchParams.append("access_token_refresh_interval", "3600");
 
     const response = await fetch(url.toString());
+    if (!response.ok) {
+      console.error(`Failed to refresh token: ${await response.text()}`);
+      return null;
+    }
 
     return await response.json();
   } catch (error) {
diff --git a/web/src/lib/constants.ts b/web/src/lib/constants.ts
index de9303c012b..eb8ac34509b 100644
--- a/web/src/lib/constants.ts
+++ b/web/src/lib/constants.ts
@@ -42,7 +42,8 @@ export const NEXT_PUBLIC_DEFAULT_SIDEBAR_OPEN =
 export const TOGGLED_CONNECTORS_COOKIE_NAME = "toggled_connectors";
 
 /* Enterprise-only settings */
-export const CUSTOM_REFRESH_URL = process.env.NEXT_PUBLIC_CUSTOM_REFRESH_URL;
+export const NEXT_PUBLIC_CUSTOM_REFRESH_URL =
+  process.env.NEXT_PUBLIC_CUSTOM_REFRESH_URL;
 
 // NOTE: this should ONLY be used on the server-side. If used client side,
 // it will not be accurate (will always be false).

From dea7a8f697794cf56430fc40f5dcb61e526fd600 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 5 Nov 2024 08:48:19 -0800
Subject: [PATCH 366/376] Clean up tooltips (#3047)

* clean up tooltips

* nit: fix delay duration
---
 web/package-lock.json                         | 463 ++----------------
 web/package.json                              |   2 +-
 .../app/admin/assistants/AssistantEditor.tsx  |  28 +-
 .../[ccPairId]/IndexingAttemptsTable.tsx      |  30 +-
 web/src/app/admin/documents/sets/page.tsx     |  69 +--
 .../status/CCPairIndexingStatusTable.tsx      |  42 +-
 web/src/app/chat/files/InputBarPreview.tsx    |  30 +-
 .../chat/files/documents/DocumentPreview.tsx  |  61 ++-
 web/src/app/chat/input/ChatInputAssistant.tsx |  52 +-
 web/src/app/chat/input/ChatInputBar.tsx       |  44 +-
 web/src/app/chat/message/Messages.tsx         |  34 +-
 web/src/app/chat/message/SearchSummary.tsx    |  32 +-
 .../admin/connectors/AdminSidebar.tsx         |   6 +-
 web/src/components/admin/connectors/Field.tsx |   8 +-
 .../components/assistants/AssistantIcon.tsx   |   1 -
 web/src/components/header/LogoType.tsx        | 104 ++--
 .../search/SearchResultsDisplay.tsx           | 102 ++--
 web/src/components/tooltip/Tooltip.tsx        |  54 --
 web/src/components/ui/tooltip.tsx             |  31 ++
 19 files changed, 438 insertions(+), 755 deletions(-)
 delete mode 100644 web/src/components/tooltip/Tooltip.tsx
 create mode 100644 web/src/components/ui/tooltip.tsx

diff --git a/web/package-lock.json b/web/package-lock.json
index 2b02ef81659..e8c0cd86f15 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -21,7 +21,7 @@
         "@radix-ui/react-separator": "^1.1.0",
         "@radix-ui/react-slot": "^1.1.0",
         "@radix-ui/react-tabs": "^1.1.1",
-        "@radix-ui/react-tooltip": "^1.0.7",
+        "@radix-ui/react-tooltip": "^1.1.3",
         "@sentry/nextjs": "^8.34.0",
         "@stripe/stripe-js": "^4.6.0",
         "@types/js-cookie": "^3.0.3",
@@ -3372,332 +3372,29 @@
       }
     },
     "node_modules/@radix-ui/react-tooltip": {
-      "version": "1.0.7",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.0.7.tgz",
-      "integrity": "sha512-lPh5iKNFVQ/jav/j6ZrWq3blfDJ0OH9R6FlNUHPMqdLuQ9vwDgFsRxvl8b7Asuy5c8xmoojHUxKHQSOAvMHxyw==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-dismissable-layer": "1.0.5",
-        "@radix-ui/react-id": "1.0.1",
-        "@radix-ui/react-popper": "1.1.3",
-        "@radix-ui/react-portal": "1.0.4",
-        "@radix-ui/react-presence": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-slot": "1.0.2",
-        "@radix-ui/react-use-controllable-state": "1.0.1",
-        "@radix-ui/react-visually-hidden": "1.0.3"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-compose-refs": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.1.tgz",
-      "integrity": "sha512-fDSBgd44FKHa1FRMU59qBMPFcl2PZE+2nmqunj+BWFyYYjnhIDWL2ItDs3rrbJDQOtzt5nIebLCQc4QRfz6LJw==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-context": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.0.1.tgz",
-      "integrity": "sha512-ebbrdFoYTcuZ0v4wG5tedGnp9tzcV8awzsxYph7gXUyvnNLuTIcCk1q17JEbnVhXAKG9oX3KtchwiMIAYp9NLg==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-dismissable-layer": {
-      "version": "1.0.5",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.0.5.tgz",
-      "integrity": "sha512-aJeDjQhywg9LBu2t/At58hCvr7pEm0o2Ke1x33B+MhjNmmZ17sy4KImo0KPLgsnc/zN7GPdce8Cnn0SWvwZO7g==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/primitive": "1.0.1",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-escape-keydown": "1.0.3"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
-      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-dismissable-layer/node_modules/@radix-ui/react-use-escape-keydown": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.0.3.tgz",
-      "integrity": "sha512-vyL82j40hcFicA+M4Ex7hVkB9vHgSse1ZWomAqV2Je3RleKGO5iM8KMOEtfoSB0PnIelMd2lATjTGMYqN5ylTg==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-id": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.0.1.tgz",
-      "integrity": "sha512-tI7sT/kqYp8p96yGWY1OAnLHrqDgzHefRBKQ2YAkBS5ja7QLcZ9Z/uY7bEjPUatf8RomoXM8/1sMj1IJaE5UzQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-id/node_modules/@radix-ui/react-use-layout-effect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
-      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper": {
       "version": "1.1.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.1.3.tgz",
-      "integrity": "sha512-cKpopj/5RHZWjrbF2846jBNacjQVwkP068DfmgrNJXpvVWrOvlAmE9xSiy5OqeE+Gi8D9fP+oDhUnPqNMY8/5w==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@floating-ui/react-dom": "^2.0.0",
-        "@radix-ui/react-arrow": "1.0.3",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-context": "1.0.1",
-        "@radix-ui/react-primitive": "1.0.3",
-        "@radix-ui/react-use-callback-ref": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1",
-        "@radix-ui/react-use-rect": "1.0.1",
-        "@radix-ui/react-use-size": "1.0.1",
-        "@radix-ui/rect": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-arrow": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.0.3.tgz",
-      "integrity": "sha512-wSP+pHsB/jQRaL6voubsQ/ZlrGBHHrOjmBnr19hxYgtS0WvAFwZhK2WP/YY5yF9uKECCEEDGxuLxq1NBK51wFA==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
-      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-layout-effect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
-      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-rect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.0.1.tgz",
-      "integrity": "sha512-Cq5DLuSiuYVKNU8orzJMbl15TXilTnJKUCltMVQg53BQOF1/C5toAaGrowkgksdBQ9H+SRL23g0HDmg9tvmxXw==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/rect": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-popper/node_modules/@radix-ui/react-use-size": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.0.1.tgz",
-      "integrity": "sha512-ibay+VqrgcaI6veAojjofPATwledXiSmX+C0KrBk/xgpX9rBzPV3OsfwlhQdUOFbh+LKQorLYT+xTXW9V8yd0g==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-portal": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.0.4.tgz",
-      "integrity": "sha512-Qki+C/EuGUVCQTOTD5vzJzJuMUlewbzuKyUy+/iHM2uwGiru9gZeBJtHAPKAEkB5KWGi9mP/CHKcY0wt1aW45Q==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-presence": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.0.1.tgz",
-      "integrity": "sha512-UXLW4UAbIY5ZjcvzjfRFo5gxva8QirC9hF7wRE4U5gz+TP0DbRk+//qyuAQ1McDxBt1xNMBTaciFGvEmJvAZCg==",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.1.3.tgz",
+      "integrity": "sha512-Z4w1FIS0BqVFI2c1jZvb/uDVJijJjJ2ZMuPV81oVgTZ7g3BZxobplnMVvXtFWgtozdvYJ+MFWtwkM5S2HnAong==",
+      "license": "MIT",
       "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1",
-        "@radix-ui/react-use-layout-effect": "1.0.1"
+        "@radix-ui/primitive": "1.1.0",
+        "@radix-ui/react-compose-refs": "1.1.0",
+        "@radix-ui/react-context": "1.1.1",
+        "@radix-ui/react-dismissable-layer": "1.1.1",
+        "@radix-ui/react-id": "1.1.0",
+        "@radix-ui/react-popper": "1.2.0",
+        "@radix-ui/react-portal": "1.1.2",
+        "@radix-ui/react-presence": "1.1.1",
+        "@radix-ui/react-primitive": "2.0.0",
+        "@radix-ui/react-slot": "1.1.0",
+        "@radix-ui/react-use-controllable-state": "1.1.0",
+        "@radix-ui/react-visually-hidden": "1.1.0"
       },
       "peerDependencies": {
         "@types/react": "*",
         "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc",
+        "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
@@ -3708,131 +3405,27 @@
         }
       }
     },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-presence/node_modules/@radix-ui/react-use-layout-effect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.0.1.tgz",
-      "integrity": "sha512-v/5RegiJWYdoCvMnITBkNNx6bCj20fiaJnWtRkU18yITptraXjffz5Qbn05uOiQnOvi+dbkznkoaMltz1GnszQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-primitive": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-1.0.3.tgz",
-      "integrity": "sha512-yi58uVyoAcK/Nq1inRY56ZSjKypBNKTa/1mcL8qdl6oJeEaDbOldlzrGn7P6Q3Id5d+SYNGc5AJgc4vGhjs5+g==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-slot": "1.0.2"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
-        }
-      }
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/primitive": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.0.tgz",
+      "integrity": "sha512-4Z8dn6Upk0qk4P74xBhZ6Hd/w0mPEzOOLxy4xiPXOXqjF7jZS0VAKk7/x/H6FyY2zCkYJqePf1G5KmkmNJ4RBA==",
+      "license": "MIT"
     },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-slot": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.0.2.tgz",
-      "integrity": "sha512-YeTpuq4deV+6DusvVUW4ivBgnkHwECUu0BiN43L5UCDFgdhsRUWAghhTF5MbvNTPzmiFOx90asDSUjWuCNapwg==",
+    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-context": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.1.tgz",
+      "integrity": "sha512-UASk9zi+crv9WteK/NU4PLvOoL3OuE6BWVKNF6hPRBtYBDXQ2u5iu3O59zUlJiTVvkyuycnqrztsHVJwcK9K+Q==",
       "license": "MIT",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-compose-refs": "1.0.1"
-      },
       "peerDependencies": {
         "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-use-controllable-state": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.0.1.tgz",
-      "integrity": "sha512-Svl5GY5FQeN758fWKrjM6Qb7asvXeiZltlT4U2gVfl8Gx5UAv2sMR0LWo8yhsIZh2oQ0eFdZ59aoOOMV7b47VA==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-use-callback-ref": "1.0.1"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-use-controllable-state/node_modules/@radix-ui/react-use-callback-ref": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.0.1.tgz",
-      "integrity": "sha512-D94LjX4Sp0xJFVaoQOd3OO9k7tpBYNOXdVhkltUbGv2Qb9OXdrg/CpsjlZv7ia14Sylv398LswWBVVu5nqKzAQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "react": "^16.8 || ^17.0 || ^18.0"
-      },
-      "peerDependenciesMeta": {
-        "@types/react": {
-          "optional": true
-        }
-      }
-    },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/react-visually-hidden": {
-      "version": "1.0.3",
-      "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.0.3.tgz",
-      "integrity": "sha512-D4w41yN5YRKtu464TLnByKzMDG/JlMPHtfZgQAu9v6mNakUqGUI9vUrfQKz8NK41VMm/xbZbh76NUTVtIYqOMA==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10",
-        "@radix-ui/react-primitive": "1.0.3"
-      },
-      "peerDependencies": {
-        "@types/react": "*",
-        "@types/react-dom": "*",
-        "react": "^16.8 || ^17.0 || ^18.0",
-        "react-dom": "^16.8 || ^17.0 || ^18.0"
+        "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc"
       },
       "peerDependenciesMeta": {
         "@types/react": {
           "optional": true
-        },
-        "@types/react-dom": {
-          "optional": true
         }
       }
     },
-    "node_modules/@radix-ui/react-tooltip/node_modules/@radix-ui/rect": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.0.1.tgz",
-      "integrity": "sha512-fyrgCaedtvMg9NK3en0pnOYJdtfwxUcNolezkNPUsoX57X8oQk+NkqcvzHXD2uKNij6GXmWU9NDru2IWjrO4BQ==",
-      "dependencies": {
-        "@babel/runtime": "^7.13.10"
-      }
-    },
     "node_modules/@radix-ui/react-use-callback-ref": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.0.tgz",
diff --git a/web/package.json b/web/package.json
index 6fb1a3b2903..6f8aed2e738 100644
--- a/web/package.json
+++ b/web/package.json
@@ -22,7 +22,7 @@
     "@radix-ui/react-separator": "^1.1.0",
     "@radix-ui/react-slot": "^1.1.0",
     "@radix-ui/react-tabs": "^1.1.1",
-    "@radix-ui/react-tooltip": "^1.0.7",
+    "@radix-ui/react-tooltip": "^1.1.3",
     "@sentry/nextjs": "^8.34.0",
     "@stripe/stripe-js": "^4.6.0",
     "@types/js-cookie": "^3.0.3",
diff --git a/web/src/app/admin/assistants/AssistantEditor.tsx b/web/src/app/admin/assistants/AssistantEditor.tsx
index 51c2cb4c3ed..b01b6f32588 100644
--- a/web/src/app/admin/assistants/AssistantEditor.tsx
+++ b/web/src/app/admin/assistants/AssistantEditor.tsx
@@ -36,7 +36,7 @@ import {
   TooltipContent,
   TooltipProvider,
   TooltipTrigger,
-} from "@radix-ui/react-tooltip";
+} from "@/components/ui/tooltip";
 import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { useEffect, useState } from "react";
@@ -565,15 +565,13 @@ export function AssistantEditor({
                   align="start"
                   side="bottom"
                 />
-                <TooltipProvider delayDuration={50}>
+                <TooltipProvider>
                   <Tooltip>
                     <TooltipTrigger>
                       <FiInfo size={12} />
                     </TooltipTrigger>
                     <TooltipContent side="top" align="center">
-                      <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-white">
-                        This icon will visually represent your Assistant
-                      </p>
+                      This icon will visually represent your Assistant
                     </TooltipContent>
                   </Tooltip>
                 </TooltipProvider>
@@ -609,16 +607,14 @@ export function AssistantEditor({
                   <div className="block  font-medium text-base">
                     Default AI Model{" "}
                   </div>
-                  <TooltipProvider delayDuration={50}>
+                  <TooltipProvider>
                     <Tooltip>
                       <TooltipTrigger>
                         <FiInfo size={12} />
                       </TooltipTrigger>
                       <TooltipContent side="top" align="center">
-                        <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-white">
-                          Select a Large Language Model (Generative AI model) to
-                          power this Assistant
-                        </p>
+                        Select a Large Language Model (Generative AI model) to
+                        power this Assistant
                       </TooltipContent>
                     </Tooltip>
                   </TooltipProvider>
@@ -706,16 +702,14 @@ export function AssistantEditor({
                   <div className="block font-medium text-base">
                     Capabilities{" "}
                   </div>
-                  <TooltipProvider delayDuration={50}>
+                  <TooltipProvider>
                     <Tooltip>
                       <TooltipTrigger>
                         <FiInfo size={12} />
                       </TooltipTrigger>
                       <TooltipContent side="top" align="center">
-                        <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-white">
-                          You can give your assistant advanced capabilities like
-                          image generation
-                        </p>
+                        You can give your assistant advanced capabilities like
+                        image generation
                       </TooltipContent>
                     </Tooltip>
                   </TooltipProvider>
@@ -726,7 +720,7 @@ export function AssistantEditor({
 
                 <div className="mt-4 flex flex-col gap-y-4  ml-1">
                   {imageGenerationTool && (
-                    <TooltipProvider delayDuration={50}>
+                    <TooltipProvider>
                       <Tooltip>
                         <TooltipTrigger asChild>
                           <div
@@ -774,7 +768,7 @@ export function AssistantEditor({
                   )}
 
                   {searchTool && (
-                    <TooltipProvider delayDuration={50}>
+                    <TooltipProvider>
                       <Tooltip>
                         <TooltipTrigger asChild>
                           <div
diff --git a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
index ac39f3c93d4..78356310730 100644
--- a/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
+++ b/web/src/app/admin/connector/[ccPairId]/IndexingAttemptsTable.tsx
@@ -23,7 +23,12 @@ import { InfoIcon, SearchIcon } from "@/components/icons/icons";
 import Link from "next/link";
 import ExceptionTraceModal from "@/components/modals/ExceptionTraceModal";
 import { useRouter } from "next/navigation";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { FiInfo } from "react-icons/fi";
 
 // This is the number of index attempts to display per page
@@ -237,15 +242,20 @@ export function IndexingAttemptsTable({ ccPair }: { ccPair: CCPairFullInfo }) {
             <TableHead>New Doc Cnt</TableHead>
             <TableHead>
               <div className="w-fit">
-                <Tooltip
-                  width="max-w-sm"
-                  content="Total number of documents replaced in the index during this indexing attempt"
-                >
-                  <span className="cursor-help flex items-center">
-                    Total Doc Cnt
-                    <InfoIcon className="ml-1 w-4 h-4" />
-                  </span>
-                </Tooltip>
+                <TooltipProvider>
+                  <Tooltip>
+                    <TooltipTrigger asChild>
+                      <span className="cursor-help flex items-center">
+                        Total Doc Cnt
+                        <InfoIcon className="ml-1 w-4 h-4" />
+                      </span>
+                    </TooltipTrigger>
+                    <TooltipContent>
+                      Total number of documents replaced in the index during
+                      this indexing attempt
+                    </TooltipContent>
+                  </Tooltip>
+                </TooltipProvider>
               </div>
             </TableHead>
             <TableHead>Error Message</TableHead>
diff --git a/web/src/app/admin/documents/sets/page.tsx b/web/src/app/admin/documents/sets/page.tsx
index 2d5296d33f2..5d443cd01ce 100644
--- a/web/src/app/admin/documents/sets/page.tsx
+++ b/web/src/app/admin/documents/sets/page.tsx
@@ -35,6 +35,12 @@ import Link from "next/link";
 import { useRouter } from "next/navigation";
 import { TableHeader } from "@/components/ui/table";
 import { Badge } from "@/components/ui/badge";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 
 const numToDisplay = 50;
 
@@ -47,8 +53,6 @@ const EditRow = ({
 }) => {
   const router = useRouter();
 
-  const [isSyncingTooltipOpen, setIsSyncingTooltipOpen] = useState(false);
-
   if (!isEditable) {
     return (
       <div className="text-emphasis font-medium my-auto p-1">
@@ -59,37 +63,36 @@ const EditRow = ({
 
   return (
     <div className="relative flex">
-      {isSyncingTooltipOpen && (
-        <div className="flex flex-nowrap absolute w-64 top-0 left-0 mt-8 border border-border bg-background px-3 py-2 rounded shadow-lg break-words z-40">
-          <InfoIcon className="mt-1 flex flex-shrink-0 mr-2" /> Cannot update
-          while syncing! Wait for the sync to finish, then try again.
-        </div>
-      )}
-      <div
-        className={`
-          text-emphasis font-medium my-auto p-1 hover:bg-hover-light flex items-center select-none
-          ${documentSet.is_up_to_date ? "cursor-pointer" : "cursor-default"}
-        `}
-        style={{ wordBreak: "normal", overflowWrap: "break-word" }}
-        onClick={() => {
-          if (documentSet.is_up_to_date) {
-            router.push(`/admin/documents/sets/${documentSet.id}`);
-          }
-        }}
-        onMouseEnter={() => {
-          if (!documentSet.is_up_to_date) {
-            setIsSyncingTooltipOpen(true);
-          }
-        }}
-        onMouseLeave={() => {
-          if (!documentSet.is_up_to_date) {
-            setIsSyncingTooltipOpen(false);
-          }
-        }}
-      >
-        <FiEdit2 className="mr-2 flex-shrink-0" />
-        <span className="font-medium">{documentSet.name}</span>
-      </div>
+      <TooltipProvider>
+        <Tooltip>
+          <TooltipTrigger asChild>
+            <div
+              className={`
+              text-emphasis font-medium my-auto p-1 hover:bg-hover-light flex items-center select-none
+              ${documentSet.is_up_to_date ? "cursor-pointer" : "cursor-default"}
+            `}
+              style={{ wordBreak: "normal", overflowWrap: "break-word" }}
+              onClick={() => {
+                if (documentSet.is_up_to_date) {
+                  router.push(`/admin/documents/sets/${documentSet.id}`);
+                }
+              }}
+            >
+              <FiEdit2 className="mr-2 flex-shrink-0" />
+              <span className="font-medium">{documentSet.name}</span>
+            </div>
+          </TooltipTrigger>
+          {!documentSet.is_up_to_date && (
+            <TooltipContent maxWidth="max-w-sm">
+              <div className="flex break-words break-keep whitespace-pre-wrap items-start">
+                <InfoIcon className="mr-2 mt-0.5" />
+                Cannot update while syncing! Wait for the sync to finish, then
+                try again.
+              </div>
+            </TooltipContent>
+          )}
+        </Tooltip>
+      </TooltipProvider>
     </div>
   );
 };
diff --git a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
index 833f1de674e..3b04e4a65d7 100644
--- a/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
+++ b/web/src/app/admin/indexing/status/CCPairIndexingStatusTable.tsx
@@ -27,7 +27,12 @@ import {
   FiRefreshCw,
   FiPauseCircle,
 } from "react-icons/fi";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { SourceIcon } from "@/components/SourceIcon";
 import { getSourceDisplayName } from "@/lib/sources";
 import { CustomTooltip } from "@/components/tooltip/CustomTooltip";
@@ -77,21 +82,26 @@ function SummaryRow({
 
       <TableCell>
         <div className="text-sm text-gray-500">Active Connectors</div>
-        <Tooltip
-          content={`${summary.active} out of ${summary.count} connectors are active`}
-        >
-          <div className="flex items-center mt-1">
-            <div className="w-full bg-gray-200 rounded-full h-2 mr-2">
-              <div
-                className="bg-green-500 h-2 rounded-full"
-                style={{ width: `${activePercentage}%` }}
-              ></div>
-            </div>
-            <span className="text-sm font-medium whitespace-nowrap">
-              {summary.active} ({activePercentage.toFixed(0)}%)
-            </span>
-          </div>
-        </Tooltip>
+        <TooltipProvider>
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <div className="flex items-center mt-1">
+                <div className="w-full bg-gray-200 rounded-full h-2 mr-2">
+                  <div
+                    className="bg-green-500 h-2 rounded-full"
+                    style={{ width: `${activePercentage}%` }}
+                  ></div>
+                </div>
+                <span className="text-sm font-medium whitespace-nowrap">
+                  {summary.active} ({activePercentage.toFixed(0)}%)
+                </span>
+              </div>
+            </TooltipTrigger>
+            <TooltipContent>
+              {summary.active} out of {summary.count} connectors are active
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
       </TableCell>
 
       {isPaidEnterpriseFeaturesEnabled && (
diff --git a/web/src/app/chat/files/InputBarPreview.tsx b/web/src/app/chat/files/InputBarPreview.tsx
index d218db04f51..6cd9478ec07 100644
--- a/web/src/app/chat/files/InputBarPreview.tsx
+++ b/web/src/app/chat/files/InputBarPreview.tsx
@@ -3,7 +3,12 @@ import { FileDescriptor } from "../interfaces";
 
 import { FiX, FiLoader, FiFileText } from "react-icons/fi";
 import { InputBarPreviewImage } from "./images/InputBarPreviewImage";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 
 function DeleteButton({ onDelete }: { onDelete: () => void }) {
   return (
@@ -138,14 +143,21 @@ export function InputBarPreview({
           </div>
         </div>
         <div className="ml-2 relative">
-          <Tooltip content={file.name} side="top" align="start">
-            <div
-              ref={fileNameRef}
-              className={`font-medium text-sm line-clamp-1 break-all ellipses max-w-48`}
-            >
-              {file.name}
-            </div>
-          </Tooltip>
+          <TooltipProvider>
+            <Tooltip>
+              <TooltipTrigger asChild>
+                <div
+                  ref={fileNameRef}
+                  className={`font-medium text-sm line-clamp-1 break-all ellipses max-w-48`}
+                >
+                  {file.name}
+                </div>
+              </TooltipTrigger>
+              <TooltipContent side="top" align="start">
+                {file.name}
+              </TooltipContent>
+            </Tooltip>
+          </TooltipProvider>
         </div>
         <button
           onClick={onDelete}
diff --git a/web/src/app/chat/files/documents/DocumentPreview.tsx b/web/src/app/chat/files/documents/DocumentPreview.tsx
index e4bc05c541f..ed1dc43e797 100644
--- a/web/src/app/chat/files/documents/DocumentPreview.tsx
+++ b/web/src/app/chat/files/documents/DocumentPreview.tsx
@@ -1,6 +1,11 @@
 import { FiFileText } from "react-icons/fi";
 import { useState, useRef, useEffect } from "react";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { ExpandTwoIcon } from "@/components/icons/icons";
 
 export function DocumentPreview({
@@ -52,16 +57,23 @@ export function DocumentPreview({
         </div>
       </div>
       <div className="ml-4 flex-grow">
-        <Tooltip content={fileName} side="top" align="start">
-          <div
-            ref={fileNameRef}
-            className={`font-medium text-sm line-clamp-1 break-all ellipsis ${
-              maxWidth ? maxWidth : "max-w-48"
-            }`}
-          >
-            {fileName}
-          </div>
-        </Tooltip>
+        <TooltipProvider>
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <div
+                ref={fileNameRef}
+                className={`font-medium text-sm line-clamp-1 break-all ellipsis ${
+                  maxWidth ? maxWidth : "max-w-48"
+                }`}
+              >
+                {fileName}
+              </div>
+            </TooltipTrigger>
+            <TooltipContent side="top" align="start">
+              {fileName}
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
         <div className="text-subtle text-xs mt-1">Document</div>
       </div>
       {open && (
@@ -128,16 +140,23 @@ export function InputDocumentPreview({
         </div>
       </div>
       <div className="ml-2 relative">
-        <Tooltip content={fileName} side="top" align="start">
-          <div
-            ref={fileNameRef}
-            className={`font-medium text-sm line-clamp-1 break-all ellipses ${
-              maxWidth ? maxWidth : "max-w-48"
-            }`}
-          >
-            {fileName}
-          </div>
-        </Tooltip>
+        <TooltipProvider>
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <div
+                ref={fileNameRef}
+                className={`font-medium text-sm line-clamp-1 break-all ellipses ${
+                  maxWidth ? maxWidth : "max-w-48"
+                }`}
+              >
+                {fileName}
+              </div>
+            </TooltipTrigger>
+            <TooltipContent side="top" align="start">
+              {fileName}
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
       </div>
     </div>
   );
diff --git a/web/src/app/chat/input/ChatInputAssistant.tsx b/web/src/app/chat/input/ChatInputAssistant.tsx
index d2d062eb2f1..26c0894f5bb 100644
--- a/web/src/app/chat/input/ChatInputAssistant.tsx
+++ b/web/src/app/chat/input/ChatInputAssistant.tsx
@@ -2,7 +2,12 @@
 
 import { Persona } from "@/app/admin/assistants/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { ForwardedRef, forwardRef, useState } from "react";
 import { FiX } from "react-icons/fi";
 
@@ -23,27 +28,30 @@ export const ChatInputAssistant = forwardRef<
       onMouseLeave={() => setIsHovered(false)}
       className="flex-none h-10 duration-300 h-10 items-center rounded-lg bg-background-150"
     >
-      <Tooltip
-        content={
-          <p className="max-w-xs flex">{alternativeAssistant.description}</p>
-        }
-      >
-        <div
-          ref={ref}
-          className="p-2 gap-x-1 relative rounded-t-lg items-center flex"
-        >
-          <AssistantIcon assistant={alternativeAssistant} border />
-          <p className="ml-1 line-clamp-1 ellipsis break-all my-auto">
-            {alternativeAssistant.name}
-          </p>
-          <div
-            className="rounded-lg rounded h-fit cursor-pointer"
-            onClick={unToggle}
-          >
-            <FiX />
-          </div>
-        </div>
-      </Tooltip>
+      <TooltipProvider>
+        <Tooltip>
+          <TooltipTrigger asChild>
+            <div
+              ref={ref}
+              className="p-2 gap-x-1 relative rounded-t-lg items-center flex"
+            >
+              <AssistantIcon assistant={alternativeAssistant} border />
+              <p className="ml-1 line-clamp-1 ellipsis break-all my-auto">
+                {alternativeAssistant.name}
+              </p>
+              <div
+                className="rounded-lg rounded h-fit cursor-pointer"
+                onClick={unToggle}
+              >
+                <FiX />
+              </div>
+            </div>
+          </TooltipTrigger>
+          <TooltipContent>
+            <p className="max-w-xs flex">{alternativeAssistant.description}</p>
+          </TooltipContent>
+        </Tooltip>
+      </TooltipProvider>
     </div>
   );
 });
diff --git a/web/src/app/chat/input/ChatInputBar.tsx b/web/src/app/chat/input/ChatInputBar.tsx
index 78afa5381d6..9dd3d5274c4 100644
--- a/web/src/app/chat/input/ChatInputBar.tsx
+++ b/web/src/app/chat/input/ChatInputBar.tsx
@@ -29,7 +29,12 @@ import { LlmTab } from "../modal/configuration/LlmTab";
 import { AssistantsTab } from "../modal/configuration/AssistantsTab";
 import { DanswerDocument } from "@/lib/search/interfaces";
 import { AssistantIcon } from "@/components/assistants/AssistantIcon";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { Hoverable } from "@/components/Hoverable";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
 import { ChatState } from "../types";
@@ -399,17 +404,20 @@ export function ChatInputBar({
                     {alternativeAssistant.name}
                   </p>
                   <div className="flex gap-x-1 ml-auto">
-                    <Tooltip
-                      content={
-                        <p className="max-w-xs flex flex-wrap">
-                          {alternativeAssistant.description}
-                        </p>
-                      }
-                    >
-                      <button>
-                        <Hoverable icon={FiInfo} />
-                      </button>
-                    </Tooltip>
+                    <TooltipProvider>
+                      <Tooltip>
+                        <TooltipTrigger asChild>
+                          <button>
+                            <Hoverable icon={FiInfo} />
+                          </button>
+                        </TooltipTrigger>
+                        <TooltipContent>
+                          <p className="max-w-xs flex flex-wrap">
+                            {alternativeAssistant.description}
+                          </p>
+                        </TooltipContent>
+                      </Tooltip>
+                    </TooltipProvider>
 
                     <Hoverable
                       icon={FiX}
@@ -613,7 +621,11 @@ export function ChatInputBar({
               chatState == "toolBuilding" ||
               chatState == "loading" ? (
                 <button
-                  className={`cursor-pointer ${chatState != "streaming" ? "bg-background-400" : "bg-background-800"}  h-[28px] w-[28px] rounded-full`}
+                  className={`cursor-pointer ${
+                    chatState != "streaming"
+                      ? "bg-background-400"
+                      : "bg-background-800"
+                  }  h-[28px] w-[28px] rounded-full`}
                   onClick={stopGenerating}
                   disabled={chatState != "streaming"}
                 >
@@ -635,7 +647,11 @@ export function ChatInputBar({
                 >
                   <SendIcon
                     size={28}
-                    className={`text-emphasis text-white p-1 rounded-full  ${chatState == "input" && message ? "bg-submit-background" : "bg-disabled-submit-background"} `}
+                    className={`text-emphasis text-white p-1 rounded-full  ${
+                      chatState == "input" && message
+                        ? "bg-submit-background"
+                        : "bg-disabled-submit-background"
+                    } `}
                   />
                 </button>
               )}
diff --git a/web/src/app/chat/message/Messages.tsx b/web/src/app/chat/message/Messages.tsx
index 2939c74d3f4..c420a3f53e9 100644
--- a/web/src/app/chat/message/Messages.tsx
+++ b/web/src/app/chat/message/Messages.tsx
@@ -45,7 +45,12 @@ import {
   TooltipGroup,
 } from "@/components/tooltip/CustomTooltip";
 import { ValidSources } from "@/lib/types";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { useMouseTracking } from "./hooks";
 import { InternetSearchIcon } from "@/components/InternetSearchIcon";
 import { SettingsContext } from "@/components/settings/SettingsProvider";
@@ -895,17 +900,22 @@ export const HumanMessage = ({
                       isHovered &&
                       !isEditing &&
                       (!files || files.length === 0) ? (
-                        <Tooltip delayDuration={1000} content={"Edit message"}>
-                          <button
-                            className="hover:bg-hover p-1.5 rounded"
-                            onClick={() => {
-                              setIsEditing(true);
-                              setIsHovered(false);
-                            }}
-                          >
-                            <FiEdit2 className="!h-4 !w-4" />
-                          </button>
-                        </Tooltip>
+                        <TooltipProvider delayDuration={1000}>
+                          <Tooltip>
+                            <TooltipTrigger>
+                              <button
+                                className="hover:bg-hover p-1.5 rounded"
+                                onClick={() => {
+                                  setIsEditing(true);
+                                  setIsHovered(false);
+                                }}
+                              >
+                                <FiEdit2 className="!h-4 !w-4" />
+                              </button>
+                            </TooltipTrigger>
+                            <TooltipContent>Edit</TooltipContent>
+                          </Tooltip>
+                        </TooltipProvider>
                       ) : (
                         <div className="w-7" />
                       )}
diff --git a/web/src/app/chat/message/SearchSummary.tsx b/web/src/app/chat/message/SearchSummary.tsx
index 6b7ea248e82..f86212fd290 100644
--- a/web/src/app/chat/message/SearchSummary.tsx
+++ b/web/src/app/chat/message/SearchSummary.tsx
@@ -4,7 +4,12 @@ import {
 } from "@/components/BasicClickable";
 import { HoverPopup } from "@/components/HoverPopup";
 import { Hoverable } from "@/components/Hoverable";
-import { Tooltip } from "@/components/tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { useEffect, useRef, useState } from "react";
 import { FiCheck, FiEdit2, FiSearch, FiX } from "react-icons/fi";
 
@@ -169,16 +174,21 @@ export function SearchSummary({
             )}
           </div>
           {handleSearchQueryEdit && (
-            <Tooltip delayDuration={1000} content={"Edit Search"}>
-              <button
-                className="my-auto hover:bg-hover p-1.5 rounded"
-                onClick={() => {
-                  setIsEditing(true);
-                }}
-              >
-                <FiEdit2 />
-              </button>
-            </Tooltip>
+            <TooltipProvider delayDuration={1000}>
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <button
+                    className="my-auto hover:bg-hover p-1.5 rounded"
+                    onClick={() => {
+                      setIsEditing(true);
+                    }}
+                  >
+                    <FiEdit2 />
+                  </button>
+                </TooltipTrigger>
+                <TooltipContent>Edit Search</TooltipContent>
+              </Tooltip>
+            </TooltipProvider>
           )}
         </>
       )}
diff --git a/web/src/components/admin/connectors/AdminSidebar.tsx b/web/src/components/admin/connectors/AdminSidebar.tsx
index e07ca690fbc..b4010d36496 100644
--- a/web/src/components/admin/connectors/AdminSidebar.tsx
+++ b/web/src/components/admin/connectors/AdminSidebar.tsx
@@ -13,7 +13,7 @@ import {
   TooltipContent,
   TooltipProvider,
   TooltipTrigger,
-} from "@radix-ui/react-tooltip";
+} from "@/components/ui/tooltip";
 
 interface Item {
   name: string | JSX.Element;
@@ -105,9 +105,7 @@ export function AdminSidebar({ collections }: { collections: Collection[] }) {
                           <WarningCircle size={18} className="text-error" />
                         </TooltipTrigger>
                         <TooltipContent>
-                          <p className="max-w-xs text-text-100 mb-1 p-2 rounded-lg bg-background-900">
-                            Navigate here to update your search settings
-                          </p>
+                          Navigate here to update your search settings
                         </TooltipContent>
                       </Tooltip>
                     </TooltipProvider>
diff --git a/web/src/components/admin/connectors/Field.tsx b/web/src/components/admin/connectors/Field.tsx
index 271a7447f3d..106478869e3 100644
--- a/web/src/components/admin/connectors/Field.tsx
+++ b/web/src/components/admin/connectors/Field.tsx
@@ -22,7 +22,7 @@ import {
   Tooltip,
   TooltipContent,
   TooltipTrigger,
-} from "@radix-ui/react-tooltip";
+} from "@/components/ui/tooltip";
 import ReactMarkdown from "react-markdown";
 import { FaMarkdown } from "react-icons/fa";
 import { useRef, useState } from "react";
@@ -107,15 +107,13 @@ export function ToolTipDetails({
   children: string | JSX.Element;
 }) {
   return (
-    <TooltipProvider delayDuration={50}>
+    <TooltipProvider>
       <Tooltip>
         <TooltipTrigger>
           <FiInfo size={12} />
         </TooltipTrigger>
         <TooltipContent side="top" align="center">
-          <p className="bg-background-900 max-w-[200px] mb-1 text-sm rounded-lg p-1.5 text-inverted">
-            {children}
-          </p>
+          {children}
         </TooltipContent>
       </Tooltip>
     </TooltipProvider>
diff --git a/web/src/components/assistants/AssistantIcon.tsx b/web/src/components/assistants/AssistantIcon.tsx
index 4126fdfca2f..9586528d0c8 100644
--- a/web/src/components/assistants/AssistantIcon.tsx
+++ b/web/src/components/assistants/AssistantIcon.tsx
@@ -1,6 +1,5 @@
 import { Persona } from "@/app/admin/assistants/interfaces";
 import React from "react";
-import { Tooltip } from "../tooltip/Tooltip";
 import { createSVG } from "@/lib/assistantIconUtils";
 import { buildImgUrl } from "@/app/chat/files/images/utils";
 import { CustomTooltip } from "../tooltip/CustomTooltip";
diff --git a/web/src/components/header/LogoType.tsx b/web/src/components/header/LogoType.tsx
index 32981bbe26c..365c734ade9 100644
--- a/web/src/components/header/LogoType.tsx
+++ b/web/src/components/header/LogoType.tsx
@@ -7,7 +7,12 @@ import {
   NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA,
 } from "@/lib/constants";
 import { LeftToLineIcon, NewChatIcon, RightToLineIcon } from "../icons/icons";
-import { Tooltip } from "../tooltip/Tooltip";
+import {
+  Tooltip,
+  TooltipContent,
+  TooltipProvider,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import { pageType } from "@/app/chat/sessionSidebar/types";
 import { Logo } from "../Logo";
 import { HeaderTitle } from "./HeaderTitle";
@@ -39,7 +44,7 @@ export default function LogoType({
     <div
       className={`${
         hideOnMobile && "mobile:hidden"
-      } z-[100] mb-auto shrink-0 flex items-center text-xl font-bold`}
+      } z-[100] mb-auto shrink-0 flex items-center text-xl`}
     >
       {toggleSidebar && page == "chat" ? (
         <button
@@ -76,51 +81,60 @@ export default function LogoType({
       </div>
 
       {page == "chat" && !showArrow && (
-        <Tooltip delayDuration={1000} content="New Chat">
-          <Link
-            className="my-auto mobile:hidden"
-            href={
-              `/${page}` +
-              (NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA && assistantId
-                ? `?assistantId=${assistantId}`
-                : "")
-            }
-            onClick={(e) => {
-              if (e.metaKey || e.ctrlKey) {
-                return;
-              }
-              if (handleNewChat) {
-                handleNewChat();
-              }
-            }}
-          >
-            <div className="cursor-pointer ml-2 flex-none text-text-700 hover:text-text-600 transition-colors duration-300">
-              <NewChatIcon size={20} />
-            </div>
-          </Link>
-        </Tooltip>
+        <TooltipProvider delayDuration={1000}>
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <Link
+                className="my-auto mobile:hidden"
+                href={
+                  `/${page}` +
+                  (NEXT_PUBLIC_NEW_CHAT_DIRECTS_TO_SAME_PERSONA && assistantId
+                    ? `?assistantId=${assistantId}`
+                    : "")
+                }
+                onClick={(e) => {
+                  if (e.metaKey || e.ctrlKey) {
+                    return;
+                  }
+                  if (handleNewChat) {
+                    handleNewChat();
+                  }
+                }}
+              >
+                <div className="cursor-pointer ml-2 flex-none text-text-700 hover:text-text-600 transition-colors duration-300">
+                  <NewChatIcon size={20} />
+                </div>
+              </Link>
+            </TooltipTrigger>
+            <TooltipContent>New Chat</TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
       )}
       {showArrow && toggleSidebar && (
-        <Tooltip
-          delayDuration={0}
-          content={toggled ? `Unpin sidebar` : "Pin sidebar"}
-        >
-          <button
-            className="mr-3 my-auto ml-auto"
-            onClick={() => {
-              toggleSidebar();
-              if (toggled) {
-                explicitlyUntoggle();
-              }
-            }}
-          >
-            {!toggled && !combinedSettings?.isMobile ? (
-              <RightToLineIcon className="text-sidebar-toggle" />
-            ) : (
-              <LeftToLineIcon className="text-sidebar-toggle" />
-            )}
-          </button>
-        </Tooltip>
+        <TooltipProvider delayDuration={0}>
+          <Tooltip>
+            <TooltipTrigger asChild>
+              <button
+                className="mr-3 my-auto ml-auto"
+                onClick={() => {
+                  toggleSidebar();
+                  if (toggled) {
+                    explicitlyUntoggle();
+                  }
+                }}
+              >
+                {!toggled && !combinedSettings?.isMobile ? (
+                  <RightToLineIcon className="text-sidebar-toggle" />
+                ) : (
+                  <LeftToLineIcon className="text-sidebar-toggle" />
+                )}
+              </button>
+            </TooltipTrigger>
+            <TooltipContent>
+              {toggled ? `Unpin sidebar` : "Pin sidebar"}
+            </TooltipContent>
+          </Tooltip>
+        </TooltipProvider>
       )}
     </div>
   );
diff --git a/web/src/components/search/SearchResultsDisplay.tsx b/web/src/components/search/SearchResultsDisplay.tsx
index 7f1036050ec..e0ef9edff60 100644
--- a/web/src/components/search/SearchResultsDisplay.tsx
+++ b/web/src/components/search/SearchResultsDisplay.tsx
@@ -7,13 +7,17 @@ import {
   SearchResponse,
 } from "@/lib/search/interfaces";
 import { usePopup } from "../admin/connectors/Popup";
-import { AlertIcon, BroomIcon, MagnifyingIcon, UndoIcon } from "../icons/icons";
+import { AlertIcon, MagnifyingIcon, UndoIcon } from "../icons/icons";
 import { AgenticDocumentDisplay, DocumentDisplay } from "./DocumentDisplay";
 import { searchState } from "./SearchSection";
-import { useContext, useEffect, useState } from "react";
-import { Tooltip } from "../tooltip/Tooltip";
+import { useEffect, useState } from "react";
+import {
+  Tooltip,
+  TooltipProvider,
+  TooltipContent,
+  TooltipTrigger,
+} from "@/components/ui/tooltip";
 import KeyboardSymbol from "@/lib/browserUtilities";
-import { SettingsContext } from "../settings/SettingsProvider";
 import { DISABLE_LLM_DOC_RELEVANCE } from "@/lib/constants";
 
 const getSelectedDocumentIds = (
@@ -169,48 +173,56 @@ export const SearchResultsDisplay = ({
             <p>Results</p>
             {!DISABLE_LLM_DOC_RELEVANCE &&
               (contentEnriched || searchResponse.additional_relevance) && (
-                <Tooltip
-                  delayDuration={1000}
-                  content={<div className="flex">{commandSymbol}O</div>}
-                >
-                  <button
-                    onClick={() => {
-                      performSweep();
-                      if (agenticResults) {
-                        setShowAll((showAll) => !showAll);
-                      }
-                    }}
-                    className={`flex items-center justify-center animate-fade-in-up rounded-lg p-1 text-xs transition-all duration-300 w-20 h-8 ${
-                      !sweep
-                        ? "bg-background-agentic-toggled text-text-agentic-toggled"
-                        : "bg-background-agentic-untoggled text-text-agentic-untoggled"
-                    }`}
-                    style={{
-                      transform: sweep ? "rotateZ(180deg)" : "rotateZ(0deg)",
-                    }}
-                  >
-                    <div
-                      className={`flex items-center ${sweep ? "rotate-180" : ""}`}
-                    >
-                      <span></span>
-                      {!sweep
-                        ? agenticResults
-                          ? "Show All"
-                          : "Focus"
-                        : agenticResults
-                          ? "Focus"
-                          : "Show All"}
+                <TooltipProvider delayDuration={1000}>
+                  <Tooltip>
+                    <TooltipTrigger asChild>
+                      <button
+                        onClick={() => {
+                          performSweep();
+                          if (agenticResults) {
+                            setShowAll((showAll) => !showAll);
+                          }
+                        }}
+                        className={`flex items-center justify-center animate-fade-in-up rounded-lg p-1 text-xs transition-all duration-300 w-20 h-8 ${
+                          !sweep
+                            ? "bg-background-agentic-toggled text-text-agentic-toggled"
+                            : "bg-background-agentic-untoggled text-text-agentic-untoggled"
+                        }`}
+                        style={{
+                          transform: sweep
+                            ? "rotateZ(180deg)"
+                            : "rotateZ(0deg)",
+                        }}
+                      >
+                        <div
+                          className={`flex items-center ${
+                            sweep ? "rotate-180" : ""
+                          }`}
+                        >
+                          <span></span>
+                          {!sweep
+                            ? agenticResults
+                              ? "Show All"
+                              : "Focus"
+                            : agenticResults
+                              ? "Focus"
+                              : "Show All"}
 
-                      <span className="ml-1">
-                        {!sweep ? (
-                          <MagnifyingIcon className="h-4 w-4" />
-                        ) : (
-                          <UndoIcon className="h-4 w-4" />
-                        )}
-                      </span>
-                    </div>
-                  </button>
-                </Tooltip>
+                          <span className="ml-1">
+                            {!sweep ? (
+                              <MagnifyingIcon className="h-4 w-4" />
+                            ) : (
+                              <UndoIcon className="h-4 w-4" />
+                            )}
+                          </span>
+                        </div>
+                      </button>
+                    </TooltipTrigger>
+                    <TooltipContent>
+                      <div className="flex">{commandSymbol}O</div>
+                    </TooltipContent>
+                  </Tooltip>
+                </TooltipProvider>
               )}
           </div>
 
diff --git a/web/src/components/tooltip/Tooltip.tsx b/web/src/components/tooltip/Tooltip.tsx
deleted file mode 100644
index d72c13dd16c..00000000000
--- a/web/src/components/tooltip/Tooltip.tsx
+++ /dev/null
@@ -1,54 +0,0 @@
-import * as TooltipPrimitive from "@radix-ui/react-tooltip";
-import { ReactNode } from "react";
-
-interface TooltipProps {
-  children: ReactNode;
-  content: ReactNode;
-  side?: "top" | "right" | "bottom" | "left";
-  align?: "start" | "center" | "end";
-}
-
-export function Tooltip({
-  children,
-  content,
-  delayDuration = 200,
-  side = "top",
-  align = "center",
-  width,
-}: {
-  children: ReactNode;
-  content: ReactNode;
-  delayDuration?: number;
-  side?: "top" | "right" | "bottom" | "left";
-  align?: "start" | "center" | "end";
-  width?: string;
-}) {
-  return (
-    <TooltipPrimitive.Provider delayDuration={delayDuration}>
-      <TooltipPrimitive.Root>
-        <TooltipPrimitive.Trigger asChild>{children}</TooltipPrimitive.Trigger>
-        <TooltipPrimitive.Content
-          side={side}
-          align={align}
-          className={`
-            bg-background-inverted 
-            text-inverted 
-            text-sm 
-            rounded 
-            ${width}
-            py-1 
-            whitespace-normal
-            break-words
-            flex
-            px-2 
-            z-10
-            shadow-lg
-          `}
-        >
-          {content}
-          <TooltipPrimitive.Arrow className="fill-black" />
-        </TooltipPrimitive.Content>
-      </TooltipPrimitive.Root>
-    </TooltipPrimitive.Provider>
-  );
-}
diff --git a/web/src/components/ui/tooltip.tsx b/web/src/components/ui/tooltip.tsx
new file mode 100644
index 00000000000..f52eaa7d997
--- /dev/null
+++ b/web/src/components/ui/tooltip.tsx
@@ -0,0 +1,31 @@
+"use client";
+
+import * as React from "react";
+import * as TooltipPrimitive from "@radix-ui/react-tooltip";
+
+import { cn } from "@/lib/utils";
+
+const TooltipProvider = TooltipPrimitive.Provider;
+
+const Tooltip = TooltipPrimitive.Root;
+
+const TooltipTrigger = TooltipPrimitive.Trigger;
+const TooltipContent = React.forwardRef<
+  React.ElementRef<typeof TooltipPrimitive.Content>,
+  React.ComponentPropsWithoutRef<typeof TooltipPrimitive.Content> & {
+    maxWidth?: string;
+  }
+>(({ className, sideOffset = 4, maxWidth, ...props }, ref) => (
+  <TooltipPrimitive.Content
+    ref={ref}
+    sideOffset={sideOffset}
+    className={cn(
+      "z-50 overflow-hidden rounded-md border border-neutral-200 text-white  bg-background-900 px-2 py-1.5 text-sm shadow-md animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 dark:border-neutral-800 dark:bg-neutral-950 dark:text-neutral-50 max-w-sm",
+      className
+    )}
+    {...props}
+  />
+));
+TooltipContent.displayName = TooltipPrimitive.Content.displayName;
+
+export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider };

From b2c55ebd71b7000ff94536e371f891be26aa7f35 Mon Sep 17 00:00:00 2001
From: pablodanswer <pablo@danswer.ai>
Date: Tue, 5 Nov 2024 08:49:04 -0800
Subject: [PATCH 367/376] ensure props aligned (#3050)

* ensure props aligned

* k

* k
---
 web/src/components/ErrorCallout.tsx | 3 ++-
 web/src/components/ui/callout.tsx   | 8 +++++---
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/web/src/components/ErrorCallout.tsx b/web/src/components/ErrorCallout.tsx
index 497412d033d..3726c09e307 100644
--- a/web/src/components/ErrorCallout.tsx
+++ b/web/src/components/ErrorCallout.tsx
@@ -1,4 +1,5 @@
 import { Callout } from "@/components/ui/callout";
+import { FiAlertTriangle } from "react-icons/fi";
 
 export function ErrorCallout({
   errorTitle,
@@ -12,7 +13,7 @@ export function ErrorCallout({
       <Callout
         className="mt-4"
         title={errorTitle || "Page not found"}
-        icon="alert"
+        icon={<FiAlertTriangle className="text-red-500 h-5 w-5" />}
         type="danger"
       >
         {errorMsg}
diff --git a/web/src/components/ui/callout.tsx b/web/src/components/ui/callout.tsx
index 99dbc31c900..5b45aab72ed 100644
--- a/web/src/components/ui/callout.tsx
+++ b/web/src/components/ui/callout.tsx
@@ -1,7 +1,7 @@
 import { cn } from "@/lib/utils";
 
 interface CalloutProps {
-  icon?: string;
+  icon?: React.ReactNode;
   children?: React.ReactNode;
   type?: "default" | "warning" | "danger" | "notice";
   className?: string;
@@ -29,8 +29,10 @@ export function Callout({
       {...props}
     >
       {icon && <span className="mr-4 text-2xl">{icon}</span>}
-      <div>
-        {title && <div className="font-medium mb-1">{title}</div>}
+      <div className="flex-1">
+        {title && (
+          <div className="font-medium mb-1 flex items-center">{title}</div>
+        )}
         {children}
       </div>
     </div>

From 62c4127bb48b6f8dd3b13617fa7a2a4413a699bb Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 6 Nov 2024 09:47:34 -0800
Subject: [PATCH 368/376] limit session scope

---
 .../background/celery/tasks/indexing/tasks.py | 53 ++++++++++---------
 .../background/celery/tasks/vespa/tasks.py    |  4 +-
 .../background/indexing/run_indexing.py       |  4 +-
 backend/danswer/db/index_attempt.py           |  4 +-
 4 files changed, 33 insertions(+), 32 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index 7d2e6e0b3dd..e2446592adc 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -489,7 +489,7 @@ def connector_indexing_task(
         f"search_settings={search_settings_id}"
     )
 
-    attempt = None
+    attempt_found = False
     n_final_progress: int | None = None
 
     redis_connector = RedisConnector(tenant_id, cc_pair_id)
@@ -557,6 +557,7 @@ def connector_indexing_task(
                 raise ValueError(
                     f"Index attempt not found: index_attempt={index_attempt_id}"
                 )
+            attempt_found = True
 
             cc_pair = get_connector_credential_pair_from_id(
                 cc_pair_id=cc_pair_id,
@@ -576,32 +577,32 @@ def connector_indexing_task(
                     f"Credential not found: cc_pair={cc_pair_id} credential={cc_pair.credential_id}"
                 )
 
-            # define a callback class
-            callback = RunIndexingCallback(
-                redis_connector.stop.fence_key,
-                redis_connector_index.generator_progress_key,
-                lock,
-                r,
-            )
+        # define a callback class
+        callback = RunIndexingCallback(
+            redis_connector.stop.fence_key,
+            redis_connector_index.generator_progress_key,
+            lock,
+            r,
+        )
 
-            logger.info(
-                f"Indexing spawned task running entrypoint: attempt={index_attempt_id} "
-                f"tenant={tenant_id} "
-                f"cc_pair={cc_pair_id} "
-                f"search_settings={search_settings_id}"
-            )
+        logger.info(
+            f"Indexing spawned task running entrypoint: attempt={index_attempt_id} "
+            f"tenant={tenant_id} "
+            f"cc_pair={cc_pair_id} "
+            f"search_settings={search_settings_id}"
+        )
 
-            run_indexing_entrypoint(
-                index_attempt_id,
-                tenant_id,
-                cc_pair_id,
-                is_ee,
-                callback=callback,
-            )
+        run_indexing_entrypoint(
+            index_attempt_id,
+            tenant_id,
+            cc_pair_id,
+            is_ee,
+            callback=callback,
+        )
 
-            # get back the total number of indexed docs and return it
-            n_final_progress = redis_connector_index.get_progress()
-            redis_connector_index.set_generator_complete(HTTPStatus.OK.value)
+        # get back the total number of indexed docs and return it
+        n_final_progress = redis_connector_index.get_progress()
+        redis_connector_index.set_generator_complete(HTTPStatus.OK.value)
     except Exception as e:
         logger.exception(
             f"Indexing spawned task failed: attempt={index_attempt_id} "
@@ -609,9 +610,9 @@ def connector_indexing_task(
             f"cc_pair={cc_pair_id} "
             f"search_settings={search_settings_id}"
         )
-        if attempt:
+        if attempt_found:
             with get_session_with_tenant(tenant_id) as db_session:
-                mark_attempt_failed(attempt, db_session, failure_reason=str(e))
+                mark_attempt_failed(index_attempt_id, db_session, failure_reason=str(e))
 
         redis_connector_index.reset()
         raise e
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 944750c7a78..1be928f5994 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -610,7 +610,7 @@ def monitor_ccpair_indexing_taskset(
             index_attempt = get_index_attempt(db_session, payload.index_attempt_id)
             if index_attempt:
                 mark_attempt_failed(
-                    index_attempt=index_attempt,
+                    index_attempt_id=payload.index_attempt_id,
                     db_session=db_session,
                     failure_reason="Connector indexing aborted or exceptioned.",
                 )
@@ -696,7 +696,7 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
                         a.connector_credential_pair_id, a.search_settings_id
                     )
                 ):
-                    mark_attempt_failed(a, db_session, failure_reason=failure_reason)
+                    mark_attempt_failed(a.id, db_session, failure_reason=failure_reason)
 
         lock_beat.reacquire()
         if r.exists(RedisConnectorCredentialPair.get_fence_key()):
diff --git a/backend/danswer/background/indexing/run_indexing.py b/backend/danswer/background/indexing/run_indexing.py
index 252309e3faa..35cb080b903 100644
--- a/backend/danswer/background/indexing/run_indexing.py
+++ b/backend/danswer/background/indexing/run_indexing.py
@@ -337,7 +337,7 @@ def _run_indexing(
                 or index_attempt.status != IndexingStatus.IN_PROGRESS
             ):
                 mark_attempt_failed(
-                    index_attempt,
+                    index_attempt.id,
                     db_session,
                     failure_reason=str(e),
                     full_exception_trace=traceback.format_exc(),
@@ -372,7 +372,7 @@ def _run_indexing(
         and index_attempt_md.num_exceptions >= batch_num
     ):
         mark_attempt_failed(
-            index_attempt,
+            index_attempt.id,
             db_session,
             failure_reason="All batches exceptioned.",
         )
diff --git a/backend/danswer/db/index_attempt.py b/backend/danswer/db/index_attempt.py
index 45a07387949..b9c3d9d4ca2 100644
--- a/backend/danswer/db/index_attempt.py
+++ b/backend/danswer/db/index_attempt.py
@@ -219,7 +219,7 @@ def mark_attempt_partially_succeeded(
 
 
 def mark_attempt_failed(
-    index_attempt: IndexAttempt,
+    index_attempt_id: int,
     db_session: Session,
     failure_reason: str = "Unknown",
     full_exception_trace: str | None = None,
@@ -227,7 +227,7 @@ def mark_attempt_failed(
     try:
         attempt = db_session.execute(
             select(IndexAttempt)
-            .where(IndexAttempt.id == index_attempt.id)
+            .where(IndexAttempt.id == index_attempt_id)
             .with_for_update()
         ).scalar_one()
 

From 93ec2a6681b3a1934689326e86a1d86459e01e0b Mon Sep 17 00:00:00 2001
From: "Richard Kuo (Danswer)" <rkuo@danswer.ai>
Date: Wed, 6 Nov 2024 09:48:36 -0800
Subject: [PATCH 369/376] abort mismatched tasks

---
 .../background/celery/tasks/indexing/tasks.py     | 12 ++++++++++--
 .../background/celery/tasks/vespa/tasks.py        | 15 ++++++++++-----
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/backend/danswer/background/celery/tasks/indexing/tasks.py b/backend/danswer/background/celery/tasks/indexing/tasks.py
index e2446592adc..cd6126223c4 100644
--- a/backend/danswer/background/celery/tasks/indexing/tasks.py
+++ b/backend/danswer/background/celery/tasks/indexing/tasks.py
@@ -173,7 +173,9 @@ def check_for_indexing(self: Task, *, tenant_id: str | None) -> int | None:
                     )
                     if attempt_id:
                         task_logger.info(
-                            f"Indexing queued: cc_pair={cc_pair.id} index_attempt={attempt_id}"
+                            f"Indexing queued: index_attempt={attempt_id} "
+                            f"cc_pair={cc_pair.id} "
+                            f"search_settings={search_settings_instance.id} "
                         )
                         tasks_created += 1
     except SoftTimeLimitExceeded:
@@ -529,6 +531,13 @@ def connector_indexing_task(
             sleep(1)
             continue
 
+        if payload.index_attempt_id != index_attempt_id:
+            raise ValueError(
+                f"connector_indexing_task - id mismatch. Task may be left over from previous run.: "
+                f"task_index_attempt={index_attempt_id} "
+                f"payload_index_attempt={payload.index_attempt_id}"
+            )
+
         logger.info(
             f"connector_indexing_task - Fence found, continuing...: fence={redis_connector_index.fence_key}"
         )
@@ -614,7 +623,6 @@ def connector_indexing_task(
             with get_session_with_tenant(tenant_id) as db_session:
                 mark_attempt_failed(index_attempt_id, db_session, failure_reason=str(e))
 
-        redis_connector_index.reset()
         raise e
     finally:
         if lock.owned():
diff --git a/backend/danswer/background/celery/tasks/vespa/tasks.py b/backend/danswer/background/celery/tasks/vespa/tasks.py
index 1be928f5994..b01a0eac815 100644
--- a/backend/danswer/background/celery/tasks/vespa/tasks.py
+++ b/backend/danswer/background/celery/tasks/vespa/tasks.py
@@ -690,12 +690,17 @@ def monitor_vespa_sync(self: Task, tenant_id: str | None) -> bool:
 
             for a in attempts:
                 # if attempts exist in the db but we don't detect them in redis, mark them as failed
-                failure_reason = f"Unknown index attempt {a.id}. Might be left over from a process restart."
-                if not r.exists(
-                    RedisConnectorIndex.fence_key_with_ids(
-                        a.connector_credential_pair_id, a.search_settings_id
+                fence_key = RedisConnectorIndex.fence_key_with_ids(
+                    a.connector_credential_pair_id, a.search_settings_id
+                )
+                if not r.exists(fence_key):
+                    failure_reason = (
+                        f"Unknown index attempt. Might be left over from a process restart: "
+                        f"index_attempt={a.id} "
+                        f"cc_pair={a.connector_credential_pair_id} "
+                        f"search_settings={a.search_settings_id}"
                     )
-                ):
+                    task_logger.warning(failure_reason)
                     mark_attempt_failed(a.id, db_session, failure_reason=failure_reason)
 
         lock_beat.reacquire()

From c08d3e73bd0e3a08f58b8af33a645e8b8c3cd599 Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Thu, 14 Nov 2024 11:48:10 +0000
Subject: [PATCH 370/376] Reinstate workflow dep

---
 .github/workflows/helm-build-push.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/helm-build-push.yml b/.github/workflows/helm-build-push.yml
index a9080239b40..a591eaecd3c 100644
--- a/.github/workflows/helm-build-push.yml
+++ b/.github/workflows/helm-build-push.yml
@@ -62,7 +62,7 @@ jobs:
     permissions:
       contents: write
     runs-on: ubuntu-latest
-    # needs: helm_chart_version_check
+    needs: helm_chart_version_check
     # if: ${{ needs.helm_chart_version_check.outputs.version_changed == 'true' }}
     steps:
       - name: Checkout

From 4c314a1af8d2400ea0be1ef3d811fab65a11d9f6 Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Thu, 14 Nov 2024 11:59:19 +0000
Subject: [PATCH 371/376] Update header fetching to use async/await

---
 web/src/app/auth/login/page.tsx | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/web/src/app/auth/login/page.tsx b/web/src/app/auth/login/page.tsx
index 4763965138d..ee3f17555f9 100644
--- a/web/src/app/auth/login/page.tsx
+++ b/web/src/app/auth/login/page.tsx
@@ -75,8 +75,8 @@ const Page = async (props: {
     return redirect(authUrl);
   }
 
-  const userHeader = headers().get('x-remote-user');
-  const groupsHeader = headers().get('x-remote-group');
+  const userHeader = (await headers()).get('x-remote-user');
+  const groupsHeader = (await headers()).get('x-remote-group');
 
   return (
     <AuthFlowContainer>

From dcfc2882c1eed5f34e559dbbd47729525b3814de Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Thu, 14 Nov 2024 12:22:12 +0000
Subject: [PATCH 372/376] Bump helm chart versions

---
 deployment/helm/charts/danswer/Chart.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/helm/charts/danswer/Chart.yaml b/deployment/helm/charts/danswer/Chart.yaml
index 490ce69a39b..c7b33c6300d 100644
--- a/deployment/helm/charts/danswer/Chart.yaml
+++ b/deployment/helm/charts/danswer/Chart.yaml
@@ -5,8 +5,8 @@ home: https://www.danswer.ai/
 sources:
   - "https://github.com/stackhpc/danswer"
 type: application
-version: 0.2.3
-appVersion: v0.7.4
+version: 0.2.4-prerelease.1
+appVersion: v0.12.0-stackhpc-prerelease.1
 dependencies:
   - name: postgresql
     version: 14.3.1

From dbb65ccf94042db51aa4d4b7a6f5b5116129b15d Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Thu, 14 Nov 2024 12:28:21 +0000
Subject: [PATCH 373/376] Fix app version tag format

---
 deployment/helm/charts/danswer/Chart.yaml  | 2 +-
 deployment/helm/charts/danswer/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/helm/charts/danswer/Chart.yaml b/deployment/helm/charts/danswer/Chart.yaml
index c7b33c6300d..b2fff4fafc2 100644
--- a/deployment/helm/charts/danswer/Chart.yaml
+++ b/deployment/helm/charts/danswer/Chart.yaml
@@ -6,7 +6,7 @@ sources:
   - "https://github.com/stackhpc/danswer"
 type: application
 version: 0.2.4-prerelease.1
-appVersion: v0.12.0-stackhpc-prerelease.1
+appVersion: v0.12.0
 dependencies:
   - name: postgresql
     version: 14.3.1
diff --git a/deployment/helm/charts/danswer/values.yaml b/deployment/helm/charts/danswer/values.yaml
index b1b6c88a148..de57f93ae6e 100644
--- a/deployment/helm/charts/danswer/values.yaml
+++ b/deployment/helm/charts/danswer/values.yaml
@@ -11,7 +11,7 @@ appVersionOverride: # e.g "v0.3.93"
 # tags to refer to downstream StackHPC-modified images.
 # The full image ref will be:
 # {{ image-name }}:{{ image-tag or appVersion }}-{{ tagSuffix }}
-tagSuffix: stackhpc.4
+tagSuffix: stackhpc-prerelease.1
 
 zenithClient:
   iconUrl: https://raw.githubusercontent.com/danswer-ai/danswer/1fabd9372d66cd54238847197c33f091a724803b/Danswer.png

From bdf8cf52fe9477bc193b04ef4c27604de9c8d0be Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Fri, 15 Nov 2024 16:50:54 +0000
Subject: [PATCH 374/376] Add pod resources option for inference model
 deployment

---
 .../charts/danswer/templates/inference-model-deployment.yaml  | 4 ++++
 deployment/helm/charts/danswer/values.yaml                    | 1 +
 2 files changed, 5 insertions(+)

diff --git a/deployment/helm/charts/danswer/templates/inference-model-deployment.yaml b/deployment/helm/charts/danswer/templates/inference-model-deployment.yaml
index 391a8e4289b..3897bba078e 100644
--- a/deployment/helm/charts/danswer/templates/inference-model-deployment.yaml
+++ b/deployment/helm/charts/danswer/templates/inference-model-deployment.yaml
@@ -25,6 +25,10 @@ spec:
         image: "{{ .Values.inferenceCapability.deployment.image.repository }}:{{ .Values.inferenceCapability.deployment.image.tag | default .Values.appVersionOverride | default .Chart.AppVersion }}"
         imagePullPolicy: {{ .Values.inferenceCapability.deployment.image.pullPolicy }}
         command: {{ toYaml .Values.inferenceCapability.deployment.command | nindent 14 }}
+        {{- if .Values.inferenceCapability.deployment.resources }}
+        resources:
+            {{- toYaml .Values.inferenceCapability.deployment.resources | nindent 10 }}
+        {{- end }}
         ports:
         - containerPort: {{ .Values.inferenceCapability.service.port }}
         envFrom:
diff --git a/deployment/helm/charts/danswer/values.yaml b/deployment/helm/charts/danswer/values.yaml
index de57f93ae6e..82e6ef892e5 100644
--- a/deployment/helm/charts/danswer/values.yaml
+++ b/deployment/helm/charts/danswer/values.yaml
@@ -39,6 +39,7 @@ inferenceCapability:
       tag:
       pullPolicy: IfNotPresent
     command: ["uvicorn", "model_server.main:app", "--host", "0.0.0.0", "--port", "9000"]
+    resources:
     port: 9000
     volumeMounts:
       - name: inference-model-storage

From 4f9b8c9e12c763fea1b2c0e6c77f06a0f66eb1ba Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Fri, 15 Nov 2024 16:52:20 +0000
Subject: [PATCH 375/376] Bump version tags for testing

---
 deployment/helm/charts/danswer/Chart.yaml  | 2 +-
 deployment/helm/charts/danswer/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/helm/charts/danswer/Chart.yaml b/deployment/helm/charts/danswer/Chart.yaml
index b2fff4fafc2..d069b7c2cfe 100644
--- a/deployment/helm/charts/danswer/Chart.yaml
+++ b/deployment/helm/charts/danswer/Chart.yaml
@@ -5,7 +5,7 @@ home: https://www.danswer.ai/
 sources:
   - "https://github.com/stackhpc/danswer"
 type: application
-version: 0.2.4-prerelease.1
+version: 0.2.4-prerelease.2
 appVersion: v0.12.0
 dependencies:
   - name: postgresql
diff --git a/deployment/helm/charts/danswer/values.yaml b/deployment/helm/charts/danswer/values.yaml
index 82e6ef892e5..e6a3c4b2a3a 100644
--- a/deployment/helm/charts/danswer/values.yaml
+++ b/deployment/helm/charts/danswer/values.yaml
@@ -11,7 +11,7 @@ appVersionOverride: # e.g "v0.3.93"
 # tags to refer to downstream StackHPC-modified images.
 # The full image ref will be:
 # {{ image-name }}:{{ image-tag or appVersion }}-{{ tagSuffix }}
-tagSuffix: stackhpc-prerelease.1
+tagSuffix: stackhpc-prerelease.2
 
 zenithClient:
   iconUrl: https://raw.githubusercontent.com/danswer-ai/danswer/1fabd9372d66cd54238847197c33f091a724803b/Danswer.png

From b4360a30d33228b52a5671d5b518d0b7714026d6 Mon Sep 17 00:00:00 2001
From: sd109 <sdavidson327@gmail.com>
Date: Mon, 25 Nov 2024 11:24:06 +0000
Subject: [PATCH 376/376] Bump to stable image and chart tags

---
 deployment/helm/charts/danswer/Chart.yaml  | 2 +-
 deployment/helm/charts/danswer/values.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/deployment/helm/charts/danswer/Chart.yaml b/deployment/helm/charts/danswer/Chart.yaml
index d069b7c2cfe..f76db88f905 100644
--- a/deployment/helm/charts/danswer/Chart.yaml
+++ b/deployment/helm/charts/danswer/Chart.yaml
@@ -5,7 +5,7 @@ home: https://www.danswer.ai/
 sources:
   - "https://github.com/stackhpc/danswer"
 type: application
-version: 0.2.4-prerelease.2
+version: 0.2.4
 appVersion: v0.12.0
 dependencies:
   - name: postgresql
diff --git a/deployment/helm/charts/danswer/values.yaml b/deployment/helm/charts/danswer/values.yaml
index e6a3c4b2a3a..e965fe0b303 100644
--- a/deployment/helm/charts/danswer/values.yaml
+++ b/deployment/helm/charts/danswer/values.yaml
@@ -11,7 +11,7 @@ appVersionOverride: # e.g "v0.3.93"
 # tags to refer to downstream StackHPC-modified images.
 # The full image ref will be:
 # {{ image-name }}:{{ image-tag or appVersion }}-{{ tagSuffix }}
-tagSuffix: stackhpc-prerelease.2
+tagSuffix: stackhpc.1
 
 zenithClient:
   iconUrl: https://raw.githubusercontent.com/danswer-ai/danswer/1fabd9372d66cd54238847197c33f091a724803b/Danswer.png