Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[FIX] Env standardization for Backend. #1050

Merged
merged 7 commits into from
Jan 9, 2025
Merged

[FIX] Env standardization for Backend. #1050

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
81d1144
env standadization for backend
harini-venkataraman Jan 7, 2025
61ce9d8
Sample envs for remote storage
harini-venkataraman Jan 8, 2025
f420143
Sample envs for remote storage
harini-venkataraman Jan 8, 2025
3aee79f
Merge branch 'main' into fix/env-standadization-backend
gaya3-zipstack Jan 9, 2025
1cc3b4c
Merge branch 'main' into fix/env-standadization-backend
gaya3-zipstack Jan 9, 2025
f0d94b3
Merge with main
gaya3-zipstack Jan 9, 2025
0bf6cc1
Resolve conflicts
gaya3-zipstack Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions backend/prompt_studio/prompt_studio_core_v2/models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@
from django.db.models import QuerySet
from file_management.file_management_helper import FileManagerHelper
from prompt_studio.prompt_studio_core_v2.constants import DefaultPrompts
from utils.file_storage.constants import FileStorageType
from utils.file_storage.helpers.common_file_helper import FileStorageHelper
from unstract.sdk.file_storage.constants import StorageType
from unstract.sdk.file_storage.env_helper import EnvHelper
from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.models.base_model import BaseModel
from utils.models.organization_mixin import (
Expand Down Expand Up @@ -155,8 +156,9 @@ def delete(self, organization_id=None, *args, **kwargs):
logger.error(f"Error: {file_path} : {e.strerror}")
# Continue with the deletion of the tool
else:
fs_instance = FileStorageHelper.initialize_file_storage(
type=FileStorageType.PERMANENT
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
file_path = PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(
organization_id,
Expand Down
10 changes: 6 additions & 4 deletions backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,12 @@
from unstract.sdk.constants import LogLevel
from unstract.sdk.exceptions import IndexingError, SdkError
from unstract.sdk.file_storage import FileStorage, FileStorageProvider
from unstract.sdk.file_storage.constants import StorageType
from unstract.sdk.file_storage.env_helper import EnvHelper
from unstract.sdk.index import Index
from unstract.sdk.prompt import PromptTool
from unstract.sdk.utils.tool_utils import ToolUtils
from utils.file_storage.constants import FileStorageType
from utils.file_storage.helpers.common_file_helper import FileStorageHelper
from utils.file_storage.constants import FileStorageKeys
from utils.file_storage.helpers.prompt_studio_file_helper import PromptStudioFileHelper
from utils.local_context import StateStore

Expand Down Expand Up @@ -782,8 +783,9 @@ def _fetch_response(
process_text=process_text,
)
else:
fs_instance = FileStorageHelper.initialize_file_storage(
type=FileStorageType.PERMANENT
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
index_result = PromptStudioHelper.dynamic_indexer(
profile_manager=profile_manager,
Expand Down
2 changes: 2 additions & 0 deletions backend/utils/file_storage/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
class FileStorageKeys:
FILE_STORAGE_PROVIDER = "FILE_STORAGE_PROVIDER"
harini-venkataraman marked this conversation as resolved.
Show resolved Hide resolved
FILE_STORAGE_CREDENTIALS = "FILE_STORAGE_CREDENTIALS"
PERMANENT_REMOTE_STORAGE = "PERMANENT_REMOTE_STORAGE"
TEMPORARY_REMOTE_STORAGE = "TEMPORARY_REMOTE_STORAGE"


class FileStorageType(Enum):
Expand Down
47 changes: 0 additions & 47 deletions backend/utils/file_storage/helpers/common_file_helper.py
View file
Open in desktop

This file was deleted.

25 changes: 15 additions & 10 deletions backend/utils/file_storage/helpers/prompt_studio_file_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@

from file_management.file_management_helper import FileManagerHelper
from unstract.sdk.file_storage import FileStorage
from utils.file_storage.constants import FileStorageConstants, FileStorageType
from utils.file_storage.helpers.common_file_helper import FileStorageHelper
from unstract.sdk.file_storage.constants import StorageType
from unstract.sdk.file_storage.env_helper import EnvHelper
from utils.file_storage.constants import FileStorageConstants, FileStorageKeys

from unstract.core.utilities import UnstractUtils

Expand Down Expand Up @@ -37,8 +38,9 @@ def get_or_create_prompt_studio_subdirectory(
extract_file_path = str(Path(file_path) / "extract")
summarize_file_path = str(Path(file_path) / "summarize")
if is_create:
fs_instance = FileStorageHelper.initialize_file_storage(
type=FileStorageType.PERMANENT
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
fs_instance.mkdir(file_path, create_parents=True)
fs_instance.mkdir(extract_file_path, create_parents=True)
Expand All @@ -57,8 +59,9 @@ def upload_for_ide(
tool_id (str): ID of the prompt studio tool
uploaded_file : File to upload to remote
"""
fs_instance = FileStorageHelper.initialize_file_storage(
type=FileStorageType.PERMANENT
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
file_system_path = (
PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(
Expand All @@ -77,8 +80,9 @@ def fetch_file_contents(
) -> Union[bytes, str]:
"""Method to fetch file contents from the remote location.
The path is constructed in runtime based on the args"""
fs_instance = FileStorageHelper.initialize_file_storage(
type=FileStorageType.PERMANENT
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
# Fetching legacy file path for lazy copy
# This has to be removed once the usage of FS APIs
Expand Down Expand Up @@ -139,8 +143,9 @@ def delete_for_ide(org_id: str, user_id: str, tool_id: str, file_name: str) -> b
"""Method to delete file in remote while the corresponsing prompt
studio project is deleted or the file is removed from the file manager.
This method handles deleted for related files as well."""
fs_instance = FileStorageHelper.initialize_file_storage(
type=FileStorageType.PERMANENT
fs_instance = EnvHelper.get_storage(
storage_type=StorageType.PERMANENT,
env_name=FileStorageKeys.PERMANENT_REMOTE_STORAGE,
)
file_system_path = (
PromptStudioFileHelper.get_or_create_prompt_studio_subdirectory(
Expand Down
Loading