diff --git a/ENV_SETUP.md b/ENV_SETUP.md index bfc30019624..8695583c12e 100644 --- a/ENV_SETUP.md +++ b/ENV_SETUP.md @@ -27,6 +27,8 @@ AWS_SECRET_ACCESS_KEY="secret-key" AWS_S3_ENDPOINT_URL="http://plane-minio:9000" # Changing this requires change in the nginx.conf for uploads if using minio setup AWS_S3_BUCKET_NAME="uploads" +AWS_S3_BUCKET_AUTH=False +AWS_QUERYSTRING_AUTH=False # Maximum file upload limit FILE_SIZE_LIMIT=5242880 ​ @@ -94,6 +96,8 @@ AWS_SECRET_ACCESS_KEY="secret-key" AWS_S3_ENDPOINT_URL="http://plane-minio:9000" # Changing this requires change in the nginx.conf for uploads if using minio setup AWS_S3_BUCKET_NAME="uploads" +AWS_S3_BUCKET_AUTH=False +AWS_QUERYSTRING_AUTH=False # Maximum file upload limit FILE_SIZE_LIMIT=5242880 ​ diff --git a/apiserver/.env.example b/apiserver/.env.example index 37178b39809..949802e83e7 100644 --- a/apiserver/.env.example +++ b/apiserver/.env.example @@ -31,6 +31,8 @@ AWS_SECRET_ACCESS_KEY="secret-key" AWS_S3_ENDPOINT_URL="http://plane-minio:9000" # Changing this requires change in the nginx.conf for uploads if using minio setup AWS_S3_BUCKET_NAME="uploads" +AWS_S3_BUCKET_AUTH=False +AWS_QUERYSTRING_AUTH=False # Maximum file upload limit FILE_SIZE_LIMIT=5242880 diff --git a/apiserver/plane/api/serializers/issue.py b/apiserver/plane/api/serializers/issue.py index ab61ae523f4..ae56c198669 100644 --- a/apiserver/plane/api/serializers/issue.py +++ b/apiserver/plane/api/serializers/issue.py @@ -21,12 +21,14 @@ IssueActivity, ProjectMember, ) + from .base import BaseSerializer from .cycle import CycleSerializer, CycleLiteSerializer from .module import ModuleSerializer, ModuleLiteSerializer from .user import UserLiteSerializer from .state import StateLiteSerializer + class IssueSerializer(BaseSerializer): assignees = serializers.ListField( child=serializers.PrimaryKeyRelatedField( @@ -67,13 +69,13 @@ def validate(self, data): and data.get("start_date", None) > data.get("target_date", None) ): raise serializers.ValidationError("Start date cannot exceed target date") - + try: - if(data.get("description_html", None) is not None): + if data.get("description_html", None) is not None: parsed = html.fromstring(data["description_html"]) - parsed_str = html.tostring(parsed, encoding='unicode') + parsed_str = html.tostring(parsed, encoding="unicode") data["description_html"] = parsed_str - + except Exception as e: raise serializers.ValidationError(f"Invalid HTML: {str(e)}") @@ -324,11 +326,11 @@ class Meta: def validate(self, data): try: - if(data.get("comment_html", None) is not None): + if data.get("comment_html", None) is not None: parsed = html.fromstring(data["comment_html"]) - parsed_str = html.tostring(parsed, encoding='unicode') + parsed_str = html.tostring(parsed, encoding="unicode") data["comment_html"] = parsed_str - + except Exception as e: raise serializers.ValidationError(f"Invalid HTML: {str(e)}") return data @@ -362,7 +364,6 @@ class Meta: class LabelLiteSerializer(BaseSerializer): - class Meta: model = Label fields = [ diff --git a/apiserver/plane/api/serializers/project.py b/apiserver/plane/api/serializers/project.py index c394a080dd9..14eafb52e16 100644 --- a/apiserver/plane/api/serializers/project.py +++ b/apiserver/plane/api/serializers/project.py @@ -2,12 +2,11 @@ from rest_framework import serializers # Module imports -from plane.db.models import Project, ProjectIdentifier, WorkspaceMember, State, Estimate +from plane.db.models import Project, ProjectIdentifier, WorkspaceMember from .base import BaseSerializer class ProjectSerializer(BaseSerializer): - total_members = serializers.IntegerField(read_only=True) total_cycles = serializers.IntegerField(read_only=True) total_modules = serializers.IntegerField(read_only=True) @@ -21,7 +20,7 @@ class Meta: fields = "__all__" read_only_fields = [ "id", - 'emoji', + "emoji", "workspace", "created_at", "updated_at", @@ -89,4 +88,4 @@ class Meta: "emoji", "description", ] - read_only_fields = fields \ No newline at end of file + read_only_fields = fields diff --git a/apiserver/plane/app/serializers/issue.py b/apiserver/plane/app/serializers/issue.py index b13d03e35a4..357f5baca83 100644 --- a/apiserver/plane/app/serializers/issue.py +++ b/apiserver/plane/app/serializers/issue.py @@ -1,5 +1,6 @@ # Django imports from django.utils import timezone +from django.conf import settings # Third Party imports from rest_framework import serializers @@ -31,9 +32,23 @@ IssueVote, IssueRelation, ) +from plane.utils.parse_html import parse_text_to_html, refresh_url_content -class IssueFlatSerializer(BaseSerializer): +class BaseIssueSerializerMixin: + """abstract class for refresh s3 link in description htlm images""" + + def refresh_html_content(self, instance, html, html_field_name="description_html"): + if settings.AWS_S3_BUCKET_AUTH: + html = parse_text_to_html(html) + refreshed, html = refresh_url_content(html) + + if refreshed: + setattr(instance, html_field_name, html) + instance.save() + + +class IssueFlatSerializer(BaseSerializer, BaseIssueSerializerMixin): ## Contain only flat fields class Meta: @@ -51,6 +66,10 @@ class Meta: "is_draft", ] + def to_representation(self, instance): + self.refresh_html_content(instance, instance.description_html) + return super().to_representation(instance) + class IssueProjectLiteSerializer(BaseSerializer): project_detail = ProjectLiteSerializer(source="project", read_only=True) @@ -100,8 +119,8 @@ class Meta: def to_representation(self, instance): data = super().to_representation(instance) - data['assignees'] = [str(assignee.id) for assignee in instance.assignees.all()] - data['labels'] = [str(label.id) for label in instance.labels.all()] + data["assignees"] = [str(assignee.id) for assignee in instance.assignees.all()] + data["labels"] = [str(label.id) for label in instance.labels.all()] return data def validate(self, data): @@ -232,7 +251,6 @@ class Meta: fields = "__all__" - class IssuePropertySerializer(BaseSerializer): class Meta: model = IssueProperty @@ -268,7 +286,6 @@ class Meta: class IssueLabelSerializer(BaseSerializer): - class Meta: model = IssueLabel fields = "__all__" @@ -283,30 +300,19 @@ class IssueRelationSerializer(BaseSerializer): class Meta: model = IssueRelation - fields = [ - "issue_detail", - "relation_type", - "related_issue", - "issue", - "id" - ] + fields = ["issue_detail", "relation_type", "related_issue", "issue", "id"] read_only_fields = [ "workspace", "project", ] + class RelatedIssueSerializer(BaseSerializer): issue_detail = IssueProjectLiteSerializer(read_only=True, source="issue") class Meta: model = IssueRelation - fields = [ - "issue_detail", - "relation_type", - "related_issue", - "issue", - "id" - ] + fields = ["issue_detail", "relation_type", "related_issue", "issue", "id"] read_only_fields = [ "workspace", "project", @@ -424,9 +430,8 @@ class Meta: class IssueReactionSerializer(BaseSerializer): - actor_detail = UserLiteSerializer(read_only=True, source="actor") - + class Meta: model = IssueReaction fields = "__all__" @@ -459,7 +464,6 @@ class Meta: class IssueVoteSerializer(BaseSerializer): - actor_detail = UserLiteSerializer(read_only=True, source="actor") class Meta: @@ -468,7 +472,7 @@ class Meta: read_only_fields = fields -class IssueCommentSerializer(BaseSerializer): +class IssueCommentSerializer(BaseSerializer, BaseIssueSerializerMixin): actor_detail = UserLiteSerializer(read_only=True, source="actor") issue_detail = IssueFlatSerializer(read_only=True, source="issue") project_detail = ProjectLiteSerializer(read_only=True, source="project") @@ -489,6 +493,10 @@ class Meta: "updated_at", ] + def to_representation(self, instance): + self.refresh_html_content(instance, instance.comment_html, "comment_html") + return super().to_representation(instance) + class IssueStateFlatSerializer(BaseSerializer): state_detail = StateLiteSerializer(read_only=True, source="state") @@ -506,7 +514,7 @@ class Meta: # Issue Serializer with state details -class IssueStateSerializer(DynamicBaseSerializer): +class IssueStateSerializer(DynamicBaseSerializer, BaseIssueSerializerMixin): label_details = LabelLiteSerializer(read_only=True, source="labels", many=True) state_detail = StateLiteSerializer(read_only=True, source="state") project_detail = ProjectLiteSerializer(read_only=True, source="project") @@ -520,15 +528,23 @@ class Meta: model = Issue fields = "__all__" + def to_representation(self, instance): + self.refresh_html_content(instance, instance.description_html) + return super().to_representation(instance) + -class IssueSerializer(BaseSerializer): +class IssueSerializer(BaseSerializer, BaseIssueSerializerMixin): project_detail = ProjectLiteSerializer(read_only=True, source="project") state_detail = StateSerializer(read_only=True, source="state") parent_detail = IssueStateFlatSerializer(read_only=True, source="parent") label_details = LabelSerializer(read_only=True, source="labels", many=True) assignee_details = UserLiteSerializer(read_only=True, source="assignees", many=True) - related_issues = IssueRelationSerializer(read_only=True, source="issue_relation", many=True) - issue_relations = RelatedIssueSerializer(read_only=True, source="issue_related", many=True) + related_issues = IssueRelationSerializer( + read_only=True, source="issue_relation", many=True + ) + issue_relations = RelatedIssueSerializer( + read_only=True, source="issue_related", many=True + ) issue_cycle = IssueCycleDetailSerializer(read_only=True) issue_module = IssueModuleDetailSerializer(read_only=True) issue_link = IssueLinkSerializer(read_only=True, many=True) @@ -548,8 +564,12 @@ class Meta: "updated_at", ] + def to_representation(self, instance): + self.refresh_html_content(instance, instance.description_html) + return super().to_representation(instance) + -class IssueLiteSerializer(DynamicBaseSerializer): +class IssueLiteSerializer(DynamicBaseSerializer, BaseIssueSerializerMixin): workspace_detail = WorkspaceLiteSerializer(read_only=True, source="workspace") project_detail = ProjectLiteSerializer(read_only=True, source="project") state_detail = StateLiteSerializer(read_only=True, source="state") @@ -577,11 +597,17 @@ class Meta: "updated_at", ] + def to_representation(self, instance): + self.refresh_html_content(instance, instance.description_html) + return super().to_representation(instance) -class IssuePublicSerializer(BaseSerializer): + +class IssuePublicSerializer(BaseSerializer, BaseIssueSerializerMixin): project_detail = ProjectLiteSerializer(read_only=True, source="project") state_detail = StateLiteSerializer(read_only=True, source="state") - reactions = IssueReactionSerializer(read_only=True, many=True, source="issue_reactions") + reactions = IssueReactionSerializer( + read_only=True, many=True, source="issue_reactions" + ) votes = IssueVoteSerializer(read_only=True, many=True) class Meta: @@ -603,6 +629,9 @@ class Meta: ] read_only_fields = fields + def to_representation(self, instance): + self.refresh_html_content(instance, instance.description_html) + return super().to_representation(instance) class IssueSubscriberSerializer(BaseSerializer): diff --git a/apiserver/plane/app/serializers/project.py b/apiserver/plane/app/serializers/project.py index aef715e33a8..e4cfff79791 100644 --- a/apiserver/plane/app/serializers/project.py +++ b/apiserver/plane/app/serializers/project.py @@ -1,3 +1,6 @@ +# Django imports +from django.conf import settings + # Third party imports from rest_framework import serializers @@ -14,9 +17,23 @@ ProjectDeployBoard, ProjectPublicMember, ) +from plane.utils.s3 import S3 + + +class BaseProjectSerializerMixin: + """abstract class for refresh cover image s3 link""" + + def refresh_cover_image(self, instance): + if settings.AWS_S3_BUCKET_AUTH: + cover_image = instance.cover_image + + if S3.verify_s3_url(cover_image) and S3.url_file_has_expired(cover_image): + s3 = S3() + instance.cover_image = s3.refresh_url(cover_image) + instance.save() -class ProjectSerializer(BaseSerializer): +class ProjectSerializer(BaseSerializer, BaseProjectSerializerMixin): workspace_detail = WorkspaceLiteSerializer(source="workspace", read_only=True) class Meta: @@ -75,8 +92,12 @@ def update(self, instance, validated_data): # If not same fail update raise serializers.ValidationError(detail="Project Identifier is already taken") + def to_representation(self, instance): + self.refresh_cover_image(instance) + return super().to_representation(instance) -class ProjectLiteSerializer(BaseSerializer): + +class ProjectLiteSerializer(BaseSerializer, BaseProjectSerializerMixin): class Meta: model = Project fields = [ @@ -90,8 +111,12 @@ class Meta: ] read_only_fields = fields + def to_representation(self, instance): + self.refresh_cover_image(instance) + return super().to_representation(instance) + -class ProjectListSerializer(DynamicBaseSerializer): +class ProjectListSerializer(DynamicBaseSerializer, BaseProjectSerializerMixin): is_favorite = serializers.BooleanField(read_only=True) total_members = serializers.IntegerField(read_only=True) total_cycles = serializers.IntegerField(read_only=True) @@ -121,8 +146,12 @@ class Meta: model = Project fields = "__all__" + def to_representation(self, instance): + self.refresh_cover_image(instance) + return super().to_representation(instance) + -class ProjectDetailSerializer(BaseSerializer): +class ProjectDetailSerializer(BaseSerializer, BaseProjectSerializerMixin): # workspace = WorkSpaceSerializer(read_only=True) default_assignee = UserLiteSerializer(read_only=True) project_lead = UserLiteSerializer(read_only=True) @@ -139,6 +168,10 @@ class Meta: model = Project fields = "__all__" + def to_representation(self, instance): + self.refresh_cover_image(instance) + return super().to_representation(instance) + class ProjectMemberSerializer(BaseSerializer): workspace = WorkspaceLiteSerializer(read_only=True) @@ -217,4 +250,4 @@ class Meta: "workspace", "project", "member", - ] \ No newline at end of file + ] diff --git a/apiserver/plane/bgtasks/export_task.py b/apiserver/plane/bgtasks/export_task.py index e895b859de8..995e0445159 100644 --- a/apiserver/plane/bgtasks/export_task.py +++ b/apiserver/plane/bgtasks/export_task.py @@ -2,7 +2,6 @@ import csv import io import json -import boto3 import zipfile # Django imports @@ -12,11 +11,11 @@ # Third party imports from celery import shared_task from sentry_sdk import capture_exception -from botocore.client import Config from openpyxl import Workbook # Module imports from plane.db.models import Issue, ExporterHistory +from plane.utils.s3 import S3 def dateTimeConverter(time): @@ -70,51 +69,17 @@ def create_zip_file(files): def upload_to_s3(zip_file, workspace_id, token_id, slug): file_name = f"{workspace_id}/export-{slug}-{token_id[:6]}-{timezone.now()}.zip" expires_in = 7 * 24 * 60 * 60 + s3 = S3() + + s3.upload_file( + zip_file, + settings.AWS_STORAGE_BUCKET_NAME, + file_name, + "public-read", + "application/zip", + ) - if settings.USE_MINIO: - s3 = boto3.client( - "s3", - endpoint_url=settings.AWS_S3_ENDPOINT_URL, - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - config=Config(signature_version="s3v4"), - ) - s3.upload_fileobj( - zip_file, - settings.AWS_STORAGE_BUCKET_NAME, - file_name, - ExtraArgs={"ACL": "public-read", "ContentType": "application/zip"}, - ) - presigned_url = s3.generate_presigned_url( - "get_object", - Params={"Bucket": settings.AWS_STORAGE_BUCKET_NAME, "Key": file_name}, - ExpiresIn=expires_in, - ) - # Create the new url with updated domain and protocol - presigned_url = presigned_url.replace( - "http://plane-minio:9000/uploads/", - f"{settings.AWS_S3_URL_PROTOCOL}//{settings.AWS_S3_CUSTOM_DOMAIN}/", - ) - else: - s3 = boto3.client( - "s3", - region_name=settings.AWS_REGION, - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - config=Config(signature_version="s3v4"), - ) - s3.upload_fileobj( - zip_file, - settings.AWS_STORAGE_BUCKET_NAME, - file_name, - ExtraArgs={"ACL": "public-read", "ContentType": "application/zip"}, - ) - - presigned_url = s3.generate_presigned_url( - "get_object", - Params={"Bucket": settings.AWS_STORAGE_BUCKET_NAME, "Key": file_name}, - ExpiresIn=expires_in, - ) + presigned_url = s3.refresh_url(file_name, expires_in) exporter_instance = ExporterHistory.objects.get(token=token_id) diff --git a/apiserver/plane/bgtasks/exporter_expired_task.py b/apiserver/plane/bgtasks/exporter_expired_task.py index 30b638c84c8..360ecb7cdd7 100644 --- a/apiserver/plane/bgtasks/exporter_expired_task.py +++ b/apiserver/plane/bgtasks/exporter_expired_task.py @@ -1,5 +1,4 @@ # Python imports -import boto3 from datetime import timedelta # Django imports @@ -9,10 +8,10 @@ # Third party imports from celery import shared_task -from botocore.client import Config # Module imports from plane.db.models import ExporterHistory +from plane.utils.s3 import S3 @shared_task @@ -21,29 +20,11 @@ def delete_old_s3_link(): expired_exporter_history = ExporterHistory.objects.filter( Q(url__isnull=False) & Q(created_at__lte=timezone.now() - timedelta(days=8)) ).values_list("key", "id") - if settings.USE_MINIO: - s3 = boto3.client( - "s3", - endpoint_url=settings.AWS_S3_ENDPOINT_URL, - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - config=Config(signature_version="s3v4"), - ) - else: - s3 = boto3.client( - "s3", - region_name=settings.AWS_REGION, - aws_access_key_id=settings.AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, - config=Config(signature_version="s3v4"), - ) + s3 = S3() for file_name, exporter_id in expired_exporter_history: # Delete object from S3 if file_name: - if settings.USE_MINIO: - s3.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=file_name) - else: - s3.delete_object(Bucket=settings.AWS_STORAGE_BUCKET_NAME, Key=file_name) + s3.delete_file(settings.AWS_STORAGE_BUCKET_NAME, file_name) ExporterHistory.objects.filter(id=exporter_id).update(url=None) diff --git a/apiserver/plane/settings/common.py b/apiserver/plane/settings/common.py index 76528176b16..cbf3316c71b 100644 --- a/apiserver/plane/settings/common.py +++ b/apiserver/plane/settings/common.py @@ -225,13 +225,18 @@ AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "access-key") AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "secret-key") AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_S3_BUCKET_NAME", "uploads") -AWS_REGION = os.environ.get("AWS_REGION", "") -AWS_DEFAULT_ACL = "public-read" -AWS_QUERYSTRING_AUTH = False -AWS_S3_FILE_OVERWRITE = False +AWS_REGION = os.environ.get("AWS_REGION") +AWS_S3_REGION_NAME = os.environ.get("AWS_REGION") +AWS_S3_SIGNATURE_VERSION = "s3v4" +AWS_S3_BUCKET_AUTH = os.environ.get("AWS_S3_BUCKET_AUTH", False) +AWS_QUERYSTRING_AUTH = os.environ.get("AWS_QUERYSTRING_AUTH", False) +AWS_DEFAULT_ACL = "private" if AWS_S3_BUCKET_AUTH else "public-read" +AWS_S3_FILE_OVERWRITE = os.environ.get("AWS_S3_FILE_OVERWRITE", False) +AWS_S3_MAX_AGE_SECONDS = 60 * 60 AWS_S3_ENDPOINT_URL = os.environ.get("AWS_S3_ENDPOINT_URL", None) or os.environ.get( "MINIO_ENDPOINT_URL", None ) + if AWS_S3_ENDPOINT_URL: parsed_url = urlparse(os.environ.get("WEB_URL", "http://localhost")) AWS_S3_CUSTOM_DOMAIN = f"{parsed_url.netloc}/{AWS_STORAGE_BUCKET_NAME}" @@ -291,7 +296,9 @@ # Sentry Settings # Enable Sentry Settings -if bool(os.environ.get("SENTRY_DSN", False)) and os.environ.get("SENTRY_DSN").startswith("https://"): +if bool(os.environ.get("SENTRY_DSN", False)) and os.environ.get( + "SENTRY_DSN" +).startswith("https://"): sentry_sdk.init( dsn=os.environ.get("SENTRY_DSN", ""), integrations=[ diff --git a/apiserver/plane/utils/parse_html.py b/apiserver/plane/utils/parse_html.py new file mode 100644 index 00000000000..43af0cc3f09 --- /dev/null +++ b/apiserver/plane/utils/parse_html.py @@ -0,0 +1,22 @@ +from bs4 import BeautifulSoup + +from plane.utils.s3 import S3 + + +def parse_text_to_html(html, features="html.parser"): + return BeautifulSoup(html, features) + + +def refresh_url_content(html): + refreshed = False + + s3 = S3() + for img_tag in html.find_all("img"): + old_src = img_tag["src"] + + if S3.verify_s3_url(old_src) and S3.url_file_has_expired(old_src): + new_url = s3.refresh_url(old_src) + img_tag["src"] = new_url + refreshed = True + + return refreshed, str(html) diff --git a/apiserver/plane/utils/s3.py b/apiserver/plane/utils/s3.py new file mode 100644 index 00000000000..733885aaf09 --- /dev/null +++ b/apiserver/plane/utils/s3.py @@ -0,0 +1,78 @@ +import re +import boto3 +from botocore.client import Config +from urllib.parse import urlparse, parse_qs +from datetime import datetime, timezone + +from django.conf import settings + + +class S3: + """class for manage s3 operations (upload, delete, refresh url file)""" + + def __init__(self): + if settings.USE_MINIO: + self.client = boto3.client( + "s3", + endpoint_url=settings.AWS_S3_ENDPOINT_URL, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=Config(signature_version=settings.AWS_S3_SIGNATURE_VERSION), + ) + else: + self.client = boto3.client( + "s3", + region_name=settings.AWS_REGION, + aws_access_key_id=settings.AWS_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, + config=Config(signature_version=settings.AWS_S3_SIGNATURE_VERSION), + ) + + def refresh_url(self, old_url, time=settings.AWS_S3_MAX_AGE_SECONDS): + path = urlparse(str(old_url)).path.lstrip("/") + + url = self.client.generate_presigned_url( + ClientMethod="get_object", + ExpiresIn=time, + Params={"Bucket": settings.AWS_STORAGE_BUCKET_NAME, "Key": path}, + ) + + if settings.USE_MINIO: + url = url.replace( + "http://plane-minio:9000/uploads/", + f"{settings.AWS_S3_URL_PROTOCOL}//{settings.AWS_S3_CUSTOM_DOMAIN}/", + ) + + return url + + def upload_file(self, file, bucket_name, file_name, acl, content_type): + self.client.upload_fileobj( + file, + bucket_name, + file_name, + ExtraArgs={"ACL": acl, "ContentType": content_type}, + ) + + def delete_file(self, bucket_name, path): + self.client.delete_object(Bucket=bucket_name, Key=path) + + @staticmethod + def verify_s3_url(url): + if url: + pattern = re.compile(r"amazonaws\.com") + return pattern.search(url) + return False + + @staticmethod + def url_file_has_expired(url, date_format="%Y%m%dT%H%M%SZ"): + parsed_url = urlparse(url) + query_params = parse_qs(parsed_url.query) + x_amz_date = query_params.get("X-Amz-Date", [None])[0] + + x_amz_date_to_date = datetime.strptime(x_amz_date, date_format).replace( + tzinfo=timezone.utc + ) + actual_date = datetime.now(timezone.utc) + seconds_difference = (actual_date - x_amz_date_to_date).total_seconds() + + return seconds_difference >= (settings.AWS_S3_MAX_AGE_SECONDS - 20) diff --git a/apiserver/requirements/base.txt b/apiserver/requirements/base.txt index 6832297e975..e33b9e36d0a 100644 --- a/apiserver/requirements/base.txt +++ b/apiserver/requirements/base.txt @@ -33,4 +33,3 @@ posthog==3.0.2 cryptography==41.0.5 lxml==4.9.3 boto3==1.28.40 - diff --git a/deploy/coolify/coolify-docker-compose.yml b/deploy/coolify/coolify-docker-compose.yml index 58e00a7a715..fd517b78bc9 100644 --- a/deploy/coolify/coolify-docker-compose.yml +++ b/deploy/coolify/coolify-docker-compose.yml @@ -54,6 +54,8 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret-key} - AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000} - AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-uploads} + - AWS_S3_BUCKET_AUTH=${AWS_S3_BUCKET_AUTH:-False} + - AWS_QUERYSTRING_AUTH=${AWS_QUERYSTRING_AUTH:-False} - FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880} - OPENAI_API_BASE=${OPENAI_API_BASE:-https://api.openai.com/v1} - OPENAI_API_KEY=${OPENAI_API_KEY:-sk-} @@ -102,6 +104,8 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret-key} - AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000} - AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-uploads} + - AWS_S3_BUCKET_AUTH=${AWS_S3_BUCKET_AUTH:-False} + - AWS_QUERYSTRING_AUTH=${AWS_QUERYSTRING_AUTH:-False} - FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880} - OPENAI_API_BASE=${OPENAI_API_BASE:-https://api.openai.com/v1} - OPENAI_API_KEY=${OPENAI_API_KEY:-sk-} @@ -148,6 +152,8 @@ services: - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-secret-key} - AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000} - AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-uploads} + - AWS_S3_BUCKET_AUTH=${AWS_S3_BUCKET_AUTH:-False} + - AWS_QUERYSTRING_AUTH=${AWS_QUERYSTRING_AUTH:-False} - FILE_SIZE_LIMIT=${FILE_SIZE_LIMIT:-5242880} - OPENAI_API_BASE=${OPENAI_API_BASE:-https://api.openai.com/v1} - OPENAI_API_KEY=${OPENAI_API_KEY:-sk-} diff --git a/deploy/selfhost/docker-compose.yml b/deploy/selfhost/docker-compose.yml index 8b4ff77ef02..e2fcec17481 100644 --- a/deploy/selfhost/docker-compose.yml +++ b/deploy/selfhost/docker-compose.yml @@ -55,6 +55,8 @@ x-app-env : &app-env - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-"secret-key"} - AWS_S3_ENDPOINT_URL=${AWS_S3_ENDPOINT_URL:-http://plane-minio:9000} - AWS_S3_BUCKET_NAME=${AWS_S3_BUCKET_NAME:-uploads} + - AWS_S3_BUCKET_AUTH=${AWS_S3_BUCKET_AUTH:-False} + - AWS_QUERYSTRING_AUTH=${AWS_QUERYSTRING_AUTH:-False} - MINIO_ROOT_USER=${MINIO_ROOT_USER:-"access-key"} - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-"secret-key"} - BUCKET_NAME=${BUCKET_NAME:-uploads} diff --git a/deploy/selfhost/variables.env b/deploy/selfhost/variables.env index 4a378181154..7df39979ffe 100644 --- a/deploy/selfhost/variables.env +++ b/deploy/selfhost/variables.env @@ -57,6 +57,8 @@ AWS_ACCESS_KEY_ID="access-key" AWS_SECRET_ACCESS_KEY="secret-key" AWS_S3_ENDPOINT_URL=http://plane-minio:9000 AWS_S3_BUCKET_NAME=uploads +AWS_S3_BUCKET_AUTH=False +AWS_QUERYSTRING_AUTH=False MINIO_ROOT_USER="access-key" MINIO_ROOT_PASSWORD="secret-key" BUCKET_NAME=uploads