feat: support zstd compression in miniostorage

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
     "requests>=2.32.3",
     "sentry-sdk>=2.13.0",
     "sqlalchemy<2",
+    "zstandard>=0.23.0",
 ]
 
 [build-system]

shared/django_apps/rollouts/models.py

@@ -23,7 +23,7 @@ class RolloutUniverse(models.TextChoices):
 
 def default_random_salt():
     # to resolve circular dependency
-    from shared.django_apps.utils.model_utils import default_random_salt
+    from shared.django_apps.utils.rollout_utils import default_random_salt
 
     return default_random_salt()
 

shared/django_apps/utils/model_utils.py

@@ -1,10 +1,8 @@
 import json
 import logging
-from random import choice
 from typing import Any, Callable, Optional
 
 from shared.api_archive.archive import ArchiveService
-from shared.django_apps.rollouts.models import RolloutUniverse
 from shared.storage.exceptions import FileNotInStorageError
 from shared.utils.ReportEncoder import ReportEncoder
 
@@ -148,24 +146,6 @@ def __set__(self, obj, value):
         setattr(obj, self.cached_value_property_name, value)
 
 
-def default_random_salt():
-    ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-    return "".join([choice(ALPHABET) for _ in range(16)])
-
-
-def rollout_universe_to_override_string(rollout_universe: RolloutUniverse):
-    if rollout_universe == RolloutUniverse.OWNER_ID:
-        return "override_owner_ids"
-    elif rollout_universe == RolloutUniverse.REPO_ID:
-        return "override_repo_ids"
-    elif rollout_universe == RolloutUniverse.EMAIL:
-        return "override_emails"
-    elif rollout_universe == RolloutUniverse.ORG_ID:
-        return "override_org_ids"
-    else:
-        return ""
-
-
 # This is the place for DB trigger logic that's been moved into code
 # Owner
 def get_ownerid_if_member(

shared/django_apps/utils/rollout_utils.py

@@ -0,0 +1,21 @@
+from random import choice
+
+from shared.django_apps.rollouts.models import RolloutUniverse
+
+
+def default_random_salt():
+    ALPHABET = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    return "".join([choice(ALPHABET) for _ in range(16)])
+
+
+def rollout_universe_to_override_string(rollout_universe: RolloutUniverse):
+    if rollout_universe == RolloutUniverse.OWNER_ID:
+        return "override_owner_ids"
+    elif rollout_universe == RolloutUniverse.REPO_ID:
+        return "override_repo_ids"
+    elif rollout_universe == RolloutUniverse.EMAIL:
+        return "override_emails"
+    elif rollout_universe == RolloutUniverse.ORG_ID:
+        return "override_org_ids"
+    else:
+        return ""

shared/rollouts/__init__.py

@@ -17,7 +17,7 @@
     Platform,
     RolloutUniverse,
 )
-from shared.django_apps.utils.model_utils import rollout_universe_to_override_string
+from shared.django_apps.utils.rollout_utils import rollout_universe_to_override_string
 
 log = logging.getLogger("__name__")
 

shared/rollouts/features.py

@@ -2,3 +2,4 @@
 
 BUNDLE_THRESHOLD_FLAG = Feature("bundle_threshold_flag")
 INCLUDE_GITHUB_COMMENT_ACTIONS_BY_OWNER = Feature("include_github_comment_actions")
+USE_NEW_MINIO = Feature("use_new_minio")

shared/storage/__init__.py

@@ -1,18 +1,22 @@
 from shared.config import get_config
+from shared.rollouts.features import USE_NEW_MINIO
 from shared.storage.aws import AWSStorageService
 from shared.storage.base import BaseStorageService
 from shared.storage.fallback import StorageWithFallbackService
 from shared.storage.gcp import GCPStorageService
 from shared.storage.minio import MinioStorageService
+from shared.storage.new_minio import NewMinioStorageService
 
 
-def get_appropriate_storage_service() -> BaseStorageService:
-    chosen_storage = get_config("services", "chosen_storage", default="minio")
-    return _get_appropriate_storage_service_given_storage(chosen_storage)
+def get_appropriate_storage_service(
+    repoid: int | None = None,
+) -> BaseStorageService:
+    chosen_storage: str = get_config("services", "chosen_storage", default="minio")  # type: ignore
+    return _get_appropriate_storage_service_given_storage(chosen_storage, repoid)
 
 
 def _get_appropriate_storage_service_given_storage(
-    chosen_storage: str,
+    chosen_storage: str, repoid: int | None
 ) -> BaseStorageService:
     if chosen_storage == "gcp":
         gcp_config = get_config("services", "gcp", default={})
@@ -28,4 +32,6 @@ def _get_appropriate_storage_service_given_storage(
         return StorageWithFallbackService(gcp_service, aws_service)
     else:
         minio_config = get_config("services", "minio", default={})
+        if repoid and USE_NEW_MINIO.check_value(repoid, default=False):
+            return NewMinioStorageService(minio_config)
         return MinioStorageService(minio_config)

shared/storage/base.py

@@ -1,6 +1,7 @@
 from typing import BinaryIO, overload
 
 CHUNK_SIZE = 1024 * 32
+PART_SIZE = 1024 * 1024 * 20  # 20MiB
 
 
 # Interface class for interfacing with codecov's underlying storage layer