Merge pull request #120 from uptick/dev-1044/fix-oom-and-rollbacks

DEV-1044 fix(server): fix oom and behave better when rollbacks are required

Author: uptickmetachu

charts/gitops/templates/deployment.yml

@@ -40,8 +40,6 @@ spec:
         resources:
           requests:
             memory: "750Mi"
-          limits:
-            memory: "1500Mi"
 
         envFrom:
         - configMapRef:

gitops_server/workers/deployer/deploy.py

@@ -11,7 +11,7 @@
 
 from gitops.common.app import App
 from gitops_server import settings
-from gitops_server.types import AppDefinitions, UpdateAppResult
+from gitops_server.types import AppDefinitions, RunOutput, UpdateAppResult
 from gitops_server.utils import get_repo_name_from_url, github, run, slack
 from gitops_server.utils.git import temp_repo
 
@@ -133,6 +133,17 @@ async def uninstall_app(self, app: App) -> UpdateAppResult:
                 )
             return update_result
 
+    async def rollback_deployment(self, app: App) -> None:
+        with tracer.start_as_current_span("rollback_deployment", attributes={"app": app.name}):
+            logger.warning(
+                "Rolling back %s deployment due to previous failed helm install",
+                app.name,
+            )
+            await run(
+                f"helm rollback --namespace={app.namespace} {app.name}",
+                suppress_errors=True,
+            )
+
     async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
         app.set_value("deployment.labels.gitops/deploy_id", self.deploy_id)
         app.set_value("deployment.labels.gitops/status", github.STATUSES.in_progress)
@@ -155,6 +166,40 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
                             os.fsync(cfg.fileno())
 
                             with tracer.start_as_current_span("helm_upgrade"):
+
+                                async def upgrade_helm_git() -> RunOutput:
+                                    result = await run(
+                                        "helm secrets upgrade --create-namespace"
+                                        f" --history-max {MAX_HELM_HISTORY}"
+                                        " --install"
+                                        " --timeout=600s"
+                                        f"{' --set skip_migrations=true' if self.skip_migrations else ''}"
+                                        f" -f {cfg.name}"
+                                        f" --namespace={app.namespace}"
+                                        f" {app.name}"
+                                        f" {chart_folder_path}",
+                                        suppress_errors=True,
+                                    )
+                                    return result
+
+                                result = await upgrade_helm_git()
+                                if result["exit_code"] != 0 and "is in progress" in result["output"]:
+                                    await self.rollback_deployment(app)
+                                    result = await upgrade_helm_git()
+
+                elif app.chart.type == "helm":
+                    span.set_attribute("gitops.chart.type", "helm")
+                    with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
+                        cfg.write(json.dumps(app.values).encode())
+                        cfg.flush()
+                        os.fsync(cfg.fileno())
+                        chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else ""
+                        with tracer.start_as_current_span("helm_repo_add"):
+                            await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}")
+
+                        with tracer.start_as_current_span("helm_upgrade"):
+
+                            async def upgrade_helm_chart() -> RunOutput:
                                 result = await run(
                                     "helm secrets upgrade --create-namespace"
                                     f" --history-max {MAX_HELM_HISTORY}"
@@ -164,32 +209,15 @@ async def update_app_deployment(self, app: App) -> UpdateAppResult | None:
                                     f" -f {cfg.name}"
                                     f" --namespace={app.namespace}"
                                     f" {app.name}"
-                                    f" {chart_folder_path}",
+                                    f" {app.chart.helm_chart} {chart_version_arguments}",
                                     suppress_errors=True,
                                 )
-                elif app.chart.type == "helm":
-                    span.set_attribute("gitops.chart.type", "helm")
-                    with tempfile.NamedTemporaryFile(suffix=".yml") as cfg:
-                        cfg.write(json.dumps(app.values).encode())
-                        cfg.flush()
-                        os.fsync(cfg.fileno())
-                        chart_version_arguments = f" --version={app.chart.version}" if app.chart.version else ""
-                        with tracer.start_as_current_span("helm_repo_add"):
-                            await run(f"helm repo add {app.chart.helm_repo} {app.chart.helm_repo_url}")
+                                return result
 
-                        with tracer.start_as_current_span("helm_upgrade"):
-                            result = await run(
-                                "helm secrets upgrade --create-namespace"
-                                f" --history-max {MAX_HELM_HISTORY}"
-                                " --install"
-                                " --timeout=600s"
-                                f"{' --set skip_migrations=true' if self.skip_migrations else ''}"
-                                f" -f {cfg.name}"
-                                f" --namespace={app.namespace}"
-                                f" {app.name}"
-                                f" {app.chart.helm_chart} {chart_version_arguments}",
-                                suppress_errors=True,
-                            )
+                            result = await upgrade_helm_chart()
+                            if result["exit_code"] != 0 and "is in progress" in result["output"]:
+                                await self.rollback_deployment(app)
+                                result = await upgrade_helm_chart()
                 else:
                     logger.warning("Local is not implemented yet")
                     return None