add secrets/updates & add force_unlock for debugging

src/_nebari/deploy.py

@@ -15,6 +15,7 @@ def deploy_configuration(
     stages: List[hookspecs.NebariStage],
     disable_prompt: bool = False,
     disable_checks: bool = False,
+    stage_force_unlock: str = None,
 ) -> Dict[str, Any]:
     if config.prevent_deploy:
         raise ValueError(
@@ -47,12 +48,26 @@ def deploy_configuration(
 
     with timer(logger, "deploying Nebari"):
         stage_outputs = {}
+        force_unlock = False
         with contextlib.ExitStack() as stack:
             for stage in stages:
                 s: hookspecs.NebariStage = stage(
                     output_directory=pathlib.Path.cwd(), config=config
                 )
-                stack.enter_context(s.deploy(stage_outputs, disable_prompt))
+                print(f"Deploying stage {s.name}")
+                print("stage_force_unlock", stage_force_unlock)
+                if stage_force_unlock:
+                    _unlock_stage_name, _unlock_state_id = stage_force_unlock.split(":")
+                    if _unlock_stage_name == s.name:
+                        force_unlock = _unlock_state_id
+                    print(f"Force unlocking stage {s.name} :: {force_unlock}")
+                stack.enter_context(
+                    s.deploy(
+                        stage_outputs=stage_outputs,
+                        disable_prompt=disable_prompt,
+                        force_unlock=force_unlock,
+                    )
+                )
 
                 if not disable_checks:
                     s.check(stage_outputs, disable_prompt)

src/_nebari/provider/terraform.py

@@ -28,6 +28,7 @@ def deploy(
     terraform_import: bool = False,
     terraform_apply: bool = True,
     terraform_destroy: bool = False,
+    terraform_lock_id: str = None,
     input_vars: Dict[str, Any] = {},
     state_imports: List[Any] = [],
 ):
@@ -46,6 +47,10 @@ def deploy(
       terraform_destroy: whether to run `terraform destroy` default
         False
 
+      terraform_lock_id: Used to toggle the force-unlock feature of a given stage based
+      on the provided lock_id.
+        None
+
       input_vars: supply values for "variable" resources within
         terraform module
 
@@ -61,6 +66,9 @@ def deploy(
         if terraform_init:
             init(directory)
 
+        if terraform_lock_id:
+            force_unlock(directory, terraform_lock_id)
+
         if terraform_import:
             for addr, id in state_imports:
                 tfimport(
@@ -139,6 +147,14 @@ def init(directory=None, upgrade=True):
         run_terraform_subprocess(command, cwd=directory, prefix="terraform")
 
 
+def force_unlock(directory=None, lock_id=None):
+    logger.info(f"terraform unlock directory={directory}")
+    with timer(logger, "terraform unlock"):
+        run_terraform_subprocess(
+            ["force-unlock", "-force", lock_id], cwd=directory, prefix="terraform"
+        )
+
+
 def apply(directory=None, targets=None, var_files=None):
     targets = targets or []
     var_files = var_files or []

src/_nebari/stages/base.py

@@ -284,13 +284,19 @@ def deploy(
         stage_outputs: Dict[str, Dict[str, Any]],
         disable_prompt: bool = False,
         terraform_init: bool = True,
+        force_unlock: bool = False,
     ):
         deploy_config = dict(
             directory=str(self.output_directory / self.stage_prefix),
             input_vars=self.input_vars(stage_outputs),
             terraform_init=terraform_init,
         )
         state_imports = self.state_imports()
+        print("Deploying terraform resources for", self.name)
+        print("Force unlock:", force_unlock)
+        if force_unlock:
+            deploy_config["terraform_lock_id"] = force_unlock
+
         if state_imports:
             deploy_config["terraform_import"] = True
             deploy_config["state_imports"] = state_imports

src/_nebari/stages/infrastructure/__init__.py

@@ -923,9 +923,12 @@ def post_deploy(
 
     @contextlib.contextmanager
     def deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self,
+        stage_outputs: Dict[str, Dict[str, Any]],
+        disable_prompt: bool = False,
+        force_unlock: bool = False,
     ):
-        with super().deploy(stage_outputs, disable_prompt):
+        with super().deploy(stage_outputs, disable_prompt, force_unlock=force_unlock):
             with kubernetes_provider_context(
                 stage_outputs["stages/" + self.name]["kubernetes_credentials"]["value"]
             ):

src/_nebari/stages/kubernetes_keycloak/__init__.py

@@ -297,9 +297,12 @@ def _attempt_keycloak_connection(
 
     @contextlib.contextmanager
     def deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self,
+        stage_outputs: Dict[str, Dict[str, Any]],
+        disable_prompt: bool = False,
+        force_unlock: bool = False,
     ):
-        with super().deploy(stage_outputs, disable_prompt):
+        with super().deploy(stage_outputs, disable_prompt, force_unlock=force_unlock):
             with keycloak_provider_context(
                 stage_outputs["stages/" + self.name]["keycloak_credentials"]["value"]
             ):

src/_nebari/stages/kubernetes_services/__init__.py

@@ -190,6 +190,14 @@ def check_default(cls, value):
         return value
 
 
+class BackupRestoreStorage(schema.Base):
+    type: str
+
+
+class BackupRestore(schema.Base):
+    storage: BackupRestoreStorage = BackupRestoreStorage(type="s3")
+
+
 class CondaEnvironment(schema.Base):
     name: str
     channels: Optional[List[str]] = None

src/_nebari/stages/kubernetes_services/template/backup-restore.tf

@@ -0,0 +1,29 @@
+variable "backup-restore-image" {
+  description = "The image to use for the backup-restore service"
+  default     = "vcerutti/nebari-backup-restore"
+}
+
+variable "backup-restore-image-tag" {
+  description = "The tag of the image to use for the backup-restore service"
+  default     = "latest"
+}
+
+variable "backup-restore-clients" {
+  description = "List of clients that can access the backup-restore server by API"
+  type        = list(string)
+  default     = ["nebari-cli"]
+}
+
+module "kubernetes-backup-restore-server" {
+  source = "./modules/kubernetes/services/backup-restore"
+
+  name      = "nebari"
+  namespace = var.environment
+
+  external-url = var.endpoint
+  realm_id     = var.realm_id
+
+  backup-restore-image     = var.backup-restore-image
+  backup-restore-image-tag = var.backup-restore-image-tag
+  clients                  = var.backup-restore-clients
+}

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/backup-restore/ingress.tf

@@ -14,7 +14,7 @@ resource "kubernetes_manifest" "backup-restore-api" {
           match = "Host(`${var.external-url}`)"
           services = [
             {
-              name      = helm_release.backup_restore.name
+              name      = kubernetes_deployment.backup_restore.metadata.0.name
               port      = 9000
               namespace = var.namespace
             }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/backup-restore/main.tf

@@ -1,9 +1,28 @@
+locals {
+  clients = {
+    for client in var.clients : client => client
+  }
+}
+
 resource "random_password" "backup_restore_service_token" {
-  for_each = var.clients
+  for_each = local.clients
   length   = 32
   special  = false
 }
 
+resource "kubernetes_secret" "backup_restore_service_token" {
+  for_each = local.clients
+  metadata {
+    name      = "backup-restore-${each.key}"
+    namespace = var.namespace
+  }
+
+  data = {
+    token = random_password.backup_restore_service_token[each.key].result
+  }
+}
+
+
 resource "kubernetes_service" "backup_restore" {
   metadata {
     name      = "backup-restore"
@@ -22,6 +41,18 @@ resource "kubernetes_service" "backup_restore" {
   }
 }
 
+resource "kubernetes_config_map" "backup-restore-etc" {
+  metadata {
+    name      = "backup-restore-etc"
+    namespace = var.namespace
+  }
+
+  data = {
+    "keycloak.json" = jsonencode({})
+    "storage.json"  = jsonencode({})
+  }
+}
+
 resource "kubernetes_service_account" "backup_restore" {
   metadata {
     name      = "backup-restore"
@@ -53,7 +84,7 @@ resource "kubernetes_manifest" "backup_restore" {
 
           services = [
             {
-              name = kubernetes_service.gateway.metadata.0.name
+              name = kubernetes_service.backup_restore.metadata.0.name
               port = 8000
             }
           ]
@@ -63,16 +94,18 @@ resource "kubernetes_manifest" "backup_restore" {
   }
 }
 
-resource "kubernetes_secret" "backup_restore_service_token" {
-  for_each = var.clients
-  metadata {
-    name      = "backup-restore-${each.key}"
-    namespace = var.namespace
-  }
 
-  data = {
-    token = random_password.backup_restore_service_token[each.key].result
-  }
+module "jupyterhub-openid-client" {
+  source = "../keycloak-client"
+
+  realm_id                 = var.realm_id
+  client_id                = "nebari-cli"
+  external-url             = var.external-url
+  role_mapping             = {}
+  client_roles             = []
+  callback-url-paths       = []
+  service-accounts-enabled = true
+  service-account-roles    = ["realm-admin"]
 }
 
 resource "kubernetes_deployment" "backup_restore" {
@@ -101,33 +134,35 @@ resource "kubernetes_deployment" "backup_restore" {
         service_account_name = kubernetes_service_account.backup_restore.metadata.0.name
 
         container {
-          name  = "backup-restore"
-          image = "nebari/backup-restore:latest"
+          name              = "backup-restore"
+          image             = "${var.backup-restore-image}:${var.backup-restore-image-tag}"
+          image_pull_policy = "Always"
 
           env {
-            name  = "BACKUP_RESTORE_TOKEN"
-            value = kubernetes_secret.backup_restore_service_token[each.key].data["token"]
+            name  = "CONFIG_DIR"
+            value = "/etc/backup-restore/config"
           }
 
-          env {
-            name  = "BACKUP_RESTORE_NAMESPACE"
-            value = var.namespace
-          }
-
-          env {
-            name  = "BACKUP_RESTORE_EXTERNAL_URL"
-            value = var.external-url
-          }
+          command = ["backup-restore", "--standalone"]
 
-          env {
-            name  = "BACKUP_RESTORE_CLIENT"
-            value = each.key
+          volume_mount {
+            name       = "config"
+            mount_path = "/etc/backup-restore/config"
+            read_only  = true
           }
 
           port {
             container_port = 8000
           }
         }
+
+        volume {
+          name = "config"
+
+          config_map {
+            name = kubernetes_config_map.backup-restore-etc.metadata.0.name
+          }
+        }
       }
     }
   }

src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/backup-restore/variables.tf

@@ -18,3 +18,18 @@ variable "clients" {
   description = "List of clients that can access the backup-restore server by API"
   type        = list(string)
 }
+
+variable "realm_id" {
+  description = "Realm ID to use for authentication"
+  type        = string
+}
+
+variable "backup-restore-image" {
+  description = "Backup-restore image"
+  type        = string
+}
+
+variable "backup-restore-image-tag" {
+  description = "Version of backup-restore to use"
+  type        = string
+}

src/_nebari/stages/terraform_state/__init__.py

@@ -232,13 +232,21 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
 
     @contextlib.contextmanager
     def deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self,
+        stage_outputs: Dict[str, Dict[str, Any]],
+        disable_prompt: bool = False,
+        force_unlock: bool = False,
     ):
         self.check_immutable_fields()
 
         # No need to run terraform init here as it's being called when running the
         # terraform show command, inside check_immutable_fields
-        with super().deploy(stage_outputs, disable_prompt, terraform_init=False):
+        with super().deploy(
+            stage_outputs,
+            disable_prompt,
+            terraform_init=False,
+            force_unlock=force_unlock,
+        ):
             env_mapping = {}
             # DigitalOcean terraform remote state using Spaces Bucket
             # assumes aws credentials thus we set them to match spaces credentials

src/_nebari/subcommands/deploy.py

@@ -59,6 +59,11 @@ def deploy(
             "--skip-remote-state-provision",
             help="Skip terraform state deployment which is often required in CI once the terraform remote state bootstrapping phase is complete",
         ),
+        stage_force_unlock: str = typer.Option(
+            None,
+            "--stage-force-unlock",
+            help="Force unlock the terraform state file for a given stage. This should be only used if you know what you are doing",
+        ),
     ):
         """
         Deploy the Nebari cluster from your [purple]nebari-config.yaml[/purple] file.
@@ -94,4 +99,5 @@ def deploy(
             stages,
             disable_prompt=disable_prompt,
             disable_checks=disable_checks,
+            stage_force_unlock=stage_force_unlock,
         )

src/nebari/hookspecs.py

@@ -28,7 +28,10 @@ def render(self) -> Dict[str, str]:
 
     @contextlib.contextmanager
     def deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self,
+        stage_outputs: Dict[str, Dict[str, Any]],
+        disable_prompt: bool = False,
+        force_unlock: str = None,
     ):
         yield