From 28db7be34d038ae486ce4fcee3bb45a89e9e0a96 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 22:25:24 -0800
Subject: [PATCH 01/42] Autoformat from Black

---
 src/_nebari/upgrade.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index b29bec03c..93cb8587f 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1000,7 +1000,6 @@ def _version_specific_upgrade(
     ):
         # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled
         if config.get("monitoring", {}).get("enabled", True):
-
             crd_urls = [
                 "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml",
                 "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.73.0/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml",

From f387e4b26e82041ed2de8ebabed02376312d9c8c Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 22:38:47 -0800
Subject: [PATCH 02/42] Add `attempt_fixes` parameter to
 `UpgradeStep.upgrade_step` and `UpgradeStep._version_specific_upgrade`

---
 src/_nebari/upgrade.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 93cb8587f..5166d5ee5 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -230,7 +230,15 @@ def requires_nebari_version_field(self):
         """
         return rounded_ver_parse(self.version) > rounded_ver_parse("0.3.13")
 
-    def upgrade_step(self, config, start_version, config_filename, *args, **kwargs):
+    def upgrade_step(
+        self,
+        config,
+        start_version,
+        config_filename,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
+    ):
         """
         Perform the upgrade from start_version to self.version.
 
@@ -415,11 +423,22 @@ def update_image_tag(
 
         # Run any version-specific tasks
         return self._version_specific_upgrade(
-            config, start_version, config_filename, *args, **kwargs
+            config,
+            start_version,
+            config_filename,
+            *args,
+            attempt_fixes=attempt_fixes,
+            **kwargs,
         )
 
     def _version_specific_upgrade(
-        self, config, start_version, config_filename, *args, **kwargs
+        self,
+        config,
+        start_version,
+        config_filename,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
     ):
         """
         Perform version-specific upgrade tasks.

From 06598c7c44c7ebef530f1a9d92ee0defb98f7990 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 23:12:07 -0800
Subject: [PATCH 03/42] Add `attempt_fixes` to `UpgradeStep.replace_image_tag`
 and `UpgradeStep.update_image_tag`

---
 src/_nebari/upgrade.py | 38 +++++++++++++++++++++++++++++---------
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 5166d5ee5..36af29cd1 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -21,7 +21,7 @@
 import rich
 from packaging.version import Version
 from pydantic import ValidationError
-from rich.prompt import Prompt
+from rich.prompt import Confirm, Prompt
 from typing_extensions import override
 
 from _nebari.config import backup_configuration
@@ -315,7 +315,12 @@ def replace_image_tag_legacy(
                 return ":".join([m.groups()[0], f"v{new_version}"])
             return None
 
-        def replace_image_tag(s: str, new_version: str, config_path: str) -> str:
+        def replace_image_tag(
+            s: str,
+            new_version: str,
+            config_path: str,
+            attempt_fixes: bool = attempt_fixes,
+        ) -> str:
             """
             Replace the image tag with the new version.
 
@@ -337,11 +342,11 @@ def replace_image_tag(s: str, new_version: str, config_path: str) -> str:
             if current_tag == new_version:
                 return s
             loc = f"{config_path}: {image_name}"
-            response = Prompt.ask(
-                f"\nDo you want to replace current tag [green]{current_tag}[/green] with [green]{new_version}[/green] for:\n[purple]{loc}[/purple]? [Y/n] ",
-                default="Y",
+            response = attempt_fixes or Confirm.ask(
+                f"\nDo you want to replace current tag [green]{current_tag}[/green] with [green]{new_version}[/green] for:\n[purple]{loc}[/purple]?",
+                default=True,
             )
-            if response.lower() in ["y", "yes", ""]:
+            if response:
                 return s.replace(current_tag, new_version)
             else:
                 return s
@@ -372,7 +377,11 @@ def set_nested_item(config: dict, config_path: list, value: str):
             config[config_path[-1]] = value
 
         def update_image_tag(
-            config: dict, config_path: str, current_image: str, new_version: str
+            config: dict,
+            config_path: str,
+            current_image: str,
+            new_version: str,
+            attempt_fixes: bool = attempt_fixes,
         ) -> dict:
             """
             Update the image tag in the configuration.
@@ -386,7 +395,12 @@ def update_image_tag(
             Returns:
                 dict: The updated configuration dictionary.
             """
-            new_image = replace_image_tag(current_image, new_version, config_path)
+            new_image = replace_image_tag(
+                current_image,
+                new_version,
+                config_path,
+                attempt_fixes,
+            )
             if new_image != current_image:
                 set_nested_item(config, config_path, new_image)
 
@@ -396,7 +410,11 @@ def update_image_tag(
         for k, v in config.get("default_images", {}).items():
             config_path = f"default_images.{k}"
             config = update_image_tag(
-                config, config_path, v, __rounded_finish_version__
+                config,
+                config_path,
+                v,
+                __rounded_finish_version__,
+                attempt_fixes,
             )
 
         # update profiles.jupyterlab images
@@ -408,6 +426,7 @@ def update_image_tag(
                     f"profiles.jupyterlab.{i}.kubespawner_override.image",
                     current_image,
                     __rounded_finish_version__,
+                    attempt_fixes,
                 )
 
         # update profiles.dask_worker images
@@ -419,6 +438,7 @@ def update_image_tag(
                     f"profiles.dask_worker.{k}.image",
                     current_image,
                     __rounded_finish_version__,
+                    attempt_fixes,
                 )
 
         # Run any version-specific tasks

From 3bf0d4d187a64a3581772d7b686494b6f5540f01 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 23:15:44 -0800
Subject: [PATCH 04/42] Add `attempt_fixes` to
 `Upgrade_2023_4_2._version_specific_upgrade`

---
 src/_nebari/upgrade.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 36af29cd1..682143021 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -663,7 +663,13 @@ class Upgrade_2023_4_2(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self, config, start_version, config_filename: Path, *args, **kwargs
+        self,
+        config,
+        start_version,
+        config_filename: Path,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
     ):
         """
         Prompt users to delete Argo CRDs
@@ -680,11 +686,11 @@ def _version_specific_upgrade(
             ""
         )
 
-        continue_ = Prompt.ask(
+        continue_ = attempt_fixes or Confirm.ask(
             "Have you deleted the Argo Workflows CRDs and service accounts? [y/N] ",
-            default="N",
+            default=False,
         )
-        if not continue_ == "y":
+        if not continue_:
             rich.print(
                 f"You must delete the Argo Workflows CRDs and service accounts before upgrading to [green]{self.version}[/green] (or later)."
             )

From a3124638c0d826745862d5f2943588bf155aed92 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 23:41:38 -0800
Subject: [PATCH 05/42] Add `attempt_fixes` to
 `Upgrade_2023_7_2._version_specific_upgrade`

---
 src/_nebari/upgrade.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 682143021..f18309e8b 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -723,15 +723,21 @@ class Upgrade_2023_7_2(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self, config, start_version, config_filename: Path, *args, **kwargs
+        self,
+        config,
+        start_version,
+        config_filename: Path,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
     ):
         argo = config.get("argo_workflows", {})
         if argo.get("enabled"):
-            response = Prompt.ask(
-                f"\nDo you want to enable the [green][link={NEBARI_WORKFLOW_CONTROLLER_DOCS}]Nebari Workflow Controller[/link][/green], required for [green][link={ARGO_JUPYTER_SCHEDULER_REPO}]Argo-Jupyter-Scheduler[/link][green]? [Y/n] ",
-                default="Y",
+            response = attempt_fixes or Confirm.ask(
+                f"\nDo you want to enable the [green][link={NEBARI_WORKFLOW_CONTROLLER_DOCS}]Nebari Workflow Controller[/link][/green], required for [green][link={ARGO_JUPYTER_SCHEDULER_REPO}]Argo-Jupyter-Scheduler[/link][green]?",
+                default=True,
             )
-            if response.lower() in ["y", "yes", ""]:
+            if response:
                 argo["nebari_workflow_controller"] = {"enabled": True}
 
         rich.print("\n ⚠️ Deprecation Warnings ⚠️")

From 0359281e985a1700aa8e72fd5b8000c218a05e6b Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 23:43:48 -0800
Subject: [PATCH 06/42] Add `attempt_fixes` to
 `Upgrade_2024_4_1._version_specific_upgrade`

---
 src/_nebari/upgrade.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index f18309e8b..88c443b30 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -995,7 +995,13 @@ class Upgrade_2024_4_1(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self, config, start_version, config_filename: Path, *args, **kwargs
+        self,
+        config,
+        start_version,
+        config_filename: Path,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
     ):
         # Default configuration for the node groups was added in this version. Therefore,
         # users upgrading who don't have any specific node groups defined on their config
@@ -1009,12 +1015,11 @@ def _version_specific_upgrade(
                     default_node_groups = provider_enum_default_node_groups_map[
                         provider
                     ]
-                    continue_ = Prompt.ask(
+                    continue_ = attempt_fixes or Confirm.ask(
                         f"Would you like to include the default configuration for the node groups in [purple]{config_filename}[/purple]?",
-                        choices=["y", "N"],
-                        default="N",
+                        default=False,
                     )
-                    if continue_ == "y":
+                    if continue_:
                         config[provider_full_name]["node_groups"] = default_node_groups
                 except KeyError:
                     pass

From a1eefe25488bea82a8c440b7d31b9a533d7d7583 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sat, 9 Nov 2024 23:59:21 -0800
Subject: [PATCH 07/42] Add `attempt_fixes` to
 `Upgrade_2024_6_1._version_specific_upgrade`

---
 src/_nebari/upgrade.py | 42 +++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 88c443b30..4e1bdd135 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1052,7 +1052,13 @@ class Upgrade_2024_6_1(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self, config, start_version, config_filename: Path, *args, **kwargs
+        self,
+        config,
+        start_version,
+        config_filename: Path,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
     ):
         # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled
         if config.get("monitoring", {}).get("enabled", True):
@@ -1085,10 +1091,9 @@ def _version_specific_upgrade(
                 "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]"
                 f"\n-> [red bold]Before upgrading, kube-prometheus-stack CRDs need to be updated and the {daemonset_name} daemonset needs to be deleted.[/red bold]"
             )
-            run_commands = Prompt.ask(
+            run_commands = attempt_fixes or Confirm.ask(
                 "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually.",
-                choices=["y", "N"],
-                default="N",
+                default=False,
             )
 
             # By default, rich wraps lines by splitting them into multiple lines. This is
@@ -1096,7 +1101,7 @@ def _version_specific_upgrade(
             # To avoid this, we use a rich console with a larger width to print the entire commands
             # and let the terminal wrap them if needed.
             console = rich.console.Console(width=220)
-            if run_commands == "y":
+            if run_commands:
                 try:
                     kubernetes.config.load_kube_config()
                 except kubernetes.config.config_exception.ConfigException:
@@ -1109,10 +1114,10 @@ def _version_specific_upgrade(
                 rich.print(
                     f"The following commands will be run for the [cyan bold]{cluster_name}[/cyan bold] cluster"
                 )
-                Prompt.ask("Hit enter to show the commands")
+                _ = attempt_fixes or Prompt.ask("Hit enter to show the commands")
                 console.print(commands)
 
-                Prompt.ask("Hit enter to continue")
+                _ = attempt_fixes or Prompt.ask("Hit enter to continue")
                 # We need to add a special constructor to the yaml loader to handle a specific
                 # tag as otherwise the kubernetes API will fail when updating the CRD.
                 yaml.constructor.add_constructor(
@@ -1154,16 +1159,15 @@ def _version_specific_upgrade(
                 rich.print(
                     "[red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]"
                 )
-                Prompt.ask("Hit enter to show the commands")
+                _ = attempt_fixes or Prompt.ask("Hit enter to show the commands")
                 console.print(commands)
 
-                Prompt.ask("Hit enter to continue")
-                continue_ = Prompt.ask(
+                _ = attempt_fixes or Prompt.ask("Hit enter to continue")
+                continue_ = attempt_fixes or Confirm.ask(
                     f"Have you backed up your custom dashboards (if necessary), deleted the {daemonset_name} daemonset and updated the kube-prometheus-stack CRDs?",
-                    choices=["y", "N"],
-                    default="N",
+                    default=False,
                 )
-                if not continue_ == "y":
+                if not continue_:
                     rich.print(
                         f"[red bold]You must back up your custom dashboards (if necessary), delete the {daemonset_name} daemonset and update the kube-prometheus-stack CRDs before upgrading to [green]{self.version}[/green] (or later).[/bold red]"
                     )
@@ -1188,12 +1192,11 @@ def _version_specific_upgrade(
                         If not, select "N" and the old default node groups will be added to the nebari config file.
                     """
                     )
-                    continue_ = Prompt.ask(
+                    continue_ = attempt_fixes or Confirm.ask(
                         text,
-                        choices=["y", "N"],
-                        default="y",
+                        default=True,
                     )
-                    if continue_ == "N":
+                    if not continue_:
                         config[provider_full_name]["node_groups"] = {
                             "general": {
                                 "instance": "n1-standard-8",
@@ -1234,8 +1237,9 @@ def _version_specific_upgrade(
                     },
                     indent=4,
                 )
-                text += "\n\nHit enter to continue"
-                Prompt.ask(text)
+                rich.print(text)
+                if not attempt_fixes:
+                    _ = Prompt.ask("\n\nHit enter to continue")
         return config
 
 

From 2ee366c47a8a004ad25487bbf0b6bbf08af148de Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sun, 10 Nov 2024 00:02:40 -0800
Subject: [PATCH 08/42] Add `attempt_fixes` to
 `Upgrade_2024_11_1._version_specific_upgrade`

---
 src/_nebari/upgrade.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 4e1bdd135..1a3773554 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1292,7 +1292,13 @@ class Upgrade_2024_11_1(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self, config, start_version, config_filename: Path, *args, **kwargs
+        self,
+        config,
+        start_version,
+        config_filename: Path,
+        *args,
+        attempt_fixes=False,
+        **kwargs,
     ):
         if config.get("provider", "") == ProviderEnum.azure.value:
             rich.print("\n ⚠️ Upgrade Warning ⚠️")
@@ -1342,13 +1348,9 @@ def _version_specific_upgrade(
         keycloak_admin = None
 
         # Prompt the user for role assignment (if yes, transforms the response into bool)
-        assign_roles = (
-            Prompt.ask(
-                "[bold]Would you like Nebari to assign the corresponding role/scopes to all of your current groups automatically?[/bold]",
-                choices=["y", "N"],
-                default="N",
-            ).lower()
-            == "y"
+        assign_roles = attempt_fixes or Confirm.ask(
+            "[bold]Would you like Nebari to assign the corresponding role/scopes to all of your current groups automatically?[/bold]",
+            default=False,
         )
 
         if assign_roles:

From a6a0364849f362fc68cdd96f92708d7e873b9808 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sun, 17 Nov 2024 18:07:11 -0800
Subject: [PATCH 09/42] Remove unnecessary argument

---
 src/_nebari/upgrade.py | 58 +++++-------------------------------------
 1 file changed, 7 insertions(+), 51 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 1a3773554..4ec46f4d8 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -230,15 +230,7 @@ def requires_nebari_version_field(self):
         """
         return rounded_ver_parse(self.version) > rounded_ver_parse("0.3.13")
 
-    def upgrade_step(
-        self,
-        config,
-        start_version,
-        config_filename,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
-    ):
+    def upgrade_step(self, config, start_version, config_filename, *args, **kwargs):
         """
         Perform the upgrade from start_version to self.version.
 
@@ -452,13 +444,7 @@ def update_image_tag(
         )
 
     def _version_specific_upgrade(
-        self,
-        config,
-        start_version,
-        config_filename,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
+        self, config, start_version, config_filename, *args, **kwargs
     ):
         """
         Perform version-specific upgrade tasks.
@@ -663,13 +649,7 @@ class Upgrade_2023_4_2(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self,
-        config,
-        start_version,
-        config_filename: Path,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
+        self, config, start_version, config_filename: Path, *args, **kwargs
     ):
         """
         Prompt users to delete Argo CRDs
@@ -723,13 +703,7 @@ class Upgrade_2023_7_2(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self,
-        config,
-        start_version,
-        config_filename: Path,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
+        self, config, start_version, config_filename: Path, *args, **kwargs
     ):
         argo = config.get("argo_workflows", {})
         if argo.get("enabled"):
@@ -995,13 +969,7 @@ class Upgrade_2024_4_1(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self,
-        config,
-        start_version,
-        config_filename: Path,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
+        self, config, start_version, config_filename: Path, *args, **kwargs
     ):
         # Default configuration for the node groups was added in this version. Therefore,
         # users upgrading who don't have any specific node groups defined on their config
@@ -1052,13 +1020,7 @@ class Upgrade_2024_6_1(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self,
-        config,
-        start_version,
-        config_filename: Path,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
+        self, config, start_version, config_filename: Path, *args, **kwargs
     ):
         # Prompt users to manually update kube-prometheus-stack CRDs if monitoring is enabled
         if config.get("monitoring", {}).get("enabled", True):
@@ -1292,13 +1254,7 @@ class Upgrade_2024_11_1(UpgradeStep):
 
     @override
     def _version_specific_upgrade(
-        self,
-        config,
-        start_version,
-        config_filename: Path,
-        *args,
-        attempt_fixes=False,
-        **kwargs,
+        self, config, start_version, config_filename: Path, *args, **kwargs
     ):
         if config.get("provider", "") == ProviderEnum.azure.value:
             rich.print("\n ⚠️ Upgrade Warning ⚠️")

From 804edfda9acf3f9d329b04d62f45c032a4c36d15 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sun, 17 Nov 2024 18:17:58 -0800
Subject: [PATCH 10/42] Remove unnecessary "[y/N]"

---
 src/_nebari/upgrade.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 4ec46f4d8..3f3b1c807 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -667,7 +667,7 @@ def _version_specific_upgrade(
         )
 
         continue_ = attempt_fixes or Confirm.ask(
-            "Have you deleted the Argo Workflows CRDs and service accounts? [y/N] ",
+            "Have you deleted the Argo Workflows CRDs and service accounts?",
             default=False,
         )
         if not continue_:

From 8f8215a323918e503daf55b9e6be0d1965e589d1 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sun, 17 Nov 2024 18:52:43 -0800
Subject: [PATCH 11/42] Use `attempt_fixes` from `kwargs`

---
 src/_nebari/upgrade.py | 43 ++++++++++++++++++++++++------------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 3f3b1c807..c53e95770 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -311,7 +311,7 @@ def replace_image_tag(
             s: str,
             new_version: str,
             config_path: str,
-            attempt_fixes: bool = attempt_fixes,
+            attempt_fixes: bool = kwargs.get("attempt_fixes", False),
         ) -> str:
             """
             Replace the image tag with the new version.
@@ -373,7 +373,7 @@ def update_image_tag(
             config_path: str,
             current_image: str,
             new_version: str,
-            attempt_fixes: bool = attempt_fixes,
+            attempt_fixes: bool = kwargs.get("attempt_fixes", False),
         ) -> dict:
             """
             Update the image tag in the configuration.
@@ -406,7 +406,7 @@ def update_image_tag(
                 config_path,
                 v,
                 __rounded_finish_version__,
-                attempt_fixes,
+                kwargs.get("attempt_fixes", False),
             )
 
         # update profiles.jupyterlab images
@@ -418,7 +418,7 @@ def update_image_tag(
                     f"profiles.jupyterlab.{i}.kubespawner_override.image",
                     current_image,
                     __rounded_finish_version__,
-                    attempt_fixes,
+                    kwargs.get("attempt_fixes", False),
                 )
 
         # update profiles.dask_worker images
@@ -430,7 +430,7 @@ def update_image_tag(
                     f"profiles.dask_worker.{k}.image",
                     current_image,
                     __rounded_finish_version__,
-                    attempt_fixes,
+                    kwargs.get("attempt_fixes", False),
                 )
 
         # Run any version-specific tasks
@@ -439,7 +439,6 @@ def update_image_tag(
             start_version,
             config_filename,
             *args,
-            attempt_fixes=attempt_fixes,
             **kwargs,
         )
 
@@ -666,7 +665,7 @@ def _version_specific_upgrade(
             ""
         )
 
-        continue_ = attempt_fixes or Confirm.ask(
+        continue_ = kwargs.get("attempt_fixes", False) or Confirm.ask(
             "Have you deleted the Argo Workflows CRDs and service accounts?",
             default=False,
         )
@@ -707,7 +706,7 @@ def _version_specific_upgrade(
     ):
         argo = config.get("argo_workflows", {})
         if argo.get("enabled"):
-            response = attempt_fixes or Confirm.ask(
+            response = kwargs.get("attempt_fixes", False) or Confirm.ask(
                 f"\nDo you want to enable the [green][link={NEBARI_WORKFLOW_CONTROLLER_DOCS}]Nebari Workflow Controller[/link][/green], required for [green][link={ARGO_JUPYTER_SCHEDULER_REPO}]Argo-Jupyter-Scheduler[/link][green]?",
                 default=True,
             )
@@ -983,7 +982,7 @@ def _version_specific_upgrade(
                     default_node_groups = provider_enum_default_node_groups_map[
                         provider
                     ]
-                    continue_ = attempt_fixes or Confirm.ask(
+                    continue_ = kwargs.get("attempt_fixes", False) or Confirm.ask(
                         f"Would you like to include the default configuration for the node groups in [purple]{config_filename}[/purple]?",
                         default=False,
                     )
@@ -1053,7 +1052,7 @@ def _version_specific_upgrade(
                 "\n-> [red bold]Nebari version 2024.6.1 comes with a new version of Grafana. Any custom dashboards that you created will be deleted after upgrading Nebari. Make sure to [link=https://grafana.com/docs/grafana/latest/dashboards/share-dashboards-panels/#export-a-dashboard-as-json]export them as JSON[/link] so you can [link=https://grafana.com/docs/grafana/latest/dashboards/build-dashboards/import-dashboards/#import-a-dashboard]import them[/link] again afterwards.[/red bold]"
                 f"\n-> [red bold]Before upgrading, kube-prometheus-stack CRDs need to be updated and the {daemonset_name} daemonset needs to be deleted.[/red bold]"
             )
-            run_commands = attempt_fixes or Confirm.ask(
+            run_commands = kwargs.get("attempt_fixes", False) or Confirm.ask(
                 "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually.",
                 default=False,
             )
@@ -1076,10 +1075,14 @@ def _version_specific_upgrade(
                 rich.print(
                     f"The following commands will be run for the [cyan bold]{cluster_name}[/cyan bold] cluster"
                 )
-                _ = attempt_fixes or Prompt.ask("Hit enter to show the commands")
+                _ = kwargs.get("attempt_fixes", False) or Prompt.ask(
+                    "Hit enter to show the commands"
+                )
                 console.print(commands)
 
-                _ = attempt_fixes or Prompt.ask("Hit enter to continue")
+                _ = kwargs.get("attempt_fixes", False) or Prompt.ask(
+                    "Hit enter to continue"
+                )
                 # We need to add a special constructor to the yaml loader to handle a specific
                 # tag as otherwise the kubernetes API will fail when updating the CRD.
                 yaml.constructor.add_constructor(
@@ -1121,11 +1124,15 @@ def _version_specific_upgrade(
                 rich.print(
                     "[red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]"
                 )
-                _ = attempt_fixes or Prompt.ask("Hit enter to show the commands")
+                _ = kwargs.get("attempt_fixes", False) or Prompt.ask(
+                    "Hit enter to show the commands"
+                )
                 console.print(commands)
 
-                _ = attempt_fixes or Prompt.ask("Hit enter to continue")
-                continue_ = attempt_fixes or Confirm.ask(
+                _ = kwargs.get("attempt_fixes", False) or Prompt.ask(
+                    "Hit enter to continue"
+                )
+                continue_ = kwargs.get("attempt_fixes", False) or Confirm.ask(
                     f"Have you backed up your custom dashboards (if necessary), deleted the {daemonset_name} daemonset and updated the kube-prometheus-stack CRDs?",
                     default=False,
                 )
@@ -1154,7 +1161,7 @@ def _version_specific_upgrade(
                         If not, select "N" and the old default node groups will be added to the nebari config file.
                     """
                     )
-                    continue_ = attempt_fixes or Confirm.ask(
+                    continue_ = kwargs.get("attempt_fixes", False) or Confirm.ask(
                         text,
                         default=True,
                     )
@@ -1200,7 +1207,7 @@ def _version_specific_upgrade(
                     indent=4,
                 )
                 rich.print(text)
-                if not attempt_fixes:
+                if not kwargs.get("attempt_fixes", False):
                     _ = Prompt.ask("\n\nHit enter to continue")
         return config
 
@@ -1304,7 +1311,7 @@ def _version_specific_upgrade(
         keycloak_admin = None
 
         # Prompt the user for role assignment (if yes, transforms the response into bool)
-        assign_roles = attempt_fixes or Confirm.ask(
+        assign_roles = kwargs.get("attempt_fixes", False) or Confirm.ask(
             "[bold]Would you like Nebari to assign the corresponding role/scopes to all of your current groups automatically?[/bold]",
             default=False,
         )

From 6fc2dea2cf2e89091249724dd4eb5c9d6e17a85f Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sun, 17 Nov 2024 19:02:59 -0800
Subject: [PATCH 12/42] Remove unnecessary default value

---
 src/_nebari/upgrade.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index c53e95770..851e08e98 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -308,10 +308,7 @@ def replace_image_tag_legacy(
             return None
 
         def replace_image_tag(
-            s: str,
-            new_version: str,
-            config_path: str,
-            attempt_fixes: bool = kwargs.get("attempt_fixes", False),
+            s: str, new_version: str, config_path: str, attempt_fixes: bool
         ) -> str:
             """
             Replace the image tag with the new version.
@@ -373,7 +370,7 @@ def update_image_tag(
             config_path: str,
             current_image: str,
             new_version: str,
-            attempt_fixes: bool = kwargs.get("attempt_fixes", False),
+            attempt_fixes: bool,
         ) -> dict:
             """
             Update the image tag in the configuration.

From 48a5ae461a37adb343cc3bfcb4703720ecf848ee Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Sun, 17 Nov 2024 19:09:48 -0800
Subject: [PATCH 13/42] Remove unnecessary inclusion of `attempt_fixes`

---
 src/_nebari/upgrade.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 851e08e98..afcfe8324 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1121,15 +1121,11 @@ def _version_specific_upgrade(
                 rich.print(
                     "[red bold]Before upgrading, you need to manually delete the prometheus-node-exporter daemonset and update the kube-prometheus-stack CRDs. To do that, please run the following commands.[/red bold]"
                 )
-                _ = kwargs.get("attempt_fixes", False) or Prompt.ask(
-                    "Hit enter to show the commands"
-                )
+                _ = Prompt.ask("Hit enter to show the commands")
                 console.print(commands)
 
-                _ = kwargs.get("attempt_fixes", False) or Prompt.ask(
-                    "Hit enter to continue"
-                )
-                continue_ = kwargs.get("attempt_fixes", False) or Confirm.ask(
+                _ = Prompt.ask("Hit enter to continue")
+                continue_ = Confirm.ask(
                     f"Have you backed up your custom dashboards (if necessary), deleted the {daemonset_name} daemonset and updated the kube-prometheus-stack CRDs?",
                     default=False,
                 )

From 38e5a48be52ac37c79d961cb6c148a7a90144534 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Mon, 18 Nov 2024 20:10:38 -0800
Subject: [PATCH 14/42] Refactor Argo delete commands

---
 src/_nebari/upgrade.py | 58 ++++++++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index afcfe8324..7975ffbc4 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -651,26 +651,52 @@ def _version_specific_upgrade(
         Prompt users to delete Argo CRDs
         """
 
-        kubectl_delete_argo_crds_cmd = "kubectl delete crds clusterworkflowtemplates.argoproj.io cronworkflows.argoproj.io workfloweventbindings.argoproj.io workflows.argoproj.io workflowtasksets.argoproj.io workflowtemplates.argoproj.io"
-
-        kubectl_delete_argo_sa_cmd = (
-            f"kubectl delete sa -n {config['namespace']} argo-admin argo-dev argo-view"
-        )
+        argo_crds = [
+            "clusterworkflowtemplates.argoproj.io",
+            "cronworkflows.argoproj.io",
+            "workfloweventbindings.argoproj.io",
+            "workflows.argoproj.io",
+            "workflowtasksets.argoproj.io",
+            "workflowtemplates.argoproj.io",
+        ]
 
-        rich.print(
-            f"\n\n[bold cyan]Note:[/] Upgrading requires a one-time manual deletion of the Argo Workflows Custom Resource Definitions (CRDs) and service accounts. \n\n[red bold]Warning:  [link=https://{config['domain']}/argo/workflows]Workflows[/link] and [link=https://{config['domain']}/argo/workflows]CronWorkflows[/link] created before deleting the CRDs will be erased when the CRDs are deleted and will not be restored.[/red bold] \n\nThe updated CRDs will be installed during the next [cyan bold]nebari deploy[/cyan bold] step. Argo Workflows will not function after deleting the CRDs until the updated CRDs and service accounts are installed in the next nebari deploy. You must delete the Argo Workflows CRDs and service accounts before upgrading to {self.version} (or later) or the deploy step will fail.  Please delete them before proceeding by generating a kubeconfig (see [link=https://www.nebari.dev/docs/how-tos/debug-nebari/#generating-the-kubeconfig]docs[/link]), installing kubectl (see [link=https://www.nebari.dev/docs/how-tos/debug-nebari#installing-kubectl]docs[/link]), and running the following two commands:\n\n\t[cyan bold]{kubectl_delete_argo_crds_cmd} [/cyan bold]\n\n\t[cyan bold]{kubectl_delete_argo_sa_cmd} [/cyan bold]"
-            ""
-        )
+        argo_sa = ["argo-admin", "argo-dev", "argo-view"]
 
-        continue_ = kwargs.get("attempt_fixes", False) or Confirm.ask(
-            "Have you deleted the Argo Workflows CRDs and service accounts?",
-            default=False,
-        )
-        if not continue_:
+        if kwargs.get("attempt_fixes", False):
+            ...
+        else:
+            kubectl_delete_argo_crds_cmd = " ".join(
+                (
+                    *("kubectl delete crds",),
+                    *argo_crds,
+                ),
+            )
+            kubectl_delete_argo_sa_cmd = " ".join(
+                (
+                    *(
+                        "kubectl delete sa",
+                        f"-n {config.get("namespace", "default")}",
+                    ),
+                    *argo_sa,
+                ),
+            )
             rich.print(
-                f"You must delete the Argo Workflows CRDs and service accounts before upgrading to [green]{self.version}[/green] (or later)."
+                f"\n\n[bold cyan]Note:[/] Upgrading requires a one-time manual deletion of the Argo Workflows Custom Resource Definitions (CRDs) and service accounts. \n\n[red bold]"
+                f"Warning:  [link=https://{config['domain']}/argo/workflows]Workflows[/link] and [link=https://{config['domain']}/argo/workflows]CronWorkflows[/link] created before deleting the CRDs will be erased when the CRDs are deleted and will not be restored.[/red bold] \n\n"
+                f"The updated CRDs will be installed during the next [cyan bold]nebari deploy[/cyan bold] step. Argo Workflows will not function after deleting the CRDs until the updated CRDs and service accounts are installed in the next nebari deploy. "
+                f"You must delete the Argo Workflows CRDs and service accounts before upgrading to {self.version} (or later) or the deploy step will fail.  "
+                f"Please delete them before proceeding by generating a kubeconfig (see [link=https://www.nebari.dev/docs/how-tos/debug-nebari/#generating-the-kubeconfig]docs[/link]), installing kubectl (see [link=https://www.nebari.dev/docs/how-tos/debug-nebari#installing-kubectl]docs[/link]), and running the following two commands:\n\n\t[cyan bold]{kubectl_delete_argo_crds_cmd} [/cyan bold]\n\n\t[cyan bold]{kubectl_delete_argo_sa_cmd} [/cyan bold]"
             )
-            exit()
+
+            continue_ = Confirm.ask(
+                "Have you deleted the Argo Workflows CRDs and service accounts?",
+                default=False,
+            )
+            if not continue_:
+                rich.print(
+                    f"You must delete the Argo Workflows CRDs and service accounts before upgrading to [green]{self.version}[/green] (or later)."
+                )
+                exit()
 
         return config
 

From 239c4193dd19895463c08c8d4c5bf6f97504fefc Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 19 Nov 2024 23:13:24 -0800
Subject: [PATCH 15/42] Fix quotation marks

---
 src/_nebari/upgrade.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 7975ffbc4..5c49b1782 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -663,7 +663,7 @@ def _version_specific_upgrade(
         argo_sa = ["argo-admin", "argo-dev", "argo-view"]
 
         if kwargs.get("attempt_fixes", False):
-            ...
+            pass
         else:
             kubectl_delete_argo_crds_cmd = " ".join(
                 (
@@ -675,7 +675,7 @@ def _version_specific_upgrade(
                 (
                     *(
                         "kubectl delete sa",
-                        f"-n {config.get("namespace", "default")}",
+                        f"-n {config.get('namespace', 'default')}",
                     ),
                     *argo_sa,
                 ),

From d7520ed73a096a923ac92f80317dc23ac0834f56 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 20 Nov 2024 19:25:35 -0800
Subject: [PATCH 16/42] Set up minimal functionality for deleting CRDs and SAs

---
 src/_nebari/upgrade.py | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 5c49b1782..237f080ee 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -650,7 +650,6 @@ def _version_specific_upgrade(
         """
         Prompt users to delete Argo CRDs
         """
-
         argo_crds = [
             "clusterworkflowtemplates.argoproj.io",
             "cronworkflows.argoproj.io",
@@ -663,7 +662,35 @@ def _version_specific_upgrade(
         argo_sa = ["argo-admin", "argo-dev", "argo-view"]
 
         if kwargs.get("attempt_fixes", False):
-            pass
+            try:
+                kubernetes.config.load_kube_config()
+            except kubernetes.config.config_exception.ConfigException:
+                rich.print(
+                    "[red bold]No default kube configuration file was found. Make sure to [link=https://www.nebari.dev/docs/how-tos/debug-nebari#generating-the-kubeconfig]have one pointing to your Nebari cluster[/link] before upgrading.[/red bold]"
+                )
+                exit()
+
+            for crd in argo_crds:
+                api_instance = kubernetes.client.ApiextensionsV1Api()
+                try:
+                    api_instance.delete_custom_resource_definition(
+                        name=crd,
+                    )
+                except kubernetes.client.exceptions.ApiException as e:
+                    if e.status != 404:
+                        raise e
+
+            namespace = config.get("namespace", "default")
+            for sa in argo_sa:
+                api_instance = kubernetes.client.CoreV1Api()
+                try:
+                    api_instance.delete_namespaced_service_account(
+                        sa,
+                        namespace,
+                    )
+                except kubernetes.client.exceptions.ApiException as e:
+                    if e.status != 404:
+                        raise e
         else:
             kubectl_delete_argo_crds_cmd = " ".join(
                 (

From 06960b29bc56b3bff85754f55811e492e4fbf026 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 20 Nov 2024 23:46:40 -0800
Subject: [PATCH 17/42] Print if 404 else raise

---
 src/_nebari/upgrade.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 237f080ee..f464ad9de 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -677,7 +677,9 @@ def _version_specific_upgrade(
                         name=crd,
                     )
                 except kubernetes.client.exceptions.ApiException as e:
-                    if e.status != 404:
+                    if e.status == 404:
+                        rich.print(f"CRD [yellow]{crd}[/yellow] not found. Ignoring.")
+                    else:
                         raise e
 
             namespace = config.get("namespace", "default")
@@ -689,7 +691,11 @@ def _version_specific_upgrade(
                         namespace,
                     )
                 except kubernetes.client.exceptions.ApiException as e:
-                    if e.status != 404:
+                    if e.status == 404:
+                        rich.print(
+                            f"Service account [yellow]{sa}[/yellow] not found. Ignoring."
+                        )
+                    else:
                         raise e
         else:
             kubectl_delete_argo_crds_cmd = " ".join(

From 46682abff1737a3ce42b6b9b40744f4412af6378 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 20 Nov 2024 23:56:52 -0800
Subject: [PATCH 18/42] Move namespace outside of if statement

---
 src/_nebari/upgrade.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index f464ad9de..f264479e3 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -661,6 +661,8 @@ def _version_specific_upgrade(
 
         argo_sa = ["argo-admin", "argo-dev", "argo-view"]
 
+        namespace = config.get("namespace", "default")
+
         if kwargs.get("attempt_fixes", False):
             try:
                 kubernetes.config.load_kube_config()
@@ -682,7 +684,6 @@ def _version_specific_upgrade(
                     else:
                         raise e
 
-            namespace = config.get("namespace", "default")
             for sa in argo_sa:
                 api_instance = kubernetes.client.CoreV1Api()
                 try:
@@ -708,7 +709,7 @@ def _version_specific_upgrade(
                 (
                     *(
                         "kubectl delete sa",
-                        f"-n {config.get('namespace', 'default')}",
+                        f"-n {namespace}",
                     ),
                     *argo_sa,
                 ),

From 88be8a4db8f2e4b6ae2dde6baf24f7c563f66d56 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Thu, 21 Nov 2024 00:07:13 -0800
Subject: [PATCH 19/42] Print successfull

---
 src/_nebari/upgrade.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index f264479e3..9737d4a6e 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -683,6 +683,8 @@ def _version_specific_upgrade(
                         rich.print(f"CRD [yellow]{crd}[/yellow] not found. Ignoring.")
                     else:
                         raise e
+                else:
+                    rich.print(f"Successfully removed CRD [green]{crd}[/green]")
 
             for sa in argo_sa:
                 api_instance = kubernetes.client.CoreV1Api()
@@ -698,6 +700,10 @@ def _version_specific_upgrade(
                         )
                     else:
                         raise e
+                else:
+                    rich.print(
+                        f"Successfully removed service account [green]{sa}[/green]"
+                    )
         else:
             kubectl_delete_argo_crds_cmd = " ".join(
                 (

From 5475e8f667b0df3977d3b716f0eb4f9c6376e2f3 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Thu, 21 Nov 2024 20:09:49 -0800
Subject: [PATCH 20/42] Make tests reflect usage of `rich.prompt.Confirm` and
 `rich.prompt.Prompt`

---
 tests/tests_unit/test_upgrade.py | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py
index fd70f367f..9505b8d55 100644
--- a/tests/tests_unit/test_upgrade.py
+++ b/tests/tests_unit/test_upgrade.py
@@ -2,7 +2,7 @@
 from pathlib import Path
 
 import pytest
-from rich.prompt import Prompt
+from rich.prompt import Confirm, Prompt
 
 from _nebari.upgrade import do_upgrade
 from _nebari.version import __version__, rounded_ver_parse
@@ -49,34 +49,31 @@ def test_upgrade_4_0(
     qhub_users_import_json,
     monkeypatch,
 ):
-
     def mock_input(prompt, **kwargs):
         # Mock different upgrade steps prompt answers
-        if (
-            prompt
-            == "Have you deleted the Argo Workflows CRDs and service accounts? [y/N] "
-        ):
-            return "y"
+        if prompt == "Have you deleted the Argo Workflows CRDs and service accounts?":
+            return True
         elif (
             prompt
             == "\nDo you want Nebari to update the kube-prometheus-stack CRDs and delete the prometheus-node-exporter for you? If not, you'll have to do it manually."
         ):
-            return "N"
+            return False
         elif (
             prompt
             == "Have you backed up your custom dashboards (if necessary), deleted the prometheus-node-exporter daemonset and updated the kube-prometheus-stack CRDs?"
         ):
-            return "y"
+            return True
         elif (
             prompt
             == "[bold]Would you like Nebari to assign the corresponding role to all of your current groups automatically?[/bold]"
         ):
-            return "N"
+            return False
         # All other prompts will be answered with "y"
         else:
-            return "y"
+            return True
 
-    monkeypatch.setattr(Prompt, "ask", mock_input)
+    monkeypatch.setattr(Confirm, "ask", mock_input)
+    monkeypatch.setattr(Prompt, "ask", lambda x: "")
 
     old_qhub_config_path = Path(__file__).parent / old_qhub_config_path_str
 

From c5f02e3f49bf28b53e493fac4e350b14fa093dea Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Mon, 2 Dec 2024 23:33:12 -0800
Subject: [PATCH 21/42] Squashed commit of the following:

commit bbbcde3c29c24fd80227c55ab6b8ce7978697bf3
Merge: ce712366 3ac01b4b
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Thu Nov 28 14:42:36 2024 -0500

    Disable `jupyterlab-jhub-apps` extension when jhub-apps is disabled (#2804)

commit ce71236648ac5f26174aba43182ccbb249b4ea67
Author: Vinicius D. Cerutti <51954708+viniciusdc@users.noreply.github.com>
Date:   Thu Nov 21 13:53:57 2024 -0300

    Add 2024.11.1 release notes and bump version (#2859)

commit d272176a83384fd06899c2916c7879eb64991a58
Merge: b442200d bbff0078
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Nov 20 08:56:51 2024 -0500

    Use tofu binary instead of terraform one (#2773)

commit bbff00786ee1388973b2a5e450ce29de8d548e04
Merge: 0ecf6ef8 b442200d
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Tue Nov 19 08:55:47 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit 3ac01b4bcefa6ff3f68557ed0f44a8e77a77ecd6
Merge: a0bceeac 621ea239
Author: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
Date:   Mon Nov 18 19:13:53 2024 +0100

    Merge branch 'main' into disable-jupyterlab-jhub-apps-extension

commit 0ecf6ef815f4fae069a81812279ea17648c0e395
Merge: a40cd40d 621ea239
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Thu Nov 14 16:32:07 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit a40cd40dd507b8699d970c8a5e7cf806ab0e2ed5
Merge: 8e59c242 e7ff0896
Author: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
Date:   Thu Nov 14 21:35:20 2024 +0100

    specify terraform registry for providers not in opentofu registry (#2852)

commit e7ff0896cf7ff70b5bce6fab66e57468a9cf9c72
Author: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
Date:   Thu Nov 14 17:56:35 2024 +0100

    only update kind

commit 6f1b8d64161e527955fad107427a4a1f4f89fdd4
Merge: b2dd11c7 8e59c242
Author: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
Date:   Wed Nov 13 18:03:11 2024 +0100

    Merge branch 'terraform-to-opentofu' into 2857-fix-providers

commit 8e59c24220ecdf85cd6ccf67f10dc0506e5bfd2b
Merge: ff299357 87ed92bd
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Nov 13 10:16:32 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit b2dd11c7f6ec61d881824625de7122de8a768691
Merge: 415b1365 ff299357
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Nov 13 10:15:25 2024 -0500

    Merge branch 'terraform-to-opentofu' into 2857-fix-providers

commit 415b136524f185ccbe86bfc34e5fb7397a90eeda
Author: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
Date:   Wed Nov 13 13:44:29 2024 +0100

    specify terraform registry for providers not in opentofu registry

commit ff2993577fc87e74422703476578190c9a60efe0
Merge: aef2796d 855aa14e
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Nov 6 13:29:43 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit aef2796d46a85721ee92e95daf310841b8a94fc2
Merge: 06f90438 34292606
Author: Chuck McAndrew <6248903+dcmcand@users.noreply.github.com>
Date:   Wed Nov 6 16:26:34 2024 +0100

    Merge branch 'main' into terraform-to-opentofu

commit a0bceeac6b73dcd32d97ed63c3fc05f78108e6f1
Author: krassowski <5832902+krassowski@users.noreply.github.com>
Date:   Thu Oct 31 12:56:20 2024 +0000

    Fix typo

commit b8530d6de8a7fbee7950a412ca1f013a06186d8f
Author: krassowski <5832902+krassowski@users.noreply.github.com>
Date:   Tue Oct 29 13:56:56 2024 +0000

    Mount `page_config.json` in top-level

    so that it applies across potential JupyterLab startup environments

    also the previous spec was incorrectly pointing to a file (not a dir)

commit d1572ebf44aa504e9fa833ef868b5653a984f123
Author: krassowski <5832902+krassowski@users.noreply.github.com>
Date:   Tue Oct 29 10:43:29 2024 +0000

    Disable `jupyterlab-jhub-apps` extension when jhub-apps is disabled (if installed)

commit 06f90438bd2ef60a0e7d9f36db72e228fc62147e
Merge: 8f709dac ba0ae822
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Fri Oct 25 03:28:49 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit 8f709dacce5d22774b6895a8207e9d81b8251102
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Thu Oct 17 08:03:09 2024 +0200

    Fix wrong function reference

commit 2b8c34775884fe61fea7f7992f692da79f5493d4
Merge: 80d67b96 c2ae895d
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Oct 16 23:36:10 2024 +0200

    Merge branch 'terraform-to-opentofu' of https://github.com/nebari-dev/nebari into terraform-to-opentofu

commit 80d67b9657a893a634b1a1163c48a271578a51d8
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Oct 16 23:36:01 2024 +0200

    Rename terraform.py file and terraform_init calls

commit c2ae895d05af3717afe08745fc2e085127beb741
Merge: 6a09d4e1 ccb8b7ef
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Wed Oct 16 08:16:50 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit 6a09d4e17aed3b44e9633e940ca4b622250ce573
Merge: d1dfabe2 38960ea4
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Tue Oct 15 04:54:16 2024 -0500

    Merge branch 'main' into terraform-to-opentofu

commit d1dfabe2a005f61d07322e83cac3e297cd298835
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Tue Oct 15 11:53:49 2024 +0200

    Remove terraform open source license test

commit 96609c8bbe86f3edff4e0ee7a827ac7953edd9de
Author: Marcelo Villa <mvilla@quansight.com>
Date:   Mon Oct 14 23:00:05 2024 +0200

    Use tofu binary instead of terraform one
---
 RELEASE.md                                    |  17 +++
 src/_nebari/constants.py                      |   7 +-
 .../provider/{terraform.py => opentofu.py}    | 119 +++++++++---------
 src/_nebari/stages/base.py                    |  34 ++---
 src/_nebari/stages/infrastructure/__init__.py |   8 +-
 .../infrastructure/template/local/main.tf     |   2 +-
 .../services/jupyterhub/configmaps.tf         |  28 +++++
 .../kubernetes/services/jupyterhub/main.tf    |   5 +
 .../stages/terraform_state/__init__.py        |  14 +--
 src/_nebari/stages/tf_objects.py              |   2 +-
 tests/tests_unit/test_dependencies.py         |  18 ---
 11 files changed, 139 insertions(+), 115 deletions(-)
 rename src/_nebari/provider/{terraform.py => opentofu.py} (62%)
 delete mode 100644 tests/tests_unit/test_dependencies.py

diff --git a/RELEASE.md b/RELEASE.md
index da6249d7d..38b44f372 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -9,6 +9,23 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. -->
 
 ---
 
+## Release 2024.11.1 - November 21, 2024 (Hotfix Release)
+
+> NOTE: This hotfix addresses several major bugs identified in the 2024.9.1 release. For a detailed overview, please refer to the related discussion at #2798. Users should upgrade directly from 2024.7.1 to 2024.11.1.
+
+## What's Changed
+
+- fix `CHECK_URL` in kuberhealthy checks to respect namespaces by @dcmcand in https://github.com/nebari-dev/nebari/pull/2779
+- fix bug where `check_immutable_fields` throws error with old version of Nebari by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2796
+- Fix immutable field validation error when a sub-schema is not Pydantic by @kenafoster in https://github.com/nebari-dev/nebari/pull/2797
+- Address issue with AWS instance type schema by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2787
+- Add broken note by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/2802
+- Refactor role creation for upgrade command path by @viniciusdc in https://github.com/nebari-dev/nebari/pull/2795
+- Allow overriding of keycloak root credentials for 2024.11.1 upgrade path #2843
+- Disable AWS `launch_template` from nebari-config schema #2856
+
+**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2024.9.1...2024.11.1
+
 ## Release 2024.9.1 - September 27, 2024 (Broken Release)
 
 > WARNING: This release was later found to have unresolved issues described further in [issue 2798](https://github.com/nebari-dev/nebari/issues/2798). We have marked this release as broken on conda-forge and yanked it on PyPI. One of the bugs prevents any upgrade from 2024.9.1 to 2024.11.1. Users should skip this release entirely and upgrade directly from 2024.7.1 to 2024.11.1.
diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py
index 6e7ab3880..2753fe792 100644
--- a/src/_nebari/constants.py
+++ b/src/_nebari/constants.py
@@ -1,11 +1,8 @@
-CURRENT_RELEASE = "2024.9.1"
+CURRENT_RELEASE = "2024.11.1"
 
 HELM_VERSION = "v3.15.3"
 KUSTOMIZE_VERSION = "5.4.3"
-# NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes
-# implemented in August 2023.
-# https://www.hashicorp.com/license-faq
-TERRAFORM_VERSION = "1.5.7"
+OPENTOFU_VERSION = "1.8.3"
 
 KUBERHEALTHY_HELM_VERSION = "100"
 
diff --git a/src/_nebari/provider/terraform.py b/src/_nebari/provider/opentofu.py
similarity index 62%
rename from src/_nebari/provider/terraform.py
rename to src/_nebari/provider/opentofu.py
index 59d88e76d..78936d180 100644
--- a/src/_nebari/provider/terraform.py
+++ b/src/_nebari/provider/opentofu.py
@@ -18,39 +18,39 @@
 logger = logging.getLogger(__name__)
 
 
-class TerraformException(Exception):
+class OpenTofuException(Exception):
     pass
 
 
 def deploy(
     directory,
-    terraform_init: bool = True,
-    terraform_import: bool = False,
-    terraform_apply: bool = True,
-    terraform_destroy: bool = False,
+    tofu_init: bool = True,
+    tofu_import: bool = False,
+    tofu_apply: bool = True,
+    tofu_destroy: bool = False,
     input_vars: Dict[str, Any] = {},
     state_imports: List[Any] = [],
 ):
-    """Execute a given terraform directory.
+    """Execute a given directory with OpenTofu infrastructure configuration.
 
     Parameters:
-      directory: directory in which to run terraform operations on
+      directory: directory in which to run tofu operations on
 
-      terraform_init: whether to run `terraform init` default True
+      tofu_init: whether to run `tofu init` default True
 
-      terraform_import: whether to run `terraform import` default
+      tofu_import: whether to run `tofu import` default
         False for each `state_imports` supplied to function
 
-      terraform_apply: whether to run `terraform apply` default True
+      tofu_apply: whether to run `tofu apply` default True
 
-      terraform_destroy: whether to run `terraform destroy` default
+      tofu_destroy: whether to run `tofu destroy` default
         False
 
       input_vars: supply values for "variable" resources within
         terraform module
 
       state_imports: (addr, id) pairs for iterate through and attempt
-        to terraform import
+        to tofu import
     """
     with tempfile.NamedTemporaryFile(
         mode="w", encoding="utf-8", suffix=".tfvars.json"
@@ -58,25 +58,25 @@ def deploy(
         json.dump(input_vars, f.file)
         f.file.flush()
 
-        if terraform_init:
+        if tofu_init:
             init(directory)
 
-        if terraform_import:
+        if tofu_import:
             for addr, id in state_imports:
                 tfimport(
                     addr, id, directory=directory, var_files=[f.name], exist_ok=True
                 )
 
-        if terraform_apply:
+        if tofu_apply:
             apply(directory, var_files=[f.name])
 
-        if terraform_destroy:
+        if tofu_destroy:
             destroy(directory, var_files=[f.name])
 
         return output(directory)
 
 
-def download_terraform_binary(version=constants.TERRAFORM_VERSION):
+def download_opentofu_binary(version=constants.OPENTOFU_VERSION):
     os_mapping = {
         "linux": "linux",
         "win32": "windows",
@@ -94,73 +94,72 @@ def download_terraform_binary(version=constants.TERRAFORM_VERSION):
         "arm64": "arm64",
     }
 
-    download_url = f"https://releases.hashicorp.com/terraform/{version}/terraform_{version}_{os_mapping[sys.platform]}_{architecture_mapping[platform.machine()]}.zip"
-    filename_directory = Path(tempfile.gettempdir()) / "terraform" / version
-    filename_path = filename_directory / "terraform"
+    download_url = f"https://github.com/opentofu/opentofu/releases/download/v{version}/tofu_{version}_{os_mapping[sys.platform]}_{architecture_mapping[platform.machine()]}.zip"
+
+    filename_directory = Path(tempfile.gettempdir()) / "opentofu" / version
+    filename_path = filename_directory / "tofu"
 
     if not filename_path.is_file():
         logger.info(
-            f"downloading and extracting terraform binary from url={download_url} to path={filename_path}"
+            f"downloading and extracting opentofu binary from url={download_url} to path={filename_path}"
         )
         with urllib.request.urlopen(download_url) as f:
             bytes_io = io.BytesIO(f.read())
         download_file = zipfile.ZipFile(bytes_io)
-        download_file.extract("terraform", filename_directory)
+        download_file.extract("tofu", filename_directory)
 
     filename_path.chmod(0o555)
     return filename_path
 
 
-def run_terraform_subprocess(processargs, **kwargs):
-    terraform_path = download_terraform_binary()
-    logger.info(f" terraform at {terraform_path}")
-    exit_code, output = run_subprocess_cmd([terraform_path] + processargs, **kwargs)
+def run_tofu_subprocess(processargs, **kwargs):
+    tofu_path = download_opentofu_binary()
+    logger.info(f" tofu at {tofu_path}")
+    exit_code, output = run_subprocess_cmd([tofu_path] + processargs, **kwargs)
     if exit_code != 0:
-        raise TerraformException("Terraform returned an error")
+        raise OpenTofuException("OpenTofu returned an error")
     return output
 
 
 def version():
-    terraform_path = download_terraform_binary()
-    logger.info(f"checking terraform={terraform_path} version")
+    tofu_path = download_opentofu_binary()
+    logger.info(f"checking opentofu={tofu_path} version")
 
-    version_output = subprocess.check_output([terraform_path, "--version"]).decode(
-        "utf-8"
-    )
+    version_output = subprocess.check_output([tofu_path, "--version"]).decode("utf-8")
     return re.search(r"(\d+)\.(\d+).(\d+)", version_output).group(0)
 
 
 def init(directory=None, upgrade=True):
-    logger.info(f"terraform init directory={directory}")
-    with timer(logger, "terraform init"):
+    logger.info(f"tofu init directory={directory}")
+    with timer(logger, "tofu init"):
         command = ["init"]
         if upgrade:
             command.append("-upgrade")
-        run_terraform_subprocess(command, cwd=directory, prefix="terraform")
+        run_tofu_subprocess(command, cwd=directory, prefix="tofu")
 
 
 def apply(directory=None, targets=None, var_files=None):
     targets = targets or []
     var_files = var_files or []
 
-    logger.info(f"terraform apply directory={directory} targets={targets}")
+    logger.info(f"tofu apply directory={directory} targets={targets}")
     command = (
         ["apply", "-auto-approve"]
         + ["-target=" + _ for _ in targets]
         + ["-var-file=" + _ for _ in var_files]
     )
-    with timer(logger, "terraform apply"):
-        run_terraform_subprocess(command, cwd=directory, prefix="terraform")
+    with timer(logger, "tofu apply"):
+        run_tofu_subprocess(command, cwd=directory, prefix="tofu")
 
 
 def output(directory=None):
-    terraform_path = download_terraform_binary()
+    tofu_path = download_opentofu_binary()
 
-    logger.info(f"terraform={terraform_path} output directory={directory}")
-    with timer(logger, "terraform output"):
+    logger.info(f"tofu={tofu_path} output directory={directory}")
+    with timer(logger, "tofu output"):
         return json.loads(
             subprocess.check_output(
-                [terraform_path, "output", "-json"], cwd=directory
+                [tofu_path, "output", "-json"], cwd=directory
             ).decode("utf8")[:-1]
         )
 
@@ -168,61 +167,61 @@ def output(directory=None):
 def tfimport(addr, id, directory=None, var_files=None, exist_ok=False):
     var_files = var_files or []
 
-    logger.info(f"terraform import directory={directory} addr={addr} id={id}")
+    logger.info(f"tofu import directory={directory} addr={addr} id={id}")
     command = ["import"] + ["-var-file=" + _ for _ in var_files] + [addr, id]
     logger.error(str(command))
-    with timer(logger, "terraform import"):
+    with timer(logger, "tofu import"):
         try:
-            run_terraform_subprocess(
+            run_tofu_subprocess(
                 command,
                 cwd=directory,
-                prefix="terraform",
+                prefix="tofu",
                 strip_errors=True,
                 timeout=30,
             )
-        except TerraformException as e:
+        except OpenTofuException as e:
             if not exist_ok:
                 raise e
 
 
-def show(directory=None, terraform_init: bool = True) -> dict:
+def show(directory=None, tofu_init: bool = True) -> dict:
 
-    if terraform_init:
+    if tofu_init:
         init(directory)
 
-    logger.info(f"terraform show directory={directory}")
+    logger.info(f"tofu show directory={directory}")
     command = ["show", "-json"]
-    with timer(logger, "terraform show"):
+    with timer(logger, "tofu show"):
         try:
             output = json.loads(
-                run_terraform_subprocess(
+                run_tofu_subprocess(
                     command,
                     cwd=directory,
-                    prefix="terraform",
+                    prefix="tofu",
                     strip_errors=True,
                     capture_output=True,
                 )
             )
             return output
-        except TerraformException as e:
+        except OpenTofuException as e:
             raise e
 
 
 def refresh(directory=None, var_files=None):
     var_files = var_files or []
 
-    logger.info(f"terraform refresh directory={directory}")
+    logger.info(f"tofu refresh directory={directory}")
     command = ["refresh"] + ["-var-file=" + _ for _ in var_files]
 
-    with timer(logger, "terraform refresh"):
-        run_terraform_subprocess(command, cwd=directory, prefix="terraform")
+    with timer(logger, "tofu refresh"):
+        run_tofu_subprocess(command, cwd=directory, prefix="tofu")
 
 
 def destroy(directory=None, targets=None, var_files=None):
     targets = targets or []
     var_files = var_files or []
 
-    logger.info(f"terraform destroy directory={directory} targets={targets}")
+    logger.info(f"tofu destroy directory={directory} targets={targets}")
     command = (
         [
             "destroy",
@@ -232,8 +231,8 @@ def destroy(directory=None, targets=None, var_files=None):
         + ["-var-file=" + _ for _ in var_files]
     )
 
-    with timer(logger, "terraform destroy"):
-        run_terraform_subprocess(command, cwd=directory, prefix="terraform")
+    with timer(logger, "tofu destroy"):
+        run_tofu_subprocess(command, cwd=directory, prefix="tofu")
 
 
 def rm_local_state(directory=None):
diff --git a/src/_nebari/stages/base.py b/src/_nebari/stages/base.py
index cef1322e9..bcc6bb82b 100644
--- a/src/_nebari/stages/base.py
+++ b/src/_nebari/stages/base.py
@@ -11,7 +11,7 @@
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
 
-from _nebari.provider import helm, kubernetes, kustomize, terraform
+from _nebari.provider import helm, kubernetes, kustomize, opentofu
 from _nebari.stages.tf_objects import NebariTerraformState
 from nebari.hookspecs import NebariStage
 
@@ -248,7 +248,7 @@ def tf_objects(self) -> List[Dict]:
 
     def render(self) -> Dict[pathlib.Path, str]:
         contents = {
-            (self.stage_prefix / "_nebari.tf.json"): terraform.tf_render_objects(
+            (self.stage_prefix / "_nebari.tf.json"): opentofu.tf_render_objects(
                 self.tf_objects()
             )
         }
@@ -283,19 +283,19 @@ def deploy(
         self,
         stage_outputs: Dict[str, Dict[str, Any]],
         disable_prompt: bool = False,
-        terraform_init: bool = True,
+        tofu_init: bool = True,
     ):
         deploy_config = dict(
             directory=str(self.output_directory / self.stage_prefix),
             input_vars=self.input_vars(stage_outputs),
-            terraform_init=terraform_init,
+            tofu_init=tofu_init,
         )
         state_imports = self.state_imports()
         if state_imports:
-            deploy_config["terraform_import"] = True
+            deploy_config["tofu_import"] = True
             deploy_config["state_imports"] = state_imports
 
-        self.set_outputs(stage_outputs, terraform.deploy(**deploy_config))
+        self.set_outputs(stage_outputs, opentofu.deploy(**deploy_config))
         self.post_deploy(stage_outputs, disable_prompt)
         yield
 
@@ -318,27 +318,27 @@ def destroy(
     ):
         self.set_outputs(
             stage_outputs,
-            terraform.deploy(
+            opentofu.deploy(
                 directory=str(self.output_directory / self.stage_prefix),
                 input_vars=self.input_vars(stage_outputs),
-                terraform_init=True,
-                terraform_import=True,
-                terraform_apply=False,
-                terraform_destroy=False,
+                tofu_init=True,
+                tofu_import=True,
+                tofu_apply=False,
+                tofu_destroy=False,
             ),
         )
         yield
         try:
-            terraform.deploy(
+            opentofu.deploy(
                 directory=str(self.output_directory / self.stage_prefix),
                 input_vars=self.input_vars(stage_outputs),
-                terraform_init=True,
-                terraform_import=True,
-                terraform_apply=False,
-                terraform_destroy=True,
+                tofu_init=True,
+                tofu_import=True,
+                tofu_apply=False,
+                tofu_destroy=True,
             )
             status["stages/" + self.name] = True
-        except terraform.TerraformException as e:
+        except opentofu.OpenTofuException as e:
             if not ignore_errors:
                 raise e
             status["stages/" + self.name] = False
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 976f15a57..7b4c1aa23 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -11,7 +11,7 @@
 from pydantic import ConfigDict, Field, field_validator, model_validator
 
 from _nebari import constants
-from _nebari.provider import terraform
+from _nebari.provider import opentofu
 from _nebari.provider.cloud import amazon_web_services, azure_cloud, google_cloud
 from _nebari.stages.base import NebariTerraformStage
 from _nebari.stages.kubernetes_services import SharedFsEnum
@@ -701,7 +701,7 @@ def state_imports(self) -> List[Tuple[str, str]]:
     def tf_objects(self) -> List[Dict]:
         if self.config.provider == schema.ProviderEnum.gcp:
             return [
-                terraform.Provider(
+                opentofu.Provider(
                     "google",
                     project=self.config.google_cloud_platform.project,
                     region=self.config.google_cloud_platform.region,
@@ -714,9 +714,7 @@ def tf_objects(self) -> List[Dict]:
             ]
         elif self.config.provider == schema.ProviderEnum.aws:
             return [
-                terraform.Provider(
-                    "aws", region=self.config.amazon_web_services.region
-                ),
+                opentofu.Provider("aws", region=self.config.amazon_web_services.region),
                 NebariTerraformState(self.name, self.config),
             ]
         else:
diff --git a/src/_nebari/stages/infrastructure/template/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf
index fb0d0997e..77aa799cb 100644
--- a/src/_nebari/stages/infrastructure/template/local/main.tf
+++ b/src/_nebari/stages/infrastructure/template/local/main.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     kind = {
-      source  = "tehcyx/kind"
+      source  = "registry.terraform.io/tehcyx/kind"
       version = "0.4.0"
     }
     docker = {
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
index bfee219e9..23f2ac933 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/configmaps.tf
@@ -60,6 +60,17 @@ resource "local_file" "overrides_json" {
   filename = "${path.module}/files/jupyterlab/overrides.json"
 }
 
+resource "local_file" "page_config_json" {
+  content = jsonencode({
+    "disabledExtensions" : {
+      "jupyterlab-jhub-apps" : !var.jhub-apps-enabled
+    },
+    # `lockedExtensions` is an empty dict to signify that `jupyterlab-jhub-apps` is not being disabled and locked (but only disabled)
+    # which means users are still allowed to disable the jupyterlab-jhub-apps extension (if they have write access to page_config).
+    "lockedExtensions" : {}
+  })
+  filename = "${path.module}/files/jupyterlab/page_config.json"
+}
 
 resource "kubernetes_config_map" "etc-ipython" {
   metadata {
@@ -92,6 +103,9 @@ locals {
   etc-jupyterlab-settings = {
     "overrides.json" = local_file.overrides_json.content
   }
+  etc-jupyterlab-page-config = {
+    "page_config.json" = local_file.page_config_json.content
+  }
 }
 
 resource "kubernetes_config_map" "etc-jupyter" {
@@ -136,6 +150,20 @@ resource "kubernetes_config_map" "jupyterlab-settings" {
   data = local.etc-jupyterlab-settings
 }
 
+
+resource "kubernetes_config_map" "jupyterlab-page-config" {
+  depends_on = [
+    local_file.page_config_json
+  ]
+
+  metadata {
+    name      = "jupyterlab-page-config"
+    namespace = var.namespace
+  }
+
+  data = local.etc-jupyterlab-page-config
+}
+
 resource "kubernetes_config_map" "git_clone_update" {
   metadata {
     name      = "git-clone-update"
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index a36090f41..9a0675fc8 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -104,6 +104,11 @@ resource "helm_release" "jupyterhub" {
               kind      = "configmap"
             }
 
+            "/etc/jupyter/labconfig" = {
+              name      = kubernetes_config_map.jupyterlab-page-config.metadata.0.name
+              namespace = kubernetes_config_map.jupyterlab-page-config.metadata.0.namespace
+              kind      = "configmap"
+            }
           }
         )
         environments = var.conda-store-environments
diff --git a/src/_nebari/stages/terraform_state/__init__.py b/src/_nebari/stages/terraform_state/__init__.py
index 690e556b0..e9a18ba7c 100644
--- a/src/_nebari/stages/terraform_state/__init__.py
+++ b/src/_nebari/stages/terraform_state/__init__.py
@@ -9,7 +9,7 @@
 from pydantic import BaseModel, field_validator
 
 from _nebari import utils
-from _nebari.provider import terraform
+from _nebari.provider import opentofu
 from _nebari.provider.cloud import azure_cloud
 from _nebari.stages.base import NebariTerraformStage
 from _nebari.stages.tf_objects import NebariConfig
@@ -162,7 +162,7 @@ def tf_objects(self) -> List[Dict]:
         resources = [NebariConfig(self.config)]
         if self.config.provider == schema.ProviderEnum.gcp:
             return resources + [
-                terraform.Provider(
+                opentofu.Provider(
                     "google",
                     project=self.config.google_cloud_platform.project,
                     region=self.config.google_cloud_platform.region,
@@ -170,9 +170,7 @@ def tf_objects(self) -> List[Dict]:
             ]
         elif self.config.provider == schema.ProviderEnum.aws:
             return resources + [
-                terraform.Provider(
-                    "aws", region=self.config.amazon_web_services.region
-                ),
+                opentofu.Provider("aws", region=self.config.amazon_web_services.region),
             ]
         else:
             return resources
@@ -217,9 +215,9 @@ def deploy(
     ):
         self.check_immutable_fields()
 
-        # No need to run terraform init here as it's being called when running the
+        # No need to run tofu init here as it's being called when running the
         # terraform show command, inside check_immutable_fields
-        with super().deploy(stage_outputs, disable_prompt, terraform_init=False):
+        with super().deploy(stage_outputs, disable_prompt, tofu_init=False):
             env_mapping = {}
 
             with modified_environ(**env_mapping):
@@ -264,7 +262,7 @@ def check_immutable_fields(self):
 
     def get_nebari_config_state(self) -> dict:
         directory = str(self.output_directory / self.stage_prefix)
-        tf_state = terraform.show(directory)
+        tf_state = opentofu.show(directory)
         nebari_config_state = None
 
         # get nebari config from state
diff --git a/src/_nebari/stages/tf_objects.py b/src/_nebari/stages/tf_objects.py
index c593eee4b..28884d478 100644
--- a/src/_nebari/stages/tf_objects.py
+++ b/src/_nebari/stages/tf_objects.py
@@ -1,4 +1,4 @@
-from _nebari.provider.terraform import Data, Provider, Resource, TerraformBackend
+from _nebari.provider.opentofu import Data, Provider, Resource, TerraformBackend
 from _nebari.utils import (
     AZURE_TF_STATE_RESOURCE_GROUP_SUFFIX,
     construct_azure_resource_group_name,
diff --git a/tests/tests_unit/test_dependencies.py b/tests/tests_unit/test_dependencies.py
deleted file mode 100644
index bcde584e0..000000000
--- a/tests/tests_unit/test_dependencies.py
+++ /dev/null
@@ -1,18 +0,0 @@
-import urllib
-
-from _nebari.provider import terraform
-
-
-def test_terraform_open_source_license():
-    tf_version = terraform.version()
-    license_url = (
-        f"https://raw.githubusercontent.com/hashicorp/terraform/v{tf_version}/LICENSE"
-    )
-
-    request = urllib.request.Request(license_url)
-    with urllib.request.urlopen(request) as response:
-        assert 200 == response.getcode()
-
-        license = str(response.read())
-        assert "Mozilla Public License" in license
-        assert "Business Source License" not in license

From 04b96e065e2e393e8e80ed2e4b3bb23255238e4a Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 15:55:49 -0800
Subject: [PATCH 22/42] Remove stage files

---
 src/_nebari/upgrade.py | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 9737d4a6e..6674c1c8e 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -214,6 +214,29 @@ def upgrade(
 
         return config
 
+    @classmethod
+    def _rm_rf_stages(cls, config_filename, dry_run: bool = False):
+        """
+        Remove stage files during and upgrade step
+
+        Args:
+            config_filename (str): The path to the configuration file.
+        Returns:
+            None
+        """
+        config_dir = Path(config_filename).resolve().parent
+
+        if Path.is_dir(config_dir):
+            stages_dir = config_dir / "stages"
+
+            stage_filenames = [d for d in stages_dir.rglob("*") if d.is_file()]
+
+            for stage_filename in stage_filenames:
+                if dry_run:
+                    rich.print(f"Dry run: Would remove {stage_filename}")
+                else:
+                    stage_filename.unlink(missing_ok=True)
+
     def get_version(self):
         """
         Returns:

From 3722cab4b60724352bc13e4c904b51596745c643 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 16:01:29 -0800
Subject: [PATCH 23/42] Remove stage directories

---
 src/_nebari/upgrade.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 6674c1c8e..7352be172 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -237,6 +237,17 @@ def _rm_rf_stages(cls, config_filename, dry_run: bool = False):
                 else:
                     stage_filename.unlink(missing_ok=True)
 
+            stage_filedirs = sorted(
+                (d for d in stages_dir.rglob("*") if d.is_dir()),
+                reverse=True,
+            )
+
+            for stage_filedir in stage_filedirs:
+                if dry_run:
+                    rich.print(f"Dry run: Would remove {stage_filedir}")
+                else:
+                    stage_filedir.rmdir()
+
     def get_version(self):
         """
         Returns:

From 62e16034fa3327826cec351c36fe214261200c6d Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 18:48:01 -0800
Subject: [PATCH 24/42] Add verbose argument

---
 src/_nebari/upgrade.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 7352be172..c6f5a8234 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -215,7 +215,7 @@ def upgrade(
         return config
 
     @classmethod
-    def _rm_rf_stages(cls, config_filename, dry_run: bool = False):
+    def _rm_rf_stages(cls, config_filename, dry_run: bool = False, verbose=False):
         """
         Remove stage files during and upgrade step
 
@@ -227,26 +227,30 @@ def _rm_rf_stages(cls, config_filename, dry_run: bool = False):
         config_dir = Path(config_filename).resolve().parent
 
         if Path.is_dir(config_dir):
-            stages_dir = config_dir / "stages"
+            stage_dir = config_dir / "stages"
 
-            stage_filenames = [d for d in stages_dir.rglob("*") if d.is_file()]
+            stage_filenames = [d for d in stage_dir.rglob("*") if d.is_file()]
 
             for stage_filename in stage_filenames:
-                if dry_run:
+                if dry_run and verbose:
                     rich.print(f"Dry run: Would remove {stage_filename}")
                 else:
                     stage_filename.unlink(missing_ok=True)
+                    if verbose:
+                        rich.print(f"Removed {stage_filename}")
 
             stage_filedirs = sorted(
-                (d for d in stages_dir.rglob("*") if d.is_dir()),
+                (d for d in stage_dir.rglob("*") if d.is_dir()),
                 reverse=True,
             )
 
             for stage_filedir in stage_filedirs:
-                if dry_run:
+                if dry_run and verbose:
                     rich.print(f"Dry run: Would remove {stage_filedir}")
                 else:
                     stage_filedir.rmdir()
+                    if verbose:
+                        rich.print(f"Removed {stage_filedir}")
 
     def get_version(self):
         """

From 28699147f1d62cbd2d6672c540b73b413156e7eb Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 19:12:43 -0800
Subject: [PATCH 25/42] Remove root stages dir

---
 src/_nebari/upgrade.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index c6f5a8234..a49da2ba2 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -252,6 +252,13 @@ def _rm_rf_stages(cls, config_filename, dry_run: bool = False, verbose=False):
                     if verbose:
                         rich.print(f"Removed {stage_filedir}")
 
+            if dry_run and verbose:
+                rich.print(f"Dry run: Would remove {stage_dir}")
+            else:
+                stage_dir.rmdir()
+                if verbose:
+                    rich.print(f"Removed {stage_dir}")
+
     def get_version(self):
         """
         Returns:

From e5614972f74ba041c850e4f800061b2b3163dee3 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 19:44:54 -0800
Subject: [PATCH 26/42] Remove Terraform scripts in 2023_10_1 upgrade

---
 src/_nebari/upgrade.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index a49da2ba2..9f6d18749 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -858,9 +858,6 @@ def _version_specific_upgrade(
         rich.print(
             "-> Data should be backed up before performing this upgrade ([green][link=https://www.nebari.dev/docs/how-tos/manual-backup]see docs[/link][/green])  The 'prevent_deploy' flag has been set in your config file and must be manually removed to deploy."
         )
-        rich.print(
-            "-> Please also run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment."
-        )
 
         # Setting the following flag will prevent deployment and display guidance to the user
         # which they can override if they are happy they understand the situation.
@@ -944,6 +941,22 @@ def _version_specific_upgrade(
             rich.print("\n ⚠️  DANGER ⚠️")
             rich.print(DESTRUCTIVE_UPGRADE_WARNING)
 
+        if kwargs.get("attempt_fixes", False) or Confirm.ask(
+            (
+                "Nebari needs to generate an updated set of Terraform scripts for your deployment and delete the old scripts.\n"
+                "Do you want Nebari to remove your [green]stages[/green] directory automatically for you? It will be recreated the next time Nebari is run.\n"
+                "[red]Warning:[/red] This will remove everything in the [green]stages[/green] directory.\n"
+                "If you do not have Nebari do it automatically here, you will need to remove the [green]stages[/green] manually with a command"
+                "like [green]rm -rf stages[/green]."
+            ),
+            default=False,
+        ):
+            self._rm_rf_stages(
+                config_filename,
+                dry_run=kwargs.get("dry_run", False),
+                verbose=True,
+            )
+
         return config
 
 

From 5fffbdedfc4f6b065d99561eba8b937a73fae413 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 19:49:18 -0800
Subject: [PATCH 27/42] Refactor confirmation string

---
 src/_nebari/upgrade.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 9f6d18749..18b5c9895 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -48,6 +48,13 @@
 
 UPGRADE_KUBERNETES_MESSAGE = "Please see the [green][link=https://www.nebari.dev/docs/how-tos/kubernetes-version-upgrade]Kubernetes upgrade docs[/link][/green] for more information."
 DESTRUCTIVE_UPGRADE_WARNING = "-> This version upgrade will result in your cluster being completely torn down and redeployed.  Please ensure you have backed up any data you wish to keep before proceeding!!!"
+TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION = (
+    "Nebari needs to generate an updated set of Terraform scripts for your deployment and delete the old scripts.\n"
+    "Do you want Nebari to remove your [green]stages[/green] directory automatically for you? It will be recreated the next time Nebari is run.\n"
+    "[red]Warning:[/red] This will remove everything in the [green]stages[/green] directory.\n"
+    "If you do not have Nebari do it automatically here, you will need to remove the [green]stages[/green] manually with a command"
+    "like [green]rm -rf stages[/green]."
+)
 
 
 def do_upgrade(config_filename, attempt_fixes=False):
@@ -942,13 +949,7 @@ def _version_specific_upgrade(
             rich.print(DESTRUCTIVE_UPGRADE_WARNING)
 
         if kwargs.get("attempt_fixes", False) or Confirm.ask(
-            (
-                "Nebari needs to generate an updated set of Terraform scripts for your deployment and delete the old scripts.\n"
-                "Do you want Nebari to remove your [green]stages[/green] directory automatically for you? It will be recreated the next time Nebari is run.\n"
-                "[red]Warning:[/red] This will remove everything in the [green]stages[/green] directory.\n"
-                "If you do not have Nebari do it automatically here, you will need to remove the [green]stages[/green] manually with a command"
-                "like [green]rm -rf stages[/green]."
-            ),
+            TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
             default=False,
         ):
             self._rm_rf_stages(

From f2088e29dadf02aadc8b355e38f55d1d1f72f3b2 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 19:50:55 -0800
Subject: [PATCH 28/42] Remove Terraform scripts in 2023_11_1 upgrade

---
 src/_nebari/upgrade.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 18b5c9895..439f92c6a 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -975,15 +975,21 @@ class Upgrade_2023_11_1(UpgradeStep):
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
-        rich.print("\n ⚠️  Warning ⚠️")
-        rich.print(
-            "-> Please run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment."
-        )
         rich.print("\n ⚠️  Deprecation Warning ⚠️")
         rich.print(
             f"-> ClearML, Prefect and kbatch are no longer supported in Nebari version [green]{self.version}[/green] and will be uninstalled."
         )
 
+        if kwargs.get("attempt_fixes", False) or Confirm.ask(
+            TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
+            default=False,
+        ):
+            self._rm_rf_stages(
+                config_filename,
+                dry_run=kwargs.get("dry_run", False),
+                verbose=True,
+            )
+
         return config
 
 

From e9699ea29cf31b10e8166fdc59755007a1693543 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 19:56:02 -0800
Subject: [PATCH 29/42] Remove Terraform scripts in 2023_12_1 upgrade

---
 src/_nebari/upgrade.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 439f92c6a..940cd911e 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1007,16 +1007,22 @@ class Upgrade_2023_12_1(UpgradeStep):
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
-        rich.print("\n ⚠️  Warning ⚠️")
-        rich.print(
-            "-> Please run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment."
-        )
         rich.print("\n ⚠️  Deprecation Warning ⚠️")
         rich.print(
             f"-> [green]{self.version}[/green] is the last Nebari version that supports the jupyterlab-videochat extension."
         )
         rich.print()
 
+        if kwargs.get("attempt_fixes", False) or Confirm.ask(
+            TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
+            default=False,
+        ):
+            self._rm_rf_stages(
+                config_filename,
+                dry_run=kwargs.get("dry_run", False),
+                verbose=True,
+            )
+
         return config
 
 

From 77ae9623c5ca52aad0b16a365b937d930a6ad9ad Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 3 Dec 2024 19:59:16 -0800
Subject: [PATCH 30/42] Remove Terraform scripts in 2024_1_1 upgrade

---
 src/_nebari/upgrade.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 940cd911e..b7144b705 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1040,10 +1040,6 @@ class Upgrade_2024_1_1(UpgradeStep):
     def _version_specific_upgrade(
         self, config, start_version, config_filename: Path, *args, **kwargs
     ):
-        rich.print("\n ⚠️  Warning ⚠️")
-        rich.print(
-            "-> Please run the [green]rm -rf stages[/green] so that we can regenerate an updated set of Terraform scripts for your deployment."
-        )
         rich.print("\n ⚠️  Deprecation Warning ⚠️")
         rich.print(
             "-> jupyterlab-videochat, retrolab, jupyter-tensorboard, jupyterlab-conda-store and jupyter-nvdashboard",
@@ -1051,6 +1047,16 @@ def _version_specific_upgrade(
         )
         rich.print()
 
+        if kwargs.get("attempt_fixes", False) or Confirm.ask(
+            TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
+            default=False,
+        ):
+            self._rm_rf_stages(
+                config_filename,
+                dry_run=kwargs.get("dry_run", False),
+                verbose=True,
+            )
+
         return config
 
 

From 40b4660b657b18939d641af89cb3969810534fa3 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 4 Dec 2024 19:08:09 -0800
Subject: [PATCH 31/42] Remove directory only if it exists and is empty

---
 src/_nebari/upgrade.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index b7144b705..c6ae78818 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -261,7 +261,7 @@ def _rm_rf_stages(cls, config_filename, dry_run: bool = False, verbose=False):
 
             if dry_run and verbose:
                 rich.print(f"Dry run: Would remove {stage_dir}")
-            else:
+            elif stage_dir.is_dir():
                 stage_dir.rmdir()
                 if verbose:
                     rich.print(f"Removed {stage_dir}")

From 52bfb1fe398175d573c9b2c847e11ad0d5184a76 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Fri, 13 Dec 2024 20:13:27 -0800
Subject: [PATCH 32/42] Add monkeypatch for remove stage dir confirmation and
 prompt

---
 tests/tests_unit/test_cli_upgrade.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tests/tests_unit/test_cli_upgrade.py b/tests/tests_unit/test_cli_upgrade.py
index 8cbac5d57..364b51b23 100644
--- a/tests/tests_unit/test_cli_upgrade.py
+++ b/tests/tests_unit/test_cli_upgrade.py
@@ -5,6 +5,7 @@
 
 import pytest
 import yaml
+from rich.prompt import Confirm, Prompt
 from typer.testing import CliRunner
 
 import _nebari.upgrade
@@ -450,6 +451,24 @@ def test_cli_upgrade_to_2023_10_1_kubernetes_validations(
         "gcp": {"incompatible": "1.23", "compatible": "1.26", "invalid": "badname"},
     }
 
+    def mock_input_ask(prompt, *args, **kwargs):
+        from _nebari.upgrade import TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION
+
+        # For more about structural pattern matching, see:
+        # https://peps.python.org/pep-0636/
+        match prompt:
+            case str(s) if s == TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION:
+                return kwargs.get("attempt_fixes", False)
+            case _:
+                return kwargs.get("default", False)
+
+    monkeypatch.setattr(Confirm, "ask", mock_input_ask)
+    monkeypatch.setattr(
+        Prompt,
+        "ask",
+        lambda x, *args, **kwargs: "",
+    )
+
     with tempfile.TemporaryDirectory() as tmp:
         tmp_file = Path(tmp).resolve() / "nebari-config.yaml"
         assert tmp_file.exists() is False

From bab0e7b266b58a1e743a81ecb4300b037019e29d Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Fri, 13 Dec 2024 20:29:44 -0800
Subject: [PATCH 33/42] Add mock input for removing stage files

---
 tests/tests_unit/test_upgrade.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py
index 9505b8d55..fe7079f75 100644
--- a/tests/tests_unit/test_upgrade.py
+++ b/tests/tests_unit/test_upgrade.py
@@ -50,6 +50,8 @@ def test_upgrade_4_0(
     monkeypatch,
 ):
     def mock_input(prompt, **kwargs):
+        from _nebari.upgrade import TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION
+
         # Mock different upgrade steps prompt answers
         if prompt == "Have you deleted the Argo Workflows CRDs and service accounts?":
             return True
@@ -68,6 +70,8 @@ def mock_input(prompt, **kwargs):
             == "[bold]Would you like Nebari to assign the corresponding role to all of your current groups automatically?[/bold]"
         ):
             return False
+        elif prompt == TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION:
+            return attempt_fixes
         # All other prompts will be answered with "y"
         else:
             return True

From 9157a27dc72e733699f791775cea8c886ca17765 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 17 Dec 2024 17:17:13 -0800
Subject: [PATCH 34/42] Monkeypatch `kubernetes.client.ApiextensionsV1Api` and
 `kubernetes.client.CoreV1Api`

---
 tests/tests_unit/test_upgrade.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py
index fe7079f75..a904fd7f3 100644
--- a/tests/tests_unit/test_upgrade.py
+++ b/tests/tests_unit/test_upgrade.py
@@ -79,6 +79,27 @@ def mock_input(prompt, **kwargs):
     monkeypatch.setattr(Confirm, "ask", mock_input)
     monkeypatch.setattr(Prompt, "ask", lambda x: "")
 
+    from kubernetes.client import ApiextensionsV1Api as _ApiextensionsV1Api
+    from kubernetes.client import CoreV1Api as _CoreV1Api
+    from kubernetes.client import V1Status as _V1Status
+
+    def monkey_patch_delete_crd(*args, **kwargs):
+        return _V1Status(code=200)
+
+    def monkey_patch_delete_namespaced_sa(*args, **kwargs):
+        return _V1Status(code=200)
+
+    monkeypatch.setattr(
+        _ApiextensionsV1Api,
+        "delete_custom_resource_definition",
+        monkey_patch_delete_crd,
+    )
+    monkeypatch.setattr(
+        _CoreV1Api,
+        "delete_namespaced_service_account",
+        monkey_patch_delete_namespaced_sa,
+    )
+
     old_qhub_config_path = Path(__file__).parent / old_qhub_config_path_str
 
     tmp_qhub_config = Path(tmp_path, old_qhub_config_path.name)

From 3c032bb2ff859cb77497138eb9303d39bf9790bc Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 17 Dec 2024 18:54:45 -0800
Subject: [PATCH 35/42] Add monkeypatches for crd and namespaced daemon set

---
 tests/tests_unit/test_upgrade.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py
index ddd2362ee..e70732759 100644
--- a/tests/tests_unit/test_upgrade.py
+++ b/tests/tests_unit/test_upgrade.py
@@ -80,6 +80,7 @@ def mock_input(prompt, **kwargs):
     monkeypatch.setattr(Prompt, "ask", lambda x: "")
 
     from kubernetes.client import ApiextensionsV1Api as _ApiextensionsV1Api
+    from kubernetes.client import AppsV1Api as _AppsV1Api
     from kubernetes.client import CoreV1Api as _CoreV1Api
     from kubernetes.client import V1Status as _V1Status
 
@@ -89,6 +90,12 @@ def monkey_patch_delete_crd(*args, **kwargs):
     def monkey_patch_delete_namespaced_sa(*args, **kwargs):
         return _V1Status(code=200)
 
+    def monkey_patch_list_namespaced_daemon_set(*args, **kwargs):
+        class MonkeypatchApiResponse:
+            items = False
+
+        return MonkeypatchApiResponse
+
     monkeypatch.setattr(
         _ApiextensionsV1Api,
         "delete_custom_resource_definition",
@@ -99,6 +106,21 @@ def monkey_patch_delete_namespaced_sa(*args, **kwargs):
         "delete_namespaced_service_account",
         monkey_patch_delete_namespaced_sa,
     )
+    monkeypatch.setattr(
+        _ApiextensionsV1Api,
+        "read_custom_resource_definition",
+        lambda x, *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        _ApiextensionsV1Api,
+        "patch_custom_resource_definition",
+        lambda x, *args, **kwargs: True,
+    )
+    monkeypatch.setattr(
+        _AppsV1Api,
+        "list_namespaced_daemon_set",
+        monkey_patch_list_namespaced_daemon_set,
+    )
 
     old_qhub_config_path = Path(__file__).parent / old_qhub_config_path_str
 

From a36df1599ab87f8423247fc0068965c7dc258ac4 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Tue, 17 Dec 2024 18:56:53 -0800
Subject: [PATCH 36/42] Clean up monkeypatch

---
 tests/tests_unit/test_upgrade.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py
index e70732759..ec4d413f8 100644
--- a/tests/tests_unit/test_upgrade.py
+++ b/tests/tests_unit/test_upgrade.py
@@ -109,12 +109,12 @@ class MonkeypatchApiResponse:
     monkeypatch.setattr(
         _ApiextensionsV1Api,
         "read_custom_resource_definition",
-        lambda x, *args, **kwargs: True,
+        lambda *args, **kwargs: True,
     )
     monkeypatch.setattr(
         _ApiextensionsV1Api,
         "patch_custom_resource_definition",
-        lambda x, *args, **kwargs: True,
+        lambda *args, **kwargs: True,
     )
     monkeypatch.setattr(
         _AppsV1Api,

From 25bb876e095013dcdf1047313065ae2902bdb6d7 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 18 Dec 2024 15:52:21 -0800
Subject: [PATCH 37/42] Add comment about failed test

---
 src/_nebari/upgrade.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index a3a52c7bb..932e8f45d 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -1447,6 +1447,7 @@ def _version_specific_upgrade(
         keycloak_admin = None
 
         # Prompt the user for role assignment (if yes, transforms the response into bool)
+        # This needs to be monkeypatched and will be addressed in a future PR. Until then, this causes test failures.
         assign_roles = kwargs.get("attempt_fixes", False) or Confirm.ask(
             "[bold]Would you like Nebari to assign the corresponding role/scopes to all of your current groups automatically?[/bold]",
             default=False,

From db894fea7c85cea221fef37f13336eff8dc83b2d Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 18 Dec 2024 19:30:09 -0800
Subject: [PATCH 38/42] Add comment to docstring of `_rm_rf_stages`

---
 src/_nebari/upgrade.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 932e8f45d..51629f02b 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -226,6 +226,10 @@ def _rm_rf_stages(cls, config_filename, dry_run: bool = False, verbose=False):
         """
         Remove stage files during and upgrade step
 
+
+        Usually used when you need files in your `stages` directory to be
+        removed in order to avoid resource conflicts
+
         Args:
             config_filename (str): The path to the configuration file.
         Returns:

From fab9548d688ff684cb0acaba1e957ef5f59c205f Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 18 Dec 2024 20:32:01 -0800
Subject: [PATCH 39/42] Add confirmation in case of non-remote
 `terraform_state` files

---
 src/_nebari/upgrade.py | 47 +++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 46 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 51629f02b..48ebb8253 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -55,6 +55,12 @@
     "If you do not have Nebari do it automatically here, you will need to remove the [green]stages[/green] manually with a command"
     "like [green]rm -rf stages[/green]."
 )
+DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE = (
+    "Nebari would like to remove your old Terraform/Opentofu [green]stages[/green] files. Your [blue]terraform_state[/blue] configuration is not set to [blue]remote[/blue], so destroying your [green]stages[/green] files could potentially be very detructive.\n"
+    "If you don't have active Terraform/Opentofu deployment state files contained within your [green]stages[/green] directory, you may proceed by entering [red]y[/red] at the prompt."
+    "If you have an active Terraform/Opentofu deployment with active state files in your [green]stages[/green] folder, you will need to either bring Nebari down temporarily to redeploy or pursue some other means to upgrade. Enter [red]n[/red] at the prompt.\n\n"
+    "Do you want to proceed by deleting your [green]stages[/green] directory and everything in it? ([red]POTENTIALLY VERY DESTRUCTIVE[/red])"
+)
 
 
 def do_upgrade(config_filename, attempt_fixes=False):
@@ -226,7 +232,6 @@ def _rm_rf_stages(cls, config_filename, dry_run: bool = False, verbose=False):
         """
         Remove stage files during and upgrade step
 
-
         Usually used when you need files in your `stages` directory to be
         removed in order to avoid resource conflicts
 
@@ -956,6 +961,16 @@ def _version_specific_upgrade(
             TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
             default=False,
         ):
+            if (
+                _terraform_state_config := config.get("terraform_state")
+                and (_terraform_state_config.get("type") != "remote")
+                and not Confirm.ask(
+                    DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
+                    default=False,
+                )
+            ):
+                exit()
+
             self._rm_rf_stages(
                 config_filename,
                 dry_run=kwargs.get("dry_run", False),
@@ -988,6 +1003,16 @@ def _version_specific_upgrade(
             TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
             default=False,
         ):
+            if (
+                _terraform_state_config := config.get("terraform_state")
+                and (_terraform_state_config.get("type") != "remote")
+                and not Confirm.ask(
+                    DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
+                    default=False,
+                )
+            ):
+                exit()
+
             self._rm_rf_stages(
                 config_filename,
                 dry_run=kwargs.get("dry_run", False),
@@ -1021,6 +1046,16 @@ def _version_specific_upgrade(
             TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
             default=False,
         ):
+            if (
+                _terraform_state_config := config.get("terraform_state")
+                and (_terraform_state_config.get("type") != "remote")
+                and not Confirm.ask(
+                    DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
+                    default=False,
+                )
+            ):
+                exit()
+
             self._rm_rf_stages(
                 config_filename,
                 dry_run=kwargs.get("dry_run", False),
@@ -1055,6 +1090,16 @@ def _version_specific_upgrade(
             TERRAFORM_REMOVE_TERRAFORM_STAGE_FILES_CONFIRMATION,
             default=False,
         ):
+            if (
+                _terraform_state_config := config.get("terraform_state")
+                and (_terraform_state_config.get("type") != "remote")
+                and not Confirm.ask(
+                    DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
+                    default=False,
+                )
+            ):
+                exit()
+
             self._rm_rf_stages(
                 config_filename,
                 dry_run=kwargs.get("dry_run", False),

From b1524cb6739750e0707f06e1e0a26439d26afeea Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 18 Dec 2024 20:57:32 -0800
Subject: [PATCH 40/42] Fix parentheses issue

---
 src/_nebari/upgrade.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index 48ebb8253..e33ca5293 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -962,7 +962,7 @@ def _version_specific_upgrade(
             default=False,
         ):
             if (
-                _terraform_state_config := config.get("terraform_state")
+                (_terraform_state_config := config.get("terraform_state"))
                 and (_terraform_state_config.get("type") != "remote")
                 and not Confirm.ask(
                     DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
@@ -1004,7 +1004,7 @@ def _version_specific_upgrade(
             default=False,
         ):
             if (
-                _terraform_state_config := config.get("terraform_state")
+                (_terraform_state_config := config.get("terraform_state"))
                 and (_terraform_state_config.get("type") != "remote")
                 and not Confirm.ask(
                     DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
@@ -1047,7 +1047,7 @@ def _version_specific_upgrade(
             default=False,
         ):
             if (
-                _terraform_state_config := config.get("terraform_state")
+                (_terraform_state_config := config.get("terraform_state"))
                 and (_terraform_state_config.get("type") != "remote")
                 and not Confirm.ask(
                     DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,
@@ -1091,7 +1091,7 @@ def _version_specific_upgrade(
             default=False,
         ):
             if (
-                _terraform_state_config := config.get("terraform_state")
+                (_terraform_state_config := config.get("terraform_state"))
                 and (_terraform_state_config.get("type") != "remote")
                 and not Confirm.ask(
                     DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE,

From eced5fd9944ac3493392ea5b28cd4f19907648e6 Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Wed, 18 Dec 2024 21:00:57 -0800
Subject: [PATCH 41/42] Add `xfail` mark to a test configuration that needs
 patching

---
 tests/tests_unit/test_upgrade.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/tests_unit/test_upgrade.py b/tests/tests_unit/test_upgrade.py
index ec4d413f8..86de650ce 100644
--- a/tests/tests_unit/test_upgrade.py
+++ b/tests/tests_unit/test_upgrade.py
@@ -34,10 +34,12 @@ def qhub_users_import_json():
             False,
             True,
         ),
-        (
+        pytest.param(
+            # We add an xfail mark until a monkey patch for the keycloak calls are implemented in upgrade steps
             "./qhub-config-yaml-files-for-upgrade/qhub-config-aws-310-customauth.yaml",
             True,
             False,
+            marks=pytest.mark.xfail,
         ),
     ],
 )

From bec3639d9eab27411e2e0c6eebf24a0950479fed Mon Sep 17 00:00:00 2001
From: smokestacklightnin
 <125844868+smokestacklightnin@users.noreply.github.com>
Date: Thu, 19 Dec 2024 12:13:59 -0800
Subject: [PATCH 42/42] Add caution label

---
 src/_nebari/upgrade.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/upgrade.py b/src/_nebari/upgrade.py
index e33ca5293..71795dfa1 100644
--- a/src/_nebari/upgrade.py
+++ b/src/_nebari/upgrade.py
@@ -56,6 +56,7 @@
     "like [green]rm -rf stages[/green]."
 )
 DESTROY_STAGE_FILES_WITH_TF_STATE_NOT_REMOTE = (
+    "⚠️ CAUTION ⚠️\n"
     "Nebari would like to remove your old Terraform/Opentofu [green]stages[/green] files. Your [blue]terraform_state[/blue] configuration is not set to [blue]remote[/blue], so destroying your [green]stages[/green] files could potentially be very detructive.\n"
     "If you don't have active Terraform/Opentofu deployment state files contained within your [green]stages[/green] directory, you may proceed by entering [red]y[/red] at the prompt."
     "If you have an active Terraform/Opentofu deployment with active state files in your [green]stages[/green] folder, you will need to either bring Nebari down temporarily to redeploy or pursue some other means to upgrade. Enter [red]n[/red] at the prompt.\n\n"