[k8s] Show hints when requested resources don't fit in Kubernetes clu…

…ster (#3590)

* Add hints when feasible resources are not found

* lint

* lint

* wip

* move to resources_utils

* fixes

* fix

* add TODO

* Add todos and fix comments

sky/clouds/aws.py

@@ -444,7 +444,7 @@ def make_deploy_resources_variables(
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
-    ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+    ) -> resources_utils.FeasibleResources:
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
             # Check the instance type is valid in the cloud
@@ -455,10 +455,12 @@ def _get_feasible_launchable_resources(
                 region=resources.region,
                 zone=resources.zone)
             if not regions:
-                return ([], [])
+                # TODO: Add hints to all return values in this method to help
+                #  users understand why the resources are not launchable.
+                return resources_utils.FeasibleResources([], [], None)
             # Treat Resources(AWS, p3.2x, V100) as Resources(AWS, p3.2x).
             resources = resources.copy(accelerators=None)
-            return ([resources], [])
+            return resources_utils.FeasibleResources([resources], [], None)
 
         def _make(instance_list):
             resource_list = []
@@ -484,9 +486,10 @@ def _make(instance_list):
                 memory=resources.memory,
                 disk_tier=resources.disk_tier)
             if default_instance_type is None:
-                return ([], [])
+                return resources_utils.FeasibleResources([], [], None)
             else:
-                return (_make([default_instance_type]), [])
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
 
         assert len(accelerators) == 1, resources
         acc, acc_count = list(accelerators.items())[0]
@@ -501,8 +504,10 @@ def _make(instance_list):
             zone=resources.zone,
             clouds='aws')
         if instance_list is None:
-            return ([], fuzzy_candidate_list)
-        return (_make(instance_list), fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
 
     @classmethod
     @functools.lru_cache(maxsize=1)  # Cache since getting identity is slow.

sky/clouds/azure.py

@@ -378,17 +378,19 @@ def _failover_disk_tier() -> Optional[resources_utils.DiskTier]:
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources.Resources'
-    ) -> Tuple[List['resources.Resources'], List[str]]:
+    ) -> 'resources_utils.FeasibleResources':
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
             ok, _ = Azure.check_disk_tier(resources.instance_type,
                                           resources.disk_tier)
             if not ok:
-                return ([], [])
+                # TODO: Add hints to all return values in this method to help
+                #  users understand why the resources are not launchable.
+                return resources_utils.FeasibleResources([], [], None)
             # Treat Resources(Azure, Standard_NC4as_T4_v3, T4) as
             # Resources(Azure, Standard_NC4as_T4_v3).
             resources = resources.copy(accelerators=None)
-            return ([resources], [])
+            return resources_utils.FeasibleResources([resources], [], None)
 
         def _make(instance_list):
             resource_list = []
@@ -418,9 +420,10 @@ def _make(instance_list):
                 memory=resources.memory,
                 disk_tier=resources.disk_tier)
             if default_instance_type is None:
-                return ([], [])
+                return resources_utils.FeasibleResources([], [], None)
             else:
-                return (_make([default_instance_type]), [])
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
 
         assert len(accelerators) == 1, resources
         acc, acc_count = list(accelerators.items())[0]
@@ -435,8 +438,10 @@ def _make(instance_list):
             zone=resources.zone,
             clouds='azure')
         if instance_list is None:
-            return ([], fuzzy_candidate_list)
-        return (_make(instance_list), fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/cloud.py

@@ -341,22 +341,26 @@ def is_label_valid(cls, label_key: str,
         return True, None
 
     def get_feasible_launchable_resources(
-        self,
-        resources: 'resources_lib.Resources',
-        num_nodes: int = 1
-    ) -> Tuple[List['resources_lib.Resources'], List[str]]:
-        """Returns ([feasible and launchable resources], [fuzzy candidates]).
+            self,
+            resources: 'resources_lib.Resources',
+            num_nodes: int = 1) -> 'resources_utils.FeasibleResources':
+        """Returns FeasibleResources for the given resources.
 
         Feasible resources refer to an offering respecting the resource
         requirements.  Currently, this function implements "filtering" the
         cloud's offerings only w.r.t. accelerators constraints.
 
         Launchable resources require a cloud and an instance type be assigned.
 
-        Fuzzy candidates example: when the requested GPU is A100:1 but is not
-        available in a cloud/region, the fuzzy candidates are results of a fuzzy
-        search in the catalog that are offered in the location. E.g.,
-          ['A100-80GB:1', 'A100-80GB:2', 'A100-80GB:4', 'A100:8']
+        The returned dataclass object FeasibleResources contains three fields:
+
+        - resources_list: a list of resources that are feasible to launch
+        - fuzzy_candidate_list: a list of resources that loosely match requested
+            resources. E.g., when A100:1 GPU is requested but is not available
+            in a cloud/region, the fuzzy candidates are results of a fuzzy
+            search in the catalog that are offered in the location. E.g.,
+            ['A100-80GB:1', 'A100-80GB:2', 'A100-80GB:4', 'A100:8']
+        - hint: an optional string hint if no feasible resources are found.
         """
         if resources.is_launchable():
             self._check_instance_type_accelerators_combination(resources)
@@ -372,13 +376,18 @@ def get_feasible_launchable_resources(
             # TODO(zhwu): The resources are now silently filtered out. We
             # should have some logging telling the user why the resources
             # are not considered.
-            return ([], [])
+            return resources_utils.FeasibleResources(resources_list=[],
+                                                     fuzzy_candidate_list=[],
+                                                     hint=None)
         return self._get_feasible_launchable_resources(resources)
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
-    ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+    ) -> 'resources_utils.FeasibleResources':
         """See get_feasible_launchable_resources()."""
+        # TODO: Currently only the Kubernetes implementation of this method
+        #  returns hints when no feasible resources are found. This should be
+        #  implemented for all clouds.
         raise NotImplementedError
 
     def get_reservations_available_resources(

sky/clouds/cudo.py

@@ -214,13 +214,16 @@ def make_deploy_resources_variables(
         }
 
     def _get_feasible_launchable_resources(
-            self, resources: 'resources_lib.Resources'):
+        self, resources: 'resources_lib.Resources'
+    ) -> 'resources_utils.FeasibleResources':
         if resources.use_spot:
-            return ([], [])
+            # TODO: Add hints to all return values in this method to help
+            #  users understand why the resources are not launchable.
+            return resources_utils.FeasibleResources([], [], None)
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
             resources = resources.copy(accelerators=None)
-            return ([resources], [])
+            return resources_utils.FeasibleResources([resources], [], None)
 
         def _make(instance_list):
             resource_list = []
@@ -243,9 +246,10 @@ def _make(instance_list):
                 memory=resources.memory,
                 disk_tier=resources.disk_tier)
             if default_instance_type is None:
-                return ([], [])
+                return resources_utils.FeasibleResources([], [], None)
             else:
-                return (_make([default_instance_type]), [])
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
 
         assert len(accelerators) == 1, resources
         acc, acc_count = list(accelerators.items())[0]
@@ -260,8 +264,10 @@ def _make(instance_list):
             zone=resources.zone,
             clouds='cudo')
         if instance_list is None:
-            return ([], fuzzy_candidate_list)
-        return (_make(instance_list), fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/fluidstack.py

@@ -211,7 +211,9 @@ def _get_feasible_launchable_resources(
             assert resources.is_launchable(), resources
             # Accelerators are part of the instance type in Fluidstack Cloud
             resources = resources.copy(accelerators=None)
-            return ([resources], [])
+            # TODO: Add hints to all return values in this method to help
+            #  users understand why the resources are not launchable.
+            return resources_utils.FeasibleResources([resources], [], None)
 
         def _make(instance_list):
             resource_list = []
@@ -239,9 +241,10 @@ def _make(instance_list):
                 memory=resources.memory,
                 disk_tier=resources.disk_tier)
             if default_instance_type is None:
-                return ([], [])
+                return resources_utils.FeasibleResources([], [], None)
             else:
-                return (_make([default_instance_type]), [])
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
 
         assert len(accelerators) == 1, resources
         acc, acc_count = list(accelerators.items())[0]
@@ -256,8 +259,10 @@ def _make(instance_list):
             zone=resources.zone,
             clouds='fluidstack')
         if instance_list is None:
-            return ([], fuzzy_candidate_list)
-        return (_make(instance_list), fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]:

sky/clouds/gcp.py

@@ -526,10 +526,10 @@ def make_deploy_resources_variables(
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources.Resources'
-    ) -> Tuple[List['resources.Resources'], List[str]]:
+    ) -> 'resources_utils.FeasibleResources':
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
-            return ([resources], [])
+            return resources_utils.FeasibleResources([resources], [], None)
 
         if resources.accelerators is None:
             # Return a default instance type with the given number of vCPUs.
@@ -538,7 +538,9 @@ def _get_feasible_launchable_resources(
                 memory=resources.memory,
                 disk_tier=resources.disk_tier)
             if host_vm_type is None:
-                return ([], [])
+                # TODO: Add hints to all return values in this method to help
+                #  users understand why the resources are not launchable.
+                return resources_utils.FeasibleResources([], [], None)
             else:
                 r = resources.copy(
                     cloud=GCP(),
@@ -547,7 +549,7 @@ def _get_feasible_launchable_resources(
                     cpus=None,
                     memory=None,
                 )
-                return ([r], [])
+                return resources_utils.FeasibleResources([r], [], None)
 
         # Find instance candidates to meet user's requirements
         assert len(resources.accelerators.items()
@@ -569,7 +571,8 @@ def _get_feasible_launchable_resources(
             clouds='gcp')
 
         if instance_list is None:
-            return ([], fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
         assert len(
             instance_list
         ) == 1, f'More than one instance type matched, {instance_list}'
@@ -584,11 +587,13 @@ def _get_feasible_launchable_resources(
                 if resources.cpus.endswith('+'):
                     cpus = float(resources.cpus[:-1])
                     if cpus > num_cpus_in_tpu_vm:
-                        return ([], fuzzy_candidate_list)
+                        return resources_utils.FeasibleResources(
+                            [], fuzzy_candidate_list, None)
                 else:
                     cpus = float(resources.cpus)
                     if cpus != num_cpus_in_tpu_vm:
-                        return ([], fuzzy_candidate_list)
+                        return resources_utils.FeasibleResources(
+                            [], fuzzy_candidate_list, None)
             # FIXME(woosuk, wei-lin): This leverages the fact that TPU VMs
             # have 334 GB RAM, and 400 GB RAM for tpu-v4. We need to move
             # this to service catalog, instead.
@@ -597,11 +602,13 @@ def _get_feasible_launchable_resources(
                 if resources.memory.endswith('+'):
                     memory = float(resources.memory[:-1])
                     if memory > memory_in_tpu_vm:
-                        return ([], fuzzy_candidate_list)
+                        return resources_utils.FeasibleResources(
+                            [], fuzzy_candidate_list, None)
                 else:
                     memory = float(resources.memory)
                     if memory != memory_in_tpu_vm:
-                        return ([], fuzzy_candidate_list)
+                        return resources_utils.FeasibleResources(
+                            [], fuzzy_candidate_list, None)
         else:
             host_vm_type = instance_list[0]
 
@@ -613,7 +620,8 @@ def _get_feasible_launchable_resources(
             cpus=None,
             memory=None,
         )
-        return ([r], fuzzy_candidate_list)
+        return resources_utils.FeasibleResources([r], fuzzy_candidate_list,
+                                                 None)
 
     @classmethod
     def get_accelerators_from_instance_type(

sky/clouds/ibm.py

@@ -266,12 +266,15 @@ def get_default_instance_type(
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
-    ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+    ) -> 'resources_utils.FeasibleResources':
         fuzzy_candidate_list: List[str] = []
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
             resources = resources.copy(accelerators=None)
-            return ([resources], fuzzy_candidate_list)
+            # TODO: Add hints to all return values in this method to help
+            #  users understand why the resources are not launchable.
+            return resources_utils.FeasibleResources([resources],
+                                                     fuzzy_candidate_list, None)
 
         def _make(instance_list):
             resource_list = []
@@ -296,9 +299,10 @@ def _make(instance_list):
                 memory=resources.memory,
                 disk_tier=resources.disk_tier)
             if default_instance_type is None:
-                return ([], [])
+                return resources_utils.FeasibleResources([], [], None)
             else:
-                return (_make([default_instance_type]), [])
+                return resources_utils.FeasibleResources(
+                    _make([default_instance_type]), [], None)
 
         assert len(accelerators) == 1, resources
         acc, acc_count = list(accelerators.items())[0]
@@ -312,8 +316,10 @@ def _make(instance_list):
             zone=resources.zone,
             clouds='ibm')
         if instance_list is None:
-            return ([], fuzzy_candidate_list)
-        return (_make(instance_list), fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([], fuzzy_candidate_list,
+                                                     None)
+        return resources_utils.FeasibleResources(_make(instance_list),
+                                                 fuzzy_candidate_list, None)
 
     @classmethod
     def get_default_image(cls, region) -> str:

sky/clouds/kubernetes.py

@@ -342,12 +342,13 @@ def make_deploy_resources_variables(
 
     def _get_feasible_launchable_resources(
         self, resources: 'resources_lib.Resources'
-    ) -> Tuple[List['resources_lib.Resources'], List[str]]:
+    ) -> 'resources_utils.FeasibleResources':
         fuzzy_candidate_list: List[str] = []
         if resources.instance_type is not None:
             assert resources.is_launchable(), resources
             resources = resources.copy(accelerators=None)
-            return ([resources], fuzzy_candidate_list)
+            return resources_utils.FeasibleResources([resources],
+                                                     fuzzy_candidate_list, None)
 
         def _make(instance_list):
             resource_list = []
@@ -403,10 +404,11 @@ def _make(instance_list):
                 logger.debug(f'Instance type {chosen_instance_type} does '
                              'not fit in the Kubernetes cluster. '
                              f'Reason: {reason}')
-                return [], []
+                return resources_utils.FeasibleResources([], [], reason)
 
         # No fuzzy lists for Kubernetes
-        return _make([chosen_instance_type]), []
+        return resources_utils.FeasibleResources(_make([chosen_instance_type]),
+                                                 [], None)
 
     @classmethod
     def check_credentials(cls) -> Tuple[bool, Optional[str]]: