[k8s] Fix ingress on kind clusters

sky/adaptors/kubernetes.py

@@ -81,6 +81,10 @@ def _load_config():
         kubernetes.config.load_incluster_config()
     except kubernetes.config.config_exception.ConfigException:
         try:
+            # Remove the environment variables set by the in-cluster config
+            # to avoid conflicts with the kubeconfig file
+            del os.environ['KUBERNETES_SERVICE_HOST']
+            del os.environ['KUBERNETES_SERVICE_PORT']
             kubernetes.config.load_kube_config()
         except kubernetes.config.config_exception.ConfigException as e:
             suffix = ''

sky/provision/kubernetes/network_utils.py

@@ -25,8 +25,7 @@ def get_port_mode(
         mode_str: Optional[str] = None) -> kubernetes_enums.KubernetesPortMode:
     """Get the port mode from the provider config."""
 
-    curr_kube_config = kubernetes_utils.get_current_kube_config_context_name()
-    running_kind = curr_kube_config == kubernetes_utils.KIND_CONTEXT_NAME
+    running_kind = kubernetes_utils.is_kind_cluster()
 
     if running_kind:
         # If running in kind (`sky local up`), use ingress mode
@@ -243,7 +242,16 @@ def get_ingress_external_ip_and_ports(
             ip = ingress_service.metadata.annotations.get(
                 'skypilot.co/external-ip', None)
         if ip is None:
-            ip = 'localhost'
+            # If running inside a Kubernetes cluster, use the internal
+            # cluster IP
+            if (kubernetes_utils.is_inside_kubernetes() and
+                    ingress_service.spec.cluster_ip is not None):
+                ip = ingress_service.spec.cluster_ip
+                http_port = 80
+                https_port = 443
+                return ip, (http_port, https_port)
+            else:
+                ip = 'localhost'
         ports = ingress_service.spec.ports
         http_port = [port for port in ports if port.name == 'http'][0].node_port
         https_port = [port for port in ports if port.name == 'https'

sky/provision/kubernetes/utils.py

@@ -55,6 +55,7 @@
                            '\nTo debug, run: {debug_cmd}')
 
 KIND_CONTEXT_NAME = 'kind-skypilot'  # Context name used by sky local up
+KIND_NODE_NAME = 'skypilot-control-plane'  # Node name used by sky local up
 
 # Port-forward proxy command constants
 PORT_FORWARD_PROXY_CMD_TEMPLATE = 'kubernetes-port-forward-proxy-command.sh'
@@ -1720,3 +1721,25 @@ def get_kubernetes_node_info() -> Dict[str, KubernetesNodeInfo]:
             free={'nvidia.com/gpu': int(accelerators_available)})
 
     return node_info_dict
+
+
+def is_inside_kubernetes() -> bool:
+    """Returns whether the caller is running inside a Kubernetes cluster"""
+    return 'KUBERNETES_SERVICE_HOST' in os.environ
+
+
+def is_kind_cluster() -> bool:
+    """Returns whether the caller is running on or in a local KinD cluster"""
+    k8s_enabled, _ = check_credentials()
+    if not k8s_enabled:
+        return False
+    curr_kube_config = get_current_kube_config_context_name()
+    if curr_kube_config is None:
+        # If kubeconfig is not present, we could be inside a KinD cluster
+        # Test if it is a kind cluster by inspecting nodes
+        nodes = get_kubernetes_nodes()
+        for node in nodes:
+            if node.metadata.name == KIND_NODE_NAME:
+                return True
+        return False
+    return curr_kube_config == KIND_CONTEXT_NAME