[Serve] Add Envoy as an alternative Sky Serve load balancer

sky/clouds/cloud.py

@@ -46,6 +46,7 @@ class CloudImplementationFeatures(enum.Enum):
     STORAGE_MOUNTING = 'storage_mounting'
     HOST_CONTROLLERS = 'host_controllers'  # Can run jobs/serve controllers
     AUTO_TERMINATE = 'auto_terminate'  # Pod/VM can stop or down itself
+    ENVOY = 'envoy_load_balancer'
 
 
 class Region(collections.namedtuple('Region', ['name'])):

sky/clouds/kubernetes.py

@@ -69,6 +69,9 @@ class Kubernetes(clouds.Cloud):
                                                              'tiers are not '
                                                              'supported in '
                                                              'Kubernetes.',
+        clouds.CloudImplementationFeatures.ENVOY: 'Envoy load balancer is not '
+                                                  'supported on Kubernetes '
+                                                  'controllers.'
     }
 
     IMAGE_CPU = 'skypilot:custom-cpu-ubuntu-2004'

sky/serve/constants.py

@@ -100,3 +100,11 @@
 TERMINATE_REPLICA_VERSION_MISMATCH_ERROR = (
     'The version of service is outdated and does not support manually '
     'terminating replicas. Please terminate the service and spin up again.')
+
+# TODO(ejj) ultimately these should be configurable by users.
+ENVOY_THREADS = '1'
+ENVOY_VERSION = '1.32.0'
+
+LB_TYPE_PYTHON = 'python'
+LB_TYPE_ENVOY = 'envoy'
+LB_TYPES = [LB_TYPE_PYTHON, LB_TYPE_ENVOY]

sky/serve/core.py

@@ -7,6 +7,7 @@
 
 import sky
 from sky import backends
+from sky import clouds
 from sky import exceptions
 from sky import sky_logging
 from sky import task as task_lib
@@ -156,6 +157,18 @@ def up(
             controller=controller_utils.Controllers.SKY_SERVE_CONTROLLER,
             task_resources=task.resources)
 
+        # Check that the Envoy load balancer isn't being used on an unsupported
+        # cloud.
+        lb_type = task_config.get('service', {}).get('load_balancer_type', None)
+        if lb_type == serve_constants.LB_TYPE_ENVOY:
+            for resource in controller_resources:
+                if resource.cloud is None:
+                    continue
+
+                requested_features = {clouds.CloudImplementationFeatures.ENVOY}
+                resource.cloud.check_features_are_supported(
+                    resource, requested_features)
+
         vars_to_fill = {
             'remote_task_yaml_path': remote_tmp_task_yaml_path,
             'local_task_yaml_path': service_file.name,