From 557d859732ae7bfd5caa2bf72c95c7190cf227d6 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 02:12:17 +0000 Subject: [PATCH 01/27] Add internal IPs and proxy command --- docs/source/reference/config.rst | 25 ++++++++ sky/backends/backend_utils.py | 13 ++-- sky/backends/cloud_vm_ray_backend.py | 22 +++++-- sky/serve/service.py | 2 + sky/skylet/events.py | 16 ++--- sky/skylet/providers/gcp/config.py | 4 +- sky/skypilot_config.py | 39 +++++------- sky/spot/controller.py | 1 + sky/templates/gcp-ray.yml.j2 | 4 ++ sky/templates/spot-controller.yaml.j2 | 1 + sky/utils/cluster_yaml_utils.py | 31 +++++++++ sky/utils/controller_utils.py | 92 +++++++++++++++------------ sky/utils/schemas.py | 3 + tests/test_config.py | 3 + 14 files changed, 165 insertions(+), 91 deletions(-) create mode 100644 sky/utils/cluster_yaml_utils.py diff --git a/docs/source/reference/config.rst b/docs/source/reference/config.rst index 51f8ef92c10..352531e8881 100644 --- a/docs/source/reference/config.rst +++ b/docs/source/reference/config.rst @@ -121,6 +121,31 @@ Available fields and semantics: # will be added. vpc_name: skypilot-vpc + # Should instances be assigned private IPs only? (optional) + # + # Set to true to use private IPs to communicate between the local client and + # any SkyPilot nodes. This requires the networking stack be properly set up. + # + # This flag is typically set together with 'vpc_name' above and + # 'ssh_proxy_command' below. + # + # Default: false. + use_internal_ips: true + # SSH proxy command (optional). + # + # Please refer to the aws.ssh_proxy_command section above for more details. + ### Format 1 ### + # A string; the same proxy command is used for all regions. + ssh_proxy_command: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no ec2-user@ + ### Format 2 ### + # A dict mapping region names to region-specific proxy commands. + # NOTE: This restricts SkyPilot's search space for this cloud to only use + # the specified regions and not any other regions in this cloud. + ssh_proxy_command: + us-east-1: ssh -W %h:%p -p 1234 -o StrictHostKeyChecking=no myself@my.us-east-1.proxy + us-east-2: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no ec2-user@ + + # Reserved capacity (optional). # # The specific reservation to be considered when provisioning clusters on GCP. diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py index d226828bba8..9e32120c18a 100644 --- a/sky/backends/backend_utils.py +++ b/sky/backends/backend_utils.py @@ -44,6 +44,7 @@ from sky.skylet import constants from sky.skylet import log_lib from sky.usage import usage_lib +from sky.utils import cluster_yaml_utils from sky.utils import command_runner from sky.utils import common_utils from sky.utils import controller_utils @@ -64,7 +65,6 @@ # NOTE: keep in sync with the cluster template 'file_mounts'. SKY_REMOTE_APP_DIR = '~/.sky/sky_app' -SKY_RAY_YAML_REMOTE_PATH = '~/.sky/sky_ray.yml' # Exclude subnet mask from IP address regex. IP_ADDR_REGEX = r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(?!/\d{1,2})\b' SKY_REMOTE_PATH = '~/.sky/wheels' @@ -1016,15 +1016,13 @@ def write_cluster_config( # execution.py::_shared_controller_env_vars). 'user': get_cleaned_username( os.environ.get(constants.USER_ENV_VAR, '')), + 'use_internal_ips': skypilot_config.get_nested( + (str(cloud).lower(), 'use_internal_ips'), False), + 'ssh_proxy_command': ssh_proxy_command, # AWS only: 'aws_vpc_name': skypilot_config.get_nested(('aws', 'vpc_name'), None), - 'use_internal_ips': skypilot_config.get_nested( - ('aws', 'use_internal_ips'), False), - # Not exactly AWS only, but we only test it's supported on AWS - # for now: - 'ssh_proxy_command': ssh_proxy_command, # User-supplied instance tags. 'instance_tags': instance_tags, @@ -1057,7 +1055,8 @@ def write_cluster_config( 'sky_remote_path': SKY_REMOTE_PATH, 'sky_local_path': str(local_wheel_path), # Add yaml file path to the template variables. - 'sky_ray_yaml_remote_path': SKY_RAY_YAML_REMOTE_PATH, + 'sky_ray_yaml_remote_path': + cluster_yaml_utils.SKY_CLUSTER_YAML_REMOTE_PATH, 'sky_ray_yaml_local_path': tmp_yaml_path if not isinstance(cloud, clouds.Local) else yaml_path, diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 7fc596d8457..10f748c6565 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -201,6 +201,7 @@ def __init__(self): # For n nodes gang scheduling. self._has_gang_scheduling = False self._num_nodes = 0 + self._provider_name = None self._has_register_run_fn = False @@ -293,6 +294,7 @@ def add_prologue(self, job_id: int, is_local: bool = False) -> None: def add_gang_scheduling_placement_group_and_setup( self, + cloud: clouds.Cloud, num_nodes: int, resources_dict: Dict[str, float], stable_cluster_internal_ips: List[str], @@ -311,6 +313,11 @@ def add_gang_scheduling_placement_group_and_setup( 'add_gang_scheduling_placement_group_and_setup().') self._has_gang_scheduling = True self._num_nodes = num_nodes + self._provider_name = str(cloud).lower() + + if envs is None: + envs = {} + envs['SKYPILOT_PROVIDER_NAME'] = self._provider_name bundles = [copy.copy(resources_dict) for _ in range(num_nodes)] # Set CPU to avoid ray hanging the resources allocation @@ -503,11 +510,12 @@ def add_ray_task(self, f'placement_group_bundle_index={gang_scheduling_id})') sky_env_vars_dict_str = [ - textwrap.dedent("""\ - sky_env_vars_dict = {} + textwrap.dedent(f"""\ + sky_env_vars_dict = {{}} sky_env_vars_dict['SKYPILOT_NODE_IPS'] = job_ip_list_str # Environment starting with `SKY_` is deprecated. sky_env_vars_dict['SKY_NODE_IPS'] = job_ip_list_str + sky_env_vars_dict['SKYPILOT_PROVIDER_NAME'] = {self._provider_name} """) ] @@ -2505,10 +2513,10 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: f'cached ({self.cached_external_ips}), new ({cluster_external_ips})' ) - is_cluster_aws = (self.launched_resources is not None and - isinstance(self.launched_resources.cloud, clouds.AWS)) - if is_cluster_aws and skypilot_config.get_nested( - keys=('aws', 'use_internal_ips'), default_value=False): + if (self.launched_resources is not None and + skypilot_config.get_nested(keys=(str( + self.launched_resources.cloud).lower(), 'use_internal_ips'), + default_value=False)): # Optimization: if we know use_internal_ips is True (currently # only exposed for AWS), then our AWS NodeProvider is # guaranteed to pick subnets that will not assign public IPs, @@ -4825,6 +4833,7 @@ def _execute_task_one_node(self, handle: CloudVmRayResourceHandle, is_local = isinstance(handle.launched_resources.cloud, clouds.Local) codegen.add_prologue(job_id, is_local=is_local) codegen.add_gang_scheduling_placement_group_and_setup( + handle.launched_resources.cloud, 1, resources_dict, stable_cluster_internal_ips=internal_ips, @@ -4892,6 +4901,7 @@ def _execute_task_n_nodes(self, handle: CloudVmRayResourceHandle, is_local = isinstance(handle.launched_resources.cloud, clouds.Local) codegen.add_prologue(job_id, is_local=is_local) codegen.add_gang_scheduling_placement_group_and_setup( + handle.launched_resources.cloud, num_actual_nodes, resources_dict, stable_cluster_internal_ips=internal_ips, diff --git a/sky/serve/service.py b/sky/serve/service.py index 1b2aaf253c0..5bacb304e59 100644 --- a/sky/serve/service.py +++ b/sky/serve/service.py @@ -28,6 +28,7 @@ from sky.serve import serve_state from sky.serve import serve_utils from sky.utils import common_utils +from sky.utils import controller_utils from sky.utils import subprocess_utils from sky.utils import ux_utils @@ -252,4 +253,5 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int): # We start process with 'spawn', because 'fork' could result in weird # behaviors; 'spawn' is also cross-platform. multiprocessing.set_start_method('spawn', force=True) + controller_utils.setup_proxy_command_on_controller() _start(args.service_name, args.task_yaml, args.job_id) diff --git a/sky/skylet/events.py b/sky/skylet/events.py index ade43104048..79aaba992c2 100644 --- a/sky/skylet/events.py +++ b/sky/skylet/events.py @@ -11,12 +11,12 @@ import yaml from sky import sky_logging -from sky.backends import backend_utils from sky.backends import cloud_vm_ray_backend from sky.serve import serve_utils from sky.skylet import autostop_lib from sky.skylet import job_lib from sky.spot import spot_utils +from sky.utils import cluster_yaml_utils from sky.utils import common_utils from sky.utils import ux_utils @@ -104,8 +104,8 @@ class AutostopEvent(SkyletEvent): def __init__(self): super().__init__() autostop_lib.set_last_active_time_to_now() - self._ray_yaml_path = os.path.abspath( - os.path.expanduser(backend_utils.SKY_RAY_YAML_REMOTE_PATH)) + self._ray_yaml_path = cluster_yaml_utils.get_cluster_yaml_absolute_path( + ) def _run(self): autostop_config = autostop_lib.get_autostop_config() @@ -139,16 +139,8 @@ def _stop_cluster(self, autostop_config): cloud_vm_ray_backend.CloudVmRayBackend.NAME): autostop_lib.set_autostopping_started() + provider_name = cluster_yaml_utils.get_provider_name() config = common_utils.read_yaml(self._ray_yaml_path) - - provider_module = config['provider']['module'] - # Examples: - # 'sky.skylet.providers.aws.AWSNodeProviderV2' -> 'aws' - # 'sky.provision.aws' -> 'aws' - provider_search = re.search(r'(?:providers|provision)\.(\w+)\.?', - provider_module) - assert provider_search is not None, config - provider_name = provider_search.group(1).lower() if provider_name in ('aws', 'gcp'): logger.info('Using new provisioner to stop the cluster.') self._stop_cluster_with_new_provisioner(autostop_config, config, diff --git a/sky/skylet/providers/gcp/config.py b/sky/skylet/providers/gcp/config.py index 68c535d73fd..c43eceb56ca 100644 --- a/sky/skylet/providers/gcp/config.py +++ b/sky/skylet/providers/gcp/config.py @@ -910,6 +910,8 @@ def _configure_subnet(config, compute): ], } ] + if config["provider"].get("use_internal_ips", False): + default_interfaces[0].pop("accessConfigs") for node_config in node_configs: # The not applicable key will be removed during node creation @@ -920,7 +922,7 @@ def _configure_subnet(config, compute): # TPU if "networkConfig" not in node_config: node_config["networkConfig"] = copy.deepcopy(default_interfaces)[0] - node_config["networkConfig"].pop("accessConfigs") + node_config["networkConfig"].pop("accessConfigs", None) return config diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py index 8c9c6fd5e5a..53155f62cea 100644 --- a/sky/skypilot_config.py +++ b/sky/skypilot_config.py @@ -49,7 +49,6 @@ import yaml from sky import sky_logging -from sky.clouds import cloud_registry from sky.utils import common_utils from sky.utils import schemas @@ -74,6 +73,7 @@ # The loaded config. _dict = None +_loaded_config_path = None def get_nested(keys: Iterable[str], default_value: Any) -> Any: @@ -119,6 +119,19 @@ def set_nested(keys: Iterable[str], value: Any) -> Dict[str, Any]: return to_return +def overwrite_config_file(config: dict) -> None: + """Overwrites the config file with the current config.""" + global _dict, _loaded_config_path + if _loaded_config_path is None: + raise RuntimeError('No config file loaded.') + common_utils.validate_schema(config, + schemas.get_config_schema(), + f'Invalid config YAML: {config!r}', + skip_none=False) + common_utils.write_yaml(_loaded_config_path, config) + _dict = config + + def to_dict() -> Dict[str, Any]: """Returns a deep-copied version of the current config.""" global _dict @@ -127,27 +140,8 @@ def to_dict() -> Dict[str, Any]: return {} -def _syntax_check_for_ssh_proxy_command(cloud: str) -> None: - ssh_proxy_command_config = get_nested((cloud.lower(), 'ssh_proxy_command'), - None) - if ssh_proxy_command_config is None or isinstance(ssh_proxy_command_config, - str): - return - - if isinstance(ssh_proxy_command_config, dict): - for region, cmd in ssh_proxy_command_config.items(): - if cmd and not isinstance(cmd, str): - raise ValueError( - f'Invalid ssh_proxy_command config for region {region!r} ' - f'(expected a str): {cmd!r}') - return - raise ValueError( - 'Invalid ssh_proxy_command config (expected a str or a dict with ' - f'region names as keys): {ssh_proxy_command_config!r}') - - def _try_load_config() -> None: - global _dict + global _dict, _loaded_config_path config_path_via_env_var = os.environ.get(ENV_VAR_SKYPILOT_CONFIG) if config_path_via_env_var is not None: config_path = config_path_via_env_var @@ -156,6 +150,7 @@ def _try_load_config() -> None: config_path = os.path.expanduser(config_path) if os.path.exists(config_path): logger.debug(f'Using config path: {config_path}') + _loaded_config_path = config_path try: _dict = common_utils.read_yaml(config_path) logger.debug(f'Config loaded:\n{pprint.pformat(_dict)}') @@ -168,8 +163,6 @@ def _try_load_config() -> None: f'Invalid config YAML ({config_path}): ', skip_none=False) - for cloud in cloud_registry.CLOUD_REGISTRY: - _syntax_check_for_ssh_proxy_command(cloud) logger.debug('Config syntax check passed.') diff --git a/sky/spot/controller.py b/sky/spot/controller.py index 8c1f30701c6..3d58424e09f 100644 --- a/sky/spot/controller.py +++ b/sky/spot/controller.py @@ -523,4 +523,5 @@ def start(job_id, dag_yaml, retry_until_up): # We start process with 'spawn', because 'fork' could result in weird # behaviors; 'spawn' is also cross-platform. multiprocessing.set_start_method('spawn', force=True) + controller_utils.setup_proxy_command_on_controller() start(args.job_id, args.dag_yaml, args.retry_until_up) diff --git a/sky/templates/gcp-ray.yml.j2 b/sky/templates/gcp-ray.yml.j2 index ca73199e317..80ae9004409 100644 --- a/sky/templates/gcp-ray.yml.j2 +++ b/sky/templates/gcp-ray.yml.j2 @@ -46,6 +46,7 @@ provider: password: {{docker_login_config.password}} server: {{docker_login_config.server}} {%- endif %} + use_internal_ips: {{use_internal_ips}} {%- if tpu_vm %} _has_tpus: True {%- endif %} @@ -59,6 +60,9 @@ provider: auth: ssh_user: gcpuser ssh_private_key: {{ssh_private_key}} +{% if ssh_proxy_command is not none %} + ssh_proxy_command: {{ssh_proxy_command}} +{% endif %} available_node_types: ray_head_default: diff --git a/sky/templates/spot-controller.yaml.j2 b/sky/templates/spot-controller.yaml.j2 index 746322fd956..184ee8980df 100644 --- a/sky/templates/spot-controller.yaml.j2 +++ b/sky/templates/spot-controller.yaml.j2 @@ -33,6 +33,7 @@ setup: | ((ps aux | grep -v nohup | grep -v grep | grep -q -- "python3 -m sky.spot.dashboard.dashboard") || (nohup python3 -m sky.spot.dashboard.dashboard >> ~/.sky/spot-dashboard.log 2>&1 &)); run: | + # Start the controller for the current spot job. python -u -m sky.spot.controller {{remote_user_yaml_path}} \ --job-id $SKYPILOT_INTERNAL_JOB_ID {% if retry_until_up %}--retry-until-up{% endif %} diff --git a/sky/utils/cluster_yaml_utils.py b/sky/utils/cluster_yaml_utils.py new file mode 100644 index 00000000000..a1a7bdbd259 --- /dev/null +++ b/sky/utils/cluster_yaml_utils.py @@ -0,0 +1,31 @@ +"""Utility functions for cluster yaml file.""" + +import os +import re + +from sky.utils import common_utils + +SKY_CLUSTER_YAML_REMOTE_PATH = '~/.sky/sky_ray.yml' + + +def get_cluster_yaml_absolute_path() -> str: + """Return the absolute path of the cluster yaml file. + + This function should be called on the remote machine. + """ + return os.path.abspath(os.path.expanduser(SKY_CLUSTER_YAML_REMOTE_PATH)) + + +def get_provider_name() -> str: + """Return the name of the provider.""" + config = common_utils.read_yaml(get_cluster_yaml_absolute_path()) + + provider_module = config['provider']['module'] + # Examples: + # 'sky.skylet.providers.aws.AWSNodeProviderV2' -> 'aws' + # 'sky.provision.aws' -> 'aws' + provider_search = re.search(r'(?:providers|provision)\.(\w+)\.?', + provider_module) + assert provider_search is not None, config + provider_name = provider_search.group(1).lower() + return provider_name diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index 4cc74ede590..1f4745efd5a 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -18,6 +18,7 @@ from sky.serve import serve_utils from sky.skylet import constants from sky.spot import spot_utils +from sky.utils import cluster_yaml_utils from sky.utils import common_utils from sky.utils import env_options from sky.utils import ux_utils @@ -219,49 +220,7 @@ def skypilot_config_setup( controller_resources_config_copied: Dict[str, Any] = copy.copy( controller_resources_config) if skypilot_config.loaded(): - # Look up the contents of the already loaded configs via the - # 'skypilot_config' module. Don't simply read the on-disk file as - # it may have changed since this process started. - # - # Set any proxy command to None, because the controller would've - # been launched behind the proxy, and in general any nodes we - # launch may not have or need the proxy setup. (If the controller - # needs to launch mew clusters in another region/VPC, the user - # should properly set up VPC peering, which will allow the - # cross-region/VPC communication. The proxy command is orthogonal - # to this scenario.) - # - # This file will be uploaded to the controller node and will be - # used throughout the spot job's / service's recovery attempts - # (i.e., if it relaunches due to preemption, we make sure the - # same config is used). - # - # NOTE: suppose that we have a controller in old VPC, then user - # changes 'vpc_name' in the config and does a 'spot launch' / - # 'serve up'. In general, the old controller may not successfully - # launch the job in the new VPC. This happens if the two VPCs don’t - # have peering set up. Like other places in the code, we assume - # properly setting up networking is user's responsibilities. - # TODO(zongheng): consider adding a basic check that checks - # controller VPC (or name) == the spot job's / service's VPC - # (or name). It may not be a sufficient check (as it's always - # possible that peering is not set up), but it may catch some - # obvious errors. - # TODO(zhwu): hacky. We should only set the proxy command of the - # cloud where the controller is launched (currently, only aws user - # uses proxy_command). - proxy_command_key = ('aws', 'ssh_proxy_command') - ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) config_dict = skypilot_config.to_dict() - if isinstance(ssh_proxy_command, str): - config_dict = skypilot_config.set_nested(proxy_command_key, None) - elif isinstance(ssh_proxy_command, dict): - # Instead of removing the key, we set the value to empty string - # so that the controller will only try the regions specified by - # the keys. - ssh_proxy_command = {k: None for k in ssh_proxy_command} - config_dict = skypilot_config.set_nested(proxy_command_key, - ssh_proxy_command) with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmpfile: common_utils.dump_yaml(tmpfile.name, config_dict) @@ -288,6 +247,55 @@ def skypilot_config_setup( return vars_to_fill, controller_resources_config_copied +def setup_proxy_command_on_controller(): + # Look up the contents of the already loaded configs via the + # 'skypilot_config' module. Don't simply read the on-disk file as + # it may have changed since this process started. + # + # Set any proxy command to None, because the controller would've + # been launched behind the proxy, and in general any nodes we + # launch may not have or need the proxy setup. (If the controller + # needs to launch mew clusters in another region/VPC, the user + # should properly set up VPC peering, which will allow the + # cross-region/VPC communication. The proxy command is orthogonal + # to this scenario.) + # + # This file will be uploaded to the controller node and will be + # used throughout the spot job's / service's recovery attempts + # (i.e., if it relaunches due to preemption, we make sure the + # same config is used). + # + # NOTE: suppose that we have a controller in old VPC, then user + # changes 'vpc_name' in the config and does a 'spot launch' / + # 'serve up'. In general, the old controller may not successfully + # launch the job in the new VPC. This happens if the two VPCs don’t + # have peering set up. Like other places in the code, we assume + # properly setting up networking is user's responsibilities. + # TODO(zongheng): consider adding a basic check that checks + # controller VPC (or name) == the spot job's / service's VPC + # (or name). It may not be a sufficient check (as it's always + # possible that peering is not set up), but it may catch some + # obvious errors. + provider_name = cluster_yaml_utils.get_provider_name() + if skypilot_config.loaded(): + # We only set the proxy command of the cloud where the controller is + # launched. + proxy_command_key = (provider_name, 'ssh_proxy_command') + ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) + config_dict = skypilot_config.to_dict() + if isinstance(ssh_proxy_command, str): + config_dict = skypilot_config.set_nested(proxy_command_key, None) + elif isinstance(ssh_proxy_command, dict): + # Instead of removing the key, we set the value to empty string + # so that the controller will only try the regions specified by + # the keys. + ssh_proxy_command = {k: None for k in ssh_proxy_command} + config_dict = skypilot_config.set_nested(proxy_command_key, + ssh_proxy_command) + + skypilot_config.overwrite_config_file(config_dict) + + def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task', path: str): """Translates local->VM mounts into Storage->VM, then syncs up any Storage. diff --git a/sky/utils/schemas.py b/sky/utils/schemas.py index 2fa7614e614..f17276f341c 100644 --- a/sky/utils/schemas.py +++ b/sky/utils/schemas.py @@ -557,6 +557,9 @@ def get_config_schema(): 'type': 'null', }], }, + 'use_internal_ips': { + 'type': 'boolean', + }, } }, 'kubernetes': { diff --git a/tests/test_config.py b/tests/test_config.py index dfe64f77f06..afa85cedf29 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -40,6 +40,7 @@ def _create_config_file(config_file_path: pathlib.Path) -> None: gcp: vpc_name: {VPC_NAME} + use_internal_ips: true kubernetes: networking: {NODEPORT_MODE_NAME} @@ -195,6 +196,7 @@ def test_config_get_set_nested(monkeypatch, tmp_path) -> None: assert skypilot_config.get_nested( ('aws', 'ssh_proxy_command'), None) is None assert skypilot_config.get_nested(('gcp', 'vpc_name'), None) == VPC_NAME + assert skypilot_config.get_nested(('gcp', 'use_internal_ips'), None) # Check config with only partial keys still works new_config3 = copy.copy(new_config2) @@ -230,3 +232,4 @@ def test_config_with_env(monkeypatch, tmp_path) -> None: assert skypilot_config.get_nested(('aws', 'ssh_proxy_command'), None) == PROXY_COMMAND assert skypilot_config.get_nested(('gcp', 'vpc_name'), None) == VPC_NAME + assert skypilot_config.get_nested(('gcp', 'use_internal_ips'), None) From 82f8e353d4b5fcc2b91b6d23dcba5d28ce850ab6 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 05:59:35 +0000 Subject: [PATCH 02/27] Fix schemas --- sky/utils/schemas.py | 74 ++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 40 deletions(-) diff --git a/sky/utils/schemas.py b/sky/utils/schemas.py index f17276f341c..28d3d47142b 100644 --- a/sky/utils/schemas.py +++ b/sky/utils/schemas.py @@ -457,6 +457,38 @@ def get_cluster_schema(): } } +_NETWORK_CONFIG_SCHEMA = { + 'vpc_name': { + 'oneOf': [{ + 'type': 'string', + }, { + 'type': 'null', + }], + }, + 'use_internal_ips': { + 'type': 'boolean', + }, + 'ssh_proxy_command': { + 'oneOf': [{ + 'type': 'string', + }, { + 'type': 'null', + }, { + 'type': 'object', + 'required': [], + 'additionalProperties': { + 'anyOf': [ + { + 'type': 'string' + }, + { + 'type': 'null' + }, + ] + } + }] + }, +} def get_config_schema(): # pylint: disable=import-outside-toplevel @@ -505,36 +537,7 @@ def get_config_schema(): 'type': 'string', }, }, - 'vpc_name': { - 'oneOf': [{ - 'type': 'string', - }, { - 'type': 'null', - }], - }, - 'use_internal_ips': { - 'type': 'boolean', - }, - 'ssh_proxy_command': { - 'oneOf': [{ - 'type': 'string', - }, { - 'type': 'null', - }, { - 'type': 'object', - 'required': [], - 'additionalProperties': { - 'anyOf': [ - { - 'type': 'string' - }, - { - 'type': 'null' - }, - ] - } - }] - }, + **_NETWORK_CONFIG_SCHEMA, } }, 'gcp': { @@ -550,16 +553,7 @@ def get_config_schema(): 'minItems': 1, 'maxItems': 1, }, - 'vpc_name': { - 'oneOf': [{ - 'type': 'string', - }, { - 'type': 'null', - }], - }, - 'use_internal_ips': { - 'type': 'boolean', - }, + **_NETWORK_CONFIG_SCHEMA, } }, 'kubernetes': { From 4851a9f9644efe0bf858b0cf8192d5602e4457c9 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 06:50:06 +0000 Subject: [PATCH 03/27] Add config --- .../cloud-setup/cloud-permissions/gcp.rst | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/source/cloud-setup/cloud-permissions/gcp.rst b/docs/source/cloud-setup/cloud-permissions/gcp.rst index 379ba7a672a..cf51793ef8a 100644 --- a/docs/source/cloud-setup/cloud-permissions/gcp.rst +++ b/docs/source/cloud-setup/cloud-permissions/gcp.rst @@ -267,3 +267,52 @@ See details in :ref:`config-yaml`. Example use cases include using a private VP VPC with fine-grained constraints, typically created via Terraform or manually. The custom VPC should contain the :ref:`required firewall rules `. + + +.. _gcp-use-internal-ips: + + +Using Internal IPs +----------------------- +For security reason, users may only want to use internal IPs for SkyPilot instances. +To do so, you can use SkyPilot's global config file ``~/.sky/config.yaml`` to specify the ``gcp.use_internal_ips`` and ``gcp.ssh_proxy_command`` field (to see the detailed syntax, see :ref:`config-yaml`): + +.. code-block:: yaml + + gcp: + use_internal_ips: true + # VPC with NAT setup, see below + vpc_name: my-vpc-name + ssh_proxy_command: ssh -W %h:%p -o StrictHostKeyChecking=no myself@my.proxy + +The ``gcp.ssh_proxy_command`` field is optional. If SkyPilot is run on a machine that can directly access the internal IPs of the instances, it can be omitted. Otherwise, it should be set to a command that can be used to proxy SSH connections to the internal IPs of the instances. + + +Cloud NAT Setup +~~~~~~~~~~~~~~~~ + +Instances created with internal IPs only on GCP cannot access public internet by default. To make sure SkyPilot can install the dependencies correctly on the instances, +cloud NAT needs to be setup for the VPC (see `GCP's documentation `__ for details). + + +Cloud NAT is a regional resource, so it will need to be created in each region that SkyPilot will be used in. To limit SkyPilot to use some specific regions only, you can specify the ``gcp.ssh_proxy_command`` to be a dict mapping from region to the SSH proxy command for that region (see :ref:`config-yaml` for details): + +.. code-block:: yaml + + gcp: + use_internal_ips: true + vpc_name: my-vpc-name + ssh_proxy_command: + us-west1: ssh -W %h:%p -o StrictHostKeyChecking=no myself@my.us-west1.proxy + us-east1: ssh -W %h:%p -o StrictHostKeyChecking=no myself@my.us-west2.proxy + +If proxy is not needed, but the regions need to be limited, you can set the ``gcp.ssh_proxy_command`` to be a dict mapping from region to ``null``: + +.. code-block:: yaml + + gcp: + use_internal_ips: true + vpc_name: my-vpc-name + ssh_proxy_command: + us-west1: null + us-east1: null From 5a18820b4537306d724c5f701b02c30ba1a6a158 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 07:54:26 +0000 Subject: [PATCH 04/27] vpc_name refactor --- sky/backends/backend_utils.py | 8 +++----- sky/templates/aws-ray.yml.j2 | 4 ++-- sky/templates/gcp-ray.yml.j2 | 4 ++-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py index 9e32120c18a..d6a3b67635f 100644 --- a/sky/backends/backend_utils.py +++ b/sky/backends/backend_utils.py @@ -1016,13 +1016,13 @@ def write_cluster_config( # execution.py::_shared_controller_env_vars). 'user': get_cleaned_username( os.environ.get(constants.USER_ENV_VAR, '')), + + # Private IPs 'use_internal_ips': skypilot_config.get_nested( (str(cloud).lower(), 'use_internal_ips'), False), 'ssh_proxy_command': ssh_proxy_command, + 'vpc_name': skypilot_config.get_nested((str(cloud).lower(), 'vpc_name'), None), - # AWS only: - 'aws_vpc_name': skypilot_config.get_nested(('aws', 'vpc_name'), - None), # User-supplied instance tags. 'instance_tags': instance_tags, @@ -1031,8 +1031,6 @@ def write_cluster_config( 'resource_group': f'{cluster_name}-{region_name}', # GCP only: - 'gcp_vpc_name': skypilot_config.get_nested(('gcp', 'vpc_name'), - None), 'gcp_project_id': gcp_project_id, 'specific_reservations': filtered_specific_reservations, 'num_specific_reserved_workers': num_specific_reserved_workers, diff --git a/sky/templates/aws-ray.yml.j2 b/sky/templates/aws-ray.yml.j2 index 1494a2c7060..efb95799366 100644 --- a/sky/templates/aws-ray.yml.j2 +++ b/sky/templates/aws-ray.yml.j2 @@ -36,9 +36,9 @@ provider: security_group: # AWS config file must include security group name GroupName: {{security_group}} -{% if aws_vpc_name is not none %} +{% if vpc_name is not none %} # NOTE: This is a new field added by SkyPilot to force use a specific VPC. - vpc_name: {{aws_vpc_name}} + vpc_name: {{vpc_name}} {% endif %} {%- if docker_login_config is not none %} # We put docker login config in provider section because ray's schema disabled diff --git a/sky/templates/gcp-ray.yml.j2 b/sky/templates/gcp-ray.yml.j2 index 80ae9004409..ab62e4f5413 100644 --- a/sky/templates/gcp-ray.yml.j2 +++ b/sky/templates/gcp-ray.yml.j2 @@ -28,9 +28,9 @@ provider: cache_stopped_nodes: True # The GCP project ID. project_id: {{gcp_project_id}} -{% if gcp_vpc_name is not none %} +{% if vpc_name is not none %} # NOTE: This is a new field added by SkyPilot to force use a specific VPC. - vpc_name: {{gcp_vpc_name}} + vpc_name: {{vpc_name}} {% endif %} # The firewall rule name for customized firewall rules. Only enabled # if we have ports requirement. From 217ed846a1b265bc20d5583c2b93133e4b6fc2c0 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 07:56:06 +0000 Subject: [PATCH 05/27] format --- sky/backends/backend_utils.py | 3 ++- sky/utils/schemas.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py index d6a3b67635f..e2c196e46cf 100644 --- a/sky/backends/backend_utils.py +++ b/sky/backends/backend_utils.py @@ -1021,7 +1021,8 @@ def write_cluster_config( 'use_internal_ips': skypilot_config.get_nested( (str(cloud).lower(), 'use_internal_ips'), False), 'ssh_proxy_command': ssh_proxy_command, - 'vpc_name': skypilot_config.get_nested((str(cloud).lower(), 'vpc_name'), None), + 'vpc_name': skypilot_config.get_nested( + (str(cloud).lower(), 'vpc_name'), None), # User-supplied instance tags. 'instance_tags': instance_tags, diff --git a/sky/utils/schemas.py b/sky/utils/schemas.py index 28d3d47142b..c1646b3fa77 100644 --- a/sky/utils/schemas.py +++ b/sky/utils/schemas.py @@ -457,6 +457,7 @@ def get_cluster_schema(): } } + _NETWORK_CONFIG_SCHEMA = { 'vpc_name': { 'oneOf': [{ @@ -490,6 +491,7 @@ def get_cluster_schema(): }, } + def get_config_schema(): # pylint: disable=import-outside-toplevel from sky.utils import kubernetes_enums From e07a67cb0ca6c6e1150490cd9e7c3c60aaf2e363 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 08:17:07 +0000 Subject: [PATCH 06/27] remove remnant --- sky/backends/cloud_vm_ray_backend.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 10f748c6565..b1d87755fc5 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -294,7 +294,6 @@ def add_prologue(self, job_id: int, is_local: bool = False) -> None: def add_gang_scheduling_placement_group_and_setup( self, - cloud: clouds.Cloud, num_nodes: int, resources_dict: Dict[str, float], stable_cluster_internal_ips: List[str], @@ -313,11 +312,9 @@ def add_gang_scheduling_placement_group_and_setup( 'add_gang_scheduling_placement_group_and_setup().') self._has_gang_scheduling = True self._num_nodes = num_nodes - self._provider_name = str(cloud).lower() if envs is None: envs = {} - envs['SKYPILOT_PROVIDER_NAME'] = self._provider_name bundles = [copy.copy(resources_dict) for _ in range(num_nodes)] # Set CPU to avoid ray hanging the resources allocation @@ -4833,7 +4830,6 @@ def _execute_task_one_node(self, handle: CloudVmRayResourceHandle, is_local = isinstance(handle.launched_resources.cloud, clouds.Local) codegen.add_prologue(job_id, is_local=is_local) codegen.add_gang_scheduling_placement_group_and_setup( - handle.launched_resources.cloud, 1, resources_dict, stable_cluster_internal_ips=internal_ips, @@ -4901,7 +4897,6 @@ def _execute_task_n_nodes(self, handle: CloudVmRayResourceHandle, is_local = isinstance(handle.launched_resources.cloud, clouds.Local) codegen.add_prologue(job_id, is_local=is_local) codegen.add_gang_scheduling_placement_group_and_setup( - handle.launched_resources.cloud, num_actual_nodes, resources_dict, stable_cluster_internal_ips=internal_ips, From 25b7a64c2ed85c97073500ae8cebbe672d253b75 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Fri, 24 Nov 2023 08:31:03 +0000 Subject: [PATCH 07/27] fix API --- sky/skypilot_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py index 53155f62cea..d105a2f246a 100644 --- a/sky/skypilot_config.py +++ b/sky/skypilot_config.py @@ -128,7 +128,7 @@ def overwrite_config_file(config: dict) -> None: schemas.get_config_schema(), f'Invalid config YAML: {config!r}', skip_none=False) - common_utils.write_yaml(_loaded_config_path, config) + common_utils.dump_yaml(_loaded_config_path, config) _dict = config From 3d7d18295236a871761bd4fdb6703ceca84a299b Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sat, 25 Nov 2023 07:34:01 +0000 Subject: [PATCH 08/27] filter by proxy command --- tests/test_smoke.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 17708ccf7ef..016ac23f997 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -194,12 +194,19 @@ def get_aws_region_for_quota_failover() -> Optional[str]: use_spot=True, region=None, zone=None) + original_resources = sky.Resources(cloud=sky.AWS(), + instance_type='p3.16xlarge', + use_spot=True) + + # Filter the regions with proxy command in ~/.sky/config.yaml. + filtered_regions = original_resources.get_valid_regions_for_launchable() + candidate_regions = [ + region for region in candidate_regions + if region.name in filtered_regions + ] for region in candidate_regions: - resources = sky.Resources(cloud=sky.AWS(), - instance_type='p3.16xlarge', - region=region.name, - use_spot=True) + resources = original_resources.copy(region=region.name) if not AWS.check_quota_available(resources): return region.name @@ -214,12 +221,21 @@ def get_gcp_region_for_quota_failover() -> Optional[str]: region=None, zone=None) + original_resources = sky.Resources(cloud=sky.GCP(), + region=region.name, + accelerators={'A100-80GB': 1}, + use_spot=True) + + # Filter the regions with proxy command in ~/.sky/config.yaml. + filtered_regions = original_resources.get_valid_regions_for_launchable() + candidate_regions = [ + region for region in candidate_regions + if region.name in filtered_regions + ] + for region in candidate_regions: if not GCP.check_quota_available( - sky.Resources(cloud=sky.GCP(), - region=region.name, - accelerators={'A100-80GB': 1}, - use_spot=True)): + original_resources.copy(region=region.name)): return region.name return None From c94ba588c9d627f843df7816a75b2bb986280498 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sat, 25 Nov 2023 09:32:24 +0000 Subject: [PATCH 09/27] fix region field --- tests/test_smoke.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 016ac23f997..bb275045eb4 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -222,7 +222,6 @@ def get_gcp_region_for_quota_failover() -> Optional[str]: zone=None) original_resources = sky.Resources(cloud=sky.GCP(), - region=region.name, accelerators={'A100-80GB': 1}, use_spot=True) From ad8ffad3546d7f7348b29ba3ba7304a778dfe036 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sat, 25 Nov 2023 13:41:08 +0000 Subject: [PATCH 10/27] Fix tpu pod --- sky/skylet/providers/gcp/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sky/skylet/providers/gcp/config.py b/sky/skylet/providers/gcp/config.py index c43eceb56ca..955b98fd880 100644 --- a/sky/skylet/providers/gcp/config.py +++ b/sky/skylet/providers/gcp/config.py @@ -922,7 +922,10 @@ def _configure_subnet(config, compute): # TPU if "networkConfig" not in node_config: node_config["networkConfig"] = copy.deepcopy(default_interfaces)[0] + # TPU doesn't have accessConfigs node_config["networkConfig"].pop("accessConfigs", None) + if config["provider"].get("use_internal_ips", False): + node_config["networkConfig"]["enableExternalIps"] = False return config From 23e0493e8e5ad8c5c37edf656fca934c11e222e2 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sat, 25 Nov 2023 14:07:32 +0000 Subject: [PATCH 11/27] Fix internal IP fetching for TPU VM --- sky/backends/cloud_vm_ray_backend.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index b1d87755fc5..becd38b4fa6 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -2486,6 +2486,9 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: return (ips is not None and len(ips) == self.num_node_ips and all(ip is not None for ip in ips)) + use_internal_ips = skypilot_config.get_nested(keys=(str( + self.launched_resources.cloud).lower(), 'use_internal_ips'), + default_value=False) if is_provided_ips_valid(external_ips): logger.debug(f'Using provided external IPs: {external_ips}') cluster_external_ips = typing.cast(List[str], external_ips) @@ -2496,7 +2499,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: handle=self, head_ip_max_attempts=max_attempts, worker_ip_max_attempts=max_attempts, - get_internal_ips=False) + get_internal_ips=use_internal_ips) if self.cached_external_ips == cluster_external_ips: logger.debug('Skipping the fetching of internal IPs as the cached ' @@ -2510,10 +2513,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: f'cached ({self.cached_external_ips}), new ({cluster_external_ips})' ) - if (self.launched_resources is not None and - skypilot_config.get_nested(keys=(str( - self.launched_resources.cloud).lower(), 'use_internal_ips'), - default_value=False)): + if (self.launched_resources is not None and use_internal_ips): # Optimization: if we know use_internal_ips is True (currently # only exposed for AWS), then our AWS NodeProvider is # guaranteed to pick subnets that will not assign public IPs, From 3a0fe9116199d985efeb7d0c98a7dbf14f3edb1b Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Sun, 26 Nov 2023 00:29:41 +0000 Subject: [PATCH 12/27] remove k80 from the resources unordered tests --- tests/test_yamls/test_multiple_accelerators_unordered.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_yamls/test_multiple_accelerators_unordered.yaml b/tests/test_yamls/test_multiple_accelerators_unordered.yaml index db0fc9c5f7c..3bb26c197ce 100644 --- a/tests/test_yamls/test_multiple_accelerators_unordered.yaml +++ b/tests/test_yamls/test_multiple_accelerators_unordered.yaml @@ -1,7 +1,7 @@ name: multi-accelerators-unordered resources: - accelerators: {'A100-40GB:1', 'T4:1', 'V100:1', 'K80:1'} + accelerators: {'A100-40GB:1', 'T4:1', 'V100:1'} run: | nvidia-smi From 1885529c4b64bf661b717817e10693f004ccb901 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Tue, 28 Nov 2023 09:04:15 +0000 Subject: [PATCH 13/27] Address comments --- sky/backends/backend_utils.py | 6 +-- sky/backends/cloud_vm_ray_backend.py | 25 ++++++----- sky/skylet/events.py | 11 ++--- sky/skylet/providers/gcp/config.py | 1 + sky/skypilot_config.py | 8 +++- sky/utils/controller_utils.py | 45 +++++++++++-------- ..._utils.py => remote_cluster_yaml_utils.py} | 18 +++++--- 7 files changed, 67 insertions(+), 47 deletions(-) rename sky/utils/{cluster_yaml_utils.py => remote_cluster_yaml_utils.py} (65%) diff --git a/sky/backends/backend_utils.py b/sky/backends/backend_utils.py index e2c196e46cf..192f8d44531 100644 --- a/sky/backends/backend_utils.py +++ b/sky/backends/backend_utils.py @@ -44,11 +44,11 @@ from sky.skylet import constants from sky.skylet import log_lib from sky.usage import usage_lib -from sky.utils import cluster_yaml_utils from sky.utils import command_runner from sky.utils import common_utils from sky.utils import controller_utils from sky.utils import env_options +from sky.utils import remote_cluster_yaml_utils from sky.utils import rich_utils from sky.utils import subprocess_utils from sky.utils import timeline @@ -1017,7 +1017,7 @@ def write_cluster_config( 'user': get_cleaned_username( os.environ.get(constants.USER_ENV_VAR, '')), - # Private IPs + # Networking configs 'use_internal_ips': skypilot_config.get_nested( (str(cloud).lower(), 'use_internal_ips'), False), 'ssh_proxy_command': ssh_proxy_command, @@ -1055,7 +1055,7 @@ def write_cluster_config( 'sky_local_path': str(local_wheel_path), # Add yaml file path to the template variables. 'sky_ray_yaml_remote_path': - cluster_yaml_utils.SKY_CLUSTER_YAML_REMOTE_PATH, + remote_cluster_yaml_utils.SKY_CLUSTER_YAML_REMOTE_PATH, 'sky_ray_yaml_local_path': tmp_yaml_path if not isinstance(cloud, clouds.Local) else yaml_path, diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index becd38b4fa6..0b41cfb2198 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -33,7 +33,6 @@ from sky import resources as resources_lib from sky import serve as serve_lib from sky import sky_logging -from sky import skypilot_config from sky import spot as spot_lib from sky import status_lib from sky import task as task_lib @@ -201,7 +200,6 @@ def __init__(self): # For n nodes gang scheduling. self._has_gang_scheduling = False self._num_nodes = 0 - self._provider_name = None self._has_register_run_fn = False @@ -313,9 +311,6 @@ def add_gang_scheduling_placement_group_and_setup( self._has_gang_scheduling = True self._num_nodes = num_nodes - if envs is None: - envs = {} - bundles = [copy.copy(resources_dict) for _ in range(num_nodes)] # Set CPU to avoid ray hanging the resources allocation # for remote functions, since the task will request 1 CPU @@ -507,12 +502,11 @@ def add_ray_task(self, f'placement_group_bundle_index={gang_scheduling_id})') sky_env_vars_dict_str = [ - textwrap.dedent(f"""\ - sky_env_vars_dict = {{}} + textwrap.dedent("""\ + sky_env_vars_dict = {} sky_env_vars_dict['SKYPILOT_NODE_IPS'] = job_ip_list_str # Environment starting with `SKY_` is deprecated. sky_env_vars_dict['SKY_NODE_IPS'] = job_ip_list_str - sky_env_vars_dict['SKYPILOT_PROVIDER_NAME'] = {self._provider_name} """) ] @@ -2379,6 +2373,14 @@ def __repr__(self): def get_cluster_name(self): return self.cluster_name + def _use_internal_ips(self): + """Returns whether to use internal IPs for SSH connections.""" + # Directly load the `use_internal_ips` flag from the cluster yaml + # instead of `skypilot_config` as the latter can be changed after the + # cluster is UP. + return common_utils.read_yaml(self.cluster_yaml).get( + 'provider', {}).get('use_internal_ips', False) + def _maybe_make_local_handle(self): """Adds local handle for the local cloud case. @@ -2486,9 +2488,8 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: return (ips is not None and len(ips) == self.num_node_ips and all(ip is not None for ip in ips)) - use_internal_ips = skypilot_config.get_nested(keys=(str( - self.launched_resources.cloud).lower(), 'use_internal_ips'), - default_value=False) + use_internal_ips = self._use_internal_ips() + if is_provided_ips_valid(external_ips): logger.debug(f'Using provided external IPs: {external_ips}') cluster_external_ips = typing.cast(List[str], external_ips) @@ -2513,7 +2514,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: f'cached ({self.cached_external_ips}), new ({cluster_external_ips})' ) - if (self.launched_resources is not None and use_internal_ips): + if use_internal_ips: # Optimization: if we know use_internal_ips is True (currently # only exposed for AWS), then our AWS NodeProvider is # guaranteed to pick subnets that will not assign public IPs, diff --git a/sky/skylet/events.py b/sky/skylet/events.py index 79aaba992c2..4287acd394b 100644 --- a/sky/skylet/events.py +++ b/sky/skylet/events.py @@ -16,8 +16,8 @@ from sky.skylet import autostop_lib from sky.skylet import job_lib from sky.spot import spot_utils -from sky.utils import cluster_yaml_utils from sky.utils import common_utils +from sky.utils import remote_cluster_yaml_utils from sky.utils import ux_utils # Seconds of sleep between the processing of skylet events. @@ -104,8 +104,8 @@ class AutostopEvent(SkyletEvent): def __init__(self): super().__init__() autostop_lib.set_last_active_time_to_now() - self._ray_yaml_path = cluster_yaml_utils.get_cluster_yaml_absolute_path( - ) + self._ray_yaml_path = ( + remote_cluster_yaml_utils.get_cluster_yaml_absolute_path()) def _run(self): autostop_config = autostop_lib.get_autostop_config() @@ -139,8 +139,9 @@ def _stop_cluster(self, autostop_config): cloud_vm_ray_backend.CloudVmRayBackend.NAME): autostop_lib.set_autostopping_started() - provider_name = cluster_yaml_utils.get_provider_name() - config = common_utils.read_yaml(self._ray_yaml_path) + config = remote_cluster_yaml_utils.load_cluster_yaml() + provider_name = remote_cluster_yaml_utils.get_provider_name(config) + if provider_name in ('aws', 'gcp'): logger.info('Using new provisioner to stop the cluster.') self._stop_cluster_with_new_provisioner(autostop_config, config, diff --git a/sky/skylet/providers/gcp/config.py b/sky/skylet/providers/gcp/config.py index 955b98fd880..781737c6cfc 100644 --- a/sky/skylet/providers/gcp/config.py +++ b/sky/skylet/providers/gcp/config.py @@ -911,6 +911,7 @@ def _configure_subnet(config, compute): } ] if config["provider"].get("use_internal_ips", False): + # Removing this key means the VM will not be assigned an external IP. default_interfaces[0].pop("accessConfigs") for node_config in node_configs: diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py index d105a2f246a..afb18a2216f 100644 --- a/sky/skypilot_config.py +++ b/sky/skypilot_config.py @@ -120,7 +120,13 @@ def set_nested(keys: Iterable[str], value: Any) -> Dict[str, Any]: def overwrite_config_file(config: dict) -> None: - """Overwrites the config file with the current config.""" + """Overwrites the config file with the current config. + + This function should only be called very carefully to avoid unexpected + behavior due to the overwrite. Currently, it is only used by the spot/serve + controllers to reconfigure the network settings before any further + operations are done. + """ global _dict, _loaded_config_path if _loaded_config_path is None: raise RuntimeError('No config file loaded.') diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index 1f4745efd5a..a0ac9dcdbaa 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -18,9 +18,9 @@ from sky.serve import serve_utils from sky.skylet import constants from sky.spot import spot_utils -from sky.utils import cluster_yaml_utils from sky.utils import common_utils from sky.utils import env_options +from sky.utils import remote_cluster_yaml_utils from sky.utils import ux_utils if typing.TYPE_CHECKING: @@ -248,6 +248,11 @@ def skypilot_config_setup( def setup_proxy_command_on_controller(): + """Sets up proxy command on the controller. + + This function should be called on the controller (remote cluster), which + has the `~/.sky/sky_ray.yaml` file. + """ # Look up the contents of the already loaded configs via the # 'skypilot_config' module. Don't simply read the on-disk file as # it may have changed since this process started. @@ -276,24 +281,26 @@ def setup_proxy_command_on_controller(): # (or name). It may not be a sufficient check (as it's always # possible that peering is not set up), but it may catch some # obvious errors. - provider_name = cluster_yaml_utils.get_provider_name() - if skypilot_config.loaded(): - # We only set the proxy command of the cloud where the controller is - # launched. - proxy_command_key = (provider_name, 'ssh_proxy_command') - ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) - config_dict = skypilot_config.to_dict() - if isinstance(ssh_proxy_command, str): - config_dict = skypilot_config.set_nested(proxy_command_key, None) - elif isinstance(ssh_proxy_command, dict): - # Instead of removing the key, we set the value to empty string - # so that the controller will only try the regions specified by - # the keys. - ssh_proxy_command = {k: None for k in ssh_proxy_command} - config_dict = skypilot_config.set_nested(proxy_command_key, - ssh_proxy_command) - - skypilot_config.overwrite_config_file(config_dict) + if not skypilot_config.loaded(): + return + provider_name = remote_cluster_yaml_utils.get_provider_name( + remote_cluster_yaml_utils.load_cluster_yaml()) + # We only set the proxy command of the cloud where the controller is + # launched. + proxy_command_key = (provider_name, 'ssh_proxy_command') + ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) + config_dict = skypilot_config.to_dict() + if isinstance(ssh_proxy_command, str): + config_dict = skypilot_config.set_nested(proxy_command_key, None) + elif isinstance(ssh_proxy_command, dict): + # Instead of removing the key, we set the value to empty string + # so that the controller will only try the regions specified by + # the keys. + ssh_proxy_command = {k: None for k in ssh_proxy_command} + config_dict = skypilot_config.set_nested(proxy_command_key, + ssh_proxy_command) + + skypilot_config.overwrite_config_file(config_dict) def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task', diff --git a/sky/utils/cluster_yaml_utils.py b/sky/utils/remote_cluster_yaml_utils.py similarity index 65% rename from sky/utils/cluster_yaml_utils.py rename to sky/utils/remote_cluster_yaml_utils.py index a1a7bdbd259..88040cf7f27 100644 --- a/sky/utils/cluster_yaml_utils.py +++ b/sky/utils/remote_cluster_yaml_utils.py @@ -1,4 +1,7 @@ -"""Utility functions for cluster yaml file.""" +"""Utility functions for cluster yaml file on remote cluster. + +This module should only be used on the remote cluster. +""" import os import re @@ -9,16 +12,17 @@ def get_cluster_yaml_absolute_path() -> str: - """Return the absolute path of the cluster yaml file. - - This function should be called on the remote machine. - """ + """Return the absolute path of the cluster yaml file.""" return os.path.abspath(os.path.expanduser(SKY_CLUSTER_YAML_REMOTE_PATH)) -def get_provider_name() -> str: +def load_cluster_yaml() -> dict: + """Load the cluster yaml file.""" + return common_utils.read_yaml(get_cluster_yaml_absolute_path()) + + +def get_provider_name(config: dict) -> str: """Return the name of the provider.""" - config = common_utils.read_yaml(get_cluster_yaml_absolute_path()) provider_module = config['provider']['module'] # Examples: From 40d913c756209223bce6e85b5b16a1e79acf5e1f Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Tue, 28 Nov 2023 09:37:19 +0000 Subject: [PATCH 14/27] add comment --- sky/backends/cloud_vm_ray_backend.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 0b41cfb2198..1040dce773a 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -2490,11 +2490,16 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: use_internal_ips = self._use_internal_ips() + # cluster_feasible_ips is the list of IPs of the nodes in the cluster + # which can be used to connect to the cluster. It is a list of external + # IPs if the cluster is assigned public IPs, otherwise it is a list of + # internal IPs. + cluster_feasible_ips: List[str] if is_provided_ips_valid(external_ips): logger.debug(f'Using provided external IPs: {external_ips}') - cluster_external_ips = typing.cast(List[str], external_ips) + cluster_feasible_ips = typing.cast(List[str], external_ips) else: - cluster_external_ips = backend_utils.get_node_ips( + cluster_feasible_ips = backend_utils.get_node_ips( self.cluster_yaml, self.launched_nodes, handle=self, @@ -2502,7 +2507,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: worker_ip_max_attempts=max_attempts, get_internal_ips=use_internal_ips) - if self.cached_external_ips == cluster_external_ips: + if self.cached_external_ips == cluster_feasible_ips: logger.debug('Skipping the fetching of internal IPs as the cached ' 'external IPs matches the newly fetched ones.') # Optimization: If the cached external IPs are the same as the @@ -2511,7 +2516,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: return logger.debug( 'Cached external IPs do not match with the newly fetched ones: ' - f'cached ({self.cached_external_ips}), new ({cluster_external_ips})' + f'cached ({self.cached_external_ips}), new ({cluster_feasible_ips})' ) if use_internal_ips: @@ -2520,7 +2525,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: # guaranteed to pick subnets that will not assign public IPs, # thus the first list of IPs returned above are already private # IPs. So skip the second query. - cluster_internal_ips = list(cluster_external_ips) + cluster_internal_ips = list(cluster_feasible_ips) elif is_provided_ips_valid(internal_ips): logger.debug(f'Using provided internal IPs: {internal_ips}') cluster_internal_ips = typing.cast(List[str], internal_ips) @@ -2533,13 +2538,13 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: worker_ip_max_attempts=max_attempts, get_internal_ips=True) - assert len(cluster_external_ips) == len(cluster_internal_ips), ( + assert len(cluster_feasible_ips) == len(cluster_internal_ips), ( f'Cluster {self.cluster_name!r}:' f'Expected same number of internal IPs {cluster_internal_ips}' - f' and external IPs {cluster_external_ips}.') + f' and external IPs {cluster_feasible_ips}.') internal_external_ips: List[Tuple[str, str]] = list( - zip(cluster_internal_ips, cluster_external_ips)) + zip(cluster_internal_ips, cluster_feasible_ips)) # Ensure head node is the first element, then sort based on the # external IPs for stableness From 3fdbdc992008b05cefa4eca8ff9e16f269e957c2 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Tue, 28 Nov 2023 09:38:52 +0000 Subject: [PATCH 15/27] Add comment --- sky/utils/remote_cluster_yaml_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sky/utils/remote_cluster_yaml_utils.py b/sky/utils/remote_cluster_yaml_utils.py index 88040cf7f27..f773f11b933 100644 --- a/sky/utils/remote_cluster_yaml_utils.py +++ b/sky/utils/remote_cluster_yaml_utils.py @@ -8,6 +8,9 @@ from sky.utils import common_utils + +# The cluster yaml used to create the current cluster where the module is +# called. SKY_CLUSTER_YAML_REMOTE_PATH = '~/.sky/sky_ray.yml' From 80c3de9b4e4df54de693fe79c0488c1f26fcfdef Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Tue, 28 Nov 2023 09:54:04 +0000 Subject: [PATCH 16/27] format --- sky/utils/remote_cluster_yaml_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sky/utils/remote_cluster_yaml_utils.py b/sky/utils/remote_cluster_yaml_utils.py index f773f11b933..6f1a044953e 100644 --- a/sky/utils/remote_cluster_yaml_utils.py +++ b/sky/utils/remote_cluster_yaml_utils.py @@ -8,7 +8,6 @@ from sky.utils import common_utils - # The cluster yaml used to create the current cluster where the module is # called. SKY_CLUSTER_YAML_REMOTE_PATH = '~/.sky/sky_ray.yml' From 08c7213aba198d496c6f405c761ab4c689fd463e Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 05:22:26 +0800 Subject: [PATCH 17/27] Update docs/source/cloud-setup/cloud-permissions/gcp.rst Co-authored-by: Zongheng Yang --- docs/source/cloud-setup/cloud-permissions/gcp.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/cloud-setup/cloud-permissions/gcp.rst b/docs/source/cloud-setup/cloud-permissions/gcp.rst index cf51793ef8a..582588fb9e8 100644 --- a/docs/source/cloud-setup/cloud-permissions/gcp.rst +++ b/docs/source/cloud-setup/cloud-permissions/gcp.rst @@ -275,7 +275,7 @@ The custom VPC should contain the :ref:`required firewall rules Date: Tue, 28 Nov 2023 22:21:00 +0000 Subject: [PATCH 18/27] Address comments --- sky/backends/cloud_vm_ray_backend.py | 17 ++++++++++------- sky/skypilot_config.py | 6 +++--- sky/utils/controller_utils.py | 2 +- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 1040dce773a..1f380dbe15f 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -2342,8 +2342,9 @@ def __init__(self, self.cluster_name_on_cloud = cluster_name_on_cloud self._cluster_yaml = cluster_yaml.replace(os.path.expanduser('~'), '~', 1) - # List of (internal_ip, external_ip) tuples for all the nodes - # in the cluster, sorted by the external ips. + # List of (internal_ip, feasible_ip) tuples for all the nodes in the + # cluster, sorted by the feasible ips. The feasible ips can be either + # internal or external ips, depending on the use_internal_ips flag. self.stable_internal_external_ips = stable_internal_external_ips self.stable_ssh_ports = stable_ssh_ports self.launched_nodes = launched_nodes @@ -2511,7 +2512,7 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: logger.debug('Skipping the fetching of internal IPs as the cached ' 'external IPs matches the newly fetched ones.') # Optimization: If the cached external IPs are the same as the - # retrieved external IPs, then we can skip retrieving internal + # retrieved feasible IPs, then we can skip retrieving internal # IPs since the cached IPs are up-to-date. return logger.debug( @@ -2521,10 +2522,9 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: if use_internal_ips: # Optimization: if we know use_internal_ips is True (currently - # only exposed for AWS), then our AWS NodeProvider is - # guaranteed to pick subnets that will not assign public IPs, - # thus the first list of IPs returned above are already private - # IPs. So skip the second query. + # only exposed for AWS and GCP), then our provisioner is guaranteed + # to not assign public IPs, thus the first list of IPs returned + # above are already private IPs. So skip the second query. cluster_internal_ips = list(cluster_feasible_ips) elif is_provided_ips_valid(internal_ips): logger.debug(f'Using provided internal IPs: {internal_ips}') @@ -2543,6 +2543,9 @@ def is_provided_ips_valid(ips: Optional[List[Optional[str]]]) -> bool: f'Expected same number of internal IPs {cluster_internal_ips}' f' and external IPs {cluster_feasible_ips}.') + # List of (internal_ip, feasible_ip) tuples for all the nodes in the + # cluster, sorted by the feasible ips. The feasible ips can be either + # internal or external ips, depending on the use_internal_ips flag. internal_external_ips: List[Tuple[str, str]] = list( zip(cluster_internal_ips, cluster_feasible_ips)) diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py index afb18a2216f..4db5c5f28bc 100644 --- a/sky/skypilot_config.py +++ b/sky/skypilot_config.py @@ -119,11 +119,11 @@ def set_nested(keys: Iterable[str], value: Any) -> Dict[str, Any]: return to_return -def overwrite_config_file(config: dict) -> None: +def unsafe_overwrite_config_file_on_controller(config: dict) -> None: """Overwrites the config file with the current config. - This function should only be called very carefully to avoid unexpected - behavior due to the overwrite. Currently, it is only used by the spot/serve + This function should be called very carefully to avoid unexpected behavior + due to the overwrite. Currently, it is only used by the spot/serve controllers to reconfigure the network settings before any further operations are done. """ diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index a0ac9dcdbaa..9f2273402df 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -300,7 +300,7 @@ def setup_proxy_command_on_controller(): config_dict = skypilot_config.set_nested(proxy_command_key, ssh_proxy_command) - skypilot_config.overwrite_config_file(config_dict) + skypilot_config.unsafe_overwrite_config_file_on_controller(config_dict) def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task', From 569101fb633b6162210657f18224839359785cfe Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Tue, 28 Nov 2023 22:30:08 +0000 Subject: [PATCH 19/27] improve docs --- .../cloud-setup/cloud-permissions/gcp.rst | 10 +++++++++- .../source/images/screenshots/gcp/cloud-nat.png | Bin 0 -> 64841 bytes docs/source/reference/config.rst | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 docs/source/images/screenshots/gcp/cloud-nat.png diff --git a/docs/source/cloud-setup/cloud-permissions/gcp.rst b/docs/source/cloud-setup/cloud-permissions/gcp.rst index 582588fb9e8..064aeb7e8d5 100644 --- a/docs/source/cloud-setup/cloud-permissions/gcp.rst +++ b/docs/source/cloud-setup/cloud-permissions/gcp.rst @@ -295,7 +295,15 @@ Instances created with internal IPs only on GCP cannot access public internet by cloud NAT needs to be setup for the VPC (see `GCP's documentation `__ for details). -Cloud NAT is a regional resource, so it will need to be created in each region that SkyPilot will be used in. To limit SkyPilot to use some specific regions only, you can specify the ``gcp.ssh_proxy_command`` to be a dict mapping from region to the SSH proxy command for that region (see :ref:`config-yaml` for details): +Cloud NAT is a regional resource, so it will need to be created in each region that SkyPilot will be used in. + + +.. image:: ../../images/screenshots/gcp/cloud-nat.png + :width: 80% + :align: center + :alt: GCP Cloud NAT + +To limit SkyPilot to use some specific regions only, you can specify the ``gcp.ssh_proxy_command`` to be a dict mapping from region to the SSH proxy command for that region (see :ref:`config-yaml` for details): .. code-block:: yaml diff --git a/docs/source/images/screenshots/gcp/cloud-nat.png b/docs/source/images/screenshots/gcp/cloud-nat.png new file mode 100644 index 0000000000000000000000000000000000000000..951056d56598749610e80baeb1ae3cb933dd5e29 GIT binary patch literal 64841 zcmdR#Wn3Ih*64u{GpJK1O7 zyU*_4`|WQiz~Rdv-l{|-}8lEK0t#Xvwnz>@tS`4Iu(St$Yn;!iXb_?^Ny z0Y(G_j4~?;2^Coh2`Uw52XiZ1GX#VWVF^j7YH{=UUrwG1c(uimb3rS}d&sf);+Rxa z63+K zWQr}s!NGBhM(Y_svOy~i6asLbu{wQ!dc~=V7dNP=e!8VkhkQaCA3oVI_&7Ry*7lZG zk0+rMnPlswtSDdhx5l5|Rw-f_uigfOlqzCH)hJb|Pe2oM+U6YhPaI zfM_a_<$FgN3@yTwWI_G!v|hY&9z=c-9cm#QVPs)qfzu96PZz>JIaXYf3XRUJHLInO zNeLD`s*Xy1CYp+m@ZKWdt_Kz2rx4=W(%TG$@4{AOZAyJl2zU(7NXL*H+i{Z-8FP8) z*HD^c!=H&Ugd#f`;;jX|j75Ayg<6bYBTCJHLe`hA)kP_e`uvRK(L=sLMHQF#I^Iu(+iqjr2d73U+-M}mN# zpQ6+wl?KPOE1vR0Nqxu?J6YH9UW$HGag7HIV024XsAl~Fs^QCh)ed?uo{&?hdi9G( zjeeg_F4`;TF0v>4wkQzU)$N+wWhC-tyX-HyqKU{ne z|48xst81id%9PBVf;z2E?EKbqQxeZ-x3~@IKAHE?;#tWm%?xMpOamx#%-cfS=-YAl zpZm4La=J9MOVskFa$P0%YBzWdEkNS&a#tvdVh<`e5jR7=et z@1i8+^9U!SCmO9f_pSG__s93;r;DbPiw5@?8qw1wlnat3VkS?<+J?qbI(|?mSNKUk zM3Qj=tcRvfCvYuaT5(w}%|NsJ$1bN8CYN&Oi*GeCOU$ykl^JDhl4hhzG&rp|m(9=3 zUkwWZE&_Op?Xt*ATXelM-IK0Eqf4XPqBSj=Mm*Cs8eR2Rm(`Y;8!cQzE`NDZc!{?# zds$rRvrUmONekqBbqZiFE-Rt@$vU}Ov|1u!ZD@^a7V9u(csdtr*~g{8v1B}F#Almp z!8L0cq?$hd%;YQkBwsf$)v4f2hTu~iahx07Fx{vkw*r5DL%v&n%n*ZlhPnJu;jn#* z$#BY!)sEn>H(` z3=ez`%nxR(qRe|oSGby7u06-xQ7)i>YA-{Bc@p+^@*(!*q=@7_@lY$|r= z-#(w8?&lu7sz1oISGTva4_VF_WlyI~uJTuEYYGsY^(|{!3S9rTR?#bQ{QIQ;s`%Px z-)y)0Mqrn_kHaFbGP+#fdl4w#Ox7Y&sW;C!e*|El^v|Tpv^U^hWmuJ4rT0Ve>-Tf> z>wJuU9DG=O#)c?{n2L1l?8+yk#B9Mg#_wph7DVN&BB^3FHfb1PDC@YW3$$Nb8g`)! zA`S@^{~;m9K=10ZyU8|L{?@2%sB;7}RfGFPtKJS8ema6?-L(Z#Ep(anP5Z zn|dVyfZ^}vWN$k9+yxv@1(=|?f4s9vx zY_9Pjh2^2Vlon%or!39ZF1P-fx1Haq`o-S8(POad&!!$$O&U_T4lHI}q-}Z+`=sPL z5|&peFD2iUsKunyX3I9qdeiQF>AZ3tO?byRQ@pFwuifB5SpDAVbFX#h^z$Or^ZN_y z8Qj_2l3VUox;Cb#hlS)CyXOMWJ)b{_K%Z1JeyK6MYi=5`TRx^77s2p9)OTp}gWa7- z-Dekd^K@@_8^{CSYWP<^{N5<2z>+0vluwuQuuIMUHMN$l z_NAR?o7>q8<2LqoQypGie{DbGS4QnxGP~9K2OBfF?W6Z6g~8+tf-@)3`na}>9+on_ zt0p&hqLar;nR>{I{!@$MG~M(ztQ=N(BfLo2(sqGxA$lZM64;H}MvhPZeHJvU`^&Ag zM7Y?uz8bqUZW@bR%P|-7Yjix(tLUx;>oR!5WG(q{E+Uh_uhm6p7ubuw6LuHZL+`=$ zku2iJ&wMk*&f#~%=ZW=+D>Lmg(L8mvJqE@8{H}6OYSUI%lX(`Q78oOu>BGW;6D;=r zZY;OD{=FN~ArO<%)kd}x@tf|S5lO7KdIGEc`$FFH59hxbOEt91ILdeoTpDMlw&$jy zMVn<&%Sy|Nv*~4Zm-Uy`S0j7EE}a)W4XFTL*5!?*Mt8w2_mlKrf;Y!3^^9%3Z8!IR zcMrTcm6VZyA#aeU+NI{U(=EXf{d8N((+riz_srvqy#p`Ovr8k9vZw8niPbjjmqIy0 zjFeMk#lH48_fxT}Dx(p%tSI{72GOnW?$AR zG4bnAXRK!~(U+qmLX*O$^xf%Rscp0gXU0X)yr&4zC>iyUI?ngrVE?fIWmA>BUF50_ z0i_mYj})c>U#B7%n`z3LD=H$qhhL*1AR*!-JcC~$!haD$tPqg@agBgL2S39XzTblp zP~m5M_@By8r2k2MR{HbV|6C*fgr^~heUgxsg`Yo}IGdT-yI4B7T4n_&A|N0MTd8Tf zYAPxSm^j$67@0a4o3Q}x9RG+Q2muA)mv&~ZMpQsMTYDD)pfJrpo)Cav|GCXdL-mhG zTy2DDG!<2-BpjU0sCZb|SlDO)7*teLLe8e<0v{!%|0xdtPngEi)zwjemDR(;gT;fB z#lhKvm7SlTpOuY+m4kyB{sgm&r@gBYklEga_ODF-Cm%^O7ZYbIM^`Hcd#XSA8W}sd zxeC+J{3+;v|NK=?GoaPKO0swPXS3i9Wc`!E%Fe>Z`hOC0wKD&|i2X_VOY9%*`l~ph zKbZ-rSOLv!wIr?V;97-O4Zy|0DfEwG{ww8Q75z){vx}Ltgo7Pi&=v46v;0%|?}`5@ z_>Uqr|5YSE7sua={9DT3B>yx)K-t*}UYgM#9Rk>eSpVzZKc5$3{iEQ&Df};M{^Ks( zoB#|V*8kl#02n+t0tyHSq6o5*VroFd!}RBApQXq`3>usA@NjJ_2%5UV$FB}Y?aMJ2 zN25oN#k;`ZSA`kgLY5T$ zw?mE^5qmL?nN{#E*4k4x<138UO!LLGNFgnw2*0 z7wT4}a@(j6aE%Jz=nfn$Fd)%T{rjO(i`KWPT{}`|&G_#LR7fN;qG0Y#r4$UN|4cE9 z_yV$-7g#MU`IoZ((wgaOcE4-sWoa zpXZp60#R{l>sI|10Z;cXWPntP0ZAb9Zq| z;m-W{-d-)<*WnGIa{Rt@tE)+!lbB(;C- zE;^bNyRdDiyg0g1jVMvS!l;#vu&(t?-FF}G!zxj)!DvN@@#Pz4*`29&#LQWe`uPPd zUUpr;^V}46RJVEv#$>&W>djZIS}lk7?z2V6aB7mpP-qD}ZRHs50}whXBA+4nrCE{9 zZ96w+JyYD^mzT&9-2PK6u-Im{bf(_PL~tXN_TKmUxamYYhC=u-P8m>fv+6HGyI+|# z8eWj$H6S97Td2A|a5d~gW34RmP;FN|F74h&9nz(K(Ne6fFU6P^tgQQku{5D*W+Aph z-gZ8md#1sSyGVocz`}9(N6j@P`EA7m*-WV=Gnq!1Z_FwMj~au)NUg1gu+LSde}m8d zv@?vi@UU$(xo(1PRQnsVBJ^Y4N6*EngiQmD>%75P&~(M9Qc0m!&A@@67dW`aW*F1^ z+-|kGu*Ndkjj1+Hga0NhR@s~rQdaFx&*auyYLBI>7Z_A&zcZszD#>C#s8L#xwV8Bh z!d0mAPDe|Jr_dPdpmXD8rxo#+}X^R>rR{5Fr> z2*zTgFILeJ=~0+%z94Vh>rUnc^}2Av%KbsN_Q$Ydx48E``~`<+>UF`JYn7VSt~H~q zwh%9y!+P%D8vSKIxcw6p(;Aipnp||;mK*f;OJzo6Qu99>&u*=5R+h?uq^D3~^bgCz z)F8Y+S{V{uyo)4#)P{v`mg34aZDxuqoHl!I?Fiqim75R$(Apk2D6``kla`+X*o^%Q zm@fQ0cwzvbyowCkT1V;ciG~G>H{&+Pu(v!jhqP;~%_a+!>$V4zARi{Or6OLN4So9+ zs^NHa3A8Ns9|@A6HT5TN?R)Y)?R?#Ib(wg*SsrwKPE>`DG3$5|{BUunpm43QI3Ubg zFrsB8WW#n`5Y1fJbNqZJ2=Z)Db(6wok?Ejeh%?rNNwaccReqfU2WG!iTsU1?@z@Fh zK>Sv{Hc8c44%B1JOSWH7$KKD}nDo;G>quaZQv`N;I!Xqs%@= zub202vJev5yfG4!wqHVZpQ9t7_h`1413^@6Y*>;u1%{Gi63IbrG(|Kf0 zI~8(lYII-8QpC5XcT$AZK=)e}O<+;(`Wsbuk+o60h1O*Ts~u%eTeF_meXvPmt8C2S z^fjNh8#?zQ#@SF={-7JiS@+YZ#$A9(0dpND1<3)K(Y9WWLy1jCdbP4d(^j83fW-5& zqxMaA(V~pU#UNvvW7~b0)AeMzRO^}arbFGZ&3p@__8NRtC3bO?b(6o))WQ>nezNDH z;l-2mdU{C7y*G@N(fIgdU6juTvtu6O?X@DgN%rB*w}`{S0{XceqM90V!IG1oL#%DP zPFA=E7emOaVq!VO2NPMr%aDD|7VB%JFR+yiYXgCpsbxrYWlLS@=A{1VsiuDvl-23! z3Q&?a?dYj{wgs(cHrJ~wJ|dfbg4yOb(eYgDnX_t|S*-9bV9!^SUJ2CXtwN?*E#?`I zS-qAoADl{;VLLAcYi*u2ozE3xGmdoI7!04QB8%ldZh?BFVxp@eqAh<^n$XCsbC`CA z6Ea4ANHUa!RNKx^DDka>l5rm%VJ8n4T)F6(_s3(@eUGkb(t?Px1eqRlW!BS$fO9&p zrE+E;eUD|g>lqz~`S^FAz=d;=yd?d7L2?tuaL;v0dTsAgf|8>&prHLIN^R6Z^Y6{$ zjsS^Q3lW*4)##j+8PvbSNspH;e>4jbY-SQrQ@BzD(wWPsxfgK_A@

5T7Lwt8_h9++ z@nTe@)^?tQjK?;4*>zH}_URGk9Ya9P33}tPKlRQ%5|Zh^a&s<4c5b_ zHii`f1Kl2m*Cs1t-cbC6zWWsoR73aY*%R2(GOympvTTJjBv`4nc<+H|Al> z&UoZOe4hIg?iIyazqH?eZ};s@_1N?8)gp(89r{{hp0MXp-+>D_4}a+$SX|&+3|cNH z3F@bt0=Xy~%A*_@9yf&8LYtcmGL2_5r-a5?&}V}x@xO-FH{PY3U7tR;)5|zG8Q+Zt zdQ$)`3YRxP2JaVf{is))R+Be`v$;%h7P3mqPP${Fs(9N{>f}*>_oDuySiN>I|De0kWr^Qm#(4G{ z+hiu}NMNRXHBGeFkE~(N|D1AGN(S<_jDo-Jb>_hi0nvgiD&nBU>$yGv)G1o{W>29B zcBt<6WC0tkmUesZ$&No*j?9$nth-eJY&xngCipBVqp6iZD}?O*Pgwlkb+5s?vQe}NuxDMO_ zaYhwGYVvGvT9V4|Zm+=aej_V{VxNM>#;3Bvr1y>z$xCij=t3bA+u`-bZFU3;$39UF zV_EYArmG(n+B0Jx7Gqvw8d5}wzPD~-TeQ0T{;D~Ck$Fau97u^3Lwb#_>?1Pet_Q>u z2b{i0u4uSB!&k%(KvLM%0fd%(qfV;LrG>3kC^AOs)E;hJ9_Ot-O^#ZpD9MxOJkENK z$2mRReNb%yp@ZD*{h!Jt1lvcrz{(S<&ufG$F^+!4y&)uIp zB*x0e+e_SjqtE)X`dHJ>YaQb#o}psPr{||oWos0ddvqdzd&)Q2{$#XELkl93Q1N;^ z7_oZ`!NP#h0v%;#ATpYx^u08;X|0P{n^mR1#b{r_aQCcOxV~TXoCPat8td3K1MWSx zw#)!q->?ag0o%Q3CL*4h($~{#w>&d?fuRP?vhhhK30$MM7LsTNV)tziLFL6q+Y~s3t z$1b5Myx!rq?CbpPAhH=6Y7XF_`_YytPZ@x8RzJ}AlQjn_$V8EDQ>N)SwA9?7am*ta#c;`aU+r&uNd1~H^oMW#jJ5C@zs|<9$wr!y zE&>y9%|m4mZ6IMw@U5FMn2?ptgvHX5z&%V=I@){H_vRKo_g?O1HIjtWM+?_qwM=a; z8Z~lrEu~vtuFQx=!d2h*YFeL)MilsgfNs9RM1~b1(4tW}6VSsCUL3sb$_cWRHl#v3 zZAOs*Z78m|PBP&<+f>1gM$_*?zJ~9Tyj7j&&60waoqgO7W`>h8DKy1HcHf91h+NN^ ztYAi2{L%MiOYUVn;Qq98g+Qi{eZ{a#m>hd=hFu1+nuZxkmp2ks&V1cVxi;MrvDAfz z5bV~)4{6Pwq$!iYsvp6v0>T=k8}j#{R9=Xi3&X?b=fza48HBE;&k={+9{Xrxh*x%9 zGyEG8WEL-!5LjoWvEJ)&-VjAPMV09;uK4G%zvPyD870aj(Bl|SKyu>W)|x}0Dp}8Z z-Ez^jj>YE)YkY*sjKIQGdb|^(hjpso>#W%6B~xy8WJ?Q|+XoHa&uZS?ZVgJxKGBpa9NS+adsA@UfMW9AaZ^vAXxSgQ0^z2^@vrO^5lmrM*I|TdQ4=yEJaOcWi$<7@!K@zZnJ(7Z5 z7xOdYlq(~Timxxw`T6*O2eL$y`;kv0 zl`e^e0xP7VmqPygYaSyoGG*(~i~dmc$nu|L_VpbR#nE#xHu8> zkV75$lX&8&;j`TddUxvGcg{7gIdu<4 z(R*du?L*mnB7ggB1zG_VQcwUCzd7~#*o^W~OWJ2Y)?+sIg){v!;#Y;k=hHrIZhMLGGQZH zk+Y9;N#1SFZ#ay;rFO|C-Xfnr4kq2Yji^2|N1&S`To1BfTfvPd)_+Ob)Pehfn7ZP3 zBns~a*2()p;1^t7&`ZuoEaew*SZiW0;?O~1VW%&pqe8em%PIAz*l3-J{qYpZ5=rDk zW#VF2mq_nJW$38aaL$~D*n6QOT?TVW6f%6{@}4~;7NAb3GP1Z(jHH?1?N;0o+%I)~ zlDsVw$Q}c7INyxx1M`O}ol6M_^{)EC>}-kjMZ_TpNCMWO9^!GglA#dw-rZL15i-Qm z0O;wmt4q6}vZhZq$@>Og+)~hXhz)zX4~b2bXT|QEIYacJuk|Rcz>@yH$O#-V+oy?9g!Hkr2(bNmVF4M2aq?NZM~YO^GAn` z&7WqZ3hi*x+a5H{=cO#WqC&R)Dpy@^(vy_SX!0H0P(kJX@IM%W)CKmOL2&e`?a_cEKO;;UqcboqBjzr!|4rMl(1UkMV9^i92? z=Y|Zu?~XmsOUP)>k`E3WpyN*0`X^UyB?`f>(%Ft%>++%fDfft120i!;24in~o4)O` zdZkG&YH8>O$5JoFpCyHn4vB@16ZaC3KAswK^l%FLFlb$y70sXOgR;>T`tFE zU?v>YcqS*fMpI90YloOI7Rt#~xn@;@h_C2P&3SOP{_~K-EqoHa8 zVI5&T{Qdz_P*Wj;`xyD`O=?||gTXE(S2Q;JWVe2%%V=qd5rVvJ-El#(0Rq5p5y*Q5 zvrHXwg6$HwJ-j)%m->PGI}`W#b4kiK90De%`$S!Cl;;H@X%!!y8#M?s@}b0q;BCYh znuDuPiS!Xcyy}U(NIwg<=({sKx)nJonF7FsK~6li74`=Gcfg|UI_K7(g;DH zsoZSF*b+M01$C~GPNA~b%~{5<4CaFrzv ztPYQ8vdsP>BYj8C6X}wlP(1zN;x>^ctvL3cv0|bUZx^SK?izBcr-$n z4O-0`JwH3fn?|fi*&<>Q1q-CCK{Sk>qTr+4jzM|UUbuhQm){b^`HA}=a6RuWathm^ z8aV;AuN5qpGwxd2zU?WmWaaiZ4*6Tn%2@}gi~IQ~UdZkIwEmY_Id}p-0V$R^+xo*&Bo(<3eT-? z0EGnhG9VyFT9dJdSYF7@lUqn2ln!jSM=rU9cRWD(H5P@LnicplFvwB=dfsJJxYPP6 z456n;>X*HM^zPeU<4CIYB(7pSkKd89Sx^%Ns_Y;AcFEQmD4uX<#r>m?-~Czd?@HOm zD6h!_StyI=hTGdG(^n&iE7L$$S69l3bu79|J`jdXolMfFY4#sVgPydantUcvd$DM< zPJIDAs3*;|&F@L=32W=@7H14}bZ;czNis*;nHewv3ezB{!>>Fq z^;lkT{N|VuA-5zeIU8u6Ap3QXxzwa(!zr4pOPMUtW9{`5Mq<@+uEN}f{Sp;c%kEHl z7BbM1L>{_wWB)7YU>ZUmJQqLDcqBuFLK=Ax%l64U$P8sTiI&qiI<<8v+No;7>qN_x zXR+Mjp5BBq$Fs`Ug&rm6gugqr_eHNV-wC-cP&*JLHwSHsZGNZ=VZpe1{-hkF% z2s!|ItmX}la+6SF*TSbR2JL<`*WpzsT)!z!RV3+*!=wj&a`=9^aCr_vYt20d%Rrw%<5n@-zq zZzF@SucMokPMl2wE;b&pV2O#!;IFsL7M5r(ug6xx{PQ`qC?$fwo{_ekMq2m_90gWn z9BJP39Iv}BeggXE+Wd!hBgeMx!w#mt=1@qBXh_Ek!< zQU>+#JSz8D`iIDl?T5+^{AH8$C6`2};+q!kY2}YHGNY*Wp{O*8=&3zTkkaBy{7@lh zd55=?B_z~MkuFp(c&&#_2PIke66ntru2sg zfo#DEs9XfX?9Xhb(A?hICTjV+5M&yO^vs_1O0si3eu_GvO-pfHK8==*7VD$U#%E>I zH*YoybvVb!Sa)RdGsQ*QBhHEgS+-qH$Y)3tqWbfp4!t4cru(rW6C#_<*2Pc31Fp!C zPF6ilgS+Vddb!fgZ8VAjdz5{;k{!isbJDrSm(C(NS_!jc8GNU99oI?GDSVrEHPMPJ zQu=$ro;!j)+#Pq8ANUV8$ta8EXTJ+GGH`CrgsB^f|ALB$2Q|F%Y%3|YL}z<987RC6 z6dQ--$-2jRUy0YeCp^r;Trl;P^{rNm>@C#n2&!mct2S)Dh|?$nwaGsTjwl1b$^Fl2 z2LXvSQJ5`KLvYuWiI8a@uTPIM!&m5rZ*H@D@J*2ZMzqKT_lCW2P99NpR@#}T-o`*A z-rdsVus*CI6&S8zeCjDym;d;XWow~pM#(GBMo@A3Bh4l=eXZLj^FqhlJ-4GT(}cG9 z+I^FgvMo0qCt!Ht$DKbiR~+|qlk>!{YeeUp>9s>_1E5dM5?5mM#%HAfkM15n9|j{Z znv;G39(PX?hwQ9IAd#d7y&l~qZyG&-GD?@;=mlf?vyQK)I20N^wP5~%uhpPun-5s) zLq<=PmtmTDEucPAaXPxZ$Z$1Hwl{&-+R%s*fuTnu;mmOh`}^8v$@_7X@SQeOvgf%_j&?WeVn-w zJ8jg13QL`)vZ2M$j(Z@zMBjUTLDHYZCk!UFdObepEbPrfK+Ln!F)H&H48gp?^OBy) zXL5;aPBg_yD}d|GtcKg2RM=%?f7eAQeWEmL#+2bu_@KhCcnI9#b+Mihis(f`LO`}s zB-fdSYH|6QdaHv^nJ(9e<||tk-g-ZL4o)ARzXh}7Eyt(mOVI^@8T(@ps|c(7PS-jB z@4lvQa#&$kz5T@7+{4s)4UgU4Q0zQhFS0Hkmap^qI1jU-#F}gvMMN8zU5hq5OeLkH zvL~jHB+o87Qmlm%(}PmYK!EJfo9*uqJ&)yWDQrjzM@QP&mg~VTj&}H?3EgrwM$Lx) zdPY`U@BxuBBS9MbB9HJQC_ChIna6Gc>Jnu^nL`i)Lk}3Z>Y7udo6Zglgb|#GNCWw; z>QNfhJ>$n?#nzme$NB@E((zh{Y!^X;y!K0c?kAP$V;;@zVN7MC;f|vd)DoeN%}&1f z`^v~Dn~gcrkhu(&Mw&L&7vs%xoz9Ka?PqRLN((RBn0@GThxk`~=4>^S!ZJfCgfc_R z?}=d>dWz=qOYV_9cKHiPywviF8jrht+X^M}vaB9H-mF@ln0M)A{n4M)X9=SHrY8w} zE)hA8@!dqXP)V7l)o zMp$BNf`kBK&&FnbnVl>xghUmpKC#`si!9pAwrEAnOS()M!~Us-AlefiM}To~hT#(k zQ3C^2iG-5S@OuvMTO*PZg8|vxmICAcM%*p|;HinvHIkVOIvNXpX~q zj3V$L4i@CiH~v7RShNkMZAYd)GbzD1u?$M$tlHessQy@}mYr~fHly&WWC{_0UlPmU zgW1;kJGzTgTfH9ndZ96T`Gn>Tk2Cc_WEEYdu?8!5@6buLH4t}6u}Yr54j-UX{IONgtIQRq%a?(aaRbPwrs=~Wx5Rs z{pSxUOx2uTufb(5f;ez5+K;lt3+XQu*Z0?z1ulB)VmNUg+v=4|$IDCCv)9?Qr*~K6 z;m3vo-_b(bQ3woyFa0aOiy=6?;mp0DCkU7j7tI3aZ7UQ~{1+G_hyI~Hae|=8I_!UV zpnrlvqE^&Y>)P5Qx%<@r3C85b!@-z$!WD&y|6*GHl8S$Yber+Hz8XPASn_8vE<>qS z&>)7v^ryeELqAXdpg2I=gsT4(4i!bjg#$tep#$Q7d;Tx(2rl)Z{uPDJ_J2`2G%5&i z;0O~TyYfHrB{+*DgoIxj)T_=P@gI%*g7y*tj_@FT`Ni~~l#(biMhv`p>lL3z{!>|0 zNY6##KokPid(8i&o^S)ev~YF(zirk)6dA9*9pYBb|DlqSKUL7;bqnZt>?8Tp}+9A=|D0``f%Sch8tsU?4bR> zllI^pNipR6#6SD;->mcJVHC+ zuv$D=NOGV#iAl>C9@o$t;0mg7n#h&C9u;|dGvqf@tmS^%fx5WS9YJ8PTiB%37W>O9I-vmMe;{6i%=6DRIP z%YHM(%YTod$-P!{wwhS-gzckq;!@W1*iu)0SDMmgyIh%ND2g6hjf5e~P|3V}x(Z zWmbtruBJXc(2?$NTTg{5tFNsZcpq`wtTZ4Q3-TW8i^csdR9`_n~7aK510W7T)oea;B= z52`W!8@@?;xd3+oxh(ny);?X$8q8!#ga|>-;3(5V10=)dPi%{*1Wjh33GTsckU7#e zxM>c(cEx}bUZj~Pm1cv^Z~WnYD_?rCU0jS&b-r^S>+$R$;ivnH3K(gYFC0=N*@lOr z6r;TD;PJ5Q(QdVyoh|q_euBP0t4&86PRX6OIMT3qTmN|AH-F&2Jul^Xyg&3E=6mY* z*Q@(w#G5(AaIJ?)6Z;!^U`ucGne#{*e~{M*6z^oE#XH($D2;#eddX!pUO1NDUAyjU zh#&lZoZZUW)#V5_35RLmg(?aS?%Q(f34>bq`LDZ$wu3fFBXyrQ!$$`_4@&A5=reur zVT`upa@!U|$woK;s38yb-FK-kD2_BSN=~P@4>zkve_*e334eH(4K67BSwc0}W6*@w zvVmJt-mse;VVs?Ca$7=4{@fRKeKoWS=RuwXKeQazZ(>(6JMkEIp$Wom?~6({i$QU+ zXx~(#vWR*c&|Y;_H43Qme|nh37P)s6JS^{gfx|dcpiH@n-D&c8z3AYLC|`|v^jd2L zjY=D@Xr1+N_{JvM&N6{9vC1Jhm(mmWdbR*9UJ}XdV?Bk}ZWf?9pG_GJOdRmq;S$Lo zj+iAXu{z$jf+pTzMuaP;W}e&|uEI;Zo&O9A75#{0V;&>l$ZY$8Uh#vDS|+1LDEk3T2b4gI?R$OsiM)kiuHwdvJGn zr2p2U{gsX_9sz?8tL5CfRhJD1@YL`-giVML9#Cw)>08hGsY=X!zG{}FSFG3M4o7v8 zv`(W{{IU(O1$L?0SVzUgnF;1E_a?DeO%`kI?BJ32D&7@fC%t@u-<$j0oEV?ovp<0( zVfB#L&Y=#vAtuxVd;mA$^FCT%{$Hp~UijZC?*3gM83%TNK8JY|Hd?GI$Er|vBL8NCM;B2nZR{o!-K*#!9Qxd>4iq&FtR*r;BOZr zbRJ4X56J2Bj}L5gw6CxI6GQD&znJ|bU_1Zr^KC1115@y{{aFhyZp`~x=WH7y2i6e=dYVcIi@WH`*XKZT>y1b%U81=F48n?dX>|`vG@mSx-VO zqo!?XzZ8_OGHr1D*%m1;#`Fme@gZ0ld@gy**9!`Q=QCk|)hvRy78 zYL}g@HxJ1!6_GiTMN?julK8Y@)a)i1_<2rHUW*HtbFfY$4}AWeV)HQYxg^2n2Ut!t z@DCgw{t#4Qj7BezV%feq4SagIY0*e@Zia)_!KYC8j=iP59=>Du?flWhy4Oj}-&2Yg zzEp11sd=W%w~l=BE^9sb%c39b=mE;og_(_iLMZ;F$D2Xk)}qc4{%#~$&*IW2DU!8# zl@&nTORM`mm|=6}lsGt;CIh?YWZCy-c?A;;09tKCDRFmsz^!Jt)*j#v$IOUFQ9zOc zSF|Es^k#c=6**yh!8w6?L06MXqTT(u-T;(>$|pbd`b%Ri0^Fm=5DAIL|y^=d{3 zK#BaqOfG~HDZOH76aXtGQUpA^S*^C3Z2ls>dq%{K%n8oCf2R?IL3qD|_zVT8Oey|+ zy|?CzszqrCxyRCY5<$h6_B_9|Pso@Q;*lfBLMk9hg2cv)5kYUn9-ecVt;^kU+I2o% zkbwT$)8kE>pDk5Z)A~Im$gc2Qm9C3$TSA4PQI=jQ+s0#WB2WMR)3w?^211zS7YQMe zTGUPLwY%=thie_mQu5MgJ+97nG)rhskk@Y--Np5R^bPcQuu#VU!7{ z?0YZ1klnC*I%97fB-lUZrt)|s=smUVxsSQm9Ci<;>I5zj9_Arl_`V92LEho+<*9@` zO)e!e8Q3`ZrPkQ(g>&;*Zhc%VcKh7)w3J`Ffb}z7z#;)Fvl>jXH$F5n*H$V#;}Ol? z%CoY=tZygs&@{zzsJ@!f3Z$LwsF)dX+yE|OPaif6@>K3%y$XwIyqOGFd5AZc+qRbd zvnPFg0}yBx#y{@6Dp+--POt5;tI2FcTJkbL;f=~0aDud~tB}q8&3EMJ;1?oHQKa1G zErnUC;v{>wX0KleZ{4d*z(|GIxR5N@Dkw!>M<&Rc0eSX(w_{>c3}0tT(~A7KY;HRR zV;X&`9aS^S(gmtpLeDpqOlSpAqA7KIl*WGwz}E^UaWrFdSXhjtdq5n^`f<#bS$*$Y zX&(mUDEDR+_=e~ziqp>rDwY%`DijcISIfzmw4u6Ld&_ku_-fokNL`_k;1T?WnrjZX zdMN@8ONd8-<}2R7KrWZ0`dUJkSOh9I4h;Rj3!IHL>s+R4UdKE*BQRs{ym-p-2;o@X=$e<&Qw z3V`(sqBR`V!3*>gmalj4rILJ+A2n-qfDHb4G!cFl*d?Nee>(@h@Nza|RryWe_61d% z_H~CS?JGQ~&L6Q7T`RleIV}j{q4w|(EY#cl9;%_#t)BVcDRKh*y6|M6jU6;A3}j}a z<-yM1P+HLjqj(njl0rtJ45v~Kg74U`ouQ}et>emoP)$h!3ic6^O*6bIH2nc7)B!k5 zTnt}MJ|>SJyxAJ1F9!0sC^0-k5=nb%E?4{v01<9tdIz4bsIu(&wPy0v}$FT$7={`H)|lgFU(!T@9z%Ej6NO4DdG0KGy$&R zS+C>Z2)136wcZsz2}^$25bh29AcK~;|G3Rd&JrI83&X}2aK_iNzj2sDt?0j?As;L` z{CzHZN|4vCseQcpbPo{$c-F6cJW91BBiEuH2t23VG7_hMbrfdl%{czpP$IEgWB+m9 zeP3<(y*h)1%<;#?umPs5+22xxu(cianF>!coo3r8yOsR#2R>V0dg;adgrnqT8=Zq5 zEpo!w2TOtbS`XTU_fVCdN$d!qQjXcYUDf`~40)Z_LwN11I(PBs962SS*=jFtMj&1k zTb_f{R6HS%*FJ{e)7srky^_>onY`j+o{QT7>;kOmPA|aTB(+$m-sI&8yR4I{bx0y@ zRdBOWD}T_tqYT+*(Zwsl;XAQq<;~{~onbrXcBf(v6%EG!m7KNB5^p@IGsFbC|Fwp` z77{0ohUAdxdwph9eHlEtDGwcMg&iM|{a$W#yNF^Jx3*k5L)Z!7`CRq}i@l37N{+OM z>nB=wR#GaeKolo9!g6VRP7+rJYdy7?>3)pE>UqmKsC(SQ9S?B+QF!>X>mIIbm4|Vcg!x<%Bf9|=C&4bJb~~_A z$n0#f=+LqGTYi0{w5N|mawQebsoL0Bwo?$ASiJEPXDA!!WjZe!E;6!oXBsRD7A@^8 zU}E0D>ak!&-{ud+Q8u_Q{>i0@2UxUEg%XM@T!Mjhmd|iw9B629NT)jlUr(sKB25-s ztH@9p9%E0if5?u~0WpkfJTlPIL`8r3HQq2&__=8JeUV;Mtq70tbJgGvGAidwlE5>c zu8DObW*pdQv$V>`%w|n4%6-DCHK{sk^?KDG)9sZs3-LjAh~Z7-H-P z%PM94x%~H(vjVdQY{-# zx41tO-}T`VHoCgsaVg!+U0c{{{%9c(wwnvyq->FCpbrf-)G0iOu|P;)9*4d2I^1<` zykR{;6mXqjhm-jO8SimQYfMLfsKMSYN+Y9^({$_}=4@staflbJ4?ifdyoI;|2M`U2J5s*~jYkiQSGnf@M*Te#$TikaPi zsUn;nup}mDa_~^zO6)SfL?~}<+-UuI3_XResWwa~AD#BY(r0VVrOPu#J8K8Hym#*c z!qKQaQY|3IxnY+g^`kSPYKbJm1TleuBfkj(=~1;hNrM;pS0dTsR6{Dr*&*oPWsu)7 z@xbB8qoCUQuv-lL($7bx$#7Pxg(@Wm1)e56CkyKdVX;I|gGxgzgr{hqXHpX+90g0z zuy-G&$3_R^363kuA?*3zIz%vqG@A+dS>8PKs7Bv)jSmoiDPhGXsH?NQ(T%gwk-GBF zTI2rM+7@cu$`*hwEU`(Zi^bqOA~Q%KeTd`(V6>MDU^ zhnHf%sWiL$2d?Ce-3`*dd!^g)Fa(SsMx+edoX^aHbyd&y%i;2Tb^wk*sDO{wR%|xK zYr5}F{qz}ot34dzf#?@}#6tUL_9SFjS92TCpwvtQ(E`p3Nieux&ZA|c?-v-GET`u= zJuFWZD5sua1afC+Iv0;53DG_Wi)l4iiYLVzn6u-lG<_ys{(IgPh^|W z`$PlR)auD4xvW&i_W2v7qRA;XAyF@v8nOG5t4Pw_&MRVGxWx@!3X^6czN22P567&Y z{Q0`S#J!_a)&W}l+)l4wZoW~@OBJyt{=jNs;9}|50QoBpQ@o@Jj`yPdsqgi@>*Vvt zvmW)!ibSI^`PBO#iJ%uhxW*{)IdRy1&t#@9rPw1`A)IKx>-=SI#Gj!NITwLjy4n@U?VKM}VPU{PIukmp@0d)=V@@kh zlb3BE$^JA-lFuh_Ik+NiHwOJ#n1cSe5mcFP!yGw8?uYYsa-#C`!`3FS*>Zec8mx+x zt3k^KvU624h4vPokoK?EwW*&eiafuBFD-R1gOteHAxW50!V3WiwdP(I18S*~7 zv`otp(USLtG{esD$Y)=jJ`S}W@+2>!$hc^izR9tGBstlLhkRXy)yxj1KUCWfMv-iP zzKBG$ifM7pWpHeC&9QLOT8FZYb4T)1H~2Yh%wir*SpTHQu8XA}p~>hz?A*aE72(Od zL|JWK_fmY>jqRkj+_l>L;zATQxDb7q@MDHf0-}-ms>2xSZn$4)eBu{6BmnngWwr@m>vSs$}0KT*Jt+X=T~TQ8|Bm|K-%P%BcJ_hrY%?9OZRn1n5S&$% z+uAR3Hc1v*D|yLw|Ne7V%sVPRY?h#lpjcn>7v{xmfusYIQibG-2<#Z>G!O%G=rf92 zcDi%m)elRK7bjckIV2y+GQ!aye`UV8)JDdhtk3{43RXL&S&a_nS$+dfXl@~HA>I7* zANmFlgY+ZQOtg7dSf_}gy#~Br*qzabke|2aO5Nez$mQ~=+nyn9VR#?k3SlX$@eCYh zjc&dqGJ%v?q+sSq6>>&GY@a?3h+TA-`q3^B$t#nr=astnt|LPp638bdkv9mFhDB0N zuDwDyUiIP{yyrjqc|z!q8ySrnoKGrL{|s1|DA{MXKpUW|iaRp-<(espaO4q&Q%UfJ zYbAe~wQ^#NIb5IE5Bv>T&B;!^Mn&}0p&HEXPct-d83;7Nefpwj_JhHPhoUzUWf_Nr zpJ{b&dG@LJA|P#7HYgXQ@k9;}MoXX9Gy@RV5HO?HT<;vbon$dK9UFZ` zJl_8L)O`|G__~y%6e&9XCpeQuGxG@J5euc}5Y42!wU6Sz=$tg&9x#KIx2Wa6## zn>$-)I8Z5B%(3(&)?Co3+t;0M2>!CWImt`?!{hVb4-=HMJyUE`!~7i9K&p7ExvOpl z&zV62?XDmal12#|!C+2B{DuS78Pg#>8-C(6%bM?rbv$GAlPENtG70Rpr45*ytmw3Z z*7N~@9q}I)0DRw%=dn8xbgz)w*}1^Z>$#NB=VcVVbs!=_`|3GacSjltUOQaskAuS8 z=wlRe;?~o?6QQpKiSzG>FZot2^hYtJ?0AgCdILzX_+GG+=ZXiUkO+529_Pvggz0K| z)LLYb#c9YJh1|Kv{#KTzU_F&{EdF4^eY0ETJxMO8iie`c#}1L(6zj>2Fbm04^{Bq8 zFcK!{lV6FvwU9aK5UwW=&Z7ActE)jk6G(_T)Paf~Men0+zNl!Y(Mu_``bfB--ukS# zqW17;f9j=PQ4gI zDA^q`_r93x=w4Dy3y$y{2a13;JBE#%7150o<8s%F7Mj~5nJ6x(EaW$a4TTMzxVuSO zvDY_9e#1SjUZqutO_0U6`aCD?eZnEk2|aQn-V4vr^3Z&#ZX`S2tG2+GFMOm^{dhD& z9CB{BJkr6-!qTQVnownzRhLt7tv~XTOhC;v?CU&lXU=rzGs_X*cvr1V+3=>$M6(7d zLydzciG4Rfp+uhx6N|Fex)@bts;{1b`(ub{-JE0cPG0YjP6Rae7`F$C@qX zsmsaVtjI^+H#qkv(+*#<_7wE)xb>xmyYO65(PIuT8ZYyu+P~d^~g~!oy0&&2POOUB{OrPCfP{0wuAB`>jO6N&u z>pXO%rjEm;$t~w2Mt{?w#hDFk8C6e`;X498m#WwsIa+-fB*08FC(&J^!1SvWhCSe; zcJ@{oYVDP}kb|zP)bCu}iU<0%^eDXWc9q?dl07s{<)Jg8g4Exl(FZB!oJYLX*XvL3 z*T*kI7nrujp4l_zkcSj@Mmr1+ zYfvVn9IEPyh*XP9u|o4gv!O#dge4+UrFzxrnmB%Oq90jiB45*BrIW!0o{h@I{zsW# zjRLClJmmEm4_xHdO3aZ9Bni}G20ts~q_ZZ2ta`gGKOIT{Z~Y%vK3pCy%N?tAfneMIp0z(0pLF~Dm^SMks9 zB)oq9xIvmwcH>X=hfD<84e*=S=fUaAbswUMxQ8d6bq6PnAaOHEU z`O}L+#oX|q>($=)y{3!#Dj;IMnK=TfaCeYNb*puGdb9?Fo(iDLy^&CrPGYFAYq=YL zP)tgsS6i24vz}xoWHA(tK7%1yUMuGtYY957%k0e5l|>Qr^UQd7|CccX=Y`wpnya(!T?a*rWqS|4U1GuDiJZMCxNVM1JF3>D&Lg43R;$e33p-C8lZIdeCDvi z>0bdo{wifT?k+~?p)dgHBoTS;Ry0%SN4%*mKWM%g0)nC-4Czt=@B2I+lU!H3%_GW(+jVz*g`sc<&Cc|I5k zqp~Ez=Lcx~!_;}*R~I`I!xK9|QKqM+Z&DhTNV?>rn%EnrnJAn?kns(HAK^4}NtSb}GsXNp zz)k)47)h1M8bF)?emsc0MGn3PW`< zD0GfOo6KSDQ>ikQC1lnYsuh6|GY}W3`cp>0jNwgk1)dV!1XsXEYR8)bbE&5`!wyew-Jd8ri@N8&^XB7mfHV$L5pK1f7yCYN-_^lJ`Q%4E+=omc>}FyTBsVfUm`CYbEgx*l6pD{lwn2I!sT z1n*7*dq&^?@1F1<3EV0~z63Ap335assX3F=jm7 zQkN(iECNzd@tteVIbO&2?(Clb`;2v!Y%O9u+O|wW;6`Ft!X&Y%Y4npH*uaN!IIlmy z(TAtXSJ)eQF$hL!)l^?(>v{?lGihRrIXc;)1`==bk0JAyXM>xp#^?Pk+;_3V;&qP3 zobyg4mv#pLbIPz%ME8fD=L6M+d-OP*=y<7y3*eDK(4zy&5y3e1x^{~^^Y z6XwUJP*XUjf7;mmShpNdXu^I?JB!k9A+oG9eOkJc0sc!ppr?p7VsG<=jSF`b4FIFh z-#6m$SfOl)c0~CtPXe&5eFhFGS$A^r zU!YB$Y?3OiFV^}savQL1eyfhzT)eTI0;wVCM84sf_v8KlAt>V+q>%YWbse@guN z*TfORfcu>O8WFDq$8S_zD4q@b9$$=^f6A^wHm$1i5LGoi`d76N74JbTH6epOTbQp# zv6EE2F;cjvRAEHlf66CVU8=#m^4%_#3T*WC1^4q@1(RtD1B|l)3CFWBd(*9@e6G1( zW30+o$z41fJMg%Na)j+oA0u*h|MO%=DYF!o*x$YW>kUy-#D2?pt>OK`(Hp3eHU}j; z>O0gb#bO-!QwtbgJ`sXn^^|8J+>+xF^i$tT}<72yd-UJ#YmSG!@wablb znoheI2n8H0YHw`?zKTs7bX=`16v|P}14(l7{bo)IkCCtUX5Dx9gP@T+txioLU zjU{7QLA#lse(fcDm8y0xGww9ZT~b_yIc8MASa&{HG;K$LGlMcLU(Osd{?U`mW!dZN zpdNJk^PeK)e%0a{kklRGz=GzDb5pe%&O&}hpIKkeVsrjH>LD2aX8!y@VOY;fg+95? zc7OhTPk6Yemj*D?nF8lwOO5T^bYHUG0(Vs}bsDVXA^O5Ma{+W2SSex+ww0f?&FtP+ zzzpd9+^EUpDesl0g&~$+GVm}A=;4Br!||5d8~7jX;l~9nf<$0L=slVrSnwxt7}U{j zkaX2O@?i4qD=M8@>(n_An>+t4FSs_Ts!oq)|J8;(|GI{L`rA#(1x}5H=9q`8m8D2G zqf}$*m=5Hy_oj5%=zChRW$*DBSsR1IkBoXZk&RxR2dBFRJT|eC&dj8&>ecnB}#D!8%7`&&b2o)?YPwdh>cEU#k{~KfSpgetIst zuvP2a<ao>?2&dGpRR-*{$^m8p!#yH?*( zbJzqUxi=l`y*|JB$#9^WW3*RiL&Q6b_!b&RGFurociAp_V>N)md$zAAaNw+`6%fK1 z_%AWq+{P53|qO=!@Nh=3-2TZ~5RJ#C=V^ zIiz0gfbVeVLl1MrPMnF=ZoJ;c)R8C0DO3v6P7iEee-d#O6B)c}^2YHG&f=46tBh=k z4Cqx&o07J!=(%b`<}*wKB$zl$-wj>KWmsLu?oBj*wh??O>!ss1^CNR3&@ADo_o9WZ zYonJN+9&(HMmB_i(_;OF=S3rTMRdq}#?@>8jx>UK58)2Mdy~wj`!NR3vxb-Bqb_*X zbIrjn_PPfk)}>$9oR00-a)VRTw83%DM|rLLVHT5!cdQ`@33nO*y4G_AA@?7O)kjgHS{h(50zG8k`mq6*&XgF|W+|M_GGNrY(FIgCu_{hWmC8$i`n_)@cCQPe?! zmDdQ2s|t{Q(Yt?}04#p4M~NCAq-V?2a$CuSI&VU5aX#m!_XQSw689eF z-gx!Ee>>6Q>$~zcZ-aXt@Jx6Cbkj)jt+Tcd@GlCqKN~&=I3YW*=CSxZ`5EJ3Z!O^7 z4XBUPlLg{Y1er6nGqyXZHRdDap1`(X0|0zJzCK78VyKEqK6lqww;J=NX*RnbpS&!8 zflY^JqEl%c_K?w5`}$Hv;*~>S26!sL&(C3WF~B#2m4ViF zeO*C=+f1o5O6_h`wX4JYfb(R2Jat0A@Vc!dE#4iwv!Zj&9go*14ktL&#~D8xct%vl zGI&M3Sn0pHKy+0iNeSi;nh7FfpeygecMxA zeHI*&ias_=aH6gT73dToyzYz98%F$^&W+~jD=-u6e8UM2xDQYsU>Ci6`(Uo|cBojR z%BvZS?^IMpac%PVn0wNhubw}42dB0xM_})a5U{azCtF05;?s-a-G9sU*XDiv5UZEI)}qLlJ$n{hul%iq%3w(_fY$0u7M6k$*{i} z&#*`4iJGqaSHXF(McE*H*bbX~SPVdo<;>$;O>Hw>5RMvOp4|KZ8mLOWVfaCSUad9O-{7GqHF25 zn$nY)6KNs@j)Iq=7r^cpn0Jlkrz8!68GkZ0Rb=49lO8pVZQXv=`kt)o(Yr~6>f3UA zEIq1|icA%_0fW=xp-OvrER{9x^DLx!15@?2t_PH)4X`i-1jSM8#C)vRp(Buz1t@nsG>^I9<;L2R(0mt+hN>Hp+Gp| zG6T${>xxuDt|CL5bUfqZT$Q4U>4JduD|_uxYF-@O1?-|qs3U~3BV{=7saypsfs(X0 zqg$p*bXcePAD(EslM-m1*RO^aZk*2W;$%4_)Mx=U|DBz9WnDUwX586^I*0i_wWY&vTE5 ztMyH3oFhanws(Vo%XYUZfbSy5FxcV7W~R0VvbqbI~;~rA&!9_ONx1=5Nvc0XFl?I z^i;kAO#D}e@D$y&^t#H$Vi7pwMQThBVdg;+35vh-Wo^J|HOgU;Z;>cS(A_Y51v7@x zW$D$T*jE`gv!PRqcHcj#(aPIFJ!uPE(Djie+H@NA?&1DccSC{TUstVWv`>1(WLB_s zL3HCH=_L^l`zy{8c^;ln@@2fkt`j1YPpZ`KDdF%X^}UP8o*~@s&6k%(_ zD%|2qv7xFU375u~93?xdHM;;vYV#G-sL@WbA4yX}a>ienHZtcDRW>!JY+Q>|9S?MV&Uq0;n#q1$c%Th<% z?9tMk9*XY46@o?17cg{2SgAFe?|&e=j-bo#m89`pD;V(bP!DSpH?B_>_$yZuX?-|j zILvyMO93}iY9!0s6=gvU_efE5IH+)Ct1HumY)=%24*-P_u0Z02eO?erS-{70 z6@ikoA1#uJue@0ek~D#X*3Em+Yhj$D;PA`AxJ-Kpe0&KI^i+J}rNlHcuP1DQw7SaB zkypuwGv+k!*vm-MkFE_9qM&HuRc6EIUqqKhZ9wd1$}}Jt5$*>7U{)B5UXZo|Hc7Ao zRdMPNJP$^@Jr48_#PO*6xG^|XJaD6{UYC|Bqf9Y(;4?BkaF2Y2ojEOdbHB$RU}5_C z6Yqo<&c>>}U;NRECy&#ua+L85E2_Z1T-=4;G|iZ{j!%u}K?UgEp!3J0#s@Wq19bCMTH{feoxJt4 zQ5MFI*_(|@y|iN$l|-~(^Z>%yHbeU@ZSbVeWKhhIi*7^kP)E36vci%8qw%F7O^W}7 zOAa9+KF8#`DX14UOzVv><%qCLc^eK^@|lLaE%HuYVh9hUEZ)scF6?`<9~!a6Cw|Kt zPnAQ7l#5(*_#9^M7+En8Dh+=PXaVXM^BqpwPA53rBUc^TNJd3>(L47$d+iZh2yI3Y zOJhX%+T)ATtxV)RxbsFS&&HGAUEMqC?Ii3i^Xm;y?q-{4EmT>G=B)C8Ul~w#F zhsj&en7z9Iv{etk-4@aNj`;TwdvYURQRiE{B~gnSIr#UOew}X$M?Z+bQ1e+bXjz}$ z1;EMEuc*1B63@d^8|Soj7?VOYr&JYcEs8n~JbRBN;A@=7ppo37?4(%!E>77g!`kyh z-+9Zh$zK=*{gO3ac&Pd%tE)WWsf+#RyBP{0}{!TOf&vZc!ZF{_7JdSu}Wx@$}lTu5D~PJKN;9ZgBQTn1Jpb&LWx?V5291(omn)IgCCjAwLRbH| z`WYRzSLjh`+L+@?O_2wgOv8?xwSgMR#H^ljBzivIJDHOPGT~6J&hYS?-~^VDX6FHx z3VCK~BjH6-ES7eoRMFQuNZD-&ywH-sSZdaGLw6jqz(Fuk2QKqAIU(`kN1`qKcC_y{n0liB$-z^Jc%2BzZ$m}u| zHCDO}gpHdfkiWnPL7oTUCvPBP70FrFY^YK<4d!`TA#orzX8T~hDqp`nQukRqhd(3) zab7FBqf!R`J)7TbW3{x?o(dO`KHz9tV{FLj&10-@S6!x-9~V>EzWy#r{0MZFPl^2!zl>ma&(A46^t1QqqegxBSj#WUg zQQ9waXLzQ@h9m2~q^I;2p_f8K((sciK0(BooHJFr7g@Qn&q-)>5eZQ+RgN-0ME~EZ z?E|T{Y8dkK9J1Mq8Rm9(ZI1a_XWxcl$*}POMT^#%XWBAi9{oEGGx{8&cxT zhLIM{Lxby{9a3++z$GT1rKIcapiS3n41&66K}FLu!fyL`{T_EGnBE&ts>JmijoBtx zM*g(KxD8aBjC<%nZoKW}DrSbvExydnO}XVE6Esf>+`QSrejQl6u*6p$Io15D=`*dM z>W5k7#655hAFfSxNE$Ft>jP5Lu@g8Z5#F%5VjLX-xZ)*C9^_9-~2#Roc72gD;u#V(&4 z85SvU)+=%LJ;hB$NHB$4n;p#BPaX9N1|vyb;$x-A)Rj4#>yC$6|=lu`jU+-gY!c_KGgzCM2%%{8m(IvK)7b zEo_EN;x<@OavAMfa!fpKJHxW$&>;>RkB^o^;!#EGSIl(^V5uf_Y{JF9F#myMn7WRe zDZ(@6zoa1&N>)0`BJ7Rs2+oG*!~9T)ftT6$JE@RlrpU)sHVy8I0`N*aJqD4qTWG42 zR%K1=W|-o!YZGq;h8^K&f(dR7GSU`TQvQ(rF%pUCzb}{C;9Eh@_X_iA!O#(Ts>r!U z21k|H8oY2uRbxt*49i@jwN`!JEGv8=^3(o9wWGF2DdycKu4w9zQyBsn`t5}C-|$uQ1{bJd7Nu4ch_`BsE#|4SZ|(zM#O8ocLau%beg?{ zEJ@KRQXgKP%Sy;pGQly>X|9||C*lrIILCtYIR|EUhn8qHtgK`cz}K7IQ-5y6-beAO-(MDn9ICUcjIf7nPDsl}M;1|iad zYBhPvhO3hm)}B8JPM`6guFI;&{otMu^Pe97H7r7J_BDY#b_9GCv42XI_3WjFxd;t| z!vTX!8NVcZYxQQh@^W-1yx;)mz>akaK1AfHmDf}0zf^c}SdYtuj$2JM5pQ?>D-f1GKWKUAZpJ-p2fd%|0Gi7s1|7kwjjlE~;3&KsOlVsU8fPOkW>+wM99-by zv7}jHDCP-b#e1z!P)}zhVu)=(6<-6>`atlubdn6Tr)YTB5|ph$o2|k{^Pg}GKN)fb zO5%w^a};|(u))HKxCO;|W-WkrADrER+^ltXBt9&FOfsas8szX?R1!e%+jRB8^)6+c z`mK)$`5@(U?i(4L-q)wcA+okv3)-E~?fXnTy2y9+$rsC;NW36MOD1iIkFfpN{<0JaLZ&{B`=Ok_p^@ z38Ww3o-%5uLJ(sNK>yQdsSO-xPmDMb6H$D@RXk?{huxs>GuuKP;*B(e)`x`4SPc_u zX0qjY_t+bOgj!Xy(m!L@V#BrIRrNrzizB}Wj1qglD`eDZqSb#|E~T+m|hc) zfD%wcj&O0CDUnH4Z9-j<>k>Fuua67wx~(jdsdFsJWTN-z5Vw60=32Y zy&FYT9ytOzjsdB9Xn0sgA!!QUq(0oUaMEx;;0gU8dN<=AF6ma_JaIqlG~PJyl1*#( zCE>!wNoq!_CuoYzZtSOf&YyID`|w(<01wz!U|q9$$G*n_&rmAC2b}?F?GlA-VX1Kz z0D~d~%K(SX^q=G%kjd}FQ=1OV`J&l2=K->vVF?D)?BLpjoj>1{{rxkhJydl6ON#}Q zj}_C;Sw)-ecaWHs^wsfN(->C2Ro>*x;YxX&VH8RpXwRwET5FVjq{c3YVD019thL5} zMR1nEnT=4O3d@M{ZWrb0}@>IYhZnt0Dj{bR`k@U7L;A)uf zF(y-&Y0T0Waz&NDq#h+$xPbBT&yJ+O=cqTmo7+4>Li02P|c4t9c?*)jh zvR9&nNW-p10tx^N2CdCp!&pyr4%#xK_Sg5=9dOd7hJk^~+5JQ5?(3<#4x%GeSR3$O=pDpIUVyu5&CG&D)djVv+Y#;7IDWc0kJF=@8+*cu5+KG;2 zmHRK(h9TURr2|T7{AHhl6x}>*u6NTX`HzM;hNitaDy7Y#6b5%XO}!fwN~L6vtv6Ic%_1R0>r0rT-Aep*G7ApwTsm&jEB39Ss6S7sMxgC4MRNC&#B|| zZQf{98v9ogev>JW;?TL+n{kWXaVXb2AdgjrgK+*-d0TSooc_Ph#8?2%5nFD<^B6R1 zX)3P`9vjar;lvSq6}9-9QI98~q&bxgwvfX2^f%Ipbcy6&6DvChgM0cbb^bN|xf6wg z>tBR)bYg=WO$5(v%hYM4I-Q5OHv(~gyZv*41AD~VP$3zt2|#y)TuhLT$5G0RR4x(! z5w`i~_VF0~=Lo!EPQCDDAg0_Q;wSr}wJK3~QPLeu2zU#3nr+!r)yj@4CVdGN62$JE zKq4GXz*0+Pko$N^NLi4#GrU|UJ@`Kp%TGp%l)5*SbuF<-kvz5o!C2HWA2cRuG~wtt zBo%cn;we2b)dA5+k36a<{jEce%J&biJ@v{<<^E?iFq9;vp)w0k?cxZ&kS_p+zEE_( z*3kyg-~5nKA^C~iLKULyl0zHh3wFeoQdY!ib-N#6qj)Wsh4b~jO|%fHg65M5dMht8 zs7@t8~YH!M3O#N^5TfYltV7Z8RTu1l? z7^J3gC7Rc(`#uXNDf9+xJ5XE%={Xjb^J%NHr;V6RbygEYJkFc3=4s7&B#%`!^=)pz zcpGD)fdC3L-km_Z-4=kpRD#<;cy;6;AfG=8*eJIcep^b}bYX~(Gf_cvlwEbN5QG|i zyH-1Sq>`v(d8O-gqhJrv?*j%Nps~R0V6aQy1k1rbSVSU9r8anbb#hm!n@UFE;b=36 zeAAJrIvofXaopEjObScgg6&S?{sCSrjH9Itdfdm(l|eOmz7MFmuM_ejAN37``_Z;0 zm{Frsz$G5A8|o0_U$nW7o0~PkGP`|thE1*sYPz#MU}94(-2|f$`gh#&UL>2`s7K2= zn!~H}#QL0pLZoV}NNonljHaM-%t~!rnZx{UG*=>~c=A+cW*tC)Iw3eGf)xM=LkN5h z^0V&)9>4{154>_xg51uV3W6Gm7EB*dq}||@w0{$QqXXsV&tiBmJdIKL4nKl@a4uga z)esn#f>FSATH>GxiokjE-kdXaSRxY70C*+NX384?MQbRDQD^&k*E@}8=6XI?Uwt0Al);0U3Zsy+6R-+Ol?$-75`%m6^l(25(?Yxo z=KK_JXUdtgyx2(1J`zjjc}!gZ%hmPc$EuEo@f*NQQA{}a$iLIw>m}u=%hMaBW4Xo( z)9k`bU_C?(IZUd@(Q^aheX`fwp>LOMc#41C8~0ec8*y4%E9d z;5zta_6Idd)kwelp#!w#_khbmeo^#p)@UF5fkYkMsC?C<{fccR1mJt9Ziy*s4nU#F zsVl>H_z?A{L~CM!_gUAh#CXLzTNx5#J!v+9&jt{4njXBJ%GAvSMi0A^e z_c11Q3Drx?_5d2mT)tcef%TBXfec;i5l_{wEi7D(Fr2{8!dKX1%w9H&!R*(>W zAhMqD1cvi24y!q)8dZ4FMKlEFk4sDAKb*{3?pam9>viuY%>#Ur=5<(cEhqzwamrur zZh>FogE(_Y?Nen5(W8D2d%vqN@a5g@6bcu=-_kQv2exoiN%`rQNCMe0rON0Oz7 z1jimCxAdcjkTk0C&UCt40R_pcH=>vcW1s@L>3!n+wRkkqdo0G8T`gRq{oI(j5*GV^G>hI zbP&0JN(@)W7+8Osdqnk~Sz$@QQ^g&e63JkI6Gq>H)%7CqC10{G`OX7JmfdTUQ8+7J zZ^0I}Z8GFXU17bVfeUJdTJA7EaZWg>+Z5CNZ+cg;ZA{z45Mx1|c*C=4q`{F34pX|^ z=Y(e)q;P?o%wd5WqRcFQL7B`i-sXmq%fKB%MW$9Co=_z$m@%^inIwh1qmC;KF+&|r zA$1O0;K1xb@TT*AUD`X#bx%U25ihh~JY`+;mv>A_p0qri7%Y(Y`U#4}fdjE8<)RKi z_sS>9I@E838K{BidtSk%Zi=3Ok-%6xoW?ag{Nm)JNo??1oVS;m=>zZ$mA3~W-MA&- z3=R93`YV!GJY)Wu&ri#go(7o_k}@TEf%atG-r$Y(Pylv7syXrW$TmuJ3Ah=JB+E!g z_{cDc(297%-%yNbq?Epp;)mO_lX7F>kA6nYURM||m)K?|CGh)9T@6|T9DRXbViIib z`_I1ukjeYF-vGO10?Zg`b$=FL;!5E$pS1L}Zo|%v5+vS#bRkPY@K%p|>~#|!JsW9r zbo{j4w?6|G!4-ZA1w;RLR+}6vZw^5@a9NqUGq zt>aom66o)lQu5mAIv=Ipbbd_X*!fXc-wSH$nNM=bdvz=5!FAtTn($IrXvdYBbm7hDom*i#wM z`;MB7fYW;O#+;!Ic5eFOyeeSiBBo);>o45>@H&$}ZWG?VX?;raAyg^+X_9A`s9w1? zbjvHZZTZr9%(*hCc29Po`|!&rNs#I+%_ZL2rp9m3mY-+zo1g1 z7Ju9o*DAvjkgNZL=~*RmyChuzT!f=AwaGJk2BZ%C9ROxHVY?Whe7<4ujHHMW=hdEw9}!Pb!EU=@N+*42!XnRQLiLEw-s8BJX2<%|HsDgd6aFVyaz@`393&v6>YD9A zHp~Uc0nG!9_Ef330VZ*MS?Q!BZ&_$f{mJb=y?i!y_!2vMLs#Ej;t)=M`EhcWkbj%t z1zf-errQav?m9s`?6w-*WG2TehT9iwO0{`Y*TR4t{Hy4ua^#wx`c; zgDb{^8Wu_3dsdEC0O^*0lUU4BuypRwpz^T+Im$5Z>`FYwq~V)4iaCL5*8V;Wqbv|9 zFyriFQt(!}Ta6JikxvJ+_vb%tBg}Ol5JnKk3OLfglu) zz?>z9XHE>3_IXQj23b(W-?^IVJ3KZPY7agES!}BK}?ZaGk zmez8-3kKC+$6?@6s!@MiG5UczvECN&+*A6M?Si^;%3MtMH*m1?(Q^ziDE3F5{&ArG zM==)t*>pC`U(6}o^j(!&vy(S8#i7)`9^`{Rf_0%W6 zgOqp|%xL%Wj-RtWEO%kh)w|=;s}7Vc5Lwf#6Sf86E?o%uMhX+yM;K%0wsaeQ)?D&} zB*M4bMK2IzN~SZ7{_wAC^Zdow&H&d>dJ|fZ1d6tbWytkmx-o046#4#RRkiwHq~7sM zZv&gIpjS+}lEaKXzLI&~f#~`UUGIk}S#{s@HB!iy27l38V8iuGpU5?3ytf2!P9bE( z1V)`{EoO)QE*s?JTGG<6$B3vBh9>3q=)3_t1gP(x6(i?En3stM^b@}-(ng%&;Oa3r zzFS6{^#e&yG}H1x@_~-zJ%lHg+s4%R6C4ahZCPROm{~LudWha~RJBWur6~8QQm z`5C=dUYra6v?71xt$~!jKl}xkHgUyIQH*<2Cdh8{A9m~ON6eUkK|MQn6xZs}!N2M@81asIKUdxO$3^G* zTBjdNY07Dk`p>10W=|1|QtFYgE8kAo>w>>k9_)Y`W|8;mq5;%V_w$BpUiViUm@n7r z0+!zoyjJCbnkbR0yM{%q-aGJRasX0AT#Ll5se@WRk^11)!;>Z7fCp|UaryRd(|xx0 zxbZU8Pw1;CP2f{z*97$Ux_O_LLRWYOobae{{J4ju&fAiup55cd>SKge(zeSyaumGe zKb}F(EJ|3zA6jLJe|@^7e*UAy&y&l9QX}Le);qeIIfP9N7*F1|=P0EzL!yc2h6W@O@q>Jynm{1S@iAc{5vm7r_@S znglIs^!f$AxUQq@_D6XAk8OWbRxaHe$s!jkWSg8_J$g)yTn>fzSA&0|T+7nifn>$G zFqN#NrsGqkK3ERN^68`K4Uyx^;zrsLu=&5W7feN?=uUjge~pbbnNC};&;f_*31Q{F z1QHbGsqbSDXC=Rv^F2`hq^mB*BTA`?OKEmnf8}!}#UJBNNOr z3JfzA#dpC5@cpzculL`ij-?j1Dh(Y(Nf}Y9i)11`r-&+{c$YbOq+5z=g{-r zlq50oY`^|!c$FA^j{w89jI{|D*%TQ!qoe=oT(ypem0bNugE$zao$wI#XHWigdNLT| zEM&=CE`~oh)l5*`j$q#u>us~dtS5W6Av@@q=b`MxXM(!cpyXLvtSx{+Q;8b?GzFei zeYpZzOiA;H?LBJ8LqR2LO!54Amb~P?8{YOPp3fj{T^t1NQ$mNNbyPft#vxib7MgO0y*Ot#Z0dZrP!6fRr|2F2H~NH3asQQc5>lm%<)Y`*v;oSGB;n(6oy5O?+ZYuNcob{XS(_umiVhEt ziuMY&(yDrVLrVXR_9o4THIf*$8vCzfv<<)9K?U-03*!HoSp!v>HynO`ed&C8&-s4D zN#K5C-uDa-XGFr=kr!!=UJ~O4`SkzpQibCCA%{8Kofe#(PhYAe3EJ86R;ft!$MYuZ z%vYojYF5r=dYPr7H0IW>xz4<-E1s$DF?};{QqtWeEeHbA-QAti-QAt%{>`=4 zf37*_IOoMad%xRb;24gf!0+k%`Cixcxk6QRpbPUY3UKh3FI$|u3hy5n?@SfQj09e6 z94{WSHbu$XPm=%R{K^+1{DeM9Qy@C~?Qmj-&0KW$+)!Sq{n_qDI}Z!KLdW?vvLfql z=5zqZuF?W$joIY_>+bcqS~iMBC9@zNx9^v|%iVnSXq~|Prog59%XNU+Yq>b}m8E6FVTj0+XD&?@cBsnZ zp{`MU_M#?JF1VoYp7HLYBhT?!2T^0%LTM(~{lkgR)RK_z!+ibZLt@iF08T1{VV)yb zn4H8cKOrW}Xh!)c4Z3>RpV!Zt8Psm2d8vQn8;i}5+P)o5dryxPjvWgeI%}W%I`zCL zy|~9P$mk64ShbLFn9FH_b+`Bfr8>qW_!MtM@DE2aD}$@}j}a8*iaZ<|sqLrz8CANi z>gQ|5@nx%wGGy(|q+1L3?3%?{rmpW7lg!uR^W5rpmsCxzSx4bgl4weYYH*8e4w|fu z+8*Hxn_DUx?gvz6>)q10T-qWXZm(m_mw%-cj(e82(6`)Tr1VY?&fo6O3y%~iYKl*5 zbxqKk#P+Q@PD-^m-stLDRWAq-Zooi^QZ{*x*BcpGnf2q~t@|_$%$Ptgs9@?_?y#Jy zKIk4Wdl0I(+YeC}$I>A35kZv@6}w)aD5X%e4VG!Q!6vcU{`3F{(>`WPiEF~RQlJVK z@6Yv=U20(iiAg+_bNcWK49LuTqOJ^YTSzKM8lKPb*yP@`(Kc#|g|i#dOl$9|AE2yl zUn)*HTJ)3lsK9iJr5JU+JTRucyJ~c6TB21wM3>iCq`n$Uy=}V8|GkuFc9)+P-HPOmCMRGU#)x5FG z`d#paa9Y|*Up6J{2p&r?i0&)SO^-;?-7#L)d9C4wauY({RNx>*l62_^`ETN23Uw;| zcy8?zSWA+ZY5_&NYArM_I$_i{6Jtng`ey%TGkRVDc=R~oER1Q+Jfx;{Vb}c&v9n;kgDqP zcJh2oo~(r9G5C|@4z8*CX!T{2swIm9n-dXr?rPL*y!1J#qsIP_5)@=&7z&H1gNS&;v!EjHwvhQLp-yiS)%(u+73`sz!~!gH%pGa@r}qsL6t!|*Sj|a zI={2*g*Tb1#(u?1DszWh4|dBm>baXPBR<(C$CkIuWPXWLH8W_U-b)eJi)HK>H^sa+ zDY+KK_FQu7V(2j9a4Ma@m5xg2_MT-H{PTJ@|m zyxZ^BieHWrIQ^1G9yb}&-5Sf`cCql?ENk?-z$vm`;^{n|SJ;Q*8MJxvI!LuN7X9gg>!gw}9x4^rw z`5w;JYxG49nAo;}#GxsOgwfxuw7n%~v@2IKPQT*$Iv%z&7<`a0hFHHJ%*e+Et zp_&}9Sru|QIWwiLvuNwWy5>kxG-0rK=B66ZB!TphCMqBr2 ze<@eG2#e^vmajQVrJh6v^f~aC5_&ctmQ~HDqD|JU*ElHU!3ymE8tcXTAS&5f>7uEr z!!)<&wqAO@ugcQxe9-x(F%0AJpfU-TwtVt-ytGQ4s~U!JjHj!kyDalJbi6{z3k%p} zr3X)4g{d6(3ba8yS3-ppH`!#2O8!XBiLr|1vpAByu{4&czO1$d8E}$J?58MN*G6aHWATglqH1``%@5kHr zCogujfP{QU-w$ET4=i|9o6j=WZKk^Hfv8>yXjy9o;e5T-u5gz8pJb_!oC-Z%kV7Gu zrM$ruQmeAE-T$2z4+Jm!hkn_gV#jfl-fv<$Sk?wpW+An&d}+!VVZdLC?tXjN+EWe(yTkNA09~d?hc`kRf-W&JD+uFNwYm%trma;s>1hX zPx?<+$2aa@pWF`*ODaxR$+D=Hv^wu0PFATqowTHC=hM-zMay4I$^TaKs2jIntMkI! zywO|RE};Z%NS#Hqfm!>Bh8rAxI>Lv6Iy6i@B-wE#!IFVRhxn6{X~Q|+4tbEv2_MF~ zcnYgZ9;+7o?JBI6wcY9zPgYBU36gf}?;K_|EpWHEbmy)=`8w#b6>N0|qmf$L<$XXe zHXWj}6r2pK8Z5nAVkF4_a1y6wR$Z^FY`Sx^9W#ErjJ$jQI>uJgDZ^911#>uclUvRB zwQ3GQI(771*%td`+^mMmnqpM-Zbe6!pz`kWwZ($nLwha4#WRYk|}8 z%0pGZ<4q_;y0U1xcUX+q~}ZP5PXD24oAbW zfWYKrV7EW>JK)(s5=&7EQ1u-^z|Dr8VB@PjCX4D$j$%>jC2R{(vb>M!sy!=-I&~qL z{Cx7P$yQ>{GFI1@tm>UOE1pg%=j}=r?80&zZd3+)Q^ij1>toGN@FW)@%-St`qd(=0 zqVG0k)lsu8)iBqd4W9JcFdo)c%f}_QoTyYfaX&r;+rod|q7v#x7s`VbbXrFfa10K% zzc*KKt|Kc}NLcUI27M-)C7_7O8uTnSlX&!lKJui!kyLr}qX8!K`lRYacAkY@o$@5lN4E=RoP(I1 zw^ieI*mEq?$YacP=jG8wEcGcj7N6(d9aMUvFxJ@bDm-Ec7(LuwYY5i@)fi$zV^NcA z^G&#FdbsJcOyzc~El?_HMUI>L+S;Q9mldnj2SLC5R@B^IWynq9!$93vBdaIw;ni*g z+y}Sm+_bz`+FB?|WjEmTQB%2DFYf3gLuIhd>#-eGRm!vR9R@Y!+C0^!oudkRkVf8&%pG0QK zQRuen6F8!(ckU)C5(;-IZ)D1opP~%j6BTX7;5Xdp?IwoHKzoA9`>LKDRwZi;^Fmvr|)yi_^E|+cvtM6+&G;$L9sF^iZriCsOk5CB9>S#!uKcK-|23Ns_xJ)?!^xpjwm@Nk@Sa zqhPS`#!p!R=Qr7k!H3f~!kLg6h%nmJD(bIghQ59~jZ4{v{p6qnd#xTC(o&&eeyE!4 z!rYDbwE+YfLpp>$bT0F`9iFexu1NAO9awRMhWNoiYg8B~sX#RHU2m@?Iz?=opn0>-y?@@kKE}t(|DC zgazJ#slm+aY(06q=zGs%uTN=;n6vlsa+Ps3)5NX6UWmP5sJY5c3TW%L>i58uxVSXZ z>|*G|I^vuyXRpE7knGhTg!uaHYs(FX6vEB#O(X}* zTe{@l!&RxXl`%YqSXhEL@uq0zeVe_HhsQS#l{h$2TV>AsL>lCpyQJu{Dzp*(-hatuDV>G5GAl5AL$)dGYL4 zc1X&d87=hqrOtPS<{@)_t7S2_q_wM=@4Vw0se2ALkWyu#d_=#(f#xl?6&bUiXj*u@ zKnE5f8ttXaIG~MX_dQesO53XM16u^3a(|Hw2U`2I@cR@!NDi#mq?qRnPt-gRm1M7a zHSF{rHZTK)lx_$Vusqi1C=?_f>V6FGdy&TYNnyF&V8||mc_)u4ZuTBJ%5^(yNa2ep z$pJdCdh3akL0;6h-OqCQPyQWGPAN7=`^C`X!lmBUSrQ^%S!~ISSS@zAp_m%w*s=cf zImE3xoY`D;`E)67<@s2&AU&(e7?%>UsAoh!>4{NZ1jj5=9!l$0NAUcNd!VD?+V6jwcTnOD@FacMj%D8LfyHS@HcLteyP8kHevCUAGfel_|5xz{OG&k3Yi@P5-2rf zM$`&m@@Q2ScqiPpi$;Tq2e2y?R*=!>7$Ia~LNuus zhE;nCo(r$f>E^;EIrhta?h#d-g=Ic}Jwk}7Pts7@QunQ6-_>;7Ji+M>cq};Yl_%e< zL49sft%TxXKRw@bw6MIjqEUl*;onHGextl1D^2yIhtp8+Ii9J4H6=aWwWQAQyZLXF zaz{Z3H`7r9$gc^!7_Lv(PbQ@Eo5%_eJ(?E$b=R!hTB06(z4n76vH?F;_LS=R4C;PL z&y&uacy2aY>BuuN)^9m0c2`j>jvBN2F7jGdDDcAV7r#$f}wi(wM2;JtPP0hieqfdwzx?Vg&*1}KPXFQ+QOVKoqNit zgZ(GeQQFU3`T}lxr-wppOCX;F46#ZFt5Wr?uOc;_Y?Keff@jUZ6?}tOx1Wextk8br z^=Zg|6sxe@?*RF}9}%-i=p&cF3;en1XZqn*^!TV?1l}h372Bv(;-!5rrA1?rTS=#d z-SMw9qyR?}BA=%$s<3L7?hO;v-WUX`cE51=k?cgN>o51&n_@7bzYBe=N2gcENG7qr zu64Tneq_^rNo#vsETknbU{GWAj4Ee&Wn-Y8QvK?}Xwlk*)ZoH6z8J=Ng1ki4f%yJ= zxI%no-7qTF4<7_2Vr@b%lJIGJsyt$ZR*Eh&S76aV!}H>xwa@{it4nXPl|m`0lFSLv z;1w^>BS5Y@j6oxuLRoL-a5vbDRv1Z~>w=!knc!VRAPn!V*^3995RI>C{M+$tAAtd` zx#9c^%Uhv-3Yr$jx?zMB`9N8LLbkGG0>(%|t+RRiJ7Rn{JnM({&Bx_1=e6D~v}d+z zJhMNwM|*{gv(bkf(IWS$-RpZY@!i~+4Be38CkkmaJR^7r6PN5SGlCcKY(3QDcZr*9 zWfv^ppL1&^AaK#te2I3!{7(EMr801A`%9Kqp zXI*5QD({?6X37lDOMj8{swVA(?ho)h7*@I6%<*LxDl=H)CmunG**wFZ2>e8%tPYIa zZm#TZ0{L;>@Ro{^L;|jlo+m*9EO{b18L8CZ++bM=3R}M`WzoHvWKlD=PHYBG&=x}k9#{5SXLd+x8yG_>8yshbCx`CswKxim6DRu_#qP9tLSZiyULPw%u+>Xa8C$#4C{rO{q zR$T?#&esfV6`Cv8#Jt_Uo+Q^3616Yeb?|TqQ8p{bsEN_UtTZ)_q&U{D+AFW}PU!`jAi+${^dR%yInCT!5Jv5 z-J!4~M@k`ok4D@_9iaz3NnZ+EWjsbogbGVVRzs~)espV%rOVrK!Q0Ip`0N6>e$H>D zge)7*6`DA5ZC?ckR$#4%>e?n2`%zmo0QyUnqD0%ILq~@t@R)LMV?Zk$7~6;{TG2|W z0Ji2THPaj|1D~kO=kG($VqI6V!tM=j9ts2+h$fD|wF4^8c~fSP`bj(IyIkA^FKMct zHFwt^-mx-j8iH;WgPYiI$LJE=qoXYLMQYZe9cMA1Ho zrpoaN&y=I63ml+S^UB#(b8w8a-`i^K*lLTs5`%MUymQpS|Qi9ul9lR%b{Q8orE* z$eXBQw!8Use$UJdE60zIs+E=T!#*vbK_iJ9kLm5AIb|(-kW{z*y8P}(d`)#e=fv$@ z(1L!xkW#&?js2>OAdXjyoRM?f=EqlI<+SB$nL|n*fqCJ?;8e)Gv1w&ivzPwx4!#$~ zS}dJLF0wbv4#!$n37UumKgfY*Zn2noV{Kg}K=6lVU|ePpa6jg1`&7inqcUclHL+N; zQTbLEMNtxycx87xBM%%$dk~|2v^wTGT~NLl`0mFo_*7w#u$?H&eq|b!oGN604T@~qF#Cza(yrNOXIE(q3l9K#u5L> z7$ZoDX72JKEB53+*ULzZLLo;(>>KnYArVuV&7hz8`j@w;U)&I%M@Ow)2(n{N9vU4i zM)sPj>Od{`P@x@)-ysE4q+=?nLuWBr!=JveO1&eO_43w@?v?yFN)qqzo-Fa#QWDuK=cq16N%5l&y!NUuBK#UMU<|(B|hiCABdzAvgdE%*62wDXxwK zW)R1AV`JiAXSDdZX%M!~lsnSV^*F}*7V>@E$f(i5JCYw(zOEAd`OGpb(1Dmyp zB3=dU2E2L)u{-A~Xh^F7nq`kq7O#%)+Efxz*I@h3UBWhJ6S8@7LgJ4~@G146p>}i` zAI0Ss)2lngy&@@1a|8GAK!#a1NH86NHK`?DHp2M z$egdd{x97Vvb%Uip05SjMn!p1fEMP63V1#KT08V%0LT3aF9sOC=0E{7W>v#1>YEzAx!uhEiPD=G^rHkre#M+>c*NXxsJ3t@Sfyg~9 za)^MGJIJ~kBGkcPmxMlv*pb&OKzNGM9T0{jN*OhCT?FpNksmygDt?oSEf`&D%8;l_ zl-XXoCzUw)t$ zdxIwK-l`<`A2L}Y3i(RCuq8<>mX%=mK6rDnZ~6=YlXGiEFbvlioZF?C)GAXTIB5!A zp34A|7*9?DXEeO|{$2b`jk&1u>5RZ=)@XvLgVJ@>b^iD(=Y61NDg}QL*l#B#zkL*n zhs}xlL`r9{!@tY&ATV2RtJ+>Gg2~)6mHQMAau@}Q+_q#4O#sgWoB!wGe{4vEx3qDbBhWa-=|A>`rH&7p8Ps!vrt@~oeZf7yT=Z2m{B`?!Wq zLb3+EgOiktD~@A4P77*}{&@oq$9>7ln~J21so=CiiApsS1UY2aMPDoJMkN$`~Pl-WGkm1oo@Zh3Wb3i;9O*nd!? za%{q4E;_@c^S-u)bSN;NEGx&0%$MQUf>1UNvqeRLT3YFDsGY09zm!A2aM;iCD#ojR zc&d9*js&Gc26qv9(VuREq1_-=1)^KhC}!eBsLo9+gE%+rtGb9@(>3C3W||hS#iT+lf@BJ$Kfu^VTH72*1$b`ZZz0C z!YQ)141Uc(Q3F6uZqb}%Y%BC>kjGU@V9-acG>k_EEb&S>0@vXtzn6F>0PcJgCkGq4 zO#%RK7tr%(za-Dw{hEQSry4&&BI{Oa`7mSVK`Z3Z!SrJ%)Xozk>`i1!Nr&6rEie2j z?T!3f+ACUGkx7n$aqIZ{6#9-I-)$YV3P!W9MO ztm(?Sb~Igq-1U1W8EhF_>VH(UX(~f?^WWcd)a_N+@-!NGiD>FBJEt3pXiMH$xcs+} z)*s~II4karHM8qxGCf;}ujZAoomOV%&y=GugG|Dzlic zRnC>kx@!c4=NSO9#+j|P&vP`~2eWrbpEm?g=t8+tP_|1mQVytI%UO?;t(@iG`=N5x3qB`w)(Wk{#_+ezEd%IXS4*N0;%H*y=E4NwS_XM2j zCu3*S%>QrNnvmz*#D4pz!vN#yK2>&d<6}x$Slw2Ps06hxRvN12B3EG=ce*tFqDN=i zee+_thnl7Cqh&*ZO4F>$IJF5xU!&>3;$fbQE1Wm?7??F!VC+ockuw7As6kmLw?~nQ z7%!$O4C5}f31X5Ojq12gTSQdrzr^pR1y8Nx^{L2CN@iy+AbTiT`p(%~AHYj~2)LnI z;v90T9u>hH*xlXaNT>Q!WYgnFh8t2ZqITfyRYC58DSDeq$+_Aa+imDpD}Q3X@uhW0 z)XiU2<8J%YY#opB?ybN8yA#XutLU!jlvTee9NEU=g?$7ODS#48aQ?+y0+c6I`9+iVs<4nsWrS5`2o z*dUzGq{C;8#|@Hn`#?QdxBI*WH8GvQXn4a8!j7A_$CQQDt%pM|JgS*B{AKm5xsuQ9 zEx`Tu8>$xX`H^bhUjC}cj6d>3S%&Y)6aqn4*U&R0KI3}BE7aMF_k)GIY6~jAd0O35 zOFE>KA`c%H%=ClSPoaa5R4->P$11O$wwgAWUi(i`N*wb`87T@_t!Dgxs#~$cK;6;} zP5#ZSrcj4{L7Qz>D(Iq6(ZVuWe%xePo_kt#tMg+X@eT;-(^lDX$Te9%#qVsR@9E6T zvN9KrPPgikw5+Vd!IWKux1|}H1eNar}wde*0`O?hW z#oMD#nr6Ue!(Dd);4ntSwS?RlB$)8|Ib=VPxy~YhKsiJzk!%DB`2$|_fYo${Mt(k< zcmVn8jr5V@-jsH=1YE1nN2gI19Yp{Ywf=NW02&b#rMypCI4&`E5xq`$fWlR`SK7A+ zNo7C}@E#-e?sBpJN8>VUPx&8>%j{ybUHi2Md&hp`E7AQ*Tgn^w)*E?*8JNlVx3}s9 zY!6jP4r22}?x@Hlwvz{}IBt^WyVJS(NDginy^lAm_DL5rl__wPJZEl2tco+HgI3sk zhxz-v+m}12nmqd^%R2||GR7`jC%@cPw+PTb-dw+t&~}?VZ$tWp!7yhNIe$=Nbs8a{ z5s80k=@wql4ZYFF05gk zb>zd=MCvTIC2?bcF1sQbIsA}xnC=F53rqoP?<-Qj?2s>F!8Bciht3S>!frtGf| zip#vpfA#(YcPqD5+|&G@*Qe~!UeR9Hwz_IZjO+bzr#izvYRd*!d9jpBLp@LSF4>NI z<&X`1oqkwSQ|_b2?t#LOuwT+s4@2Y=%jdOYLN6nEzmI1eMZmyxL}NS0C{tzC1Bq!J zjait z0rvRHgaTlPm7IIkC$h-eNv@0`+VwcpYFN4ZcvD|GMbesQM!nmmsw8QtJW3fXM)R;u zy0YaGZXLJ}wLQ`pl19bZq;C8Uadk7{*iECD=MIlH`eG2rmtR~yWm1eboA8mzBZ*q6 zX6(gUpLvb?`);-^%fLFM8>L>tvrXd2)|KfjdveD;hr^Mmq1ea8)${#5<}9;1+2_Ix zzA_SK<|4Zt>?&nCPdBsB?v+v8xzZPHYaWs|p`#Yn5hm$9)@dM~9`zm8qF zF_6kNdpQLE>?5~UE(sg{J<~lbT1+=shw1}9NSud81WXt=X&!u>9@T(VtLVCwkeC_> zkgq%58g^1WN;+G_Ga`4{lO{*=BY_dJ=oS8rr*h2rO)lfwG?%m?!eOd!R z9&o>f;eonqk$rc!^-Wn&$%ObD`HKQw(6!u{0@mQj^Z7$yHPwScwr7y@r?A9Ps4ZRB zqAU-sE!F1zpcJ1F#V15a7LRE5y7_KZ?Q5Eq!idml@t3!4P$)`l8?x< zlXW0n)r>okK=6CL2yYAr@R|)@n9nC#+|E$la{pRCzEtVCmk$=1IGJJplrECdd~HSg z*LD7ub>u2`n89k*YFF!PC0W#xzs)-W%`G-caczNtD>IqW zpKeV?9Vc?M-DbDctTAacOhVs`2MZ_e9C<_lv}PZ`ro%NQmw_)BL}gTzy!A$5-Pq_dy&03JYk+KSrtbKD!~v`g(0C5%tdRwDe3w^aPUpAFoav0uIs$a{0(VANQ{HXu`A1~>4lXB z-EZSRhKe@wil! zMj|kMR!_6FJWi|{_@fJPlg7_8GY>*2WGndDk}3>J8d%Ov`_kSQak(ha=BZ@VSL7~C zE!?eoILu}b;o0pZl2GE6Q`G@axGiKiXAKcJ^qm^Kk`m>AW=x2EJ~NSItn<*Nw`R-6X%4_#_eKAL?rc#4}yA z30t&af+<9(;o~CSsZp7!#y`dRV&BotNCjO2)*sRCf<1mKYtTk488Z%fO={;HW38E| z`V)jK4__immg5P5SW`xE)=P_FhYUaZBMf14x=D#RGp`(*QKA+lQBa2+2aF`}PZ zS^};ZB~sNWWe((MnSA$3PHbViHnmpaQViDb{os4@T79RqrWl2rx%6~c$={|UoJZUR z1_m#;`QP?zXNHY5Lc`#@xFm30FW4-GN2-PK^O0$Lj@w`kA*id7;!}VW#m<5FUFukh z%dQXkvBug^18m${H0|=J=%RXGO`ife4i8^I zm#9kIx^`*g1;eq>;!R;vKfl9aC`--Hp9^cL&$`CRO=Q(xKi@YTwI9_yrk`qI8qKz} z>17@5_bpo@`C!wk5}(#`PJSS;LfNXrkY|@a#*l7I_N#%1z^ovT=>+~85qSYRcfo=+ zd>8Shr#0&f-Qs=>w`Xqj^m%-DrZ)DOo9FAM`#*jGZU%&r2Y z9V{RH&D(QEDh(Ys?w5TY$0B&4gUK!U<_OiEL8u6a(P1`UF6SL@{oj?5A}ToLEX(*L z(K8-admJbDy}qQ4GNvPZ`%2KJRcjp}LQWY^_@+U%)^IWIyXAe*(|FQ< zwQ3TAkw9Qf7&*gxUtO0@Ja(Zqz@e6!U`-lK%BYBlh=Oid289_V+$!iuX|AjYqjO~ z)%_ifI@??|Lx^eD)fVes+@~b#FSEc)&sxkO(B6(L8l5tR1^otNqTx~FWl$L-c?0R9 zK}ZqT$L}E(eiTd7b>wm#(pdsn+GBOKsH^Q|u#^-Qiq$WpjRLrnY-PjT&Q{3`tDE1R z^&on&(!Ql&Q|=z@KMgQsOkPH(sJ=Q@Ga$H(V4Yn1=+zC=TNY9BWOrrd-flZ9-cKQK zNEl_ymXAbrixi&YB5C_pV1WSC;i<|6-A1Dt`taCTJY=*cP%p@E*3+$QXM0^c|Q7ozz zo*<%`CJ=>oTc?`Hp6BZ0;xVWz`DLr`fL5L$50T*3ZQ~|*yI;@g>uJ`lmUX{%@IApO z&Qo=O4|;z)Xh{G)3Ovm_u>zfCXIN+1XL;!pZF-cV{!nL>Vt&g4ff}CntmDtv@FXIa zKXA+tr)x=Q=bd5o>R4K!eZGea9k;@-{`D<8qS4rxugrdYA}?7#c~~< zYu)y_5wFVZy4Q}!(uC5Ep2m7`pLs-k@R4(Hpt6%K*f2Q>JKQ$m`Rd%BS+ebIsBMXe zHBBYy;IsE#H3{>q@yXuKt*o0ixSIxzt8=5zt#OA8v+p`;;PjKrI&W?Ok>=GV>|5)> zk2qC8!(!dMK;Og+mb9~IWYwQeH?wg#DtkRRNeJD83Ds@PTYO`7M*foihq6x-mlqFeKu=xg|+$E1bUN1mc_5tVS!edml=N1ny%f!WK?vv}6Gf;37xn^_1S&L)uK4#| zKGibFb|NFYW807MG~y1Qf3%HSj>5g=YuYCrU1BHw9LqTmQUYhxbVsM{GqtJ^Z{2vF1!532R~gu!>tgE-NREiPHB; zM!FyseY#HFR>C&nTF$;k=7ht^Pizd*yi7CjSE|0LzkLOZ%^2v7RGP2N=7&^>h5D-7 z1ZHk%vhML`=6v-Nw09r+skf~Jy0#|d4C-Ym+uKItgk<__N(E-Ia$UEWG z?02F3SP+&dyTn`XsrSm}Tf9#c=`i1aHs)U@{_JPgl8bS-WBm*v6AbPigUH$lZ^n78KXn_^^{$ZTOx9?`0o zv(_Iyb<4a^-ht!mn5;O`rRQ6jV6T8s{aX27Y8>wjyzA-;i!{t(1FQ9wG#^%#@aRuh zZHrQ4p6IZCGnheRyNnd6vRtHV^?!?2jhEKyXOteZ+H9Sj28GJ`ygilERWx{){I z1^Zrej;eXOi2^CsVJ!^-UtXY6zmjaHg7%7y-1ExC*mN+ZN*M#VBM&lRCcr|B%^05| za65|(&+dRqPX3OF(uYRwt#2~JyGOB4Qi;LmDaiL$13^HA2nxlDW@DmAljpH1@(s~l znvGfh`c%@-tm_z_+BHH)+cTw}F5WGuq+wX+rxpXGvqBr$4&ieB3=QKpv3(FDE(j(YT7eF4n znW+Mu%4CIu>(dg(j`iXx++z>lZo-E+s~m?kIMN3^!D%^_EGHjo8hBI7x14{Odlr5i#SH!9yx%;n(tUq73C7Yo1mxs@2x#4vIdOthu|k*1yl z3Z6g81X`pdcv?KC5WrKJ^}11XmkkX3Qk_=dE#G#UD+QTC{ltA-2GjHp8p;9;tI!;I z#j+fKIDXiGIvxEk^PVWe9$t&2^Sj`4-@d1IN;JfzL8h6MeMv0gDNe4GrW4ian#|c2 z`|sXiaO}+%lhV_@pM9r5R2o&{KUHdz7eqw-TR8`U@E>15Hl_#D16LJ6kAec# zN>;VW{L_ER^LlTg=u4{oPROJPn^Gp3xXUcI9Nhi{6aPsl7J|_!tp04`Q~5tK@}IvC zMBV9y2hsP}K)oIj3I`O-q3C~a8$Tf>YJhWyse(@?b1<*N+wb|U)9VsD?0^0rLSWs^ zvl^^jtX26(*xm*bwy#nB3HuKw?jVFyW;k!DUhoe!o(-bL&r!>IZ%WnaJ$XX=MVOyY z?oX$~zbp@s-@1}mc2HFaKxhBsg81VvrMOSY3zZMza$6vBDY`gBeCjZ#lKFHRtTO_>sM{_h=F!@g(t_Tz~}^UE{(u6St9<6V!o0#z^0zs1F?A! zcmX`58qnTLltbjslg%NIVeWs>KRxqB00)(BwbW#FY~}!hcExvIdzA+OY39?V9e`Y@ z1<3?fST1rG$m841K}x6~!RgS~^ef-pjHjE4n*kD`#|3=XEL2oI+)vDGiWDHlCP0IwCsw{24{_7>; zY4CzghhP zG32?M%frKhBC|U{x8949l_&uWvKdeyXHb1l1o2*~gKka;4RH!!w++D*5M%&d`VUu! zz`q%q&y>@LOo1xP{CS%`YWUk!=MC&bX0>byJeMU;I5Qw5y8|QCtt%JU6{na>fGSOl z8$Ax!+h(!U?AiEw(s$N=Ng~8I=0cm=ZP^#gH333`Ok_4UseZF))rM4QyD8CxXG(1^ z{eC|i_v17~(k{k#1o%zM9}SLk*8Y~1_PWO%^KY&J;E}5ch~SG9ayeQ>4Ht=!RG~Y8 z2PTc05peNTfRy%LdhVAhK$lajIcABH*Xv>q04$!4E%M?_AaZ zYQh71Qvfht!&;=AKBXy!LcX`K@y5i`T%e?WBhfS;RD3%?`okEtEB1is%>_=liS$o! zMc*KlMSv&Wz$gUFfMH4pvLcVi`zipQ>#IAmp;J)HkFK#2s8577z~6}uwR ziE!A=npaH&f$hGeT6xiFHPoHx$!AA<44rw^AoD(`h$X{Pm#(?wf>J zFr)J;;1VD49t`!mxjk%}h45cZfy@9$UVsY<9!tE#1_F?j2uq;C0TA_{d{IrY49hqYSX#HSsuv5zAM6CBjcTxVVCT6_m?Kf?5zjw_y2Lf{a z?>m57lwjrR$tmu(!CxTg>2ss9umhaWK$LVD8^MT8D zk*M+*K&S(%xf)n>UUlsBAmot?heLRrkh*(F|KtTwEMFgIrlYop+n5nLpKhFs&;~66 zEu(%VbOnp1{m!oziN$2R+bNO??s5yv)hR~aR-gRjcfUVZJUnn2mlhPD!k$6DcXsW{YEI7RJp>{VS6y$k<3Z#ua_Z2o1CPjclr^jt#FxE1dRk*uP%3 zo_fzS>sHWyko(n( z&Hc_v^JUf*xR0Nw0TS)rd}f-<0oyxwECi|vxkB1Rl%bGkmq4zKPqO+T1X=CWt4yNFNn_QuV7cf?Z0} zY`?l@ze=<`%3d~Qw|DULY1JsQerx!tNO>XYR`C@(jDg# zYY^R9i}>sff^s%L$y$CKq0a)a+Avu-Br#X2g-`VpUn}{1lUuQpwsx@=_4*B zc1$$toUimjw2W0WyT zKIgk7ZV_v18ohgg5$NCgj7o`FcEu0J-K_kO%(b3~q`37)R8?w;QD$kVCJ2kxeb_ia zbN!AA3!OqKOLzs{L_GUtZ)7tC8z24Ww_PbJHdoK-*jo6#|C@gWcvNpZ5$UZnzWGuX zwzCZax%Cu&;vUf#Blv1!`1F{Koa9xX!54~xWMmTB#0;!o=rYVdVDcP@vtmyL>frQf zUdG3iJA&NO!g`{jiW$Q6r7ePlrG0DYTOA+`j=Hgl>1NZykj~ zn3GfvZR2YHb;f$afHf$Z)Ry;iJxj=v`tttw?>}DvcpXS14gl|pkxD)3@2l4U-aU9B z>rgJ+6zRXOQTYimz;gH}R9w*C7q(xYx2J-o=oI55_TN{5yf88l3o?aPZqEPxRsVMf z{&`OQubvh*qpN*@-E0ufdD6`%Zw}najgJp5 zj~Uc17kg|uV)@Kg55Qg6wPVNrudbl~ym5nzj98S=vXo@BrWvxpgw%6ZK%l(Lpc66) z;0PYw!8%a|gm*`;L*NvvTrMpjR>Of5j!Ce1Uc%@EMD^LL<5ge&liIqr8Xh+yqjwmn^D z_&N2n{XbrsgW%ictOdxEKqtWdT+JAGLofi#^Z%#4EB~hIfA=NIm}H)tP-da{80$nC z3YjtwCo)f`WFD%cB2wlE$s9*Gh9WZ0QxYLUjxx&-j^RGL@9I;!|G>TL-hODEwU@Qe z*?Yg=`*~ik=XqYQ<>7oOJ&^n5RTa4ERKB2`cPK@1q9bXzWFOfIG}#v7Ov=((Oud*l z5X*9xk?LY}!~QZL0du{7%&tH@$oOIVI7j5egj#`E|B0PZ;SfofqPht-AD4nZf{1A& zz05)Hf;k|pZ^=F1o*u3o2HPAoSlwpn8%mo&U(^hEPT-p&_|D62Lao&f6qxZ*5RG5o zn6kG2kh3uU@&O>HM-;ua%D5>=^5%)XxkeL=BS^d9jJS2{{6GXz8?FVc0f^He$e%Bb zA*_x;!ZWR;9+f@mDyrK+reInXKvNx!_^ z`^-DtTTli~Jl%VHd=0W4^}f43L})+#J`7S~SG}QVFcXUo%LmDbj}WA18U5TXi)O(h z1Q0_@nB2-0Z$uV#vh%;E?iwVl*`4y?$vzko_Vc~YawQ=TQcZNFXkK*xZm_!uX zm&$ROH=dC@0X{^m5HhKmF7>ktBUv2fA=N`O4R$_5o)c@9KJ~e=_Ee<$=7Q;sh-&~M zF^_A6t0{!o#lz3eLI3Ry^diIP)D0rwq>Wjm8PIQQ?HUn8^BqBz2+1%9z!`h`hpXHb zdq6}EbE6Li#Cq@E3VqxUUbKi2&zpj#+A|GdoHK~yOPX4YAtaPD(MC=xHi$=)17P7T z86slqfGFxXRNtX6eMS>$JQ|S2&U<`iehuIa{bbjFzY!vcKB2H7zVW5hk=lok=lSl< zIdr999S2Lh?ya4BNuE+(fA#~;#2(2mnQ9{woWC?aa<3v|_Gqz%wnj&gFBnQ;s~zXn zZ76jen}*I~JoJUL1s}&%0zgrn3_nK%a6UI?l7A2_Hq)Dzq(2WbTobuTfG6C}u5CNd z4LwiepHc|(S6LtlP9ZtMlDwNgCDubggfy1-+304l+EG%i?)Pc8;%u2Bk39?T=NGX` z$9gL1oggsMpV28VE9WykB6{ZBd%udLVdoBKmI;mt>TANEi3U?H*4A?$JBIS4{paK6 z7G=wzu{F!9A2t$5@SW3q#&b`c)EgwHY!zRtv-#oc7T{tIM0n0dv8r%&`j zbQ0&yK{0)kafh3LhsP7?9bWVxYbZPY;J@B;8!s-o4}JnfEYe_Km9Rd|!P=7hACQytV5p z7jKUO+oB}Q)yHz_k8!VeFkw?3#};)xKc8`^F??u4xy}vxTWy^L06niyojDGj;NCdZ zaxkkgH?{LSI#{ra@dZ8RLHa4Cd`ogsjJ1Jn+^!~1&E-^RRg=*T1=R2<6@{V3MXeRz zaUl`IxWxAZ(-I!U4{%PH2Alq*9E`ux4>w+vizW&r2ar%6|Mg!eg-+^dv#~m%_I*3)S~+z6ePeKG;)H26L}UwU$jvA$EuqGxC@Zwa;K$aARYi(<9; zGh5A`idvj?YLwg^A8f15-Z1)vmC(XWnZPYXlvR}*LhPpBXKH0#ZgKR1xra!q?`d2$ zo4?k;&(#oknUtT&_*+7F?O>xrlu4rau((m`eb3xh=UjJU6|LWa+xuwu6R-sZv+zRT z0ud_>j3iYDR#a^lkGb|4o;w;K*l}8a8N|>+f%{3#KHhUayUE~aY2|!1)i+=244$t9 zkuQ-mF`MaL{lJmnGwDObv{>8N77Zw#=%^Sp24mx|ER0}z!(`c#WLP^{C(5mlOl!+zbbKIbNg{=o6Gf}a8v)y1^n0A*z=o0X8>HxbaHC>gU8m3cB8lzG)G_TV7`$4+i zxKGH(UprAeBPx3#SlGI5dA=rwLf~EIS@9oHWO)wDLT_Ujd`2v^fA1o5D|(xi&#w!1 zdsI;RXpvIm%px`Vr*wY(fu3AoFg2s;a%}mLfA`&g?ztfai?KbAbLEcf2_WHt!)lgp zeE)yy zmy!F5c8d`2Tn}6_J-pQJ!@NsrS501F`#Y-`xeXogUCI~zoVABV(e)3mfQg(J zd(|s&fm%otk(B-Pqo~(31WU!N42f%ezi#2oHiYZc=4Cx2M@I0w@hs$w+4BbcRK(zS zfpxLmljTwlWV&Gi_~zPE0}k}@j>QIr+{1gKh0R@rJ~hiR{u}xx)sjc>dIZmdw}Oix ziEe;7A?|&g9-8x|FSX(}A}bY5TYT8d!jxI6hP0V3Yh8CXFJtiB--$bi#=%*O8%w~GYHZren01VEOVx+3nW_hvY?l9!sQ}_ zg%76QGdAGc_vtgh6lu6Il^$Onoqs3oJH|eI52_Y&r#@Zpj+Jt?q{Ktd_~rikTbD`m zbLavzq{kzdA{CpAO@d_RAU9utG&{8yJXZ!vfi6aGtS-2MAy0a5rfB!R9#j4Z00g$~ zjpKgw>jsVdi-usCK9>Cgr^=}AevF*Ut?_V{Mv$y{No>6=I*5=69Ctog%+!N+RSHz5 z7ho@?&un1c?%r3+4a?O0>(YLi>2U8&u)b60f}zNVz~-k0Bsm`Q(cbL@ne6nNA4655 zlhrH0#W!ec$T1GDwcm7$&nyk&kUj|~t5{;|-QJw@S1^%jN4zkdR#!mAHga7KQuyb< zpH)q0d6?)MyDG7rY;*NYC<>D8Go5}drI7%`sSL1M<|*|*6fVw}{E=W#N+c=P-4-aR z{044G2TXrJtX--pzg&F*0$Kc=s@~@dF!8NIsZ4r-&EkRGH22p0U8(PlN6`R~Ua5}2 z*5QCMjR~#P)yLQ-(vi*ItuN%jVqgFGw4f8X<0tW5qG!-FWxS76mjNxFLEZ* zqGfwyLIKmz#wD5>w@V$=qY9&$svcF4`$I|)cIoLqX~ExaRuVsbzFRMJ&lD`IVqx@k zW5(HEZBf4$TyjbQ3v|2R5@HURQ0W?}^x6G(CXAAq7dHXbjK7C$F>tJCk3Nbj~{mJdufs7)!6x=r4Fo1!h9awUdm8N zb2Q##b_@^Yx)Kpu>yE^yBy(71(^z#W0*sM%i#k^y3qira>hTX((4h?9$t$s;*i=fC z{g%9smSTVc{JQSpJ^n{^^y6ycC7G%%x5Or(kkpXoy=$T)kn6OQarQa%*{@$(0_$bj zUJJ_?I!I0xeK36CRH5F)v61^{OrdK_WXPHvFWK}pvVJ@H)+cYDl zV9SN%YD{wwfhm#p#n{2?P-RLCVxmtVcSV4ChWUkE9VCZ!-&U^xACS)Pw54gdVqxQy zugv2C0NS-ni8*cZSd{VlFgfN7)7p3WOLYp}S>$ppcHVm41LEdm=s$&R{4 z5o!>NCT40qf9EtfbZ=MV{NRQ%PqYYcL4C|F1ZJ-+iB}qsepA?9(_qtk>UcF#h)N5z zCfXF7NqBmjH9k<5XuFPYuzff0dKrWhkV&un9TylhljX^t$dx+1tlF!T7pa6T6{1Ax zux=T85Lro_{jTtPg9I4wuk?w=L<4VGdddO~1dX7I%=k_8>*cY=!%}iJqLz7E}Dz-9{)afNS z<>YNRCYYhPT63*EF!@t~8ROPNT3C+;t@E&+dCB-N~))45mP7>POel z8#p~$ob%t7F`=Kh_|=+RG)gWAUyw;JROU#73Nzr0RhB|$E`e>bcKC#^V0&#sN44@> z^aY{7Smg`j#s3K6@9>=o?eWTz`d)M7ypKw!W=&E++hCui8N=0vR4;XG1U5R>UR5%( zE@5lyM`ea*L4tS`dVS0lfY?dFgtd$WdRAT_mT=RbvGF@4SHNkZlW!^^CX8(P%paP@&|s)kBNG^{%5x?h zi!0qHBtTCupT%%llINWE2`j86;h}X+r1p8T?U!CBXTbc8zwdjchBWV@IYpGuN5!-l z%=r41*kI4h1`dq+lkMh-riAgsa8Pdq2c+IJ` z<;w0o7<(SwTPJ4lKDcA?yu!K`+NAX2Z(zQg%zwO__9u!@cX$ zn;#avS5KzpD-}+z4a%MV_udDJ6e}v-5=a${teLqgpGP-en#9K_7EeN{bLs78LusRj zzZajTL7+7^f5KCzBzfO=j@ktib$;4b=zK1ey%?w29|#J8N{Y!J^%>i*S()^O zpEuZjm(u@3t^RMAtG7Qeo3uCTc6FhSc@L9lDXu>$Pe0?=PRVH|VT>Pp&HvnUC%z0I z>7f@b)$8a_{C%dMpPCPdmPQ8>jQ%>D&1deCn^+|G*N=`o3_0kYwvvDOg8yI}9Syi= za`xVrf1$f=>PM?s!4le`F?I|q&8>F6u!&b>_ zO#iNypI;I<36S>1HAju#fbQofD{>a&7R5!OzYhOTy5Vmpd%jLSe<&Xeq;Ty6R`JU> zVZ2!l+_m}i%--;S(jPlGgZk@6o+T05BHF&_%tunMp8yQlo-tst&2aJ1AX*2R+m94_LSZKCJ>JP5xe~ zdMbgzQL=8zI?TmVzi2Y|t(b+N#Qvrpm5oRC)SZwi~!4MlR0b+-8$z8aGQ#M%Xua!VUv zKtSG(w0X8VimW8R-wb}!yMyUd4D@0GB2tJt*T}I*cN+5N6D$WQlc237L2abs6%C_@ z;<6i1SZ6wAJ_qc%Obe7^Sj~PJ!6CTt*ne)=8QpH$BjzvB1URq@5;2}$|1rD4WQs8!Xk#9dBw`YKE5PxdUeyCqW z@x%}j!Oc*b-~c=n0<(!H{kDn26UE~S2rL^*jkwhOMgn?@`h0(U>Rx-uB6ft#;S|&C zkAdN>Q^kOCY@nGa`;b3a4SZwa2BDS%ZlGbYD4|1)sob!d?UVl(%>I0=lUm7kRZ(wa zC7JBAb&tx@vY!&vHS+%+kyI-)R~~U|66xv!rq&bzE?rsL8GSc;O~yb>_C8Sa=@w;B zXfckst{Vfqu&&PCN^gBh zD70CkXBfcuDNYRJ=3xkOaBLZV7`y@vRLqs@J$a*n`?xODJaH~xsK;}o!r)3qBr1OZ zZ*XH{{;mRb5bwb6WeJ}Gy%>>&U@W$Z)C#B$(~(l>t>wB5Z#YJI$hwZ!vdFkQ48l|I zdWD%_*b6jx%@wyV?uBBs~Cto?_*D<+5XIAtYArk*Hd^cp3+wd`&}OE7qlX-aPwJj)(dUJm_`V$_qS2C>2-m46lXCM^e!IL3|u@ z*^$u6L@3*CmQ` z+=N!4S7Y=;h?}^_?5l})v^{+eLwvh`7!9xMAu#5;p+Ud~}Raj5i=tDOTWxi`7X~Q}OaSf`B(nK(F*Xyqjw7WJ;s>6EIe zhW(JfBWS&Up;&io3P^(OID6}==MG)ZH$p)>S9_~FSe{oCBg0-j31z$6N+Gy6EBz83 zQ5IEqh+ny?Wi@Jj!{D>d(3*07+RUF zSoUgTxH8e42pccPukVp`c5Bq;yO`*07FI&*s5+wP^3M?PIh+VS_fogX@M@KIr8OZe z;_~a+Y~#0ghVbR%n)w4G4#gj>ZNqpNt8%m)(^O-T#w&KxdOJx12J{}^AS6<%>ZGB4 zZWIZ};(3Z5s6u)O8WbFBJIaGc`mzucbG)6idzg+$Rz2bLSxcb27R^*cWGhgL!nCS$eovgeOY0+X!RwtPNZB@@qSph?{qOi16#F0Se+Aveo83&^CG5~T#e>)Pw#cboocCNNjYWNe_s`@F zn?ExHiHNt~(dzNvl0RCIE9;yxxR?1-EOHfymONXRl+iv-n{gispNkZc*xp#VjA}zc ze8JQCkMHICwinL|H1I|;QRO8H)xmZdvCgLS_kcde=16s&f(6Q-}*13hqEjI literal 0 HcmV?d00001 diff --git a/docs/source/reference/config.rst b/docs/source/reference/config.rst index 352531e8881..16a22350209 100644 --- a/docs/source/reference/config.rst +++ b/docs/source/reference/config.rst @@ -136,14 +136,14 @@ Available fields and semantics: # Please refer to the aws.ssh_proxy_command section above for more details. ### Format 1 ### # A string; the same proxy command is used for all regions. - ssh_proxy_command: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no ec2-user@ + ssh_proxy_command: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no gcpuser@ ### Format 2 ### # A dict mapping region names to region-specific proxy commands. # NOTE: This restricts SkyPilot's search space for this cloud to only use # the specified regions and not any other regions in this cloud. ssh_proxy_command: - us-east-1: ssh -W %h:%p -p 1234 -o StrictHostKeyChecking=no myself@my.us-east-1.proxy - us-east-2: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no ec2-user@ + us-central1: ssh -W %h:%p -p 1234 -o StrictHostKeyChecking=no myself@my.us-central1.proxy + us-west1: ssh -W %h:%p -i ~/.ssh/sky-key -o StrictHostKeyChecking=no gcpuser@ # Reserved capacity (optional). From ee727beec2f331530cfed114b602f9ba092716e0 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 05:54:03 +0000 Subject: [PATCH 20/27] refactor skypilot_config upload --- sky/backends/cloud_vm_ray_backend.py | 10 +-- sky/execution.py | 23 ++----- sky/serve/core.py | 23 ++----- sky/serve/service.py | 2 - sky/skypilot_config.py | 19 ------ sky/spot/controller.py | 1 - sky/task.py | 5 +- sky/templates/sky-serve-controller.yaml.j2 | 4 +- sky/templates/spot-controller.yaml.j2 | 4 +- sky/utils/controller_utils.py | 73 +++++++++++++--------- 10 files changed, 67 insertions(+), 97 deletions(-) diff --git a/sky/backends/cloud_vm_ray_backend.py b/sky/backends/cloud_vm_ray_backend.py index 1f380dbe15f..6007bf37726 100644 --- a/sky/backends/cloud_vm_ray_backend.py +++ b/sky/backends/cloud_vm_ray_backend.py @@ -17,7 +17,7 @@ import threading import time import typing -from typing import Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union import colorama import filelock @@ -1459,7 +1459,7 @@ def _retry_zones( cloud_user_identity: Optional[List[str]], prev_cluster_status: Optional[status_lib.ClusterStatus], prev_handle: Optional['CloudVmRayResourceHandle'], - ): + ) -> Dict[str, Any]: """The provision retry loop.""" style = colorama.Style fore = colorama.Fore @@ -2179,7 +2179,7 @@ def provision_with_retries( to_provision_config: ToProvisionConfig, dryrun: bool, stream_logs: bool, - ): + ) -> Dict[str, Any]: """Provision with retries for all launchable resources.""" cluster_name = to_provision_config.cluster_name to_provision = to_provision_config.resources @@ -2217,7 +2217,7 @@ def provision_with_retries( prev_cluster_status=prev_cluster_status, prev_handle=prev_handle) if dryrun: - return + return config_dict except (exceptions.InvalidClusterNameError, exceptions.NotSupportedError, exceptions.CloudUserIdentityError) as e: @@ -3218,6 +3218,8 @@ def _sync_file_mounts( storage_mounts: Optional[Dict[Path, storage_lib.Storage]], ) -> None: """Mounts all user files to the remote nodes.""" + controller_utils.replace_skypilot_config_path_in_file_mounts( + handle.launched_resources.cloud, all_file_mounts) self._execute_file_mounts(handle, all_file_mounts) self._execute_storage_mounts(handle, storage_mounts) self._set_storage_mounts_metadata(handle.cluster_name, storage_mounts) diff --git a/sky/execution.py b/sky/execution.py index affe3c1e90d..aa5076bd6c4 100644 --- a/sky/execution.py +++ b/sky/execution.py @@ -23,7 +23,6 @@ from sky.clouds import gcp from sky.skylet import constants from sky.usage import usage_lib -from sky.utils import common_utils from sky.utils import controller_utils from sky.utils import dag_utils from sky.utils import env_options @@ -667,21 +666,10 @@ def spot_launch( prefix = spot.SPOT_TASK_YAML_PREFIX remote_user_yaml_path = f'{prefix}/{dag.name}-{dag_uuid}.yaml' remote_user_config_path = f'{prefix}/{dag.name}-{dag_uuid}.config_yaml' - extra_vars, controller_resources_config = ( - controller_utils.skypilot_config_setup( - controller_type='spot', - controller_resources_config=spot.constants.CONTROLLER_RESOURCES, - remote_user_config_path=remote_user_config_path)) - try: - controller_resources = sky.Resources.from_yaml_config( - controller_resources_config) - except ValueError as e: - with ux_utils.print_exception_no_traceback(): - raise ValueError( - controller_utils.CONTROLLER_RESOURCES_NOT_VALID_MESSAGE. - format(controller_type='spot', - err=common_utils.format_exception( - e, use_bracket=True))) from e + controller_resources = (controller_utils.get_controller_resources( + controller_type='spot', + controller_resources_config=spot.constants.CONTROLLER_RESOURCES)) + vars_to_fill = { 'remote_user_yaml_path': remote_user_yaml_path, 'user_yaml_path': f.name, @@ -691,7 +679,8 @@ def spot_launch( 'google_sdk_installation_commands': gcp.GOOGLE_SDK_INSTALLATION_COMMAND, 'retry_until_up': retry_until_up, - **extra_vars, + 'remote_user_config_path': remote_user_config_path, + 'controller_envs': controller_utils.shared_controller_env_vars(), } yaml_path = os.path.join(spot.SPOT_CONTROLLER_YAML_PREFIX, diff --git a/sky/serve/core.py b/sky/serve/core.py index 44ae04c4c02..3fe9dd069fe 100644 --- a/sky/serve/core.py +++ b/sky/serve/core.py @@ -84,22 +84,10 @@ def up( serve_utils.generate_remote_config_yaml_file_name(service_name)) controller_log_file = ( serve_utils.generate_remote_controller_log_file_name(service_name)) - extra_vars, controller_resources_config = ( - controller_utils.skypilot_config_setup( - controller_type='serve', - controller_resources_config=serve_constants. - CONTROLLER_RESOURCES, - remote_user_config_path=remote_config_yaml_path)) - try: - controller_resources = sky.Resources.from_yaml_config( - controller_resources_config) - except ValueError as e: - with ux_utils.print_exception_no_traceback(): - raise ValueError( - controller_utils.CONTROLLER_RESOURCES_NOT_VALID_MESSAGE. - format(controller_type='serve', - err=common_utils.format_exception( - e, use_bracket=True))) from e + controller_resources = (controller_utils.get_controller_resources( + controller_type='serve', + controller_resources_config=serve_constants.CONTROLLER_RESOURCES)) + vars_to_fill = { 'remote_task_yaml_path': remote_tmp_task_yaml_path, 'local_task_yaml_path': service_file.name, @@ -107,7 +95,8 @@ def up( gcp.GOOGLE_SDK_INSTALLATION_COMMAND, 'service_name': service_name, 'controller_log_file': controller_log_file, - **extra_vars, + 'remote_user_config_path': remote_config_yaml_path, + 'controller_envs': controller_utils.shared_controller_env_vars(), } backend_utils.fill_template(serve_constants.CONTROLLER_TEMPLATE, vars_to_fill, diff --git a/sky/serve/service.py b/sky/serve/service.py index 5bacb304e59..1b2aaf253c0 100644 --- a/sky/serve/service.py +++ b/sky/serve/service.py @@ -28,7 +28,6 @@ from sky.serve import serve_state from sky.serve import serve_utils from sky.utils import common_utils -from sky.utils import controller_utils from sky.utils import subprocess_utils from sky.utils import ux_utils @@ -253,5 +252,4 @@ def _start(service_name: str, tmp_task_yaml: str, job_id: int): # We start process with 'spawn', because 'fork' could result in weird # behaviors; 'spawn' is also cross-platform. multiprocessing.set_start_method('spawn', force=True) - controller_utils.setup_proxy_command_on_controller() _start(args.service_name, args.task_yaml, args.job_id) diff --git a/sky/skypilot_config.py b/sky/skypilot_config.py index 4db5c5f28bc..78fe4cf6ca5 100644 --- a/sky/skypilot_config.py +++ b/sky/skypilot_config.py @@ -119,25 +119,6 @@ def set_nested(keys: Iterable[str], value: Any) -> Dict[str, Any]: return to_return -def unsafe_overwrite_config_file_on_controller(config: dict) -> None: - """Overwrites the config file with the current config. - - This function should be called very carefully to avoid unexpected behavior - due to the overwrite. Currently, it is only used by the spot/serve - controllers to reconfigure the network settings before any further - operations are done. - """ - global _dict, _loaded_config_path - if _loaded_config_path is None: - raise RuntimeError('No config file loaded.') - common_utils.validate_schema(config, - schemas.get_config_schema(), - f'Invalid config YAML: {config!r}', - skip_none=False) - common_utils.dump_yaml(_loaded_config_path, config) - _dict = config - - def to_dict() -> Dict[str, Any]: """Returns a deep-copied version of the current config.""" global _dict diff --git a/sky/spot/controller.py b/sky/spot/controller.py index 3d58424e09f..8c1f30701c6 100644 --- a/sky/spot/controller.py +++ b/sky/spot/controller.py @@ -523,5 +523,4 @@ def start(job_id, dag_yaml, retry_until_up): # We start process with 'spawn', because 'fork' could result in weird # behaviors; 'spawn' is also cross-platform. multiprocessing.set_start_method('spawn', force=True) - controller_utils.setup_proxy_command_on_controller() start(args.job_id, args.dag_yaml, args.retry_until_up) diff --git a/sky/task.py b/sky/task.py index b2af3b1ae24..d7e22323aa1 100644 --- a/sky/task.py +++ b/sky/task.py @@ -759,8 +759,9 @@ def set_file_mounts(self, file_mounts: Optional[Dict[str, str]]) -> 'Task': raise ValueError( 'File mount destination paths cannot be cloud storage') if not data_utils.is_cloud_store_url(source): - if not os.path.exists( - os.path.abspath(os.path.expanduser(source))): + if (not os.path.exists( + os.path.abspath(os.path.expanduser(source))) and + not source.startswith('skypilot:')): with ux_utils.print_exception_no_traceback(): raise ValueError( f'File mount source {source!r} does not exist ' diff --git a/sky/templates/sky-serve-controller.yaml.j2 b/sky/templates/sky-serve-controller.yaml.j2 index 0c3378afc56..41c37e8f0f6 100644 --- a/sky/templates/sky-serve-controller.yaml.j2 +++ b/sky/templates/sky-serve-controller.yaml.j2 @@ -14,9 +14,7 @@ setup: | file_mounts: {{remote_task_yaml_path}}: {{local_task_yaml_path}} -{% if user_config_path is not none %} - {{remote_user_config_path}}: {{user_config_path}} -{% endif %} + {{remote_user_config_path}}: skypilot:local_skypilot_config_path run: | # Start sky serve service. diff --git a/sky/templates/spot-controller.yaml.j2 b/sky/templates/spot-controller.yaml.j2 index 184ee8980df..0bb165f981f 100644 --- a/sky/templates/spot-controller.yaml.j2 +++ b/sky/templates/spot-controller.yaml.j2 @@ -4,9 +4,7 @@ name: {{dag_name}} file_mounts: {{remote_user_yaml_path}}: {{user_yaml_path}} -{% if user_config_path is not none %} - {{remote_user_config_path}}: {{user_config_path}} -{% endif %} + {{remote_user_config_path}}: skypilot:local_skypilot_config_path setup: | # Install cli dependencies diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index 9f2273402df..acd2c3aa091 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -6,11 +6,12 @@ import os import tempfile import typing -from typing import Any, Dict, Optional, Tuple +from typing import Any, Dict, Optional import colorama from sky import exceptions +from sky import resources from sky import sky_logging from sky import skypilot_config from sky.data import data_utils @@ -20,10 +21,10 @@ from sky.spot import spot_utils from sky.utils import common_utils from sky.utils import env_options -from sky.utils import remote_cluster_yaml_utils from sky.utils import ux_utils if typing.TYPE_CHECKING: + from sky import clouds from sky import task as task_lib from sky.backends import cloud_vm_ray_backend @@ -37,6 +38,9 @@ '{controller_type}.controller.resources is a valid resources spec. ' 'Details:\n {err}') +# The placeholder for the local skypilot config path in file mounts. +LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER = 'skypilot:local_skypilot_config_path' + @dataclasses.dataclass class _ControllerSpec: @@ -187,7 +191,7 @@ def download_and_stream_latest_job_log( return log_file -def _shared_controller_env_vars() -> Dict[str, str]: +def shared_controller_env_vars() -> Dict[str, str]: env_vars: Dict[str, str] = { env.value: '1' for env in env_options.Options if env.get() } @@ -202,11 +206,10 @@ def _shared_controller_env_vars() -> Dict[str, str]: return env_vars -def skypilot_config_setup( +def get_controller_resources( controller_type: str, controller_resources_config: Dict[str, Any], - remote_user_config_path: str, -) -> Tuple[Dict[str, Any], Dict[str, Any]]: +) -> resources.Resources: """Read the skypilot config and setup the controller resources. Returns: @@ -216,21 +219,9 @@ def skypilot_config_setup( used to launch the controller. """ vars_to_fill: Dict[str, Any] = {} - controller_envs = _shared_controller_env_vars() controller_resources_config_copied: Dict[str, Any] = copy.copy( controller_resources_config) if skypilot_config.loaded(): - config_dict = skypilot_config.to_dict() - - with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmpfile: - common_utils.dump_yaml(tmpfile.name, config_dict) - controller_envs[skypilot_config.ENV_VAR_SKYPILOT_CONFIG] = ( - remote_user_config_path) - vars_to_fill.update({ - 'user_config_path': tmpfile.name, - 'remote_user_config_path': remote_user_config_path, - }) - # Override the controller resources with the ones specified in the # config. custom_controller_resources_config = skypilot_config.get_nested( @@ -243,11 +234,22 @@ def skypilot_config_setup( # so that the template won't render this. vars_to_fill['user_config_path'] = None - vars_to_fill['controller_envs'] = controller_envs - return vars_to_fill, controller_resources_config_copied + try: + controller_resources = resources.Resources.from_yaml_config( + controller_resources_config_copied) + except ValueError as e: + with ux_utils.print_exception_no_traceback(): + raise ValueError( + CONTROLLER_RESOURCES_NOT_VALID_MESSAGE.format( + controller_type=controller_type, + err=common_utils.format_exception(e, + use_bracket=True))) from e + return controller_resources -def setup_proxy_command_on_controller(): + +def _get_skypilot_config_for_controller_task( + cloud: 'clouds.Cloud') -> Dict[str, Any]: """Sets up proxy command on the controller. This function should be called on the controller (remote cluster), which @@ -281,13 +283,7 @@ def setup_proxy_command_on_controller(): # (or name). It may not be a sufficient check (as it's always # possible that peering is not set up), but it may catch some # obvious errors. - if not skypilot_config.loaded(): - return - provider_name = remote_cluster_yaml_utils.get_provider_name( - remote_cluster_yaml_utils.load_cluster_yaml()) - # We only set the proxy command of the cloud where the controller is - # launched. - proxy_command_key = (provider_name, 'ssh_proxy_command') + proxy_command_key = (str(cloud).lower(), 'ssh_proxy_command') ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) config_dict = skypilot_config.to_dict() if isinstance(ssh_proxy_command, str): @@ -300,7 +296,26 @@ def setup_proxy_command_on_controller(): config_dict = skypilot_config.set_nested(proxy_command_key, ssh_proxy_command) - skypilot_config.unsafe_overwrite_config_file_on_controller(config_dict) + return config_dict + + +def replace_skypilot_config_path_in_file_mounts( + cloud: 'clouds.Cloud', file_mounts: Optional[Dict[str, str]]): + """Replaces the SkyPilot config path in file mounts with the real path.""" + if file_mounts is None or not skypilot_config.loaded(): + return + replaced = False + with tempfile.NamedTemporaryFile('w', delete=False) as f: + new_skypilot_config = _get_skypilot_config_for_controller_task(cloud) + if new_skypilot_config is not None: + common_utils.dump_yaml(f.name, new_skypilot_config) + for remote_path, local_path in file_mounts.items(): + if local_path == LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER: + file_mounts[remote_path] = f.name + replaced = True + if replaced: + logger.debug(f'Replaced {LOCAL_SKYPILOT_CONFIG_PATH_PLACEHOLDER} with ' + f'the real path in file mounts: {file_mounts}') def maybe_translate_local_file_mounts_and_sync_up(task: 'task_lib.Task', From 423b83c17990ff926336a1f1e268fc1f9a83c80e Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 06:04:09 +0000 Subject: [PATCH 21/27] fix merge issue --- sky/execution.py | 2 +- sky/serve/core.py | 2 +- sky/utils/controller_utils.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sky/execution.py b/sky/execution.py index 589989be9a9..9a6f5e98cd3 100644 --- a/sky/execution.py +++ b/sky/execution.py @@ -677,7 +677,7 @@ def spot_launch( 'dag_name': dag.name, 'retry_until_up': retry_until_up, 'remote_user_config_path': remote_user_config_path, - **controller_utils.shared_controller_vars_to_fill(), + **controller_utils.shared_controller_vars_to_fill('spot'), } yaml_path = os.path.join(spot.SPOT_CONTROLLER_YAML_PREFIX, diff --git a/sky/serve/core.py b/sky/serve/core.py index 786eeb9d3b2..e285492987f 100644 --- a/sky/serve/core.py +++ b/sky/serve/core.py @@ -93,7 +93,7 @@ def up( 'service_name': service_name, 'controller_log_file': controller_log_file, 'remote_user_config_path': remote_config_yaml_path, - **controller_utils.shared_controller_vars_to_fill(), + **controller_utils.shared_controller_vars_to_fill('serve'), } backend_utils.fill_template(serve_constants.CONTROLLER_TEMPLATE, vars_to_fill, diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index 76afb063a54..ad87dd7c706 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -6,7 +6,7 @@ import os import tempfile import typing -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import colorama From c0731e2d5dbb8d64ed9dd3d236cca7e34dc16647 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 06:05:10 +0000 Subject: [PATCH 22/27] fix --- sky/utils/controller_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index ad87dd7c706..ec27f121697 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -242,7 +242,7 @@ def shared_controller_vars_to_fill(controller_type: str) -> Dict[str, str]: env_options.Options.SKIP_CLOUD_IDENTITY_CHECK.value: '1', }) vars_to_fill['controller_envs'] = env_vars - return env_vars + return vars_to_fill def get_controller_resources( From 9675edd5e4cd90019511ce6e8c3570d1dd0154cd Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 08:58:22 +0000 Subject: [PATCH 23/27] Adopt test fixes from #2681 --- examples/job_queue/job.yaml | 2 +- tests/test_smoke.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/job_queue/job.yaml b/examples/job_queue/job.yaml index c54e3d9a173..aa9c3502247 100644 --- a/examples/job_queue/job.yaml +++ b/examples/job_queue/job.yaml @@ -17,7 +17,7 @@ setup: | run: | timestamp=$(date +%s) conda env list - for i in {1..120}; do + for i in {1..140}; do echo "$timestamp $i" sleep 1 done diff --git a/tests/test_smoke.py b/tests/test_smoke.py index cd6dc4542f7..cbeb3350826 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -1626,7 +1626,7 @@ def test_autostop(generic_cloud: str): f'sky status | grep {name} | grep "1m"', # Ensure the cluster is not stopped early. - 'sleep 45', + 'sleep 30', f's=$(sky status {name} --refresh); echo "$s"; echo; echo; echo "$s" | grep {name} | grep UP', # Ensure the cluster is STOPPED. @@ -1684,7 +1684,7 @@ def test_autodown(generic_cloud: str): # Ensure autostop is set. f'sky status | grep {name} | grep "1m (down)"', # Ensure the cluster is not terminated early. - 'sleep 45', + 'sleep 30', f's=$(sky status {name} --refresh); echo "$s"; echo; echo; echo "$s" | grep {name} | grep UP', # Ensure the cluster is terminated. f'sleep {autodown_timeout}', From b90df17e8b3978913b5c6086f65404d716b58239 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 11:04:25 +0000 Subject: [PATCH 24/27] fix test for quota --- tests/test_smoke.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index cbeb3350826..353a61de3ab 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -222,6 +222,7 @@ def get_gcp_region_for_quota_failover() -> Optional[str]: zone=None) original_resources = sky.Resources(cloud=sky.GCP(), + instance_type='a2-ultragpu-1g', accelerators={'A100-80GB': 1}, use_spot=True) From 304fc9eb621cabccecc2e5fe1ccb5604c3ee13a1 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Wed, 29 Nov 2023 11:06:50 +0000 Subject: [PATCH 25/27] longer timeout --- tests/test_smoke.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 353a61de3ab..3c6df59faf1 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -1242,7 +1242,7 @@ def test_large_job_queue(generic_cloud: str): ], ], f'sky down -y {name}', - timeout=20 * 60, + timeout=25 * 60, ) run_one_test(test) From b8cd23b75039d5165365cfe26298b2fc87f5fdf3 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Thu, 30 Nov 2023 22:34:33 +0000 Subject: [PATCH 26/27] Address comments --- sky/utils/controller_utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index ec27f121697..3465c8abead 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -282,8 +282,8 @@ def get_controller_resources( return controller_resources -def _get_skypilot_config_for_controller_task( - cloud: 'clouds.Cloud') -> Dict[str, Any]: +def _setup_proxy_command_on_controller( + controller_launched_cloud: 'clouds.Cloud') -> Dict[str, Any]: """Sets up proxy command on the controller. This function should be called on the controller (remote cluster), which @@ -317,7 +317,7 @@ def _get_skypilot_config_for_controller_task( # (or name). It may not be a sufficient check (as it's always # possible that peering is not set up), but it may catch some # obvious errors. - proxy_command_key = (str(cloud).lower(), 'ssh_proxy_command') + proxy_command_key = (str(controller_launched_cloud).lower(), 'ssh_proxy_command') ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) config_dict = skypilot_config.to_dict() if isinstance(ssh_proxy_command, str): @@ -336,11 +336,16 @@ def _get_skypilot_config_for_controller_task( def replace_skypilot_config_path_in_file_mounts( cloud: 'clouds.Cloud', file_mounts: Optional[Dict[str, str]]): """Replaces the SkyPilot config path in file mounts with the real path.""" + # TODO(zhwu): This function can be moved to `backend_utils` once we have + # more predefined file mounts that needs to be replaced after the cluster + # is provisioned, e.g., we may need to decide which cloud to create a bucket + # to be mounted to the cluster based on the cloud the cluster is actually + # launched on (after failover). if file_mounts is None or not skypilot_config.loaded(): return replaced = False with tempfile.NamedTemporaryFile('w', delete=False) as f: - new_skypilot_config = _get_skypilot_config_for_controller_task(cloud) + new_skypilot_config = _setup_proxy_command_on_controller(cloud) if new_skypilot_config is not None: common_utils.dump_yaml(f.name, new_skypilot_config) for remote_path, local_path in file_mounts.items(): From 51ad80f40783588a4787c933dbee7db1e3dacce9 Mon Sep 17 00:00:00 2001 From: Zhanghao Wu Date: Thu, 30 Nov 2023 22:34:54 +0000 Subject: [PATCH 27/27] format --- sky/utils/controller_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sky/utils/controller_utils.py b/sky/utils/controller_utils.py index 3465c8abead..bdf001965ff 100644 --- a/sky/utils/controller_utils.py +++ b/sky/utils/controller_utils.py @@ -317,7 +317,8 @@ def _setup_proxy_command_on_controller( # (or name). It may not be a sufficient check (as it's always # possible that peering is not set up), but it may catch some # obvious errors. - proxy_command_key = (str(controller_launched_cloud).lower(), 'ssh_proxy_command') + proxy_command_key = (str(controller_launched_cloud).lower(), + 'ssh_proxy_command') ssh_proxy_command = skypilot_config.get_nested(proxy_command_key, None) config_dict = skypilot_config.to_dict() if isinstance(ssh_proxy_command, str):