[Logging] Symlink to latest logs

sky/backends/cloud_vm_ray_backend.py

@@ -1307,7 +1307,8 @@ def _retry_zones(
         log_path = os.path.join(self.log_dir, 'provision.log')
         log_abs_path = os.path.abspath(log_path)
         if not dryrun:
-            os.makedirs(os.path.expanduser(self.log_dir), exist_ok=True)
+            log_utils.create_and_symlink_log_dir(
+                os.path.expanduser(self.log_dir))
             os.system(f'touch {log_path}')
         tail_cmd = f'tail -n100 -f {log_path}'
         logger.info('To view detailed progress: '
@@ -3052,7 +3053,7 @@ def _sync_workdir_node(runner: command_runner.CommandRunner) -> None:
             f'{style.BRIGHT}{workdir}{style.RESET_ALL}'
             f' -> '
             f'{style.BRIGHT}{SKY_REMOTE_WORKDIR}{style.RESET_ALL}')
-        os.makedirs(os.path.expanduser(self.log_dir), exist_ok=True)
+        log_utils.create_and_symlink_log_dir(os.path.expanduser(self.log_dir))
         os.system(f'touch {log_path}')
         tail_cmd = f'tail -n100 -f {log_path}'
         logger.info('To view detailed progress: '

sky/callbacks/sky_callback/base.py

@@ -9,6 +9,8 @@
 
 import psutil
 
+from sky.utils import log_utils
+
 # NOTE: This must be the same as _SKY_REMOTE_BENCHMARK_DIR_SYMLINK
 # in sky/benchmark/benchmark_utils.py.
 _SKY_REMOTE_BENCHMARK_DIR = '~/sky_benchmark_dir'
@@ -39,7 +41,7 @@ def __init__(self,
             log_dir, 'sky-callback-' +
             datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f'))
         log_dir = os.path.expanduser(log_dir)
-        os.makedirs(log_dir, exist_ok=True)
+        log_utils.create_and_symlink_log_dir(log_dir)
 
         # TODO(woosuk): Do not store the entire timestamps.
         self._step_begins = []

sky/provision/logging.py

@@ -9,6 +9,7 @@
 import threading
 
 from sky import sky_logging
+from sky.utils import log_utils
 
 
 @dataclasses.dataclass
@@ -24,7 +25,7 @@ def setup_provision_logging(log_dir: str):
     try:
         # Redirect underlying provision logs to file.
         log_path = os.path.expanduser(os.path.join(log_dir, 'provision.log'))
-        os.makedirs(os.path.dirname(log_path), exist_ok=True)
+        log_utils.create_and_symlink_log_dir(os.path.dirname(log_path))
         log_abs_path = pathlib.Path(log_path).expanduser().absolute()
         fh = logging.FileHandler(log_abs_path)
         fh.setFormatter(sky_logging.FORMATTER)

sky/skylet/constants.py

@@ -6,6 +6,7 @@
 import sky
 
 SKY_LOGS_DIRECTORY = '~/sky_logs'
+SKY_LATEST_LINK = 'sky_latest'
 SKY_REMOTE_WORKDIR = '~/sky_workdir'
 
 # Default Ray port is 6379. Default Ray dashboard port is 8265.

sky/skylet/log_lib.py

@@ -163,7 +163,7 @@ def run_with_log(
 
     log_path = os.path.expanduser(log_path)
     dirname = os.path.dirname(log_path)
-    os.makedirs(dirname, exist_ok=True)
+    log_utils.create_and_symlink_log_dir(dirname)
     # Redirect stderr to stdout when using ray, to preserve the order of
     # stdout and stderr.
     stdout_arg = stderr_arg = None

sky/utils/command_runner.py

@@ -11,6 +11,7 @@
 from sky.skylet import constants
 from sky.skylet import log_lib
 from sky.utils import common_utils
+from sky.utils import log_utils
 from sky.utils import subprocess_utils
 from sky.utils import timeline
 
@@ -571,7 +572,7 @@ def run(
         command = base_ssh_command + [shlex.quote(command_str)]
 
         log_dir = os.path.expanduser(os.path.dirname(log_path))
-        os.makedirs(log_dir, exist_ok=True)
+        log_utils.create_and_symlink_log_dir(log_dir)
 
         executable = None
         if not process_stream:
@@ -750,7 +751,7 @@ def run(
         ]
 
         log_dir = os.path.expanduser(os.path.dirname(log_path))
-        os.makedirs(log_dir, exist_ok=True)
+        log_utils.create_and_symlink_log_dir(log_dir)
 
         executable = None
         if not process_stream:

sky/utils/log_utils.py

@@ -1,12 +1,14 @@
 """Logging utils."""
 import enum
+import os
 from typing import List, Optional
 
 import colorama
 import pendulum
 import prettytable
 
 from sky import sky_logging
+from sky.skylet import constants
 from sky.utils import rich_utils
 
 logger = sky_logging.init_logger(__name__)
@@ -191,3 +193,36 @@ def readable_time_duration(start: Optional[float],
         diff = diff.replace('hour', 'hr')
 
     return diff
+
+
+def create_and_symlink_log_dir(log_dir: str):
+    """Create a log directory and symlink to it from parent directory.
+
+    If file operations fail, will log a warning and return without error.
+
+    Args:
+        log_dir (str): Expanded log directory path.
+    """
+    try:
+        os.makedirs(log_dir, exist_ok=True)
+    except OSError:
+        logger.warning(f'Failed to create log directory at {log_dir!r}.'
+                       'Check if there is a file with the same name.')
+        return
+    symlink_path = os.path.join(os.path.dirname(os.path.abspath(log_dir)),
+                                constants.SKY_LATEST_LINK)
+    if os.path.exists(symlink_path):
+        if os.path.islink(symlink_path):
+            try:
+                os.remove(symlink_path)
+            except OSError:
+                return
+        else:
+            logger.warning(
+                (f'Failed to symlink to latest logs at {symlink_path!r}.'
+                 'Please remove the existing file/directory.'))
+            return
+    try:
+        os.symlink(log_dir, symlink_path)
+    except OSError:
+        return

tests/test_smoke.py

@@ -1317,6 +1317,24 @@ def test_scp_logs():
     run_one_test(test)
 
 
+@pytest.mark.kubernetes
+def test_symlink_latest_logs():
+    test = Test(
+        'symlink_latest_logs',
+        [
+            'sky launch -c test2 --cloud kubernetes -- echo hi || true',
+            'eval symlink_path="~/sky_logs/sky_latest"; [ -L "$symlink_path" ] || exit 1 ;'
+            'target_file=$(readlink -f "$symlink_path") || exit 1 ;'
+            'sky local down || true ;'
+            '[ -L "$symlink_path" ] || exit 1 ;'
+            'target_file2=$(readlink -f "$symlink_path") ;'
+            '[ "$target_file" != "$target_file2" ] || exit 1'
+        ],
+        'sky local down',
+    )
+    run_one_test(test)
+
+
 # ---------- Job Queue. ----------
 @pytest.mark.no_fluidstack  # FluidStack DC has low availability of T4 GPUs
 @pytest.mark.no_lambda_cloud  # Lambda Cloud does not have T4 gpus