[Storage] gsutil rsync failing to sync source with dangling symlink (#…

…2165) * update_gsutil_rsync_cmd * fixes * format * nit * nit
skypilot-org · Jul 6, 2023 · ac9ba6a · ac9ba6a
1 parent 9bfa771
commit ac9ba6a
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 4 deletions.
diff --git a/sky/cloud_stores.py b/sky/cloud_stores.py
@@ -133,7 +133,7 @@ def is_directory(self, url: str) -> bool:
     def make_sync_dir_command(self, source: str, destination: str) -> str:
         """Downloads a directory using gsutil."""
         download_via_gsutil = (
-            f'{self._GSUTIL} -m rsync -r {source} {destination}')
+            f'{self._GSUTIL} -m rsync -e -r {source} {destination}')
         all_commands = [self._GET_GSUTIL]
         all_commands.append(download_via_gsutil)
         return ' && '.join(all_commands)

diff --git a/sky/data/storage.py b/sky/data/storage.py
@@ -1507,14 +1507,15 @@ def batch_gsutil_rsync(self,
 
         def get_file_sync_command(base_dir_path, file_names):
             sync_format = '|'.join(file_names)
-            sync_command = (f'gsutil -m rsync -x \'^(?!{sync_format}$).*\' '
+            sync_command = (f'gsutil -m rsync -e -x \'^(?!{sync_format}$).*\' '
                             f'{base_dir_path} gs://{self.name}')
             return sync_command
 
         def get_dir_sync_command(src_dir_path, dest_dir_name):
             # we exclude .git directory from the sync
-            sync_command = (f'gsutil -m rsync -r -x \'.git/*\' {src_dir_path} '
-                            f'gs://{self.name}/{dest_dir_name}')
+            sync_command = (
+                f'gsutil -m rsync -e -r -x \'.git/*\' {src_dir_path} '
+                f'gs://{self.name}/{dest_dir_name}')
             return sync_command
 
         # Generate message for upload