[Tests] Fix storage mount smoke test for k8s

sky/cloud_stores.py

@@ -52,7 +52,12 @@ class S3CloudStorage(CloudStorage):
 
     # List of commands to install AWS CLI
     _GET_AWSCLI = [
-        'aws --version >/dev/null 2>&1 || pip3 install awscli',
+        'aws --version >/dev/null 2>&1 || '
+        # We use SKY_UV_PIP_CMD to install awscli, instead of raw pip command,
+        # as on some clouds, pip may not be available, e.g., on Kubernetes, our
+        # conda pacakge is installing in the background, and may not be ready
+        # to use yet when we reach here.
+        f'{constants.SKY_UV_PIP_CMD} install awscli',
     ]
 
     def is_directory(self, url: str) -> bool:
@@ -82,7 +87,8 @@ def make_sync_dir_command(self, source: str, destination: str) -> str:
         # AWS Sync by default uses 10 threads to upload files to the bucket.
         # To increase parallelism, modify max_concurrent_requests in your
         # aws config file (Default path: ~/.aws/config).
-        download_via_awscli = ('aws s3 sync --no-follow-symlinks '
+        download_via_awscli = (f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws s3 '
+                               'sync --no-follow-symlinks '
                                f'{source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
@@ -91,7 +97,8 @@ def make_sync_dir_command(self, source: str, destination: str) -> str:
 
     def make_sync_file_command(self, source: str, destination: str) -> str:
         """Downloads a file using AWS CLI."""
-        download_via_awscli = f'aws s3 cp {source} {destination}'
+        download_via_awscli = (f'{constants.SKY_REMOTE_PYTHON_ENV}/bin/aws '
+                               f's3 cp {source} {destination}')
 
         all_commands = list(self._GET_AWSCLI)
         all_commands.append(download_via_awscli)

sky/data/mounting_utils.py

@@ -67,7 +67,9 @@ def get_az_mount_install_cmd() -> str:
     install_cmd = ('sudo apt-get update; '
                    'sudo apt-get install -y '
                    '-o Dpkg::Options::="--force-confdef" '
-                   'fuse3 libfuse3-dev && '
+                   'fuse3 libfuse3-dev || { echo "Failed to install fuse3 '
+                   'libfuse3-dev. Check if the OS is Ubuntu 22.04 or later." '
+                   '&& exit 1; } && '
                    'wget -nc https://github.com/Azure/azure-storage-fuse'
                    f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
                    f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '

tests/smoke_tests/test_mount_and_storage.py

@@ -328,15 +328,15 @@ def test_docker_storage_mounts(generic_cloud: str, image_id: str):
     azure_blob_command = TestStorageWithCredentials.cli_ls_cmd(
         storage_lib.StoreType.AZURE, storage_name, suffix='hello.txt')
     if azure_mount_unsupported_ubuntu_version in image_id:
-        # The store for mount_private_mount is not specified in the template.
-        # If we're running on Azure, the private mount will be created on
-        # azure blob. That will not be supported on the ubuntu 18.04 image
-        # and thus fail. For other clouds, the private mount on other
-        # storage types (GCS/S3) should succeed.
-        include_private_mount = False if generic_cloud == 'azure' else True
+        # Set the store for the private mount to None, when using AWS or GCP, as
+        # SkyPilot should automatically pick the right store on the same cloud.
+        # For other clouds, since SkyPilot may choose azure as the store, which
+        # is not supported on ubuntu 18.04, we set the store to s3.
+        private_mount_store = (None
+                               if generic_cloud in ['aws', 'gcp'] else 's3')
         content = template.render(storage_name=storage_name,
                                   include_azure_mount=False,
-                                  include_private_mount=include_private_mount)
+                                  private_mount_store=private_mount_store)
     else:
         content = template.render(storage_name=storage_name,)
     with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:

tests/test_yamls/test_storage_mounting.yaml.j2

@@ -28,12 +28,13 @@ file_mounts:
     source: ['~/tmp-workdir/tmp file', '~/tmp-workdir/tmp file2']
     mode: {% if only_mount | default(false) %}MOUNT{% else %}COPY{% endif %}
 
-  {% if include_private_mount | default(True) %}
   # Mounting private buckets in MOUNT mode
   /mount_private_mount:
     name: {{storage_name}}
     source: ~/tmp-workdir
     mode: MOUNT
+  {% if private_mount_store | default(None) %}
+    store: {{private_mount_store}}
   {% endif %}
 
 run: |
@@ -51,16 +52,12 @@ run: |
   ls -ltr /mount_private_copy/tmp\ file
   ls -ltr /mount_private_copy_lof/tmp\ file
   ls -ltr /mount_private_copy_lof/tmp\ file2
-  {% if include_private_mount | default(True) %}
   ls -ltr /mount_private_mount/foo
   ls -ltr /mount_private_mount/tmp\ file
-  {% endif %}
 
   # Symlinks are not copied to buckets
   ! ls /mount_private_copy/circle-link
-  {% if include_private_mount | default(True) %}
   ! ls /mount_private_mount/circle-link
 
   # Write to private bucket in MOUNT mode should pass
   echo "hello" > /mount_private_mount/hello.txt
-  {% endif %}