Skip to content

Commit

Permalink
[Tests] Fix test_docker_storage_mounts for azure blob (#3784)
Browse files Browse the repository at this point in the history
* fix azure smoke tests

* lint

* Add comment
  • Loading branch information
romilbhardwaj authored Jul 25, 2024
1 parent eb03d8f commit 93b319e
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 4 deletions.
28 changes: 24 additions & 4 deletions tests/test_smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
from sky import global_user_state
from sky import jobs
from sky import serve
from sky import skypilot_config
from sky.adaptors import cloudflare
from sky.adaptors import ibm
from sky.clouds import AWS
Expand Down Expand Up @@ -1189,9 +1190,22 @@ def test_docker_storage_mounts(generic_cloud: str, image_id: str):
template = jinja2.Template(template_str)
# ubuntu 18.04 does not support fuse3, and blobfuse2 depends on fuse3.
azure_mount_unsupported_ubuntu_version = '18.04'
# Commands to verify bucket upload. We need to check all three
# storage types because the optimizer may pick any of them.
s3_command = f'aws s3 ls {storage_name}/hello.txt'
gsutil_command = f'gsutil ls gs://{storage_name}/hello.txt'
azure_blob_command = TestStorageWithCredentials.cli_ls_cmd(
storage_lib.StoreType.AZURE, storage_name, suffix='hello.txt')
if azure_mount_unsupported_ubuntu_version in image_id:
# The store for mount_private_mount is not specified in the template.
# If we're running on Azure, the private mount will be created on
# azure blob. That will not be supported on the ubuntu 18.04 image
# and thus fail. For other clouds, the private mount on other
# storage types (GCS/S3) should succeed.
include_private_mount = False if generic_cloud == 'azure' else True
content = template.render(storage_name=storage_name,
include_azure_mount=False)
include_azure_mount=False,
include_private_mount=include_private_mount)
else:
content = template.render(storage_name=storage_name,)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
Expand All @@ -1202,8 +1216,10 @@ def test_docker_storage_mounts(generic_cloud: str, image_id: str):
*STORAGE_SETUP_COMMANDS,
f'sky launch -y -c {name} --cloud {generic_cloud} --image-id {image_id} {file_path}',
f'sky logs {name} 1 --status', # Ensure job succeeded.
f'aws s3 ls {storage_name}/hello.txt || '
f'gsutil ls gs://{storage_name}/hello.txt',
# Check AWS, GCP, or Azure storage mount.
f'{s3_command} || '
f'{gsutil_command} || '
f'{azure_blob_command}',
]
test = Test(
'docker_storage_mounts',
Expand Down Expand Up @@ -4277,7 +4293,11 @@ def cli_ls_cmd(store_type, bucket_name, suffix=''):
return f'gsutil ls {url}'
if store_type == storage_lib.StoreType.AZURE:
default_region = 'eastus'
storage_account_name = (
config_storage_account = skypilot_config.get_nested(
('azure', 'storage_account'), None)
storage_account_name = config_storage_account if (
config_storage_account is not None
) else (
storage_lib.AzureBlobStore.DEFAULT_STORAGE_ACCOUNT_NAME.format(
region=default_region,
user_hash=common_utils.get_user_hash()))
Expand Down
6 changes: 6 additions & 0 deletions tests/test_yamls/test_storage_mounting.yaml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@ file_mounts:
source: ['~/tmp-workdir/tmp file', '~/tmp-workdir/tmp file2']
mode: COPY

{% if include_private_mount | default(True) %}
# Mounting private buckets in MOUNT mode
/mount_private_mount:
name: {{storage_name}}
source: ~/tmp-workdir
mode: MOUNT
{% endif %}

run: |
set -ex
Expand All @@ -49,12 +51,16 @@ run: |
ls -ltr /mount_private_copy/tmp\ file
ls -ltr /mount_private_copy_lof/tmp\ file
ls -ltr /mount_private_copy_lof/tmp\ file2
{% if include_private_mount | default(True) %}
ls -ltr /mount_private_mount/foo
ls -ltr /mount_private_mount/tmp\ file
{% endif %}

# Symlinks are not copied to buckets
! ls /mount_private_copy/circle-link
{% if include_private_mount | default(True) %}
! ls /mount_private_mount/circle-link

# Write to private bucket in MOUNT mode should pass
echo "hello" > /mount_private_mount/hello.txt
{% endif %}

0 comments on commit 93b319e

Please sign in to comment.