Skip to content

Commit

Permalink
[Storage] Storage tests cleanup and fixes (#2774)
Browse files Browse the repository at this point in the history 
* storage delete confirmation prompt override for tests

* remove kubernetes stop test

* commit msg

* remove redundant storage tests

* repr

* revert eksctl changes
  • Loading branch information
@romilbhardwaj
romilbhardwaj authored Nov 13, 2023
1 parent 1ce2dc1 commit 7f9a305
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 107 deletions.
2 changes: 1 addition & 1 deletion sky/data/storage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -731,7 +731,7 @@ def _add_store_from_metadata(
# Following error is raised from _get_bucket and caught only when
# an externally removed storage is attempted to be fetched.
except exceptions.StorageExternalDeletionError:
logger.debug(f'Storage object, {self.name}, was attempted to '
logger.debug(f'Storage object {self.name!r} was attempted to '
'be reconstructed while the corresponding bucket'
' was externally deleted.')
continue
Expand Down
123 changes: 17 additions & 106 deletions tests/test_smoke.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -804,7 +804,7 @@ def test_using_file_mounts_with_env_vars(generic_cloud: str):

# ---------- storage ----------
@pytest.mark.aws
def test_aws_storage_mounts():
def test_aws_storage_mounts_with_stop():
name = _get_cluster_name()
storage_name = f'sky-test-{int(time.time())}'
template_str = pathlib.Path(
Expand All @@ -820,18 +820,23 @@ def test_aws_storage_mounts():
f'sky launch -y -c {name} --cloud aws {file_path}',
f'sky logs {name} 1 --status', # Ensure job succeeded.
f'aws s3 ls {storage_name}/hello.txt',
f'sky stop -y {name}',
f'sky start -y {name}',
# Check if hello.txt from mounting bucket exists after restart in
# the mounted directory
f'sky exec {name} -- "set -ex; ls /mount_private_mount/hello.txt"'
]
test = Test(
'aws_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
f'sky down -y {name}; sky storage delete -y {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)


@pytest.mark.gcp
def test_gcp_storage_mounts():
def test_gcp_storage_mounts_with_stop():
name = _get_cluster_name()
storage_name = f'sky-test-{int(time.time())}'
template_str = pathlib.Path(
Expand All @@ -847,11 +852,16 @@ def test_gcp_storage_mounts():
f'sky launch -y -c {name} --cloud gcp {file_path}',
f'sky logs {name} 1 --status', # Ensure job succeeded.
f'gsutil ls gs://{storage_name}/hello.txt',
f'sky stop -y {name}',
f'sky start -y {name}',
# Check if hello.txt from mounting bucket exists after restart in
# the mounted directory
f'sky exec {name} -- "set -ex; ls /mount_private_mount/hello.txt"'
]
test = Test(
'gcp_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
f'sky down -y {name}; sky storage delete -y {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)
Expand Down Expand Up @@ -881,7 +891,7 @@ def test_kubernetes_storage_mounts():
test = Test(
'kubernetes_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
f'sky down -y {name}; sky storage delete -y {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)
Expand Down Expand Up @@ -910,7 +920,7 @@ def test_cloudflare_storage_mounts(generic_cloud: str):
test = Test(
'cloudflare_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
f'sky down -y {name}; sky storage delete -y {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)
Expand Down Expand Up @@ -939,106 +949,7 @@ def test_ibm_storage_mounts():
test = Test(
'ibm_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)


@pytest.mark.aws
def test_aws_storage_mounts_with_stop():
name = _get_cluster_name()
storage_name = f'sky-test-{int(time.time())}'
template_str = pathlib.Path(
'tests/test_yamls/test_storage_mounting.yaml.j2').read_text()
template = jinja2.Template(template_str)
content = template.render(storage_name=storage_name)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
f.write(content)
f.flush()
file_path = f.name
test_commands = [
*storage_setup_commands,
f'sky launch -y -c {name} --cloud aws {file_path}',
f'sky logs {name} 1 --status', # Ensure job succeeded.
f'aws s3 ls {storage_name}/hello.txt',
f'sky stop -y {name}',
f'sky start -y {name}',
# Check if hello.txt from mounting bucket exists after restart in
# the mounted directory
f'sky exec {name} -- "set -ex; ls /mount_private_mount/hello.txt"'
]
test = Test(
'aws_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)


@pytest.mark.gcp
def test_gcp_storage_mounts_with_stop():
name = _get_cluster_name()
storage_name = f'sky-test-{int(time.time())}'
template_str = pathlib.Path(
'tests/test_yamls/test_storage_mounting.yaml.j2').read_text()
template = jinja2.Template(template_str)
content = template.render(storage_name=storage_name)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
f.write(content)
f.flush()
file_path = f.name
test_commands = [
*storage_setup_commands,
f'sky launch -y -c {name} --cloud gcp {file_path}',
f'sky logs {name} 1 --status', # Ensure job succeeded.
f'gsutil ls gs://{storage_name}/hello.txt',
f'sky stop -y {name}',
f'sky start -y {name}',
# Check if hello.txt from mounting bucket exists after restart in
# the mounted directory
f'sky exec {name} -- "set -ex; ls /mount_private_mount/hello.txt"'
]
test = Test(
'gcp_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)


@pytest.mark.kubernetes
def test_kubernetes_storage_mounts_with_stop():
# Tests bucket mounting on k8s, assuming S3 is configured.
# This test will fail if run on non x86_64 architecture, since goofys is
# built for x86_64 only.
name = _get_cluster_name()
storage_name = f'sky-test-{int(time.time())}'
template_str = pathlib.Path(
'tests/test_yamls/test_storage_mounting.yaml.j2').read_text()
template = jinja2.Template(template_str)
content = template.render(storage_name=storage_name)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as f:
f.write(content)
f.flush()
file_path = f.name
test_commands = [
*storage_setup_commands,
f'sky launch -y -c {name} --cloud kubernetes {file_path}',
f'sky logs {name} 1 --status', # Ensure job succeeded.
f'aws s3 ls {storage_name}/hello.txt',
f'sky stop -y {name}',
f'sky start -y {name}',
# Check if hello.txt from mounting bucket exists after restart in
# the mounted directory
f'sky exec {name} -- "set -ex; ls /mount_private_mount/hello.txt"'
]
test = Test(
'kubernetes_storage_mounts',
test_commands,
f'sky down -y {name}; sky storage delete {storage_name}',
f'sky down -y {name}; sky storage delete -y {storage_name}',
timeout=20 * 60, # 20 mins
)
run_one_test(test)
Expand Down

0 comments on commit 7f9a305

Please sign in to comment.