Skip to content

Commit

Permalink
[Core] Deactivate SkyPilot runtime env for jobs (#3639)
Browse files Browse the repository at this point in the history
* [Core] Deactivate SkyPilot runtime env for jobs

* Add smoke test for python path

* Update sky/skylet/log_lib.py

Co-authored-by: Zongheng Yang <[email protected]>

* fix

---------

Co-authored-by: Zongheng Yang <[email protected]>
  • Loading branch information
Michaelvll and concretevitamin committed Aug 23, 2024
1 parent ebcad96 commit d47b016
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 0 deletions.
5 changes: 5 additions & 0 deletions sky/skylet/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@
SKY_REMOTE_PYTHON_ENV_NAME = 'skypilot-runtime'
SKY_REMOTE_PYTHON_ENV = f'~/{SKY_REMOTE_PYTHON_ENV_NAME}'
ACTIVATE_SKY_REMOTE_PYTHON_ENV = f'source {SKY_REMOTE_PYTHON_ENV}/bin/activate'
# Deleting the SKY_REMOTE_PYTHON_ENV_NAME from the PATH to deactivate the
# environment. `deactivate` command does not work when conda is used.
DEACTIVATE_SKY_REMOTE_PYTHON_ENV = (
'export PATH='
f'$(echo $PATH | sed "s|$(echo ~)/{SKY_REMOTE_PYTHON_ENV_NAME}/bin:||")')

# The name for the environment variable that stores the unique ID of the
# current task. This will stay the same across multiple recoveries of the
Expand Down
4 changes: 4 additions & 0 deletions sky/skylet/log_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,6 +263,9 @@ def make_task_bash_script(codegen: str,
# set -a is used for exporting all variables functions to the environment
# so that bash `user_script` can access `conda activate`. Detail: #436.
# Reference: https://www.gnu.org/software/bash/manual/html_node/The-Set-Builtin.html # pylint: disable=line-too-long
# DEACTIVATE_SKY_REMOTE_PYTHON_ENV: Deactivate the SkyPilot runtime env, as
# the ray cluster is started within the runtime env, which may cause the
# user program to run in that env as well.
# PYTHONUNBUFFERED is used to disable python output buffering.
script = [
textwrap.dedent(f"""\
Expand All @@ -271,6 +274,7 @@ def make_task_bash_script(codegen: str,
set -a
. $(conda info --base 2> /dev/null)/etc/profile.d/conda.sh > /dev/null 2>&1 || true
set +a
{constants.DEACTIVATE_SKY_REMOTE_PYTHON_ENV}
export PYTHONUNBUFFERED=1
cd {constants.SKY_REMOTE_WORKDIR}"""),
]
Expand Down
10 changes: 10 additions & 0 deletions tests/test_smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
from sky.data import data_utils
from sky.data import storage as storage_lib
from sky.data.data_utils import Rclone
from sky.skylet import constants
from sky.skylet import events
from sky.utils import common_utils
from sky.utils import resources_utils
Expand Down Expand Up @@ -362,6 +363,9 @@ def test_aws_region():
f'sky status --all | grep {name} | grep us-east-2', # Ensure the region is correct.
f'sky exec {name} \'echo $SKYPILOT_CLUSTER_INFO | jq .region | grep us-east-2\'',
f'sky logs {name} 2 --status', # Ensure the job succeeded.
# A user program should not access SkyPilot runtime env python by default.
f'sky exec {name} \'which python | grep {constants.SKY_REMOTE_PYTHON_ENV_NAME} || exit 1\'',
f'sky logs {name} 3 --status', # Ensure the job succeeded.
],
f'sky down -y {name}',
)
Expand All @@ -382,6 +386,9 @@ def test_gcp_region_and_service_account():
f'sky status --all | grep {name} | grep us-central1', # Ensure the region is correct.
f'sky exec {name} \'echo $SKYPILOT_CLUSTER_INFO | jq .region | grep us-central1\'',
f'sky logs {name} 3 --status', # Ensure the job succeeded.
# A user program should not access SkyPilot runtime env python by default.
f'sky exec {name} \'which python | grep {constants.SKY_REMOTE_PYTHON_ENV_NAME} || exit 1\'',
f'sky logs {name} 4 --status', # Ensure the job succeeded.
],
f'sky down -y {name}',
)
Expand Down Expand Up @@ -419,6 +426,9 @@ def test_azure_region():
f'sky logs {name} 2 --status', # Ensure the job succeeded.
f'sky exec {name} \'echo $SKYPILOT_CLUSTER_INFO | jq .zone | grep null\'',
f'sky logs {name} 3 --status', # Ensure the job succeeded.
# A user program should not access SkyPilot runtime env python by default.
f'sky exec {name} \'which python | grep {constants.SKY_REMOTE_PYTHON_ENV_NAME} || exit 1\'',
f'sky logs {name} 4 --status', # Ensure the job succeeded.
],
f'sky down -y {name}',
)
Expand Down

0 comments on commit d47b016

Please sign in to comment.