Skip to content

Commit

Permalink
rename
Browse files Browse the repository at this point in the history
  • Loading branch information
Michaelvll committed Mar 1, 2024
1 parent 22e57ce commit c3635fb
Showing 1 changed file with 5 additions and 6 deletions.
11 changes: 5 additions & 6 deletions sky/provision/instance_setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,17 +200,16 @@ def _setup_node(runner: command_runner.SSHCommandRunner,
stream_logs=False,
log_path=log_path,
require_outputs=True)
max_retry = 3
cnt = 0
while returncode == 255 and cnt < max_retry:
# Network connection issue occur during setup. This could happen
# when a setup step requires a reboot, e.g. nvidia-driver
retry_cnt = 0
while returncode == 255 and retry_cnt < _MAX_RETRY:
# Got network connection issue occur during setup. This could
# happen when a setup step requires a reboot, e.g. nvidia-driver
# installation (happens for fluidstack). We should retry for it.
logger.info('Network connection issue during setup, this is '
'likely due to the reboot of the instance. '
'Retrying setup in 10 seconds.')
time.sleep(10)
cnt += 1
retry_cnt += 1
returncode, stdout, stderr = runner.run(cmd,
stream_logs=False,
log_path=log_path,
Expand Down

0 comments on commit c3635fb

Please sign in to comment.