Skip to content

Commit

Permalink
hi
Browse files Browse the repository at this point in the history
Signed-off-by: Judy Ng <[email protected]>
  • Loading branch information
judysng committed Mar 26, 2024
1 parent b7761cf commit 66a4346
Show file tree
Hide file tree
Showing 3 changed files with 7 additions and 7 deletions.
4 changes: 2 additions & 2 deletions tests/integration-tests/configs/scaling_stress_test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ test-suites:
performance_tests:
test_scaling.py::test_scaling_stress_test:
dimensions:
- regions: [ "euw1-az3" ]
instances: [ "p3.2xlarge" ]
- regions: [ "us-east-1" ]
instances: [ "c5.large" ]
oss: [ "alinux2" ]
schedulers: [ "slurm" ]
4 changes: 2 additions & 2 deletions tests/integration-tests/tests/common/scaling_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,8 @@ def get_bootstrap_errors(remote_command_executor: RemoteCommandExecutor, cluster
Filters=[{"Name": "private-ip-address", "Values": [ip_address]}]
)["Reservations"][0]["Instances"][0]["InstanceId"]
logging.info(f"Instance {instance_id} had bootstrap errors. Check the logs for details.")
compute_node_log = client.get_console_output(InstanceId=instance_id)["Output"]
with open(f"bootstrap_errors/{ip_address}-{cluster_name}-{instance_id}-bootstrap-error.txt", "w") as file:
compute_node_log = client.get_console_output(InstanceId=instance_id, Latest=True)["Output"]
with open(f"bootstrap_errors/{region}-{cluster_name}-{instance_id}-bootstrap-error.txt", "w") as file:
file.write(compute_node_log)


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def test_scaling_stress_test(


@pytest.mark.usefixtures("scheduler")
@pytest.mark.parametrize("scaling_strategy", ["all-or-nothing", "best-effort"])
@pytest.mark.parametrize("scaling_strategy", ["all-or-nothing"])
def test_static_scaling_stress_test(
test_datadir,
instance,
Expand Down Expand Up @@ -287,8 +287,8 @@ def _scale_up_and_down(
# Get the compute node logs for bootstrap errors if compute nodes did not scale up to scaling target within time
if scaling_target not in compute_nodes_time_series_up:
get_bootstrap_errors(remote_command_executor, cluster.name, region)
raise Exception(f"Cluster did not scale up to {scaling_target} nodes."
f"Check the compute node logs for any bootstrap errors.")
raise Exception(f"Cluster did not scale up to {scaling_target} nodes. "
f"Check the compute node logs for any bootstrap errors in the test artifacts.")

# Extract scale up duration and timestamp from the monitoring metrics collected above
_, scale_up_time_ec2 = _get_scaling_time(ec2_capacity_time_series_up, timestamps, scaling_target, start_time)
Expand Down

0 comments on commit 66a4346

Please sign in to comment.