Skip to content

Commit

Permalink
Add pytest_runtest_makereport
Browse files Browse the repository at this point in the history
s5u13b committed Nov 27, 2024
1 parent fa6174a commit d160224
Showing 4 changed files with 40 additions and 35 deletions.
38 changes: 38 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -11,10 +11,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from datetime import datetime
import shutil
import os
import subprocess
import ray
import pytest

from llumnix.utils import random_uuid


def pytest_sessionstart(session):
subprocess.run(["ray", "stop"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
subprocess.run(["ray", "start", "--head", "--disable-usage-stats", "--port=6379"], check=False,
@@ -46,3 +52,35 @@ def setup_ray_env():
# pylint: disable=bare-except
except:
pass

def backup_error_log(func_name):
curr_time = datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dst_dir = os.path.expanduser(f'/mnt/error_log/{curr_time}_{random_uuid()}')
os.makedirs(dst_dir, exist_ok=True)

src_dir = os.getcwd()

for filename in os.listdir(src_dir):
if filename.startswith("instance_"):
src_file = os.path.join(src_dir, filename)
shutil.copy(src_file, dst_dir)
print(f"Backup error instance log: {src_file} to {dst_dir}")

if filename.startswith("bench_"):
src_file = os.path.join(src_dir, filename)
shutil.copy(src_file, dst_dir)
print(f"Backup error bench log: {src_file} to {dst_dir}")

file_path = os.path.join(dst_dir, 'test.info')
print(f"Backup error test.info: {file_path}")
with open(file_path, 'w', encoding='utf-8') as file:
file.write(f'{func_name}')

@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
outcome = yield
report = outcome.get_result()

if report.when == "call" and report.failed:
func_name = item.name
backup_error_log(func_name)
8 changes: 1 addition & 7 deletions tests/e2e_test/test_bench.py
Original file line number Diff line number Diff line change
@@ -22,8 +22,7 @@

# pylint: disable=unused-import
from .utils import (generate_launch_command, generate_bench_command, to_markdown_table,
cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service,
backup_error_log)
cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service)

BENCH_TEST_TIMEOUT_MINS = 30

@@ -99,7 +98,6 @@ def run_bench_command(command):
)
tasks.append(bench_command)

test_mode = "e2e_tests"
with ThreadPoolExecutor() as executor:
future_to_command = {executor.submit(run_bench_command, command): command for command in tasks}

@@ -108,14 +106,10 @@ def run_bench_command(command):
process = future.result()
process.wait(timeout=60*BENCH_TEST_TIMEOUT_MINS)

if process.returncode != 0:
backup_error_log(test_mode)

assert process.returncode == 0, "bench_test failed with return code {}.".format(process.returncode)
# pylint: disable=broad-except
except subprocess.TimeoutExpired:
process.kill()
backup_error_log(test_mode)
print("bench_test timed out after 30 minutes.")

with open("performance.txt", "w", encoding="utf-8") as f:
8 changes: 1 addition & 7 deletions tests/e2e_test/test_migration.py
Original file line number Diff line number Diff line change
@@ -22,8 +22,7 @@

# pylint: disable=unused-import
from .utils import (generate_launch_command, generate_bench_command, to_markdown_table,
cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service,
backup_error_log)
cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service)

size_pattern = re.compile(r'total_kv_cache_size:\s*([\d.]+)\s*(B|KB|MB|GB|KB|TB)')
speed_pattern = re.compile(r'speed:\s*([\d.]+)GB/s')
@@ -125,7 +124,6 @@ def run_bench_command(command):
)
tasks.append(bench_command)

test_mode = f"migration_tests/{migration_backend}/{migrated_request_status}"
# Execute the commands concurrently using ThreadPoolExecutor
with ThreadPoolExecutor() as executor:
future_to_command = {executor.submit(run_bench_command, command): command for command in tasks}
@@ -135,14 +133,10 @@ def run_bench_command(command):
process = future.result()
process.wait(timeout=60*MIGRATION_BENCH_TIMEOUT_MINS)

if process.returncode != 0:
backup_error_log(test_mode)

assert process.returncode == 0, "migration_test failed with return code {}.".format(process.returncode)
# pylint: disable=broad-except
except subprocess.TimeoutExpired:
process.kill()
backup_error_log(test_mode)
print("bench_test timed out after {} minutes.".format(MIGRATION_BENCH_TIMEOUT_MINS))

await asyncio.sleep(3)
21 changes: 0 additions & 21 deletions tests/e2e_test/utils.py
Original file line number Diff line number Diff line change
@@ -12,10 +12,7 @@
# limitations under the License.

import time
import os
import subprocess
from datetime import datetime
import shutil
import pytest
import ray
import requests
@@ -181,21 +178,3 @@ def to_markdown_table(data):

table = f"{header_row}\n{separator_row}\n" + "\n".join(data_rows) + "\n\n"
return table

def backup_error_log(test_mode):
curr_time = datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dst_dir = os.path.expanduser(f'/mnt/error_log/{test_mode}/{curr_time}/')
os.makedirs(dst_dir, exist_ok=True)

src_dir = os.getcwd()

for filename in os.listdir(src_dir):
if filename.startswith("instance_"):
src_file = os.path.join(src_dir, filename)
shutil.copy(src_file, dst_dir)
print(f"Copied instance log: {src_file} to {dst_dir}")

if filename.startswith("bench_"):
src_file = os.path.join(src_dir, filename)
shutil.copy(src_file, dst_dir)
print(f"Copied bench log: {src_file} to {dst_dir}")

0 comments on commit d160224

Please sign in to comment.