From d16022423854d7f2ad5278ba2b087a314b688bf8 Mon Sep 17 00:00:00 2001 From: s5u13b Date: Wed, 27 Nov 2024 12:02:00 +0000 Subject: [PATCH] Add pytest_runtest_makereport --- tests/conftest.py | 38 ++++++++++++++++++++++++++++++++ tests/e2e_test/test_bench.py | 8 +------ tests/e2e_test/test_migration.py | 8 +------ tests/e2e_test/utils.py | 21 ------------------ 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 00977327..655de403 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,10 +11,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from datetime import datetime +import shutil +import os import subprocess import ray import pytest +from llumnix.utils import random_uuid + + def pytest_sessionstart(session): subprocess.run(["ray", "stop"], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) subprocess.run(["ray", "start", "--head", "--disable-usage-stats", "--port=6379"], check=False, @@ -46,3 +52,35 @@ def setup_ray_env(): # pylint: disable=bare-except except: pass + +def backup_error_log(func_name): + curr_time = datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + dst_dir = os.path.expanduser(f'/mnt/error_log/{curr_time}_{random_uuid()}') + os.makedirs(dst_dir, exist_ok=True) + + src_dir = os.getcwd() + + for filename in os.listdir(src_dir): + if filename.startswith("instance_"): + src_file = os.path.join(src_dir, filename) + shutil.copy(src_file, dst_dir) + print(f"Backup error instance log: {src_file} to {dst_dir}") + + if filename.startswith("bench_"): + src_file = os.path.join(src_dir, filename) + shutil.copy(src_file, dst_dir) + print(f"Backup error bench log: {src_file} to {dst_dir}") + + file_path = os.path.join(dst_dir, 'test.info') + print(f"Backup error test.info: {file_path}") + with open(file_path, 'w', encoding='utf-8') as file: + file.write(f'{func_name}') + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + + if report.when == "call" and report.failed: + func_name = item.name + backup_error_log(func_name) diff --git a/tests/e2e_test/test_bench.py b/tests/e2e_test/test_bench.py index c3c7b671..9cb2a9cd 100644 --- a/tests/e2e_test/test_bench.py +++ b/tests/e2e_test/test_bench.py @@ -22,8 +22,7 @@ # pylint: disable=unused-import from .utils import (generate_launch_command, generate_bench_command, to_markdown_table, - cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service, - backup_error_log) + cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service) BENCH_TEST_TIMEOUT_MINS = 30 @@ -99,7 +98,6 @@ def run_bench_command(command): ) tasks.append(bench_command) - test_mode = "e2e_tests" with ThreadPoolExecutor() as executor: future_to_command = {executor.submit(run_bench_command, command): command for command in tasks} @@ -108,14 +106,10 @@ def run_bench_command(command): process = future.result() process.wait(timeout=60*BENCH_TEST_TIMEOUT_MINS) - if process.returncode != 0: - backup_error_log(test_mode) - assert process.returncode == 0, "bench_test failed with return code {}.".format(process.returncode) # pylint: disable=broad-except except subprocess.TimeoutExpired: process.kill() - backup_error_log(test_mode) print("bench_test timed out after 30 minutes.") with open("performance.txt", "w", encoding="utf-8") as f: diff --git a/tests/e2e_test/test_migration.py b/tests/e2e_test/test_migration.py index aa05100b..6a71de59 100644 --- a/tests/e2e_test/test_migration.py +++ b/tests/e2e_test/test_migration.py @@ -22,8 +22,7 @@ # pylint: disable=unused-import from .utils import (generate_launch_command, generate_bench_command, to_markdown_table, - cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service, - backup_error_log) + cleanup_ray_env, wait_for_llumnix_service_ready, shutdown_llumnix_service) size_pattern = re.compile(r'total_kv_cache_size:\s*([\d.]+)\s*(B|KB|MB|GB|KB|TB)') speed_pattern = re.compile(r'speed:\s*([\d.]+)GB/s') @@ -125,7 +124,6 @@ def run_bench_command(command): ) tasks.append(bench_command) - test_mode = f"migration_tests/{migration_backend}/{migrated_request_status}" # Execute the commands concurrently using ThreadPoolExecutor with ThreadPoolExecutor() as executor: future_to_command = {executor.submit(run_bench_command, command): command for command in tasks} @@ -135,14 +133,10 @@ def run_bench_command(command): process = future.result() process.wait(timeout=60*MIGRATION_BENCH_TIMEOUT_MINS) - if process.returncode != 0: - backup_error_log(test_mode) - assert process.returncode == 0, "migration_test failed with return code {}.".format(process.returncode) # pylint: disable=broad-except except subprocess.TimeoutExpired: process.kill() - backup_error_log(test_mode) print("bench_test timed out after {} minutes.".format(MIGRATION_BENCH_TIMEOUT_MINS)) await asyncio.sleep(3) diff --git a/tests/e2e_test/utils.py b/tests/e2e_test/utils.py index 146f6dfe..9b346014 100644 --- a/tests/e2e_test/utils.py +++ b/tests/e2e_test/utils.py @@ -12,10 +12,7 @@ # limitations under the License. import time -import os import subprocess -from datetime import datetime -import shutil import pytest import ray import requests @@ -181,21 +178,3 @@ def to_markdown_table(data): table = f"{header_row}\n{separator_row}\n" + "\n".join(data_rows) + "\n\n" return table - -def backup_error_log(test_mode): - curr_time = datetime.now().strftime('%Y_%m_%d_%H_%M_%S') - dst_dir = os.path.expanduser(f'/mnt/error_log/{test_mode}/{curr_time}/') - os.makedirs(dst_dir, exist_ok=True) - - src_dir = os.getcwd() - - for filename in os.listdir(src_dir): - if filename.startswith("instance_"): - src_file = os.path.join(src_dir, filename) - shutil.copy(src_file, dst_dir) - print(f"Copied instance log: {src_file} to {dst_dir}") - - if filename.startswith("bench_"): - src_file = os.path.join(src_dir, filename) - shutil.copy(src_file, dst_dir) - print(f"Copied bench log: {src_file} to {dst_dir}")