Fix lint

Makefile

@@ -32,6 +32,7 @@ test: check_pytest_installed
 	@pytest -x -v --ignore=third_party/ --ignore=tests/e2e_test --disable-warnings
 	@python examlpes/offline_inference.py
 	@pytest -v tests/e2e_test/test_e2e.py
+	@pytest -v -x ./tests/e2e_test/test_migration.py
 
 .PHONY: unit_test
 unit_test: check_pytest_installed

llumnix/llm_engine_manager.py

@@ -77,8 +77,6 @@ def __init__(self,
         self.instance_migrating: Dict[str, bool] = {}
         self.pending_rebuild_migration_instances = 0
         self.global_scheduler = GlobalScheduler(global_scheduler_config)
-        # When manager starts, it automatically connects to all existing instances.
-        self._connect_to_instances()
 
         self.polling_interval = engine_manager_args.polling_interval
         asyncio.create_task(self._update_instance_info_loop(self.polling_interval))
@@ -108,6 +106,9 @@ def __init__(self,
             self._init_instance_info_csv(engine_manager_args)
             self.instance_last_logged_empty = {}
 
+        # When manager starts, it automatically connects to all existing instances.
+        self._connect_to_instances()
+
     async def generate(
             self,
             request_id: str,

tests/e2e_test/test_bench.py

@@ -16,7 +16,6 @@
 import os
 import subprocess
 import pytest
-import ray
 import torch
 import numpy as np
 

tests/e2e_test/test_e2e.py

@@ -122,7 +122,7 @@ def generate_launch_command(result_filename: str = "", launch_ray_cluster: bool
 def launch_llumnix_service(model: str, max_model_len: int, port: int, migration_backend: str, launch_mode: str):
     command = generate_launch_command(model=model, max_model_len=max_model_len,
                                       port=port, migration_backend=migration_backend,
-                                      launch_mode=launch_mode)    
+                                      launch_mode=launch_mode)
     subprocess.run(command, shell=True, check=True)
 
 def shutdown_llumnix_service():

tests/e2e_test/test_migration.py

@@ -111,7 +111,7 @@ async def run_bench_command(command):
         assert process.returncode == 0
 
     for i in range(device_count//2):
-        bench_command = generate_bench_command(ip_ports=f"127.0.0.1:{base_port+i}", model=model, num_prompts=10,
+        bench_command = generate_bench_command(ip_ports=f"127.0.0.1:{base_port+i}", model=model, num_prompts=300,
                                                dataset_type="sharegpt",
                                                dataset_path="/mnt/dataset/sharegpt_gpt4/sharegpt_gpt4.jsonl" ,
                                                qps=10)