diff --git a/docs/Arguments.md b/docs/Arguments.md index dfe06df9..d6c8b7fe 100644 --- a/docs/Arguments.md +++ b/docs/Arguments.md @@ -147,13 +147,13 @@ usage: -m llumnix.entrypoints.vllm.api_server [-h] `--migration-backend` - Communication backend of migration. -- Possible choices: gloo, rayrpc, grpc, kvtransfer. [gloo, rayrpc] are available for vllm and [grpc, kvtransfer] are available for bladellm. -- Default: "rayrpc" +- Possible choices: gloo, rayrpc, nccl, grpc, kvtransfer. [gloo, rayrpc, nccl] are available for vllm and [grpc, kvtransfer] are available for bladellm. +- Default: "gloo" `--migration-backend-transfer-type` -- Transfer type for migration backend grpc and kvTransfer. -- Possible choices: cuda_ipc, rdma, "" -- Default: "" +- Transfer type for migration backend kvTransfer. +- Possible choices: cuda_ipc, rdma +- Default: "rdma" `--migration-backend-server-address` - Address of grpc server for migration backend @@ -161,7 +161,7 @@ usage: -m llumnix.entrypoints.vllm.api_server [-h] `--migration-backend-kvtransfer-naming-url` - URL of naming server for kvtransfer migration backend -- Default: "" +- Default: "file:/tmp/llumnix/naming/" `--migration-buffer-blocks` - Number of buffer blocks in migration. diff --git a/llumnix/arg_utils.py b/llumnix/arg_utils.py index 4bad0cd4..2386fa4f 100644 --- a/llumnix/arg_utils.py +++ b/llumnix/arg_utils.py @@ -318,7 +318,7 @@ def add_cli_args(parser: argparse.ArgumentParser) -> argparse.ArgumentParser: parser.add_argument('--migration-backend', type=str, choices=['gloo','nccl','rayrpc','grpc','kvtransfer'], - help='communication backend of migration, [gloo, rayrpc] are available for vllm \ + help='communication backend of migration, [gloo, rayrpc, nccl] are available for vllm \ and [grpc, kvtransfer] are available for bladellm') parser.add_argument('--migration-backend-transfer-type', type=str, diff --git a/llumnix/config/default.py b/llumnix/config/default.py index 729cf76d..aa76ba30 100644 --- a/llumnix/config/default.py +++ b/llumnix/config/default.py @@ -105,12 +105,12 @@ # Communication backend of migration _C.MANAGER.MIGRATION_BACKEND = "gloo" -# Transfer type for migration backend grpc and kvTransfer -_C.MANAGER.MIGRATION_BACKEND_TRANSFER_TYPE = "" +# Transfer type for migration backend kvTransfer +_C.MANAGER.MIGRATION_BACKEND_TRANSFER_TYPE = "rdma" # Address of grpc server for migration backend _C.MANAGER.MIGRATION_BACKEND_SERVER_ADDRESS = "127.0.0.1:50051" # URL of naming server for kvtransfer migration backend -_C.MANAGER.MIGRATION_BACKEND_KVTRANSFER_NAMING_URL = "" +_C.MANAGER.MIGRATION_BACKEND_KVTRANSFER_NAMING_URL = "file:/tmp/llumnix/naming/" # Timeout(s) for initializing migration backend _C.MANAGER.MIGRATION_BACKEND_INIT_TIMEOUT = 10.0 # Number of cache blocks in migration