From 04376dcf7e490bf1d0909d930450cf98990dd864 Mon Sep 17 00:00:00 2001 From: Joe Runde Date: Fri, 18 Oct 2024 22:26:36 -0500 Subject: [PATCH] [Doc] update gpu-memory-utilization flag docs (#9507) Signed-off-by: Joe Runde Signed-off-by: Alvant --- vllm/engine/arg_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 480d3709224ba..56582ab618797 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -428,7 +428,11 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: help='The fraction of GPU memory to be used for the model ' 'executor, which can range from 0 to 1. For example, a value of ' '0.5 would imply 50%% GPU memory utilization. If unspecified, ' - 'will use the default value of 0.9.') + 'will use the default value of 0.9. This is a global gpu memory ' + 'utilization limit, for example if 50%% of the gpu memory is ' + 'already used before vLLM starts and --gpu-memory-utilization is ' + 'set to 0.9, then only 40%% of the gpu memory will be allocated ' + 'to the model executor.') parser.add_argument( '--num-gpu-blocks-override', type=int,