From c885b8a7bf075fa9356ea537eedf02cbef432975 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Thu, 31 Oct 2024 05:31:59 +0000 Subject: [PATCH 1/8] Update doc --- docs/source/models/supported_models.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index ff893b613f150..dfd9c9730bcbe 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -277,7 +277,7 @@ Text Generation * - :code:`QWenLMHeadModel` - Qwen - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc. - - + - ✅︎ - ✅︎ * - :code:`Qwen2ForCausalLM` - Qwen2 @@ -414,7 +414,7 @@ Text Generation - Models - Inputs - Example HF Models - - :ref:`LoRA ` + - :ref:`LoRA (see note) ` - :ref:`PP ` * - :code:`Blip2ForConditionalGeneration` - BLIP-2 @@ -516,7 +516,7 @@ Text Generation - Qwen-VL - T + I\ :sup:`E+` - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc. - - + - ✅︎ - ✅︎ * - :code:`Qwen2AudioForConditionalGeneration` - Qwen2-Audio @@ -541,8 +541,9 @@ Text Generation | :sup:`+` Multiple items can be inputted per text prompt for this modality. .. note:: - For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. + - For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 + - Regarding multimodal models, vLLM currently only supports adding LoRA to language model. Multimodal Embedding -------------------- From a1e875bc25663029d381ecfa4889a61ae3db197b Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Thu, 31 Oct 2024 05:40:51 +0000 Subject: [PATCH 2/8] Update code --- vllm/model_executor/models/qwen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/model_executor/models/qwen.py b/vllm/model_executor/models/qwen.py index 0a1b40927e9f9..998016ea28c26 100644 --- a/vllm/model_executor/models/qwen.py +++ b/vllm/model_executor/models/qwen.py @@ -1048,7 +1048,7 @@ def get_mm_mapping(self) -> MultiModelKeys: @MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS) @INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen) @INPUT_REGISTRY.register_input_processor(input_processor_for_qwen) -class QWenLMHeadModel(QWenBaseModel): +class QWenLMHeadModel(QWenBaseModel, SupportsLoRA): """ QWenLMHeadModel is not only applicable to LLM but also to VL, which is not conducive to the current integration logic of LoRA in vLLM. Therefore, it From 3dfd27892ca82105e80c2d4f2df4a702d369d318 Mon Sep 17 00:00:00 2001 From: Jee Jee Li Date: Thu, 31 Oct 2024 05:49:01 +0000 Subject: [PATCH 3/8] Update note --- docs/source/models/supported_models.rst | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index dfd9c9730bcbe..3af104bdf907e 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -541,9 +541,12 @@ Text Generation | :sup:`+` Multiple items can be inputted per text prompt for this modality. .. note:: - - For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. - For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 - - Regarding multimodal models, vLLM currently only supports adding LoRA to language model. +- Regarding multimodal models, vLLM currently only supports adding LoRA to language model. + + +.. note:: +- For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. + For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 Multimodal Embedding -------------------- From 430ed2fdb122c9e1f1d85e038441785492a17358 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 31 Oct 2024 13:52:07 +0800 Subject: [PATCH 4/8] Update docs/source/models/supported_models.rst --- docs/source/models/supported_models.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index 3af104bdf907e..2085b81c996b6 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -541,11 +541,11 @@ Text Generation | :sup:`+` Multiple items can be inputted per text prompt for this modality. .. note:: -- Regarding multimodal models, vLLM currently only supports adding LoRA to language model. + Regarding multimodal models, vLLM currently only supports adding LoRA to language model. .. note:: -- For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. + For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 Multimodal Embedding From ef3b211a685e89c7db83601476f7a28af13ead30 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 31 Oct 2024 13:52:31 +0800 Subject: [PATCH 5/8] Update docs/source/models/supported_models.rst --- docs/source/models/supported_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index 2085b81c996b6..ce596226dfb1f 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -546,7 +546,7 @@ Text Generation .. note:: For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. - For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 + For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 Multimodal Embedding -------------------- From e9f48594a64247fdfeb8f57995b69c37a64e8048 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 31 Oct 2024 13:53:41 +0800 Subject: [PATCH 6/8] Update docs/source/models/supported_models.rst --- docs/source/models/supported_models.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index ce596226dfb1f..5fc521850cae7 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -541,8 +541,7 @@ Text Generation | :sup:`+` Multiple items can be inputted per text prompt for this modality. .. note:: - Regarding multimodal models, vLLM currently only supports adding LoRA to language model. - + vLLM currently only supports adding LoRA to the language backbone of multimodal models. .. note:: For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now. From bc76f090693da21e7cdfc48e5c015f5792ac3980 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 31 Oct 2024 13:54:44 +0800 Subject: [PATCH 7/8] Update docs/source/models/supported_models.rst --- docs/source/models/supported_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index 5fc521850cae7..88998aae19ec1 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -414,7 +414,7 @@ Text Generation - Models - Inputs - Example HF Models - - :ref:`LoRA (see note) ` + - :ref:`LoRA ` (see note) - :ref:`PP ` * - :code:`Blip2ForConditionalGeneration` - BLIP-2 From 94aae9011e2a88cd34f3f7fa9a58ce5193ec8850 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 31 Oct 2024 14:11:20 +0800 Subject: [PATCH 8/8] Update docs/source/models/supported_models.rst --- docs/source/models/supported_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/models/supported_models.rst b/docs/source/models/supported_models.rst index 88998aae19ec1..3279e7a108232 100644 --- a/docs/source/models/supported_models.rst +++ b/docs/source/models/supported_models.rst @@ -414,7 +414,7 @@ Text Generation - Models - Inputs - Example HF Models - - :ref:`LoRA ` (see note) + - :ref:`LoRA ` - :ref:`PP ` * - :code:`Blip2ForConditionalGeneration` - BLIP-2