From 1f845f2ad9aa91e7ca5e68455b709251c9cc40bd Mon Sep 17 00:00:00 2001 From: hezhihui Date: Thu, 25 Jul 2024 21:01:54 +0800 Subject: [PATCH 1/3] fix openai server image placeholder --- examples/minicpmv_example.py | 1 + vllm/entrypoints/chat_utils.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/examples/minicpmv_example.py b/examples/minicpmv_example.py index 52366a7030ad0..387786f1cc114 100644 --- a/examples/minicpmv_example.py +++ b/examples/minicpmv_example.py @@ -4,6 +4,7 @@ from vllm.assets.image import ImageAsset # 2.0 +# For temporary using # MODEL_NAME = "HwwwH/MiniCPM-V-2" # 2.5 MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5" diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index dca4523d1a27d..53e934b724efe 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -100,6 +100,8 @@ def _image_token_str(model_config: ModelConfig, if model_type == "phi3_v": # Workaround since this token is not defined in the tokenizer return "<|image_1|>" + if model_type == "minicpmv": + return "(./)" if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"): # These models do not use image tokens in the prompt return None From 8e1f3935139389db65b20d7543e7141f705a50cd Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 25 Jul 2024 21:35:26 +0800 Subject: [PATCH 2/3] Update vllm/entrypoints/chat_utils.py --- vllm/entrypoints/chat_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 53e934b724efe..1f6d77b828459 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -102,7 +102,7 @@ def _image_token_str(model_config: ModelConfig, return "<|image_1|>" if model_type == "minicpmv": return "(./)" - if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"): + if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"): # These models do not use image tokens in the prompt return None if model_type.startswith("llava"): From 9f9529f2da78933ac3477db8dd96bffa41550861 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Thu, 25 Jul 2024 22:45:05 +0800 Subject: [PATCH 3/3] Update comment --- examples/minicpmv_example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/minicpmv_example.py b/examples/minicpmv_example.py index 387786f1cc114..bf20a7ea04ad4 100644 --- a/examples/minicpmv_example.py +++ b/examples/minicpmv_example.py @@ -4,7 +4,8 @@ from vllm.assets.image import ImageAsset # 2.0 -# For temporary using +# The official repo doesn't work yet, so we need to use a fork for now +# For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630 # MODEL_NAME = "HwwwH/MiniCPM-V-2" # 2.5 MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"