[Bugfix] Add image placeholder for OpenAI Compatible Server of MiniCP…

…M-V (vllm-project#6787) Co-authored-by: hezhihui <[email protected]> Co-authored-by: Cyrus Leung <[email protected]> Signed-off-by: Alvant <[email protected]>
compressa-ai · Oct 26, 2024 · 7c63149 · 7c63149
1 parent 1652610
commit 7c63149
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 1 deletion.
diff --git a/examples/minicpmv_example.py b/examples/minicpmv_example.py
@@ -4,6 +4,8 @@
 from vllm.assets.image import ImageAsset
 
 # 2.0
+# The official repo doesn't work yet, so we need to use a fork for now
+# For more details, please see: See: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
 # MODEL_NAME = "HwwwH/MiniCPM-V-2"
 # 2.5
 MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"

diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py
@@ -100,7 +100,9 @@ def _image_token_str(model_config: ModelConfig,
     if model_type == "phi3_v":
         # Workaround since this token is not defined in the tokenizer
         return "<|image_1|>"
-    if model_type in ("blip-2", "chatglm", "fuyu", "minicpmv", "paligemma"):
+    if model_type == "minicpmv":
+        return "(<image>./</image>)"
+    if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
         # These models do not use image tokens in the prompt
         return None
     if model_type.startswith("llava"):