[Bugfix][Harmless] Fix float16 dtype for model_is_embedding (vllm-pro…

…ject#7566)
rasmith · Aug 16, 2024 · 9c8e2d1 · 9c8e2d1
1 parent 21313e0
commit 9c8e2d1
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py
@@ -66,7 +66,7 @@ def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool:
                        tokenizer_mode="auto",
                        trust_remote_code=trust_remote_code,
                        seed=0,
-                       dtype="float16").embedding_mode
+                       dtype="auto").embedding_mode
 
 
 @asynccontextmanager