From bd210a6cf6e4805c5d192022eb00cab3e9df8210 Mon Sep 17 00:00:00 2001 From: AlpinDale Date: Mon, 2 Sep 2024 04:33:51 +0000 Subject: [PATCH] fix: use args.trust_remote_code --- aphrodite/endpoints/openai/api_server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aphrodite/endpoints/openai/api_server.py b/aphrodite/endpoints/openai/api_server.py index 951bee65e..d491f615f 100644 --- a/aphrodite/endpoints/openai/api_server.py +++ b/aphrodite/endpoints/openai/api_server.py @@ -73,11 +73,11 @@ _running_tasks: Set[asyncio.Task] = set() -def model_is_embedding(model_name: str) -> bool: +def model_is_embedding(model_name: str, trust_remote_code: bool) -> bool: return ModelConfig(model=model_name, tokenizer=model_name, tokenizer_mode="auto", - trust_remote_code=False, + trust_remote_code=trust_remote_code, seed=0, dtype="float16").embedding_mode @@ -110,7 +110,7 @@ async def build_async_engine_client(args) -> AsyncIterator[AsyncEngineClient]: # If manually triggered or embedding model, use AsyncAphrodite in process. # TODO: support embedding model via RPC. - if (model_is_embedding(args.model) + if (model_is_embedding(args.model, args.trust_remote_code) or args.disable_frontend_multiprocessing): async_engine_client = AsyncAphrodite.from_engine_args(engine_args) yield async_engine_client