Skip to content

Commit

Permalink
Avoid the same session id for openai endpoint (#1995)
Browse files Browse the repository at this point in the history
* Avoid the same session id for openai endpoint

* move to check_request
  • Loading branch information
AllentDan authored Jul 15, 2024
1 parent 93e1082 commit 9c8e38e
Showing 1 changed file with 12 additions and 4 deletions.
16 changes: 12 additions & 4 deletions lmdeploy/serve/openai/api_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,12 @@ async def check_request(request) -> Optional[JSONResponse]:
return create_error_response(
HTTPStatus.BAD_REQUEST,
f'The temperature `{request.temperature}` must be in [0, 2]')
if hasattr(request,
'session_id') and VariableInterface.async_engine.id2step.get(
str(request.session_id), 0) != 0:
return create_error_response(
HTTPStatus.BAD_REQUEST,
f'The session_id `{request.session_id}` is occupied.')
return


Expand Down Expand Up @@ -416,8 +422,9 @@ async def chat_completions_v1(request: ChatCompletionRequest,
- presence_penalty (replaced with repetition_penalty)
- frequency_penalty (replaced with repetition_penalty)
"""
VariableInterface.session_id += 1
request.session_id = VariableInterface.session_id
if request.session_id == -1:
VariableInterface.session_id += 1
request.session_id = VariableInterface.session_id
error_check_ret = await check_request(request)
if error_check_ret is not None:
return error_check_ret
Expand Down Expand Up @@ -785,8 +792,9 @@ async def completions_v1(request: CompletionRequest,
- presence_penalty (replaced with repetition_penalty)
- frequency_penalty (replaced with repetition_penalty)
"""
VariableInterface.session_id += 1
request.session_id = VariableInterface.session_id
if request.session_id == -1:
VariableInterface.session_id += 1
request.session_id = VariableInterface.session_id
error_check_ret = await check_request(request)
if error_check_ret is not None:
return error_check_ret
Expand Down

0 comments on commit 9c8e38e

Please sign in to comment.