Skip to content

Commit

Permalink
Add __del__ at Vllm for deleting model (#10733)
Browse files Browse the repository at this point in the history
* Add __del__ at Vllm for deleting model

* cr
  • Loading branch information
hatianzhang authored Feb 14, 2024
1 parent f7b3fee commit c6b43b2
Showing 1 changed file with 13 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,17 @@ def _model_kwargs(self) -> Dict[str, Any]:
}
return {**base_kwargs}

def __del__(self) -> None:
import torch
from vllm.model_executor.parallel_utils.parallel_state import (
destroy_model_parallel,
)

destroy_model_parallel()
del self._client
if torch.cuda.is_available():
torch.cuda.synchronize()

def _get_all_kwargs(self, **kwargs: Any) -> Dict[str, Any]:
return {
**self._model_kwargs,
Expand Down Expand Up @@ -262,7 +273,8 @@ async def achat(
async def acomplete(
self, prompt: str, formatted: bool = False, **kwargs: Any
) -> CompletionResponse:
raise (ValueError("Not Implemented"))
kwargs = kwargs if kwargs else {}
return self.complete(prompt, **kwargs)

@llm_chat_callback()
async def astream_chat(
Expand Down

0 comments on commit c6b43b2

Please sign in to comment.