From fc6dba3236a747ef222baf4911d8228944e989ae Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Tue, 21 May 2024 16:00:30 +0800 Subject: [PATCH 1/4] Bump version --- .../embeddings/llama-index-embeddings-ipex-llm/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/pyproject.toml b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/pyproject.toml index a4596dcc17716..3db75ee5d7fa5 100644 --- a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/pyproject.toml +++ b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/pyproject.toml @@ -30,7 +30,7 @@ license = "MIT" name = "llama-index-embeddings-ipex-llm" packages = [{include = "llama_index/"}] readme = "README.md" -version = "0.1.1" +version = "0.1.2" [tool.poetry.dependencies] python = ">=3.9,<4.0" From afcb121d71c017d763b2bb06b7e41698feb87bc6 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Tue, 21 May 2024 16:25:04 +0800 Subject: [PATCH 2/4] Update device related logic for ipex-llm embedding --- .../llama_index/embeddings/ipex_llm/base.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py index b5b102f291a6c..404594176f476 100644 --- a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py +++ b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py @@ -11,7 +11,7 @@ ) from llama_index.core.bridge.pydantic import Field, PrivateAttr from llama_index.core.callbacks import CallbackManager -from llama_index.core.utils import get_cache_dir, infer_torch_device +from llama_index.core.utils import get_cache_dir from llama_index.embeddings.ipex_llm.utils import ( DEFAULT_HUGGINGFACE_EMBEDDING_MODEL, BGE_MODELS, @@ -57,7 +57,8 @@ def __init__( callback_manager: Optional[CallbackManager] = None, **model_kwargs, ): - self._device = device or infer_torch_device() + # Make default device be cpu + self._device = device or "cpu" cache_folder = cache_folder or get_cache_dir() @@ -84,13 +85,11 @@ def __init__( **model_kwargs, ) - if self._device == "cpu": - self._model = _optimize_pre(self._model) - self._model = _optimize_post(self._model) - # TODO: optimize using ipex-llm optimize_model - elif self._device == "xpu": - self._model = _optimize_pre(self._model) - self._model = _optimize_post(self._model) + # Apply ipex-llm optimizations + self._model = _optimize_pre(self._model) + self._model = _optimize_post(self._model) + if self._device == "xpu": + # TODO: apply `ipex_llm.optimize_model` self._model = self._model.half().to(self._device) if max_length: From 3689a3195e53312c78bd1b8b9a745c8e91cb51bc Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Wed, 22 May 2024 15:03:07 +0800 Subject: [PATCH 3/4] Limit device to be cpu or xpu --- .../llama_index/embeddings/ipex_llm/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py index 404594176f476..fa8edf5478c26 100644 --- a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py +++ b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py @@ -57,9 +57,15 @@ def __init__( callback_manager: Optional[CallbackManager] = None, **model_kwargs, ): - # Make default device be cpu + # Set "cpu" as default device self._device = device or "cpu" + if self._device not in ["cpu", "xpu"]: + logger.warning( + "IpexLLMEmbedding currently only supports device to be 'cpu' or 'xpu'," + f"but you have: {self._device}; Use 'cpu' instead." + ) + cache_folder = cache_folder or get_cache_dir() if model_name is None: From 4bc7180977746adc698a6e3a09c6570bec4ac835 Mon Sep 17 00:00:00 2001 From: Yuwen Hu Date: Wed, 22 May 2024 15:27:34 +0800 Subject: [PATCH 4/4] Small fixes --- .../llama_index/embeddings/ipex_llm/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py index fa8edf5478c26..47b48470a7b7c 100644 --- a/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py +++ b/llama-index-integrations/embeddings/llama-index-embeddings-ipex-llm/llama_index/embeddings/ipex_llm/base.py @@ -62,9 +62,10 @@ def __init__( if self._device not in ["cpu", "xpu"]: logger.warning( - "IpexLLMEmbedding currently only supports device to be 'cpu' or 'xpu'," + "IpexLLMEmbedding currently only supports device to be 'cpu' or 'xpu', " f"but you have: {self._device}; Use 'cpu' instead." ) + self._device = "cpu" cache_folder = cache_folder or get_cache_dir()