fix llava

InternLM · Dec 10, 2024 · 92b09d0 · 92b09d0
1 parent 88f99d4
commit 92b09d0
Show file tree

Hide file tree

Showing 4 changed files with 7 additions and 29 deletions.
diff --git a/lmdeploy/pytorch/configurations/llava.py b/lmdeploy/pytorch/configurations/llava.py
diff --git a/lmdeploy/pytorch/engine/engine.py b/lmdeploy/pytorch/engine/engine.py
@@ -831,6 +831,9 @@ async def _async_loop_preprocess_message(self, inque, outque):
                 req_data = req.data
                 if req_data.get('input_multimodals', None) is None:
                     continue
+                elif self.input_processor is None:
+                    logger.warning('Do not support Multimodal inputs.')
+                    continue
                 input_ids = req_data['token_ids']
                 input_multimodals = req_data['input_multimodals']
                 if len(input_multimodals) == 0:

diff --git a/lmdeploy/pytorch/models/llava.py b/lmdeploy/pytorch/models/llava.py
@@ -902,7 +902,8 @@ def prepare_inputs_for_generation(
             if len(img_mms) > 0:
                 image_token_id = img_mms[0].meta['image_token_id']
                 image_mask = input_ids == image_token_id
-                pixel_values = torch.cat([data.data for data in img_mms])
+                pixel_values = torch.cat(
+                    [data.data.flatten(0, 1) for data in img_mms])
                 image_sizes = torch.cat(
                     [data.meta['image_sizes'] for data in img_mms])
             else:

diff --git a/lmdeploy/pytorch/supported_models.py b/lmdeploy/pytorch/supported_models.py
@@ -47,9 +47,9 @@
     # cogvlm-chat
     CogVLMForCausalLM=True,
     # llava
-    LlavaLlamaForCausalLM=True,
+    LlavaLlamaForCausalLM=False,
     # llava mistral
-    LlavaMistralForCausalLM=True,
+    LlavaMistralForCausalLM=False,
     # deepseekvl
     MultiModalityCausalLM=False,
     # StarCoder2