0.11.0 +Mini-Gemini renamed MGM

Dockerfile

@@ -5,18 +5,18 @@ RUN pip install --no-cache-dir --upgrade pip
 
 RUN mkdir -p /app
 RUN git clone https://github.com/01-ai/Yi --single-branch /app/Yi
-RUN git clone https://github.com/dvlab-research/MiniGemini.git --single-branch /app/MiniGemini
+RUN git clone https://github.com/dvlab-research/MGM.git --single-branch /app/MGM
 
 WORKDIR /app
 COPY requirements.txt .
 ARG VERSION=latest
-# transformers==4.36.2 supports most models except Mini-Gemini-2B, llava-1.6, nanollava
+# transformers==4.36.2 supports most models except MGM-2B, llava-1.6, nanollava
 RUN if [ "$VERSION" = "alt" ]; then echo "transformers==4.36.2" >> requirements.txt; else echo "transformers>=4.39.0" >> requirements.txt ; fi
 # TODO: nvidia apex wheel
 RUN pip install --no-cache-dir -U -r requirements.txt \
     https://github.com/Dao-AILab/flash-attention/releases/download/v2.5.7/flash_attn-2.5.7+cu122torch2.2cxx11abiFALSE-cp311-cp311-linux_x86_64.whl
 
-WORKDIR /app/MiniGemini
+WORKDIR /app/MGM
 RUN pip install --no-cache-dir --no-deps -e .
 
 WORKDIR /app

README.md

@@ -41,16 +41,16 @@ An OpenAI API compatible vision server, it functions like `gpt-4-vision-preview`
 - - [X] [MiniCPM-V-2](https://huggingface.co/openbmb/MiniCPM-V-2)
 - - [X] [MiniCPM-V aka. OmniLMM-3B](https://huggingface.co/openbmb/MiniCPM-V)
 - - [ ] [OmniLMM-12B](https://huggingface.co/openbmb/OmniLMM-12B)
-- [X] [MiniGemini](https://huggingface.co/collections/YanweiLi/) (more complex setup, see: `prepare_minigemini.sh`)
-- - [X] [MiniGemini-2B](https://huggingface.co/YanweiLi/Mini-Gemini-2B) (main docker only)
-- - [X] [MiniGemini-7B](https://huggingface.co/YanweiLi/Mini-Gemini-7B) (alternate docker only)
-- - [X] [MiniGemini-13B](https://huggingface.co/YanweiLi/Mini-Gemini-13B) (alternate docker only)
-- - [X] [MiniGemini-34B](https://huggingface.co/YanweiLi/Mini-Gemini-34B) (alternate docker only)
-- - [X] [MiniGemini-8x7B](https://huggingface.co/YanweiLi/Mini-Gemini-8x7B) (alternate docker only)
-- - [X] [MiniGemini-7B-HD](https://huggingface.co/YanweiLi/Mini-Gemini-7B-HD) (alternate docker only)
-- - [X] [MiniGemini-13B-HD](https://huggingface.co/YanweiLi/Mini-Gemini-13B-HD) (alternate docker only)
-- - [X] [MiniGemini-34B-HD](https://huggingface.co/YanweiLi/Mini-Gemini-34B-HD) (alternate docker only)
-- - [X] [MiniGemini-8x7B-HD](https://huggingface.co/YanweiLi/Mini-Gemini-8x7B-HD) (alternate docker only)
+- [X] [YanweiLi/MGM](https://huggingface.co/collections/YanweiLi/) (aka Mini-Gemini, more complex setup, see: `prepare_minigemini.sh`)
+- - [X] [MGM-2B](https://huggingface.co/YanweiLi/MGM-2B) (main docker only)
+- - [X] [MGM-7B](https://huggingface.co/YanweiLi/MGM-7B) (alternate docker only)
+- - [X] [MGM-13B](https://huggingface.co/YanweiLi/MGM-13B) (alternate docker only)
+- - [X] [MGM-34B](https://huggingface.co/YanweiLi/MGM-34B) (alternate docker only)
+- - [X] [MGM-8x7B](https://huggingface.co/YanweiLi/MGM-8x7B) (alternate docker only)
+- - [X] [MGM-7B-HD](https://huggingface.co/YanweiLi/MGM-7B-HD) (alternate docker only)
+- - [X] [MGM-13B-HD](https://huggingface.co/YanweiLi/MGM-13B-HD) (alternate docker only)
+- - [X] [MGM-34B-HD](https://huggingface.co/YanweiLi/MGM-34B-HD) (alternate docker only)
+- - [X] [MGM-8x7B-HD](https://huggingface.co/YanweiLi/MGM-8x7B-HD) (alternate docker only)
 - [X] [qnguyen3/nanoLLaVA](https://huggingface.co/qnguyen3/nanoLLaVA) (main docker only, wont gpu split)
 - [ ] [Deepseek-VL-7b-chat](https://huggingface.co/deepseek-ai/deepseek-vl-7b-chat)
 - [ ] [Deepseek-VL-1.3b-chat](https://huggingface.co/deepseek-ai/deepseek-vl-1.3b-chat)
@@ -64,6 +64,7 @@ See: [OpenVLM Leaderboard](https://huggingface.co/spaces/opencompass/open_vlm_le
 Version: 0.11.0
 
 - new model support: OpenGVLab/InternVL-Chat-V1-5, up to 4k resolution, top opensource model
+- MiniGemini renamed MGM upstream
 
 Version: 0.10.0
 

backend/minigemini.py

@@ -1,29 +1,27 @@
 import re
 from accelerate import infer_auto_device_map
 
-
-
 import transformers
 import warnings
 # disable some warnings
 transformers.logging.set_verbosity_error()
 warnings.filterwarnings('ignore')
 
-from minigemini.constants import IMAGE_TOKEN_INDEX
-from minigemini.model.builder import load_pretrained_model
-from minigemini.mm_utils import process_images, tokenizer_image_token
+from mgm.constants import IMAGE_TOKEN_INDEX
+from mgm.model.builder import load_pretrained_model
+from mgm.mm_utils import process_images, tokenizer_image_token
 
 from vision_qna import *
 
-# YanweiLi/Mini-Gemini-2B
-# YanweiLi/Mini-Gemini-7B
-# YanweiLi/Mini-Gemini-7B-HD
-# YanweiLi/Mini-Gemini-13B
-# YanweiLi/Mini-Gemini-34B
-# YanweiLi/Mini-Gemini-34B-HD
-# YanweiLi/Mini-Gemini-13B-HDs
-# YanweiLi/Mini-Gemini-8x7B-HD
-# YanweiLi/Mini-Gemini-8x7B
+# YanweiLi/MGM-2B
+# YanweiLi/MGM-7B
+# YanweiLi/MGM-7B-HD
+# YanweiLi/MGM-13B
+# YanweiLi/MGM-34B
+# YanweiLi/MGM-34B-HD
+# YanweiLi/MGM-13B-HDs
+# YanweiLi/MGM-8x7B-HD
+# YanweiLi/MGM-8x7B
 
 class VisionQnA(VisionQnABase):
     model_name: str = "minigemini"

model_conf_tests.alt.json

@@ -7,15 +7,15 @@
   ["THUDM/cogvlm-chat-hf"],
   ["THUDM/cogagent-chat-hf"],
   ["Qwen/Qwen-VL-Chat"],
-  ["YanweiLi/Mini-Gemini-2B", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-7B", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-7B-HD", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-13B", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-13B-HD", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-34B", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-34B-HD", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-8x7B", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-8x7B-HD", "--use-flash-attn"],
+  ["YanweiLi/MGM-2B", "--use-flash-attn"],
+  ["YanweiLi/MGM-7B", "--use-flash-attn"],
+  ["YanweiLi/MGM-7B-HD", "--use-flash-attn"],
+  ["YanweiLi/MGM-13B", "--use-flash-attn"],
+  ["YanweiLi/MGM-13B-HD", "--use-flash-attn"],
+  ["YanweiLi/MGM-34B", "--use-flash-attn"],
+  ["YanweiLi/MGM-34B-HD", "--use-flash-attn"],
+  ["YanweiLi/MGM-8x7B", "--use-flash-attn"],
+  ["YanweiLi/MGM-8x7B-HD", "--use-flash-attn"],
   ["adept/fuyu-8b", "--device-map", "cuda:0"],
   ["internlm/internlm-xcomposer2-7b", "--use-flash-attn", "--device-map", "cuda:0"],
   ["internlm/internlm-xcomposer2-vl-7b", "--use-flash-attn", "--device-map", "cuda:0"],
@@ -25,7 +25,7 @@
   ["llava-hf/llava-1.5-7b-hf", "--use-flash-attn", "--device-map", "cuda:0"],
   ["llava-hf/llava-1.5-13b-hf", "--use-flash-attn", "--device-map", "cuda:0"],
 
-  ["OpenGVLab/InternVL-Chat-V1-5", , "--load-in-4bit", "--device-map", "cuda:0"],
+  ["OpenGVLab/InternVL-Chat-V1-5", "--load-in-4bit", "--device-map", "cuda:0"],
   ["THUDM/cogvlm-chat-hf", "--load-in-4bit"],
   ["THUDM/cogagent-chat-hf", "--load-in-4bit"],
   ["internlm/internlm-xcomposer2-7b-4bit", "--use-flash-attn", "--device", "cuda:0"],
@@ -34,12 +34,12 @@
   ["llava-hf/llava-1.5-7b-hf", "--load-in-4bit", "--use-flash-attn"],
   ["llava-hf/llava-1.5-13b-hf", "--load-in-4bit", "--use-flash-attn"],
 
-  ["YanweiLi/Mini-Gemini-7B", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-7B-HD", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-13B", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-13B-HD", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-34B", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-34B-HD", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-8x7B", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-8x7B-HD", "--load-in-4bit", "--use-flash-attn"]
+  ["YanweiLi/MGM-7B", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-7B-HD", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-13B", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-13B-HD", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-34B", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-34B-HD", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-8x7B", "--load-in-4bit", "--use-flash-attn"],
+  ["YanweiLi/MGM-8x7B-HD", "--load-in-4bit", "--use-flash-attn"]
 ]

model_conf_tests.json

@@ -22,7 +22,7 @@
   ["llava-hf/llava-v1.6-34b-hf", "--use-flash-attn"],
   ["01-ai/Yi-VL-6B", "--use-flash-attn"],
   ["01-ai/Yi-VL-34B", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-2B", "--use-flash-attn"],
+  ["YanweiLi/MGM-2B", "--use-flash-attn"],
 
   ["OpenGVLab/InternVL-Chat-V1-5", "--load-in-4bit", "--device-map", "cuda:0"],
   ["qnguyen3/nanoLLaVA", "--use-flash-attn", "--load-in-4bit"],
@@ -39,5 +39,5 @@
   ["llava-hf/llava-v1.6-34b-hf", "--load-in-4bit", "--use-flash-attn"],
   ["01-ai/Yi-VL-6B", "--load-in-4bit", "--use-flash-attn"],
   ["01-ai/Yi-VL-34B", "--load-in-4bit", "--use-flash-attn"],
-  ["YanweiLi/Mini-Gemini-2B", "--use-flash-attn", "--load-in-4bit"]
+  ["YanweiLi/MGM-2B", "--use-flash-attn", "--load-in-4bit"]
 ]

prepare_minigemini.sh

@@ -12,15 +12,15 @@ huggingface-cli download OpenAI/clip-vit-large-patch14-336 --local-dir model_zoo
 huggingface-cli download laion/CLIP-convnext_large_d_320.laion2B-s29B-b131K-ft-soup --local-dir model_zoo/OpenAI/openclip-convnext-large-d-320-laion2B-s29B-b131K-ft-soup
 
 # Select the model(s) of your choice and download them before starting the server
-#huggingface-cli download YanweiLi/Mini-Gemini-2B --local-dir YanweiLi/Mini-Gemini-2B # main image
-#huggingface-cli download YanweiLi/Mini-Gemini-7B --local-dir YanweiLi/Mini-Gemini-7B # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-7B-HD --local-dir YanweiLi/Mini-Gemini-7B-HD # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-13B --local-dir YanweiLi/Mini-Gemini-13B # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-13B-HD --local-dir YanweiLi/Mini-Gemini-13B-HD # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-34B --local-dir YanweiLi/Mini-Gemini-34B # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-34B-HD --local-dir YanweiLi/Mini-Gemini-34B-HD # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-8x7B --local-dir YanweiLi/Mini-Gemini-8x7B # alt image
-#huggingface-cli download YanweiLi/Mini-Gemini-8x7B-HD --local-dir YanweiLi/Mini-Gemini-8x7B-HD # alt image
+huggingface-cli download YanweiLi/MGM-2B --local-dir YanweiLi/MGM-2B # main image
+huggingface-cli download YanweiLi/MGM-7B --local-dir YanweiLi/MGM-7B # alt image
+huggingface-cli download YanweiLi/MGM-7B-HD --local-dir YanweiLi/MGM-7B-HD # alt image
+huggingface-cli download YanweiLi/MGM-13B --local-dir YanweiLi/MGM-13B # alt image
+huggingface-cli download YanweiLi/MGM-13B-HD --local-dir YanweiLi/MGM-13B-HD # alt image
+huggingface-cli download YanweiLi/MGM-34B --local-dir YanweiLi/MGM-34B # alt image
+huggingface-cli download YanweiLi/MGM-34B-HD --local-dir YanweiLi/MGM-34B-HD # alt image
+huggingface-cli download YanweiLi/MGM-8x7B --local-dir YanweiLi/MGM-8x7B # alt image
+huggingface-cli download YanweiLi/MGM-8x7B-HD --local-dir YanweiLi/MGM-8x7B-HD # alt image
 
 #huggingface-cli download 01-ai/Yi-VL-6B --local-dir Yi/VL/01-ai/Yi-VL-6B
 #huggingface-cli download 01-ai/Yi-VL-34B --local-dir Yi/VL/01-ai/Yi-VL-34B

requirements.txt

@@ -15,7 +15,7 @@ uvicorn
 xformers
 
 # moondream
-deepspeed
+deepspeed==0.11.1
 einops
 einops-exts
 httpx