diff --git a/test_models.py b/test_models.py index 4e76f97..944707f 100755 --- a/test_models.py +++ b/test_models.py @@ -136,7 +136,7 @@ def test(cmd_args: list[str]) -> int: tok_total += tok tim_total += tim if tim_total > 0.0: - note += f', {tok_total/tim_total:0.1f} T/s ({tok_total}/{tim_total:0.1f}s)' + note += f', ({tok_total}/{tim_total:0.1f}s) {tok_total/tim_total:0.1f} T/s' print(f"\n\n###\n\nTest complete.\nResult: {green_pass if result else red_fail}, time: {t:.1f}s") diff --git a/vision.sample.env b/vision.sample.env index 539d134..58d7316 100644 --- a/vision.sample.env +++ b/vision.sample.env @@ -6,120 +6,118 @@ HF_HUB_ENABLE_HF_TRANSFER=1 #CUDA_VISIBLE_DEVICES=1,0 #OPENEDAI_DEVICE_MAP="sequential" -#CLI_COMMAND="python vision.py -m AIDC-AI/Ovis1.5-Gemma2-9B -A flash_attention_2" # test pass✅, time: 9.2s, mem: 23.2GB, 13/13 tests passed, 12.8 T/s -#CLI_COMMAND="python vision.py -m AIDC-AI/Ovis1.5-Llama3-8B -A flash_attention_2" # test pass✅, time: 5.9s, mem: 19.2GB, 13/13 tests passed, 22.2 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-2B-zh --load-in-4bit" # test pass✅, time: 8.2s, mem: 9.6GB, 13/13 tests passed, 24.2 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-2B-zh" # test pass✅, time: 5.8s, mem: 10.8GB, 13/13 tests passed, 34.9 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-3B --load-in-4bit" # test pass✅, time: 11.3s, mem: 8.5GB, 13/13 tests passed, 20.2 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-3B" # test pass✅, time: 9.5s, mem: 12.0GB, 13/13 tests passed, 30.0 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-3B-zh" # test pass✅, time: 8.1s, mem: 12.7GB, 13/13 tests passed, 20.0 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-4B --load-in-4bit" # test pass✅, time: 9.0s, mem: 5.1GB, 13/13 tests passed, 19.4 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-4B" # test pass✅, time: 6.9s, mem: 12.2GB, 13/13 tests passed, 23.2 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_1-4B --load-in-4bit" # test pass✅, time: 10.4s, mem: 5.8GB, 13/13 tests passed, 14.4 T/s -#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_1-4B" # test pass✅, time: 8.4s, mem: 13.0GB, 13/13 tests passed, 15.3 T/s -#CLI_COMMAND="python vision.py -m BAAI/Emu2-Chat --load-in-4bit" # test pass✅, time: 30.3s, mem: 29.3GB, 13/13 tests passed, 8.1 T/s -#CLI_COMMAND="python vision.py -m BAAI/Emu2-Chat --max-memory=0:78GiB,1:20GiB" # test pass✅, time: 21.8s, mem: 71.8GB, 13/13 tests passed, 12.1 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 22.8s, mem: 27.5GB, 13/13 tests passed, 8.5 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0 --max-tiles 40 --load-in-4bit" # test pass✅, time: 29.3s, mem: 30.8GB, 13/13 tests passed, 6.2 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0 --max-tiles 40" # test pass✅, time: 25.3s, mem: 55.7GB, 13/13 tests passed, 5.7 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0" # test pass✅, time: 18.8s, mem: 52.6GB, 13/13 tests passed, 8.6 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-1B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 30.8s, mem: 1.9GB, 13/13 tests passed, 28.2 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-1B --device-map cuda:0" # test pass✅, time: 8.2s, mem: 2.6GB, 13/13 tests passed, 35.4 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-2B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 16.8s, mem: 5.4GB, 13/13 tests passed, 31.1 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-2B --device-map cuda:0" # test pass✅, time: 8.5s, mem: 7.2GB, 13/13 tests passed, 38.4 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-8B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 9.8s, mem: 9.1GB, 13/13 tests passed, 15.3 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-8B --device-map cuda:0" # test pass✅, time: 8.0s, mem: 19.1GB, 13/13 tests passed, 19.8 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-26B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 25.9s, mem: 27.7GB, 13/13 tests passed, 9.2 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-26B --device-map cuda:0" # test pass✅, time: 20.5s, mem: 53.0GB, 13/13 tests passed, 9.3 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-40B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 39.9s, mem: 31.7GB, 13/13 tests passed, 6.6 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-40B --device-map cuda:0" # test pass✅, time: 47.7s, mem: 76.8GB, 13/13 tests passed, 9.1 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-Llama3-76B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 41.1s, mem: 51.7GB, 13/13 tests passed, 3.1 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5 --load-in-4bit" # test pass✅, time: 6.9s, mem: 6.1GB, 13/13 tests passed, 23.0 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5 --max-tiles 40 --load-in-4bit" # test pass✅, time: 7.7s, mem: 8.2GB, 13/13 tests passed, 20.9 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5 --max-tiles 40" # test pass✅, time: 6.9s, mem: 10.1GB, 13/13 tests passed, 26.0 T/s -#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5" # test pass✅, time: 6.0s, mem: 7.8GB, 13/13 tests passed, 31.2 T/s -#CLI_COMMAND="python vision.py -m Qwen/Qwen-VL-Chat --load-in-4bit" # test pass✅, time: 10.2s, mem: 11.4GB, 13/13 tests passed -#CLI_COMMAND="python vision.py -m Qwen/Qwen-VL-Chat" # test pass✅, time: 6.9s, mem: 19.7GB, 13/13 tests passed -#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-2B-Instruct-AWQ -A flash_attention_2" # test pass✅, time: 13.2s, mem: 7.1GB, 13/13 tests passed, 11.9 T/s -#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-2B-Instruct -A flash_attention_2" # test pass✅, time: 15.6s, mem: 16.5GB, 13/13 tests passed, 8.1 T/s -#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-7B-Instruct-AWQ -A flash_attention_2" # test pass✅, time: 18.9s, mem: 18.7GB, 13/13 tests passed, 6.3 T/s -#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-7B-Instruct -A flash_attention_2" # test pass✅, time: 16.5s, mem: 27.7GB, 13/13 tests passed, 6.2 T/s -#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-72B-Instruct-AWQ -A flash_attention_2" # test pass✅, time: 36.6s, mem: 45.3GB, 13/13 tests passed, 2.7 T/s -#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-dpo-r-v1.5" # test pass✅, time: 9.4s, mem: 9.4GB, 13/13 tests passed, 26.5 T/s -#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5" # test pass✅, time: 3.8s, mem: 9.4GB, 13/13 tests passed, 12.9 T/s -#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-singleimg-r-v1.5" # test pass✅, time: 9.8s, mem: 9.4GB, 13/13 tests passed, 27.0 T/s -#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-r-v1" # test pass✅, time: 9.8s, mem: 9.9GB, 13/13 tests passed, 26.8 T/s -#CLI_COMMAND="python vision.py -m THUDM/glm-4v-9b --device-map cuda:0 --load-in-4bit" # test fail❌, time: 1.6s, mem: 15.1GB, Test failed with Exception: Error code: 500 - {'message': 'InternalServerError', 'code': 500} -#CLI_COMMAND="python vision.py -m THUDM/glm-4v-9b --device-map cuda:0" # test fail❌, time: 1.6s, mem: 26.8GB, Test failed with Exception: Error code: 500 - {'message': 'InternalServerError', 'code': 500} -#CLI_COMMAND="python vision.py -m TIGER-Lab/Mantis-8B-Fuyu --device-map cuda:0 --load-in-4bit" # test pass✅, time: 9.9s, mem: 11.5GB, 13/13 tests passed, 5.0 T/s -#CLI_COMMAND="python vision.py -m TIGER-Lab/Mantis-8B-Fuyu --device-map cuda:0" # test pass✅, time: 9.2s, mem: 20.8GB, 13/13 tests passed, 8.6 T/s -#CLI_COMMAND="python vision.py -m adept/fuyu-8b --device-map cuda:0 --load-in-4bit" # test pass✅, time: 23.7s, mem: 16.0GB, 13/13 tests passed, 12.2 T/s -#CLI_COMMAND="python vision.py -m adept/fuyu-8b --device-map cuda:0" # test pass✅, time: 17.4s, mem: 26.0GB, 13/13 tests passed, 14.8 T/s -#CLI_COMMAND="python vision.py -m allenai/MolmoE-1B-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 131.1s, mem: 6.1GB, 13/13 tests passed, 2.4 T/s -#CLI_COMMAND="python vision.py -m allenai/MolmoE-1B-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 98.4s, mem: 6.3GB, 13/13 tests passed, 2.6 T/s -#CLI_COMMAND="python vision.py -m allenai/MolmoE-1B-0924 -A flash_attention_2" # test pass✅, time: 27.6s, mem: 15.3GB, 13/13 tests passed, 4.7 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-D-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 49.2s, mem: 8.0GB, 13/13 tests passed, 20.4 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-D-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 41.1s, mem: 8.4GB, 13/13 tests passed, 22.9 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-D-0924 -A flash_attention_2" # test pass✅, time: 32.6s, mem: 18.2GB, 13/13 tests passed, 28.8 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-O-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 41.4s, mem: 8.6GB, 13/13 tests passed, 16.2 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-O-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 38.6s, mem: 8.9GB, 13/13 tests passed, 17.3 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-O-0924 -A flash_attention_2" # test pass✅, time: 30.2s, mem: 18.6GB, 13/13 tests passed, 21.7 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-72B-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 130.9s, mem: 43.3GB, 13/13 tests passed, 6.7 T/s -#CLI_COMMAND="python vision.py -m allenai/Molmo-72B-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 113.7s, mem: 48.3GB, 13/13 tests passed, 7.4 T/s -#CLI_COMMAND="python vision.py -m echo840/Monkey-Chat --load-in-4bit" # test pass✅, time: 15.7s, mem: 15.9GB, 13/13 tests passed, 10.4 T/s -#CLI_COMMAND="python vision.py -m echo840/Monkey-Chat" # test pass✅, time: 11.9s, mem: 21.8GB, 13/13 tests passed, 9.3 T/s -#CLI_COMMAND="python vision.py -m failspy/Phi-3-vision-128k-instruct-abliterated-alpha -A flash_attention_2 --load-in-4bit" # test pass✅, time: 11.5s, mem: 7.6GB, 13/13 tests passed, 11.4 T/s -#CLI_COMMAND="python vision.py -m failspy/Phi-3-vision-128k-instruct-abliterated-alpha -A flash_attention_2" # test pass✅, time: 9.5s, mem: 12.5GB, 13/13 tests passed, 13.8 T/s -#CLI_COMMAND="python vision.py -m fancyfeast/joy-caption-pre-alpha --load-in-4bit -A flash_attention_2" # test pass✅, time: 126.4s, mem: 8.6GB, 13/13 tests passed, 16.3 T/s -#CLI_COMMAND="python vision.py -m fancyfeast/joy-caption-pre-alpha -A flash_attention_2" # test pass✅, time: 68.0s, mem: 18.2GB, 13/13 tests passed, 31.2 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2d5-7b -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test fail❌, time: 3.1s, mem: 9.0GB, 1/13 tests passed -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2d5-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 32.9s, mem: 30.1GB, 13/13 tests passed, 7.3 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-4khd-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 22.2s, mem: 22.1GB, 13/13 tests passed, 5.9 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 27.6s, mem: 18.4GB, 13/13 tests passed, 7.6 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-7b-4bit -A flash_attention_2" # test pass✅, time: 60.6s, mem: 8.9GB, 13/13 tests passed, 3.8 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-vl-1_8b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 9.0s, mem: 7.8GB, 13/13 tests passed, 11.2 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-vl-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 25.2s, mem: 20.8GB, 13/13 tests passed, 6.7 T/s -#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-vl-7b-4bit -A flash_attention_2" # test pass✅, time: 46.1s, mem: 10.8GB, 13/13 tests passed, 3.3 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-13b-hf -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 16.1s, mem: 9.4GB, 13/13 tests passed, 12.4 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-13b-hf -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 11.0s, mem: 26.6GB, 13/13 tests passed, 18.6 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-7b-hf -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 13.7s, mem: 5.6GB, 13/13 tests passed, 15.0 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-7b-hf -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 10.5s, mem: 14.4GB, 13/13 tests passed, 22.1 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-34b-hf -A flash_attention_2 --load-in-4bit" # test pass✅, time: 71.9s, mem: 22.4GB, 13/13 tests passed, 8.0 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-34b-hf -A flash_attention_2" # test pass✅, time: 88.3s, mem: 67.5GB, 13/13 tests passed, 8.6 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-13b-hf -A flash_attention_2 --load-in-4bit" # test pass✅, time: 19.9s, mem: 12.6GB, 13/13 tests passed, 9.1 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-13b-hf -A flash_attention_2" # test pass✅, time: 17.8s, mem: 29.6GB, 13/13 tests passed, 10.2 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-7b-hf -A flash_attention_2 --load-in-4bit" # test pass✅, time: 22.8s, mem: 7.9GB, 13/13 tests passed, 12.5 T/s -#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-7b-hf -A flash_attention_2" # test pass✅, time: 16.6s, mem: 16.7GB, 13/13 tests passed, 16.1 T/s -#CLI_COMMAND="python vision.py -m lmms-lab/llava-onevision-qwen2-0.5b-ov -A flash_attention_2" # test pass✅, time: 8.9s, mem: 23.6GB, 13/13 tests passed, 15.3 T/s -#CLI_COMMAND="python vision.py -m lmms-lab/llava-onevision-qwen2-7b-ov -A flash_attention_2" # test pass✅, time: 19.2s, mem: 36.7GB, 13/13 tests passed, 8.8 T/s -#CLI_COMMAND="python vision.py -m meta-llama/Llama-3.2-11B-Vision-Instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 57.1s, mem: 8.9GB, 13/13 tests passed, 15.6 T/s -#CLI_COMMAND="python vision.py -m meta-llama/Llama-3.2-11B-Vision-Instruct -A flash_attention_2" # test pass✅, time: 28.2s, mem: 22.6GB, 13/13 tests passed, 22.2 T/s -#CLI_COMMAND="python vision.py -m meta-llama/Llama-3.2-90B-Vision-Instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 146.9s, mem: 51.1GB, 13/13 tests passed, 5.7 T/s -#CLI_COMMAND="python vision.py -m microsoft/Florence-2-base-ft -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 3.3s, mem: 1.1GB, 13/13 tests passed -#CLI_COMMAND="python vision.py -m microsoft/Florence-2-base-ft -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 2.5s, mem: 1.4GB, 13/13 tests passed -#CLI_COMMAND="python vision.py -m microsoft/Florence-2-large-ft -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 3.6s, mem: 1.6GB, 13/13 tests passed -#CLI_COMMAND="python vision.py -m microsoft/Florence-2-large-ft -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 2.5s, mem: 2.5GB, 13/13 tests passed -#CLI_COMMAND="python vision.py -m microsoft/Phi-3-vision-128k-instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 14.0s, mem: 7.6GB, 13/13 tests passed, 12.7 T/s -#CLI_COMMAND="python vision.py -m microsoft/Phi-3-vision-128k-instruct -A flash_attention_2" # test pass✅, time: 10.0s, mem: 12.5GB, 13/13 tests passed, 13.5 T/s -#CLI_COMMAND="python vision.py -m microsoft/Phi-3.5-vision-instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 9.8s, mem: 4.7GB, 13/13 tests passed, 13.6 T/s -#CLI_COMMAND="python vision.py -m microsoft/Phi-3.5-vision-instruct -A flash_attention_2" # test pass✅, time: 8.7s, mem: 9.7GB, 13/13 tests passed, 17.3 T/s -#CLI_COMMAND="python vision.py -m mistralai/Pixtral-12B-2409" # test pass✅, time: 17.2s, mem: 35.8GB, 13/13 tests passed, 12.7 T/s (manual calc) -#CLI_COMMAND="python vision.py -m mx262/MiniMonkey -A flash_attention_2 --load-in-4bit" # test pass✅, time: 11.1s, mem: 14.3GB, 13/13 tests passed, 11.5 T/s -#CLI_COMMAND="python vision.py -m mx262/MiniMonkey -A flash_attention_2" # test pass✅, time: 10.0s, mem: 16.6GB, 13/13 tests passed, 13.0 T/s -#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-V-2_6-int4 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 18.1s, mem: 9.6GB, 13/13 tests passed, 17.3 T/s -#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-V-2_6 -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 15.4s, mem: 9.8GB, 13/13 tests passed, 22.0 T/s -#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-V-2_6 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 12.8s, mem: 19.2GB, 13/13 tests passed, 29.3 T/s -#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-Llama3-V-2_5 -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 27.7s, mem: 9.4GB, 13/13 tests passed, 8.4 T/s -#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-Llama3-V-2_5 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 25.0s, mem: 19.4GB, 13/13 tests passed, 9.1 T/s -#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 9.3s, mem: 8.3GB, 13/13 tests passed, 21.9 T/s -#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 6.9s, mem: 8.7GB, 13/13 tests passed, 32.6 T/s -#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA-1.5 -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 7.8s, mem: 8.3GB, 13/13 tests passed, 22.6 T/s -#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA-1.5 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 7.8s, mem: 8.7GB, 13/13 tests passed, 36.5 T/s -#CLI_COMMAND="python vision.py -m qresearch/llama-3-vision-alpha-hf --device cuda:0 --load-in-4bit" # test pass✅, time: 13.3s, mem: 6.9GB, 13/13 tests passed, 20.2 T/s -#CLI_COMMAND="python vision.py -m qresearch/llama-3-vision-alpha-hf --device cuda:0" # test pass✅, time: 8.6s, mem: 16.8GB, 13/13 tests passed, 27.0 T/s -#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-Med-v1 --load-in-4bit" # test pass✅, time: 11.3s, mem: 8.2GB, 13/13 tests passed, 13.7 T/s -#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-Med-v1" # test pass✅, time: 9.7s, mem: 18.6GB, 13/13 tests passed, 19.0 T/s -#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-v1 --load-in-4bit" # test pass✅, time: 14.1s, mem: 8.2GB, 13/13 tests passed, 15.5 T/s -#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-v1" # test pass✅, time: 13.4s, mem: 18.6GB, 13/13 tests passed, 24.4 T/s -#CLI_COMMAND="python vision.py -m vikhyatk/moondream2 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 6.5s, mem: 2.8GB, 13/13 tests passed, 35.0 T/s -#CLI_COMMAND="python vision.py -m vikhyatk/moondream2 -A flash_attention_2" # test pass✅, time: 5.0s, mem: 4.7GB, 13/13 tests passed, 48.1 T/s +#CLI_COMMAND="python vision.py -m AIDC-AI/Ovis1.5-Gemma2-9B -A flash_attention_2" # test pass✅, time: 10.0s, mem: 23.2GB, 13/13 tests passed, (32/2.7s) 12.0 T/s +#CLI_COMMAND="python vision.py -m AIDC-AI/Ovis1.5-Llama3-8B -A flash_attention_2" # test pass✅, time: 6.1s, mem: 19.2GB, 13/13 tests passed, (32/1.5s) 21.6 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-2B-zh --load-in-4bit" # test pass✅, time: 8.8s, mem: 9.5GB, 13/13 tests passed, (39/1.6s) 24.0 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-2B-zh" # test pass✅, time: 5.8s, mem: 10.8GB, 13/13 tests passed, (38/1.2s) 31.8 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-3B --load-in-4bit" # test pass✅, time: 11.3s, mem: 8.5GB, 13/13 tests passed, (59/3.0s) 20.0 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-3B" # test pass✅, time: 9.2s, mem: 12.0GB, 13/13 tests passed, (70/2.3s) 30.1 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-3B-zh" # test pass✅, time: 8.3s, mem: 12.7GB, 13/13 tests passed, (37/1.9s) 19.6 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-4B --load-in-4bit" # test pass✅, time: 9.4s, mem: 5.1GB, 13/13 tests passed, (48/2.5s) 18.8 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_0-4B" # test pass✅, time: 7.3s, mem: 12.2GB, 13/13 tests passed, (44/1.9s) 22.7 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_1-4B --load-in-4bit" # test pass✅, time: 11.0s, mem: 5.8GB, 13/13 tests passed, (44/3.1s) 14.1 T/s +#CLI_COMMAND="python vision.py -m BAAI/Bunny-v1_1-4B" # test pass✅, time: 8.6s, mem: 13.0GB, 13/13 tests passed, (35/2.4s) 14.9 T/s +#CLI_COMMAND="python vision.py -m BAAI/Emu2-Chat --load-in-4bit" # test pass✅, time: 35.4s, mem: 29.3GB, 13/13 tests passed, (98/12.1s) 8.1 T/s +#CLI_COMMAND="python vision.py -m BAAI/Emu2-Chat --max-memory=0:78GiB,1:20GiB" # test pass✅, time: 20.7s, mem: 71.9GB, 13/13 tests passed, (76/6.2s) 12.3 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 23.7s, mem: 27.5GB, 13/13 tests passed, (60/7.3s) 8.2 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0 --max-tiles 40 --load-in-4bit" # test pass✅, time: 30.6s, mem: 30.9GB, 13/13 tests passed, (58/9.6s) 6.1 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0 --max-tiles 40" # test pass✅, time: 25.4s, mem: 55.8GB, 13/13 tests passed, (45/8.0s) 5.6 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL-Chat-V1-5 --device-map cuda:0" # test pass✅, time: 19.0s, mem: 52.7GB, 13/13 tests passed, (50/5.9s) 8.5 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-1B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 31.3s, mem: 1.9GB, 13/13 tests passed, (271/10.0s) 27.0 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-1B --device-map cuda:0" # test pass✅, time: 8.0s, mem: 2.8GB, 13/13 tests passed, (77/2.3s) 33.5 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-2B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 17.2s, mem: 5.5GB, 13/13 tests passed, (156/5.3s) 29.5 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-2B --device-map cuda:0" # test pass✅, time: 8.2s, mem: 7.4GB, 13/13 tests passed, (90/2.3s) 39.5 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-8B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 9.9s, mem: 9.3GB, 13/13 tests passed, (43/2.8s) 15.6 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-8B --device-map cuda:0" # test pass✅, time: 8.0s, mem: 19.0GB, 13/13 tests passed, (43/2.2s) 19.5 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-26B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 26.5s, mem: 27.5GB, 13/13 tests passed, (75/8.1s) 9.2 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-26B --device-map cuda:0" # test pass✅, time: 20.3s, mem: 52.7GB, 13/13 tests passed, (59/6.3s) 9.4 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-40B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 38.4s, mem: 31.5GB, 13/13 tests passed, (82/12.0s) 6.8 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-40B --device-map cuda:0" # test pass✅, time: 47.3s, mem: 76.5GB, 13/13 tests passed, (140/15.2s) 9.2 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/InternVL2-Llama3-76B --device-map cuda:0 --load-in-4bit" # test pass✅, time: 40.9s, mem: 51.4GB, 13/13 tests passed, (40/12.8s) 3.1 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5 --load-in-4bit" # test pass✅, time: 6.8s, mem: 5.8GB, 13/13 tests passed, (42/1.8s) 23.9 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5 --max-tiles 40 --load-in-4bit" # test pass✅, time: 7.6s, mem: 8.1GB, 13/13 tests passed, (42/2.0s) 20.8 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5 --max-tiles 40" # test pass✅, time: 6.8s, mem: 10.0GB, 13/13 tests passed, (48/1.8s) 26.7 T/s +#CLI_COMMAND="python vision.py -m OpenGVLab/Mini-InternVL-Chat-2B-V1-5" # test pass✅, time: 6.0s, mem: 7.7GB, 13/13 tests passed, (48/1.5s) 31.5 T/s +#CLI_COMMAND="python vision.py -m Qwen/Qwen-VL-Chat --load-in-4bit" # test pass✅, time: 10.0s, mem: 11.1GB, 13/13 tests passed +#CLI_COMMAND="python vision.py -m Qwen/Qwen-VL-Chat" # test pass✅, time: 7.0s, mem: 19.5GB, 13/13 tests passed +#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-2B-Instruct-AWQ -A flash_attention_2" # test pass✅, time: 13.0s, mem: 6.8GB, 13/13 tests passed, (44/3.7s) 11.9 T/s +#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-2B-Instruct -A flash_attention_2" # test pass✅, time: 18.1s, mem: 16.2GB, 13/13 tests passed, (36/5.3s) 6.8 T/s +#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-7B-Instruct-AWQ -A flash_attention_2" # test pass✅, time: 23.1s, mem: 18.4GB, 13/13 tests passed, (36/6.9s) 5.2 T/s +#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-7B-Instruct -A flash_attention_2" # test pass✅, time: 19.7s, mem: 27.4GB, 13/13 tests passed, (31/5.8s) 5.4 T/s +#CLI_COMMAND="python vision.py -m Qwen/Qwen2-VL-72B-Instruct-AWQ -A flash_attention_2" # test pass✅, time: 37.6s, mem: 45.2GB, 13/13 tests passed, (31/11.9s) 2.6 T/s +#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-dpo-r-v1.5" # test pass✅, time: 9.2s, mem: 9.3GB, 13/13 tests passed, (68/2.6s) 26.6 T/s +#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-interleave-r-v1.5" # test pass✅, time: 3.9s, mem: 9.3GB, 13/13 tests passed, (10/0.8s) 12.9 T/s +#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-singleimg-r-v1.5" # test pass✅, time: 9.6s, mem: 9.3GB, 13/13 tests passed, (73/2.6s) 28.2 T/s +#CLI_COMMAND="python vision.py -m Salesforce/xgen-mm-phi3-mini-instruct-r-v1" # test pass✅, time: 9.8s, mem: 9.8GB, 13/13 tests passed, (74/2.8s) 26.2 T/s +#CLI_COMMAND="python vision.py -m TIGER-Lab/Mantis-8B-Fuyu --device-map cuda:0 --load-in-4bit" # test pass✅, time: 9.9s, mem: 11.4GB, 13/13 tests passed, (14/2.7s) 5.1 T/s +#CLI_COMMAND="python vision.py -m TIGER-Lab/Mantis-8B-Fuyu --device-map cuda:0" # test pass✅, time: 9.2s, mem: 20.7GB, 13/13 tests passed, (22/2.6s) 8.4 T/s +#CLI_COMMAND="python vision.py -m adept/fuyu-8b --device-map cuda:0 --load-in-4bit" # test pass✅, time: 23.5s, mem: 15.8GB, 13/13 tests passed, (92/7.4s) 12.5 T/s +#CLI_COMMAND="python vision.py -m adept/fuyu-8b --device-map cuda:0" # test pass✅, time: 17.0s, mem: 25.9GB, 13/13 tests passed, (79/5.4s) 14.7 T/s +#CLI_COMMAND="python vision.py -m allenai/MolmoE-1B-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 127.7s, mem: 6.0GB, 13/13 tests passed, (104/41.9s) 2.5 T/s +#CLI_COMMAND="python vision.py -m allenai/MolmoE-1B-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 95.6s, mem: 6.2GB, 13/13 tests passed, (84/31.0s) 2.7 T/s +#CLI_COMMAND="python vision.py -m allenai/MolmoE-1B-0924 -A flash_attention_2" # test pass✅, time: 26.5s, mem: 15.2GB, 13/13 tests passed, (40/8.3s) 4.8 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-D-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 48.0s, mem: 7.8GB, 13/13 tests passed, (318/15.7s) 20.3 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-D-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 41.5s, mem: 8.2GB, 13/13 tests passed, (310/13.4s) 23.2 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-D-0924 -A flash_attention_2" # test pass✅, time: 32.0s, mem: 18.0GB, 13/13 tests passed, (298/10.2s) 29.1 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-O-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 42.2s, mem: 8.5GB, 13/13 tests passed, (214/13.7s) 15.6 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-O-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 36.6s, mem: 8.7GB, 13/13 tests passed, (214/11.7s) 18.3 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-7B-O-0924 -A flash_attention_2" # test pass✅, time: 29.9s, mem: 18.4GB, 13/13 tests passed, (209/9.3s) 22.4 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-72B-0924 -A flash_attention_2 --load-in-4bit --use-double-quant" # test pass✅, time: 129.6s, mem: 43.1GB, 13/13 tests passed, (285/42.3s) 6.7 T/s +#CLI_COMMAND="python vision.py -m allenai/Molmo-72B-0924 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 111.1s, mem: 48.0GB, 13/13 tests passed, (271/36.0s) 7.5 T/s +#CLI_COMMAND="python vision.py -m echo840/Monkey-Chat --load-in-4bit" # test pass✅, time: 15.1s, mem: 15.6GB, 13/13 tests passed, (49/4.5s) 10.8 T/s +#CLI_COMMAND="python vision.py -m echo840/Monkey-Chat" # test pass✅, time: 11.8s, mem: 21.6GB, 13/13 tests passed, (32/3.5s) 9.2 T/s +#CLI_COMMAND="python vision.py -m failspy/Phi-3-vision-128k-instruct-abliterated-alpha -A flash_attention_2 --load-in-4bit" # test pass✅, time: 11.2s, mem: 7.3GB, 13/13 tests passed, (37/3.2s) 11.7 T/s +#CLI_COMMAND="python vision.py -m failspy/Phi-3-vision-128k-instruct-abliterated-alpha -A flash_attention_2" # test pass✅, time: 9.4s, mem: 12.3GB, 13/13 tests passed, (37/2.6s) 14.0 T/s +#CLI_COMMAND="python vision.py -m fancyfeast/joy-caption-pre-alpha --load-in-4bit -A flash_attention_2" # test pass✅, time: 117.7s, mem: 8.3GB, 13/13 tests passed, (619/36.8s) 16.8 T/s +#CLI_COMMAND="python vision.py -m fancyfeast/joy-caption-pre-alpha -A flash_attention_2" # test pass✅, time: 64.6s, mem: 18.0GB, 13/13 tests passed, (697/21.6s) 32.2 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2d5-7b -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test fail❌, time: 3.0s, mem: 8.8GB, 1/13 tests passed +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2d5-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 21.2s, mem: 29.8GB, 13/13 tests passed, (21/4.6s) 4.6 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-4khd-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 20.0s, mem: 22.0GB, 13/13 tests passed, (35/5.6s) 6.2 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 27.8s, mem: 18.2GB, 13/13 tests passed, (71/9.4s) 7.6 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-7b-4bit -A flash_attention_2" # test pass✅, time: 57.8s, mem: 8.7GB, 13/13 tests passed, (65/18.2s) 3.6 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-vl-1_8b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 7.7s, mem: 7.6GB, 13/13 tests passed, (8/1.3s) 6.4 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-vl-7b -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 25.1s, mem: 20.6GB, 13/13 tests passed, (56/7.8s) 7.2 T/s +#CLI_COMMAND="python vision.py -m internlm/internlm-xcomposer2-vl-7b-4bit -A flash_attention_2" # test pass✅, time: 51.2s, mem: 10.4GB, 13/13 tests passed, (64/17.2s) 3.7 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-13b-hf -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 15.4s, mem: 9.2GB, 13/13 tests passed, (58/4.7s) 12.5 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-13b-hf -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 10.7s, mem: 26.3GB, 13/13 tests passed, (59/3.1s) 19.0 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-7b-hf -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 13.4s, mem: 5.4GB, 13/13 tests passed, (62/3.9s) 15.9 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-1.5-7b-hf -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 9.9s, mem: 14.2GB, 13/13 tests passed, (65/2.7s) 23.8 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-34b-hf -A flash_attention_2 --load-in-4bit" # test pass✅, time: 70.7s, mem: 22.2GB, 13/13 tests passed, (184/22.6s) 8.2 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-34b-hf -A flash_attention_2" # test pass✅, time: 88.3s, mem: 67.3GB, 13/13 tests passed, (246/28.7s) 8.6 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-13b-hf -A flash_attention_2 --load-in-4bit" # test pass✅, time: 20.0s, mem: 12.4GB, 13/13 tests passed, (55/6.0s) 9.2 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-13b-hf -A flash_attention_2" # test pass✅, time: 17.8s, mem: 29.4GB, 13/13 tests passed, (55/5.3s) 10.3 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-7b-hf -A flash_attention_2 --load-in-4bit" # test pass✅, time: 22.1s, mem: 7.7GB, 13/13 tests passed, (88/6.8s) 12.9 T/s +#CLI_COMMAND="python vision.py -m llava-hf/llava-v1.6-vicuna-7b-hf -A flash_attention_2" # test pass✅, time: 16.4s, mem: 16.5GB, 13/13 tests passed, (82/5.0s) 16.5 T/s +#CLI_COMMAND="python vision.py -m lmms-lab/llava-onevision-qwen2-0.5b-ov -A flash_attention_2" # test pass✅, time: 8.8s, mem: 23.5GB, 13/13 tests passed, (37/2.3s) 15.9 T/s +#CLI_COMMAND="python vision.py -m lmms-lab/llava-onevision-qwen2-7b-ov -A flash_attention_2" # test pass✅, time: 19.0s, mem: 36.6GB, 13/13 tests passed, (51/5.8s) 8.8 T/s +#CLI_COMMAND="python vision.py -m meta-llama/Llama-3.2-11B-Vision-Instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 50.5s, mem: 8.8GB, 13/13 tests passed, (69/5.4s) 12.8 T/s +#CLI_COMMAND="python vision.py -m meta-llama/Llama-3.2-11B-Vision-Instruct -A flash_attention_2" # test pass✅, time: 33.0s, mem: 22.4GB, 13/13 tests passed, (175/8.0s) 21.9 T/s +#CLI_COMMAND="python vision.py -m meta-llama/Llama-3.2-90B-Vision-Instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 107.2s, mem: 50.8GB, 13/13 tests passed, (244/43.0s) 5.7 T/s +#CLI_COMMAND="python vision.py -m microsoft/Florence-2-base-ft -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 3.2s, mem: 0.9GB, 13/13 tests passed +#CLI_COMMAND="python vision.py -m microsoft/Florence-2-base-ft -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 2.6s, mem: 1.2GB, 13/13 tests passed +#CLI_COMMAND="python vision.py -m microsoft/Florence-2-large-ft -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 3.4s, mem: 1.5GB, 13/13 tests passed +#CLI_COMMAND="python vision.py -m microsoft/Florence-2-large-ft -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 2.5s, mem: 2.3GB, 13/13 tests passed +#CLI_COMMAND="python vision.py -m microsoft/Phi-3-vision-128k-instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 13.4s, mem: 7.4GB, 13/13 tests passed, (51/4.0s) 12.9 T/s +#CLI_COMMAND="python vision.py -m microsoft/Phi-3-vision-128k-instruct -A flash_attention_2" # test pass✅, time: 9.4s, mem: 12.3GB, 13/13 tests passed, (37/2.7s) 13.9 T/s +#CLI_COMMAND="python vision.py -m microsoft/Phi-3.5-vision-instruct -A flash_attention_2 --load-in-4bit" # test pass✅, time: 9.5s, mem: 4.6GB, 13/13 tests passed, (37/2.6s) 14.0 T/s +#CLI_COMMAND="python vision.py -m microsoft/Phi-3.5-vision-instruct -A flash_attention_2" # test pass✅, time: 8.6s, mem: 9.5GB, 13/13 tests passed, (41/2.3s) 17.5 T/s +#CLI_COMMAND="python vision.py -m mistralai/Pixtral-12B-2409" # test pass✅, time: 16.7s, mem: 35.7GB, 13/13 tests passed, (manual calc) 12.7 T/s +#CLI_COMMAND="python vision.py -m mx262/MiniMonkey -A flash_attention_2 --load-in-4bit" # test pass✅, time: 11.2s, mem: 14.1GB, 13/13 tests passed, (37/3.3s) 11.2 T/s +#CLI_COMMAND="python vision.py -m mx262/MiniMonkey -A flash_attention_2" # test pass✅, time: 10.0s, mem: 16.5GB, 13/13 tests passed, (37/2.8s) 13.0 T/s +#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-V-2_6-int4 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 18.9s, mem: 9.4GB, 13/13 tests passed, (92/5.0s) 18.4 T/s +#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-V-2_6 -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 15.6s, mem: 9.7GB, 13/13 tests passed, (100/4.6s) 21.9 T/s +#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-V-2_6 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 12.9s, mem: 19.1GB, 13/13 tests passed, (108/3.7s) 29.2 T/s +#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-Llama3-V-2_5 -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 30.2s, mem: 9.2GB, 13/13 tests passed, (69/9.0s) 7.7 T/s +#CLI_COMMAND="python vision.py -m openbmb/MiniCPM-Llama3-V-2_5 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 27.2s, mem: 19.2GB, 13/13 tests passed, (85/8.5s) 10.0 T/s +#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 9.3s, mem: 8.2GB, 13/13 tests passed, (46/2.1s) 22.3 T/s +#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 6.7s, mem: 8.6GB, 13/13 tests passed, (52/1.5s) 33.8 T/s +#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA-1.5 -A flash_attention_2 --device-map cuda:0 --load-in-4bit" # test pass✅, time: 8.0s, mem: 8.2GB, 13/13 tests passed, (49/2.1s) 22.9 T/s +#CLI_COMMAND="python vision.py -m qnguyen3/nanoLLaVA-1.5 -A flash_attention_2 --device-map cuda:0" # test pass✅, time: 7.3s, mem: 8.5GB, 13/13 tests passed, (52/1.4s) 35.9 T/s +#CLI_COMMAND="python vision.py -m qresearch/llama-3-vision-alpha-hf --device cuda:0 --load-in-4bit" # test pass✅, time: 13.2s, mem: 6.8GB, 13/13 tests passed, (80/4.2s) 18.9 T/s +#CLI_COMMAND="python vision.py -m qresearch/llama-3-vision-alpha-hf --device cuda:0" # test pass✅, time: 8.2s, mem: 16.7GB, 13/13 tests passed, (63/2.3s) 27.9 T/s +#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-Med-v1 --load-in-4bit" # test pass✅, time: 11.3s, mem: 8.1GB, 13/13 tests passed, (43/3.3s) 13.1 T/s +#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-Med-v1" # test pass✅, time: 9.6s, mem: 18.6GB, 13/13 tests passed, (51/2.6s) 19.9 T/s +#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-v1 --load-in-4bit" # test pass✅, time: 13.8s, mem: 8.1GB, 13/13 tests passed, (61/4.0s) 15.4 T/s +#CLI_COMMAND="python vision.py -m togethercomputer/Llama-3-8B-Dragonfly-v1" # test pass✅, time: 13.3s, mem: 18.6GB, 13/13 tests passed, (95/3.8s) 24.9 T/s +#CLI_COMMAND="python vision.py -m vikhyatk/moondream2 -A flash_attention_2 --load-in-4bit" # test pass✅, time: 6.4s, mem: 2.6GB, 13/13 tests passed, (63/1.8s) 36.0 T/s +#CLI_COMMAND="python vision.py -m vikhyatk/moondream2 -A flash_attention_2" # test pass✅, time: 5.0s, mem: 4.5GB, 13/13 tests passed, (63/1.3s) 49.2 T/s