Skip to content

Commit

Permalink
Improve qwen vl impl (#2943)
Browse files Browse the repository at this point in the history 
* feat: refactor model, improve startup and re enable tests

* fix: improve multimodal rotary embed caching

* fix: limit vision flop calc to qwen2 vl models and update config typing

* fix: include clippy lint

* feat: refactor position ids in warmup and bump tests

* fix: prefer default dtype

* fix: enable all cuda graphs and bump snapshots

* fix: adjust rotaty init path

* fix: simplify get position ids and remove usused vision config

* fix: update position ids so first dim is batch, simplify rotary and bump vlm default token limit

* fix: improve position id init during cuda warmup for mrope and simplfy rotary forward

* fix: check existance before accessing rope type in cuda warmup

* fix: check key before access

* fix: improve mrope check in cuda graph warmup

* fix: remove check for default rope type

* fix: add more test and improve model generation

* fix: improve and simplify get_cos_sin, refactors and cleanup  get_position_ids

* fix: adjust signatures with types
  • Loading branch information
@drbh
drbh authored Feb 4, 2025
1 parent dd2bd5f commit c1cf36c
Show file tree
Hide file tree
Showing 15 changed files with 850 additions and 309 deletions.
422 changes: 409 additions & 13 deletions ...ssed_tensors_w8a8_int_dynamic_weight/test_compressed_tensors_w8a8_int_dynamic_weight.json
View file

Large diffs are not rendered by default.

50 changes: 25 additions & 25 deletions ...s_w8a8_int_dynamic_weight/test_compressed_tensors_w8a8_int_dynamic_weight_all_params.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,67 +7,67 @@
"seed": 0,
"tokens": [
{
"id": 1939,
"logprob": -2.2460938,
"id": 5267,
"logprob": -1.1464844,
"special": false,
"text": "?\n\n"
"text": "?\n"
},
{
"id": 33464,
"logprob": 0.0,
"logprob": -0.83203125,
"special": false,
"text": "Deep"
},
{
"id": 20909,
"logprob": -0.48608398,
"logprob": -0.5625,
"special": false,
"text": " Learning"
},
{
"id": 4102,
"logprob": -2.265625,
"id": 320,
"logprob": -2.1464844,
"special": false,
"text": " "
"text": " ("
},
{
"id": 285,
"id": 16524,
"logprob": 0.0,
"special": false,
"text": "is"
"text": "DL"
},
{
"id": 458,
"logprob": -0.6328125,
"id": 701,
"logprob": -2.2089844,
"special": false,
"text": " an"
"text": "),"
},
{
"id": 20443,
"logprob": -0.1796875,
"id": 476,
"logprob": -0.27368164,
"special": false,
"text": " artificial"
"text": " or"
},
{
"id": 11229,
"logprob": 0.0,
"id": 20443,
"logprob": -0.09442139,
"special": false,
"text": " intelligence"
"text": " artificial"
},
{
"id": 320,
"logprob": -0.37695312,
"id": 29728,
"logprob": 0.0,
"special": false,
"text": " ("
"text": " neural"
},
{
"id": 15469,
"id": 14155,
"logprob": 0.0,
"special": false,
"text": "AI"
"text": " networks"
}
],
"top_tokens": null
},
"generated_text": "What is deep learning?\n\nDeep Learning is an artificial intelligence (AI"
"generated_text": "What is deep learning?\nDeep Learning (DL), or artificial neural networks"
}
80 changes: 40 additions & 40 deletions ...tensors_w8a8_int_dynamic_weight/test_compressed_tensors_w8a8_int_dynamic_weight_load.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,61 +9,61 @@
"tokens": [
{
"id": 18183,
"logprob": -1.4912109,
"logprob": -1.5195312,
"special": false,
"text": " Deep"
},
{
"id": 6832,
"logprob": -0.075683594,
"logprob": -0.06817627,
"special": false,
"text": " learning"
},
{
"id": 374,
"logprob": -0.12408447,
"logprob": -0.13122559,
"special": false,
"text": " is"
},
{
"id": 264,
"logprob": -0.12768555,
"logprob": -0.13415527,
"special": false,
"text": " a"
},
{
"id": 25993,
"logprob": -0.82128906,
"logprob": -0.87353516,
"special": false,
"text": " subset"
},
{
"id": 315,
"logprob": -0.0012636185,
"logprob": -0.0011396408,
"special": false,
"text": " of"
},
{
"id": 5662,
"logprob": -0.12878418,
"logprob": -0.16442871,
"special": false,
"text": " machine"
},
{
"id": 6832,
"logprob": -0.0015888214,
"logprob": -0.0026416779,
"special": false,
"text": " learning"
},
{
"id": 429,
"logprob": -0.49194336,
"logprob": -0.48754883,
"special": false,
"text": " that"
},
{
"id": 5711,
"logprob": -1.2626953,
"logprob": -1.2294922,
"special": false,
"text": " uses"
}
Expand All @@ -82,61 +82,61 @@
"tokens": [
{
"id": 18183,
"logprob": -1.4912109,
"logprob": -1.5195312,
"special": false,
"text": " Deep"
},
{
"id": 6832,
"logprob": -0.075683594,
"logprob": -0.06817627,
"special": false,
"text": " learning"
},
{
"id": 374,
"logprob": -0.12408447,
"logprob": -0.13122559,
"special": false,
"text": " is"
},
{
"id": 264,
"logprob": -0.12768555,
"logprob": -0.13415527,
"special": false,
"text": " a"
},
{
"id": 25993,
"logprob": -0.82128906,
"logprob": -0.87353516,
"special": false,
"text": " subset"
},
{
"id": 315,
"logprob": -0.0012636185,
"logprob": -0.0011396408,
"special": false,
"text": " of"
},
{
"id": 5662,
"logprob": -0.12878418,
"logprob": -0.16442871,
"special": false,
"text": " machine"
},
{
"id": 6832,
"logprob": -0.0015888214,
"logprob": -0.0026416779,
"special": false,
"text": " learning"
},
{
"id": 429,
"logprob": -0.49194336,
"logprob": -0.48754883,
"special": false,
"text": " that"
},
{
"id": 5711,
"logprob": -1.2626953,
"logprob": -1.2294922,
"special": false,
"text": " uses"
}
Expand All @@ -155,61 +155,61 @@
"tokens": [
{
"id": 18183,
"logprob": -1.4912109,
"logprob": -1.5195312,
"special": false,
"text": " Deep"
},
{
"id": 6832,
"logprob": -0.075683594,
"logprob": -0.06817627,
"special": false,
"text": " learning"
},
{
"id": 374,
"logprob": -0.12408447,
"logprob": -0.13122559,
"special": false,
"text": " is"
},
{
"id": 264,
"logprob": -0.12768555,
"logprob": -0.13415527,
"special": false,
"text": " a"
},
{
"id": 25993,
"logprob": -0.82128906,
"logprob": -0.87353516,
"special": false,
"text": " subset"
},
{
"id": 315,
"logprob": -0.0012636185,
"logprob": -0.0011396408,
"special": false,
"text": " of"
},
{
"id": 5662,
"logprob": -0.12878418,
"logprob": -0.16442871,
"special": false,
"text": " machine"
},
{
"id": 6832,
"logprob": -0.0015888214,
"logprob": -0.0026416779,
"special": false,
"text": " learning"
},
{
"id": 429,
"logprob": -0.49194336,
"logprob": -0.48754883,
"special": false,
"text": " that"
},
{
"id": 5711,
"logprob": -1.2626953,
"logprob": -1.2294922,
"special": false,
"text": " uses"
}
Expand All @@ -228,61 +228,61 @@
"tokens": [
{
"id": 18183,
"logprob": -1.4912109,
"logprob": -1.5195312,
"special": false,
"text": " Deep"
},
{
"id": 6832,
"logprob": -0.075683594,
"logprob": -0.06817627,
"special": false,
"text": " learning"
},
{
"id": 374,
"logprob": -0.12408447,
"logprob": -0.13122559,
"special": false,
"text": " is"
},
{
"id": 264,
"logprob": -0.12768555,
"logprob": -0.13415527,
"special": false,
"text": " a"
},
{
"id": 25993,
"logprob": -0.82128906,
"logprob": -0.87353516,
"special": false,
"text": " subset"
},
{
"id": 315,
"logprob": -0.0012636185,
"logprob": -0.0011396408,
"special": false,
"text": " of"
},
{
"id": 5662,
"logprob": -0.12878418,
"logprob": -0.16442871,
"special": false,
"text": " machine"
},
{
"id": 6832,
"logprob": -0.0015888214,
"logprob": -0.0026416779,
"special": false,
"text": " learning"
},
{
"id": 429,
"logprob": -0.49194336,
"logprob": -0.48754883,
"special": false,
"text": " that"
},
{
"id": 5711,
"logprob": -1.2626953,
"logprob": -1.2294922,
"special": false,
"text": " uses"
}
Expand Down
Loading

0 comments on commit c1cf36c

Please sign in to comment.