fix dtype set

Signed-off-by: Isotr0py <[email protected]>
vllm-project · Nov 26, 2024 · a58c4b8 · a58c4b8
1 parent 1f55028
commit a58c4b8
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/vllm/model_executor/layers/quantization/gguf.py b/vllm/model_executor/layers/quantization/gguf.py
@@ -169,9 +169,10 @@ def materialize_nested(self) -> Parameter:
         dtype = {data.dtype for data in self.data_container}
         assert len(dtype) == 1, ValueError(
             f"Data container has mixed dtypes: {dtype}")
+        dtype = next(iter(dtype))
         nested_data = torch.nested.nested_tensor(self.data_container,
                                                  device=self.device,
-                                                 dtype=dtype[0])
+                                                 dtype=dtype)
         self.data_container.clear()
         param = torch.Tensor._make_subclass(self.cls_to_become,
                                             nested_data,