add assertion and helpful message

vllm-project · Dec 21, 2024 · aa87178 · aa87178
1 parent ca5f54a
commit aa87178
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/...el_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py b/...el_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
@@ -61,6 +61,10 @@ def create_weights(self, layer: torch.nn.Module, input_size: int,
                        params_dtype: torch.dtype, weight_loader: Callable,
                        **kwargs):
 
+        assert params_dtype == torch.float16, (
+            "float16 is required for marlin24 compressd models. Set dtype=torch.float16"  # noqa: E501
+        )
+
         pack_factor = 32 // self.quant_type.size_bits
         output_size_per_partition = sum(output_partition_sizes)