From d25b24b5fee8162fd03bfbf0af43ddc802341d83 Mon Sep 17 00:00:00 2001 From: sfc-gh-zhwang Date: Tue, 20 Aug 2024 15:44:45 -0700 Subject: [PATCH] commit --- vllm/model_executor/model_loader/loader.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm/model_executor/model_loader/loader.py b/vllm/model_executor/model_loader/loader.py index d0427fb9b16af..2f6cdbc6ce3e9 100644 --- a/vllm/model_executor/model_loader/loader.py +++ b/vllm/model_executor/model_loader/loader.py @@ -579,6 +579,10 @@ def load_model(self, *, model_config: ModelConfig, with torch.device(device_config.device): model = _initialize_model(model_config, self.load_config, lora_config, cache_config) + for _, module in model.named_modules(): + quant_method = getattr(module, "quant_method", None) + if quant_method is not None: + quant_method.process_weights_after_loading(module) rank = get_tensor_model_parallel_rank() pattern = os.path.join( local_model_path,