Skip to content

Commit

Permalink
make max the default
Browse files Browse the repository at this point in the history
  • Loading branch information
kylesayrs committed Oct 2, 2024
1 parent f12b3c7 commit d643095
Showing 1 changed file with 5 additions and 3 deletions.
8 changes: 5 additions & 3 deletions src/llmcompressor/transformers/compression/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,7 +205,7 @@ def custom_offload_device_map(
def calculate_offload_device_map(
model_stub: str,
reserve_for_hessians=False,
num_gpus: int = 1,
num_gpus: Optional[int] = None,
torch_dtype: torch.dtype = torch.float16,
**model_kwargs,
) -> Dict[Union[int, str], Union[int, str]]:
Expand All @@ -215,14 +215,16 @@ def calculate_offload_device_map(
:param model_stub: local path or HF stub to calculate mapping for
:param reserve_for_hessians: whether to reserve memory for GPTQ
:param num_gpus: number of gpus to utilize
:param num_gpus: number of gpus to utilize, defaults to max available
:param model_kwargs: keyword arguments to pass to model initializer
:return: memory mapping for layers of model_stub to be passed to from_pretrained()
"""
max_cpu_memory = psutil.virtual_memory().available
max_gpu_memory = torch.cuda.mem_get_info(0)[0]
available_gpus = torch.cuda.device_count()
if available_gpus < num_gpus:
if num_gpus is None:
num_gpus = available_gpus
elif num_gpus >= available_gpus:
raise ValueError(
f"Requested {num_gpus} GPUs but only {available_gpus} are available."
)
Expand Down

0 comments on commit d643095

Please sign in to comment.