diff --git a/exllamav2/lora.py b/exllamav2/lora.py
index 1dd99cbe..b5675935 100644
--- a/exllamav2/lora.py
+++ b/exllamav2/lora.py
@@ -81,6 +81,8 @@ def __init__(self,
             f = load_file(self.lora_path, map_location = "cpu")
 
         for key in f.keys():
+            if any(key.endswith(x) for x in [".original_module.weight", ".modules_to_save.weight"]):
+                continue
             tensor = f[key]
 
             # Find target