default clip_loss to 0 when finetuning

mlfoundations · May 15, 2023 · fb72f4d · fb72f4d
1 parent a09d519
commit fb72f4d
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/src/open_clip/loss.py b/src/open_clip/loss.py
@@ -158,8 +158,12 @@ def __init__(
         self.caption_loss = nn.CrossEntropyLoss(ignore_index=pad_id)
 
     def forward(self, image_features, text_features, logits, labels, logit_scale, output_dict=False):
-        clip_loss = super().forward(image_features, text_features, logit_scale)
-        clip_loss = self.clip_loss_weight * clip_loss
+
+        clip_loss = 0
+
+        if self.clip_loss_weight:
+            clip_loss = super().forward(image_features, text_features, logit_scale)
+            clip_loss = self.clip_loss_weight * clip_loss
 
         caption_loss = self.caption_loss(
             logits.permute(0, 2, 1),