Made litellm judge backend more robust.

src/lighteval/metrics/llm_as_judge.py

@@ -28,6 +28,7 @@
 
 from tqdm import tqdm
 
+from lighteval.models.model_output import ModelResponse
 from lighteval.utils.imports import is_litellm_available, is_openai_available, is_vllm_available
 
 
@@ -195,20 +196,28 @@ def __call_litellm(self, prompts):
         def __call_api(prompt):
             for _ in range(self.API_MAX_RETRY):
                 try:
-                    response = litellm.completion(
-                        model=self.model,
-                        messages=prompt,
-                        response_format={"type": "text"},
-                        max_tokens=512,
-                        n=1,
-                        caching=True,
-                    )
+                    kwargs = {
+                        "model": self.model,
+                        "messages": prompt,
+                        "response_format": {"type": "text"},
+                        "max_tokens": 512,
+                        "n": 1,
+                        "caching": True,
+                    }
+                    response = litellm.completion(**kwargs)
                     text = response.choices[0].message.content
+                    if text is None:
+                        kwargs["caching"] = False
+                        response = litellm.completion(**kwargs)
+                        text = response.choices[0].message.content
+                        if text is None:
+                            # Just return an error response if the second attempt fails too
+                            return ModelResponse(text="Failed to get response from the API.", model=self.model)
                     return text
                 except Exception as e:
                     logger.warning(f"{type(e), e}")
                     time.sleep(self.API_RETRY_SLEEP)
-            raise Exception("Failed to get response from the API")
+            return ModelResponse(text="Failed to get response from the API.", model=self.model)
 
         results = []
         with ThreadPoolExecutor(100) as executor: