diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..65727e0 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,28 @@ +from wandbot.configs.chat_config import ChatConfig + +class TestConfig(ChatConfig): + """Test configuration with minimal retry settings for faster tests""" + + # Override LLM retry settings + llm_max_retries: int = 1 + llm_retry_min_wait: int = 1 + llm_retry_max_wait: int = 2 + llm_retry_multiplier: int = 1 + + # Override Embedding retry settings + embedding_max_retries: int = 1 + embedding_retry_min_wait: int = 1 + embedding_retry_max_wait: int = 2 + embedding_retry_multiplier: int = 1 + + # Override Reranker retry settings + reranker_max_retries: int = 1 + reranker_retry_min_wait: int = 1 + reranker_retry_max_wait: int = 2 + reranker_retry_multiplier: int = 1 + + # Override retry settings for faster tests + max_retries: int = 1 # Only try once + retry_min_wait: int = 1 # Wait 1 second minimum + retry_max_wait: int = 2 # Wait 2 seconds maximum + retry_multiplier: int = 1 # No exponential increase \ No newline at end of file diff --git a/tests/test_llm.py b/tests/test_llm.py index 7046f6a..5fb1332 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -3,15 +3,14 @@ from dotenv import load_dotenv from pydantic import BaseModel import asyncio -import json from wandbot.models.llm import ( AsyncOpenAILLMModel, AsyncAnthropicLLMModel, LLMModel, extract_system_and_messages, - LLMError ) +from wandbot.utils import ErrorInfo # Load environment variables from .env load_dotenv() @@ -77,10 +76,11 @@ async def test_openai_llm_creation(model_name): @pytest.mark.parametrize("model_name", openai_models) async def test_openai_llm_create(model_name): model = AsyncOpenAILLMModel(model_name=model_name, temperature=0) - response = await model.create([ + result, error_info = await model.create([ {"role": "user", "content": "What is 2+2? Answer with just the number."} ]) - assert response.strip() == "4" + assert result.strip() == "4" + assert not error_info.has_error @pytest.mark.asyncio @pytest.mark.parametrize("model_name", openai_models) @@ -90,18 +90,24 @@ async def test_openai_llm_create_with_response_model(model_name): temperature=0, response_model=SimpleResponse ) - response = await model.create([ + result, error_info = await model.create([ {"role": "user", "content": "Return the number 4 as a JSON object with the key 'answer'. Respond with only valid JSON."} ]) - assert isinstance(response, SimpleResponse) - assert response.answer == 4 + assert isinstance(result, SimpleResponse) + assert result.answer == 4 + assert not error_info.has_error @pytest.mark.asyncio async def test_openai_invalid_model(): - model = AsyncOpenAILLMModel(model_name="invalid-model", temperature=0) - with pytest.raises(Exception): # OpenAI will raise an error on API call - await model.create([{"role": "user", "content": "test"}]) + client = AsyncOpenAILLMModel(model_name="invalid-model") + result = await client.create([{"role": "user", "content": "test"}]) + assert isinstance(result, tuple) + assert result[0] is None + assert isinstance(result[1], ErrorInfo) + assert result[1].has_error is True + assert "model_not_found" in result[1].error_message + assert result[1].component == "openai" # Anthropic model tests @pytest.mark.asyncio @@ -116,16 +122,22 @@ async def test_anthropic_llm_creation(model_name): @pytest.mark.parametrize("model_name", anthropic_models) async def test_anthropic_llm_create(model_name): model = AsyncAnthropicLLMModel(model_name=model_name, temperature=0) - response = await model.create([ + result, error_info = await model.create([ {"role": "user", "content": "What is 2+2? Answer with just the number."} ], max_tokens=4000) - assert response.strip() == "4" + assert result.strip() == "4" + assert not error_info.has_error @pytest.mark.asyncio async def test_anthropic_invalid_model(): - model = AsyncAnthropicLLMModel(model_name="invalid-model", temperature=0) - with pytest.raises(Exception): # Anthropic will raise an error on API call - await model.create([{"role": "user", "content": "test"}], max_tokens=4000) + client = AsyncAnthropicLLMModel(model_name="invalid-model") + result = await client.create([{"role": "user", "content": "test"}]) + assert isinstance(result, tuple) + assert result[0] is None + assert isinstance(result[1], ErrorInfo) + assert result[1].has_error is True + assert "not_found_error" in result[1].error_message + assert result[1].component == "anthropic" @pytest.mark.asyncio async def test_anthropic_llm_create_with_response_model(): @@ -134,12 +146,13 @@ async def test_anthropic_llm_create_with_response_model(): temperature=0, response_model=SimpleResponse ) - response = await model.create([ + result, error_info = await model.create([ {"role": "user", "content": "Return the number 4 as a JSON object with the key 'answer'. Respond with only valid JSON."} ]) - assert isinstance(response, SimpleResponse) - assert response.answer == 4 + assert isinstance(result, SimpleResponse) + assert result.answer == 4 + assert not error_info.has_error # LLMModel wrapper tests def test_llm_model_invalid_provider(): @@ -149,18 +162,38 @@ def test_llm_model_invalid_provider(): @pytest.mark.asyncio async def test_llm_model_invalid_openai_model(): model = LLMModel(provider="openai", model_name="invalid-model") - response = await model.create([{"role": "user", "content": "test"}]) - assert isinstance(response, LLMError) - assert response.error - assert "model_not_found" in response.error_message + response, error_info = await model.create([{"role": "user", "content": "test"}]) + assert response is None + assert isinstance(error_info, ErrorInfo) + assert error_info.has_error is True + assert "model_not_found" in error_info.error_message + assert error_info.component == "openai" + assert error_info.error_type is not None + assert error_info.stacktrace is not None + assert error_info.file_path is not None @pytest.mark.asyncio async def test_llm_model_invalid_anthropic_model(): model = LLMModel(provider="anthropic", model_name="invalid-model") - response = await model.create([{"role": "user", "content": "test"}]) - assert isinstance(response, LLMError) - assert response.error - assert "not_found_error" in response.error_message + response, error_info = await model.create([{"role": "user", "content": "test"}]) + assert response is None + assert isinstance(error_info, ErrorInfo) + assert error_info.has_error is True + assert "not_found_error" in error_info.error_message + assert error_info.component == "anthropic" + assert error_info.error_type is not None + assert error_info.stacktrace is not None + assert error_info.file_path is not None + +@pytest.mark.asyncio +async def test_successful_call_error_info(): + model = LLMModel(provider="openai", model_name="gpt-4-1106-preview") + result, error_info = await model.create([{"role": "user", "content": "Say 'test'"}]) + assert result is not None + assert isinstance(error_info, ErrorInfo) + assert error_info.has_error is False + assert error_info.error_message is None + assert error_info.component == "llm" @pytest.mark.parametrize("provider,model_name", [ ("openai", "gpt-4-1106-preview"), @@ -174,14 +207,16 @@ def test_llm_model_valid_models(provider, model_name): @pytest.mark.asyncio async def test_llm_model_create_with_response_model(): model = LLMModel( - provider="openai", + provider="openai", model_name="gpt-4o-2024-08-06", response_model=SimpleResponse ) - response = await model.create([ + response, error_info = await model.create([ {"role": "user", "content": "Return the number 4 as a JSON object with the key 'answer'. Respond with only valid JSON."} ]) assert isinstance(response, SimpleResponse) + assert isinstance(error_info, ErrorInfo) + assert error_info.has_error is False assert response.answer == 4 @pytest.mark.asyncio @@ -194,5 +229,6 @@ async def test_parallel_api_calls(): ])) responses = await asyncio.gather(*tasks) - for i, response in enumerate(responses): - assert response.strip() == str(i + i) \ No newline at end of file + for i, (result, error_info) in enumerate(responses): + assert result.strip() == str(i + i) + assert not error_info.has_error \ No newline at end of file