experimenting litellm

litellm/batch_prompts.json

@@ -0,0 +1,25 @@
+{
+  "model": "claude-3-sonnet-20240229",
+  "messages": [
+    {
+      "role": "user",
+      "content": "What is the capital of France?"
+    },
+    {
+      "role": "user",
+      "content": "Write a hello world program in Python"
+    },
+    {
+      "role": "user",
+      "content": "Explain what is machine learning in simple terms"
+    },
+    {
+      "role": "user",
+      "content": "What is the difference between SQL and NoSQL databases?"
+    },
+    {
+      "role": "user",
+      "content": "How do I make a chocolate cake?"
+    }
+  ]
+}

litellm/config.yaml

@@ -0,0 +1,25 @@
+model_list:
+  # For response gathering
+  - model_name: "claude-3-sonnet-20240229"
+    litellm_params:
+      model: "anthropic/claude-3-sonnet-20240229"
+      api_key: "os.environ/ANTHROPIC_API_KEY"
+      batch_size: 20  # Enable batch API with max 20 messages per batch
+
+  # For evaluation models
+  - model_name: "gpt-4"
+    litellm_params:
+      model: "openai/gpt-4"
+      api_key: "os.environ/OPENAI_API_KEY"
+
+  - model_name: "gemini-pro"
+    litellm_params:
+      model: "vertex_ai/gemini-pro"
+      project_id: "os.environ/VERTEX_PROJECT"
+      location: "os.environ/VERTEX_LOCATION"
+
+general_settings:
+  # Enable detailed logging for debugging
+  debug: true
+  # Set a master key for proxy authentication
+  master_key: "os.environ/LITELLM_MASTER_KEY"

litellm/generate_prompts.py

@@ -0,0 +1,18 @@
+import json
+
+prompts = [
+    {"role": "user", "content": "What is the capital of France?"},
+    {"role": "user", "content": "Write a hello world program in Python"},
+    {"role": "user", "content": "Explain what is machine learning in simple terms"},
+    {
+        "role": "user",
+        "content": "What is the difference between SQL and NoSQL databases?",
+    },
+    {"role": "user", "content": "How do I make a chocolate cake?"},
+]
+
+# Create a single batch request
+batch_request = {"model": "claude-3-sonnet-20240229", "messages": prompts}
+
+with open("batch_prompts.json", "w") as f:
+    json.dump(batch_request, f, indent=2)

litellm/start_proxy.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# # Check if required environment variables are set
+# if [ -z "$ANTHROPIC_API_KEY" ]; then
+#     echo "Error: ANTHROPIC_API_KEY is not set"
+#     exit 1
+# fi
+
+# if [ -z "$OPENAI_API_KEY" ]; then
+#     echo "Error: OPENAI_API_KEY is not set"
+#     exit 1
+# fi
+
+# if [ -z "$VERTEX_PROJECT" ]; then
+#     echo "Error: VERTEXAI_PROJECT is not set"
+#     exit 1
+# fi
+
+# if [ -z "$VERTEX_LOCATION" ]; then
+#     echo "Error: VERTEXAI_LOCATION is not set"
+#     exit 1
+# fi
+
+# if [ -z "$LITELLM_MASTER_KEY" ]; then
+#     echo "Error: LITELLM_MASTER_KEY is not set"
+#     exit 1
+# fi
+
+# Start the LiteLLM proxy with the config file
+litellm --config config.yaml --detailed_debug

litellm/test_prompts.jsonl

@@ -0,0 +1,5 @@
+{"model": "claude-3-sonnet-20240229", "messages": [{"role": "user", "content": "What is the capital of France?"}]}
+{"model": "claude-3-sonnet-20240229", "messages": [{"role": "user", "content": "Write a hello world program in Python"}]}
+{"model": "claude-3-sonnet-20240229", "messages": [{"role": "user", "content": "Explain what is machine learning in simple terms"}]}
+{"model": "claude-3-sonnet-20240229", "messages": [{"role": "user", "content": "What is the difference between SQL and NoSQL databases?"}]}
+{"model": "claude-3-sonnet-20240229", "messages": [{"role": "user", "content": "How do I make a chocolate cake?"}]}