I figured actual code might be useful here.
import os
import time
from tenacity import (
retry,
stop_after_attempt,
wait_exponential,
retry_if_exception_type,
before_sleep_log
)
import logging
import openai
from openai import RateLimitError, APIConnectionError, InternalServerError
# Optional: Set up basic logging so you can see the retries happening in your console
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
# Initialize the LLM client
# Ensure you have your environment variable set: export OPENAI_API_KEY="sk-..."
client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "your_dummy_key_here"))
# Configure the robust retry decorator:
# 1. retry_if_exception_type: ONLY retries on transient network errors or rate limits.
# (It will immediately fail on AuthenticationError or BadRequestError so you don't waste time/money).
# 2. wait_exponential: Waits 2^x * 1 second between retries (e.g., 2s, 4s, 8s...). Max wait is 10s.
# 3. stop_after_attempt: Gives up completely after 5 tries to prevent infinite loops.
# 4. reraise=True: If it fails 5 times, it throws the actual LLM error rather than a Tenacity error.
@retry(
retry=retry_if_exception_type((RateLimitError, APIConnectionError, InternalServerError)),
wait=wait_exponential(multiplier=1, min=2, max=10),
stop=stop_after_attempt(5),
before_sleep=before_sleep_log(logger, logging.WARNING),
reraise=True
)
def generate_text_with_retry(prompt: str, model: str = "gpt-3.5-turbo") -> str:
print(f"\n[Action] Sending prompt to {model}...")
response = client.chat.completions.create(
model=model,
messages=[{"role": "user", "content": prompt}],
timeout=15.0 # Always set a timeout so hanging requests trigger the retry!
)
return response.choices[0].message.content
if __name__ == "__main__":
user_prompt = "Explain the importance of exponential backoff in API calls."
try:
# Call the decorated function exactly as you normally would
result = generate_text_with_retry(user_prompt)
print("\n✅ Success! LLM Response:")
print(result)
except Exception as e:
# If it reaches here, it either hit an unrecoverable error (like a bad API key)
# OR it failed 5 times in a row due to rate limits.
print(f"\n❌ Operation failed. Final error: {e}")