LiteLLM Complete Guide 2025: Unify 100+ LLMs with a Single API Proxy Server

# Problem: Each provider has a different API format
# OpenAI
import openai
response = openai.ChatCompletion.create(model="gpt-4", messages=[...])

# Anthropic
import anthropic
response = anthropic.messages.create(model="claude-3-opus", messages=[...])

# Google
import google.generativeai as genai
response = genai.GenerativeModel("gemini-pro").generate_content(...)

┌──────────────────────────────────────────────────┐
│                  Your Application                │
│              (OpenAI SDK Format)                 │
└─────────────────────┬────────────────────────────┘
                      │
                      ▼
┌──────────────────────────────────────────────────┐
│              LiteLLM Proxy Server                │
│  ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │
│  │  Cost    │ │  Load    │ │  Virtual Keys    │ │
│  │  Tracking│ │  Balance │ │  + Budget Mgmt   │ │
│  └──────────┘ └──────────┘ └──────────────────┘ │
└───┬──────────────┬──────────────┬────────────────┘
    │              │              │
    ▼              ▼              ▼
┌────────┐  ┌──────────┐  ┌───────────┐
│ OpenAI │  │ Anthropic│  │  Google   │
│ GPT-4o │  │ Claude   │  │  Gemini   │
└────────┘  └──────────┘  └───────────┘
    ▲              ▲              ▲
    │              │              │
┌────────┐  ┌──────────┐  ┌───────────┐
│  AWS   │  │  Azure   │  │  Ollama   │
│Bedrock │  │ OpenAI   │  │  (Local)  │
└────────┘  └──────────┘  └───────────┘

# pip install litellm
from litellm import completion

# Call OpenAI
response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Hello!"}]
)

# Call Anthropic (same format!)
response = completion(
    model="anthropic/claude-sonnet-4-20250514",
    messages=[{"role": "user", "content": "Hello!"}]
)

# Call Google Gemini (same format!)
response = completion(
    model="gemini/gemini-2.0-flash",
    messages=[{"role": "user", "content": "Hello!"}]
)

# Start proxy server
litellm --config config.yaml --port 4000

# Any language can call using OpenAI SDK format
import openai

client = openai.OpenAI(
    api_key="sk-litellm-your-virtual-key",
    base_url="http://localhost:4000"
)

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Hello!"}]
)

from litellm import completion
import os

# Set API keys via environment variables
os.environ["OPENAI_API_KEY"] = "sk-..."
os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
os.environ["GEMINI_API_KEY"] = "AI..."

# Same function, different models
models = [
    "gpt-4o",
    "anthropic/claude-sonnet-4-20250514",
    "gemini/gemini-2.0-flash",
    "groq/llama-3.1-70b-versatile",
    "mistral/mistral-large-latest",
]

for model in models:
    response = completion(
        model=model,
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Explain quantum computing in 2 sentences."}
        ],
        temperature=0.7,
        max_tokens=200,
    )
    print(f"[{model}]: {response.choices[0].message.content}")

from litellm import completion

response = completion(
    model="anthropic/claude-sonnet-4-20250514",
    messages=[{"role": "user", "content": "Write a short poem about coding."}],
    stream=True,
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

from litellm import embedding

# OpenAI Embedding
response = embedding(
    model="text-embedding-3-small",
    input=["Hello world", "How are you?"]
)

# Cohere Embedding (same format!)
response = embedding(
    model="cohere/embed-english-v3.0",
    input=["Hello world", "How are you?"]
)

print(f"Embedding dimension: {len(response.data[0]['embedding'])}")

from litellm import image_generation

response = image_generation(
    model="dall-e-3",
    prompt="A futuristic city powered by AI, digital art style",
    n=1,
    size="1024x1024",
)

print(f"Image URL: {response.data[0]['url']}")

import asyncio
from litellm import acompletion

async def generate_responses():
    tasks = [
        acompletion(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": f"Tell me fact #{i} about space."}]
        )
        for i in range(5)
    ]
    responses = await asyncio.gather(*tasks)
    for i, resp in enumerate(responses):
        print(f"Fact #{i}: {resp.choices[0].message.content[:100]}")

asyncio.run(generate_responses())

model_list:
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY

  - model_name: claude-sonnet
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY

  - model_name: gemini-flash
    litellm_params:
      model: gemini/gemini-2.0-flash
      api_key: os.environ/GEMINI_API_KEY

  - model_name: llama-local
    litellm_params:
      model: ollama/llama3.1:70b
      api_base: http://localhost:11434

litellm_settings:
  drop_params: true
  set_verbose: false
  num_retries: 3
  request_timeout: 120

general_settings:
  master_key: sk-master-key-1234
  database_url: os.environ/DATABASE_URL

# Docker Compose file
cat > docker-compose.yaml << 'EOF'
version: "3.9"
services:
  litellm:
    image: ghcr.io/berriai/litellm:main-latest
    ports:
      - "4000:4000"
    volumes:
      - ./config.yaml:/app/config.yaml
    command: ["--config", "/app/config.yaml", "--port", "4000"]
    environment:
      - OPENAI_API_KEY
      - ANTHROPIC_API_KEY
      - GEMINI_API_KEY
      - DATABASE_URL=postgresql://user:pass@db:5432/litellm
    depends_on:
      - db

  db:
    image: postgres:16
    environment:
      POSTGRES_USER: user
      POSTGRES_PASSWORD: pass
      POSTGRES_DB: litellm
    volumes:
      - pgdata:/var/lib/postgresql/data

volumes:
  pgdata:
EOF

# Launch
docker compose up -d

# Health check
curl http://localhost:4000/health

# Chat Completion call
curl -X POST http://localhost:4000/v1/chat/completions \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gpt-4o",
    "messages": [{"role": "user", "content": "Hello!"}]
  }'

# List models
curl http://localhost:4000/v1/models \
  -H "Authorization: Bearer sk-master-key-1234"

┌────────────────────┬──────────────────────────────────────┐
│ Provider           │ Supported Models (examples)          │
├────────────────────┼──────────────────────────────────────┤
│ OpenAI             │ gpt-4o, gpt-4o-mini, o1, o3-mini    │
│ Anthropic          │ claude-opus-4, claude-sonnet, haiku  │
│ Google (Gemini)    │ gemini-2.0-flash, gemini-1.5-pro    │
│ AWS Bedrock        │ Claude, Titan, Llama via Bedrock     │
│ Azure OpenAI       │ gpt-4o (Azure hosted)               │
│ Mistral AI         │ mistral-large, mistral-small         │
│ Groq               │ llama-3.1-70b, mixtral-8x7b         │
│ Together AI        │ llama-3.1, CodeLlama, Qwen          │
│ Ollama (Local)     │ llama3.1, codellama, mistral         │
│ vLLM (Self-hosted) │ Any HuggingFace model               │
│ Cohere             │ command-r-plus, embed models         │
│ Deepseek           │ deepseek-chat, deepseek-coder        │
│ Fireworks AI       │ llama, mixtral, etc.                 │
│ Perplexity         │ pplx-70b-online, etc.               │
└────────────────────┴──────────────────────────────────────┘

model_list:
  - model_name: bedrock-claude
    litellm_params:
      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
      aws_region_name: us-east-1

  - model_name: bedrock-llama
    litellm_params:
      model: bedrock/meta.llama3-1-70b-instruct-v1:0
      aws_region_name: us-west-2

model_list:
  - model_name: azure-gpt4
    litellm_params:
      model: azure/gpt-4o-deployment-name
      api_base: https://your-resource.openai.azure.com/
      api_key: os.environ/AZURE_API_KEY
      api_version: "2024-06-01"

model_list:
  - model_name: local-llama
    litellm_params:
      model: ollama/llama3.1:70b
      api_base: http://localhost:11434
      stream: true

  - model_name: local-codellama
    litellm_params:
      model: ollama/codellama:34b
      api_base: http://localhost:11434

router_settings:
  routing_strategy: cost-based-routing

model_list:
  - model_name: general-chat
    litellm_params:
      model: gpt-4o-mini
      # Input: $0.15/1M tokens, Output: $0.60/1M tokens

  - model_name: general-chat
    litellm_params:
      model: anthropic/claude-3-haiku-20240307
      # Input: $0.25/1M tokens, Output: $1.25/1M tokens

  - model_name: general-chat
    litellm_params:
      model: gemini/gemini-2.0-flash
      # Input: $0.075/1M tokens, Output: $0.30/1M tokens

router_settings:
  routing_strategy: latency-based-routing
  routing_strategy_args:
    ttl: 60  # Re-measure latency every 60 seconds

router_settings:
  routing_strategy: usage-based-routing

model_list:
  - model_name: fast-model
    litellm_params:
      model: gpt-4o-mini
      rpm: 500  # Max 500 requests per minute

  - model_name: fast-model
    litellm_params:
      model: gemini/gemini-2.0-flash
      rpm: 1000  # Max 1000 requests per minute

model_list:
  # Distribute across multiple API keys for the same model
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: sk-key-1
      rpm: 100

  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: sk-key-2
      rpm: 100

  # Distribute across different providers
  - model_name: fast-model
    litellm_params:
      model: openai/gpt-4o-mini
      rpm: 500

  - model_name: fast-model
    litellm_params:
      model: anthropic/claude-3-haiku-20240307
      rpm: 300

router_settings:
  routing_strategy: simple-shuffle  # Round-robin style
  allowed_fails: 3  # Deactivate deployment after 3 failures
  cooldown_time: 60  # Reactivate after 60 seconds

litellm_settings:
  fallbacks:
    - model_name: gpt-4o
      fallback_models:
        - anthropic/claude-sonnet-4-20250514
        - gemini/gemini-1.5-pro

    - model_name: claude-sonnet
      fallback_models:
        - gpt-4o
        - gemini/gemini-1.5-pro

  # Fallback on content policy violations
  content_policy_fallbacks:
    - model_name: gpt-4o
      fallback_models:
        - anthropic/claude-sonnet-4-20250514

  # Fallback when context window is exceeded
  context_window_fallbacks:
    - model_name: gpt-4o-mini
      fallback_models:
        - gpt-4o
        - anthropic/claude-sonnet-4-20250514

User Request: Chat Completion with "gpt-4o"
    |
    v
[1] Try gpt-4o
    |
    +-- Success --> Return response
    |
    +-- Failure (429 Rate Limit / 500 Error)
        |
        v
[2] Try claude-sonnet-4 (1st fallback)
    |
    +-- Success --> Return response
    |
    +-- Failure
        |
        v
[3] Try gemini-1.5-pro (2nd fallback)
    |
    +-- Success --> Return response
    |
    +-- Failure --> Return error

# Generate a new virtual key with the master key
curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "key_alias": "frontend-team-key",
    "duration": "30d",
    "max_budget": 100.0,
    "models": ["gpt-4o", "claude-sonnet"],
    "max_parallel_requests": 10,
    "tpm_limit": 100000,
    "rpm_limit": 100,
    "metadata": {
      "team": "frontend",
      "environment": "production"
    }
  }'

# Create a team
curl -X POST http://localhost:4000/team/new \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "team_alias": "ml-engineering",
    "max_budget": 500.0,
    "models": ["gpt-4o", "claude-sonnet", "gemini-flash"],
    "members_with_roles": [
      {"role": "admin", "user_id": "user-alice@company.com"},
      {"role": "user", "user_id": "user-bob@company.com"}
    ],
    "metadata": {
      "department": "engineering",
      "cost_center": "ENG-001"
    }
  }'

# Assign keys to a team
curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "team_id": "team-ml-engineering-id",
    "key_alias": "ml-team-prod-key",
    "max_budget": 200.0
  }'

# Model access groups in config.yaml
model_list:
  - model_name: premium-model
    litellm_params:
      model: openai/gpt-4o
    model_info:
      access_groups: ["premium-tier"]

  - model_name: basic-model
    litellm_params:
      model: openai/gpt-4o-mini
    model_info:
      access_groups: ["basic-tier", "premium-tier"]

general_settings:
  max_budget: 10000.0  # Total monthly budget $10,000
  budget_duration: 1m   # Monthly reset

# Update key budget
curl -X POST http://localhost:4000/key/update \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "key": "sk-virtual-key-xxx",
    "max_budget": 50.0,
    "budget_duration": "1m",
    "soft_budget": 40.0
  }'

# Query total spend
curl http://localhost:4000/spend/logs \
  -H "Authorization: Bearer sk-master-key-1234" \
  -G \
  -d "start_date=2025-01-01" \
  -d "end_date=2025-01-31"

# Query spend by model
curl http://localhost:4000/spend/tags \
  -H "Authorization: Bearer sk-master-key-1234" \
  -G \
  -d "group_by=model"

# Query spend by team
curl http://localhost:4000/spend/tags \
  -H "Authorization: Bearer sk-master-key-1234" \
  -G \
  -d "group_by=team"

general_settings:
  alerting:
    - slack
  alerting_threshold: 300  # Alert when 80% of budget is consumed
  alert_types:
    - budget_alerts
    - spend_reports
    - failed_tracking

environment_variables:
  SLACK_WEBHOOK_URL: os.environ/SLACK_WEBHOOK_URL

curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "key_alias": "rate-limited-key",
    "rpm_limit": 60,
    "tpm_limit": 100000,
    "max_parallel_requests": 5
  }'

model_list:
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY
      rpm: 200    # Max requests per minute for this deployment
      tpm: 400000 # Max tokens per minute for this deployment

general_settings:
  global_max_parallel_requests: 100  # Global concurrent request limit

litellm_settings:
  guardrails:
    - guardrail_name: pii-masking
      litellm_params:
        guardrail: presidio
        mode: during_call  # Mask PII before sending to LLM
        output_parse_pii: true  # Also mask PII in responses

# Request containing PII
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{
        "role": "user",
        "content": "My email is john@example.com and phone is 555-123-4567"
    }],
    extra_body={
        "metadata": {
            "guardrails": ["pii-masking"]
        }
    }
)
# LLM receives: "My email is [EMAIL] and phone is [PHONE]"

litellm_settings:
  guardrails:
    - guardrail_name: content-filter
      litellm_params:
        guardrail: lakera
        mode: pre_call  # Filter before sending request
        api_key: os.environ/LAKERA_API_KEY

litellm_settings:
  guardrails:
    - guardrail_name: prompt-injection
      litellm_params:
        guardrail: lakera
        mode: pre_call
        prompt_injection: true

# custom_guardrail.py
from litellm.proxy.guardrails.guardrail_hooks import GuardrailHook

class CustomContentFilter(GuardrailHook):
    def __init__(self):
        self.blocked_words = ["harmful", "dangerous", "illegal"]

    async def pre_call_hook(self, data, call_type):
        user_message = data.get("messages", [])[-1].get("content", "")
        for word in self.blocked_words:
            if word.lower() in user_message.lower():
                raise ValueError(f"Blocked content detected: {word}")
        return data

    async def post_call_hook(self, data, response, call_type):
        return response

litellm_settings:
  success_callback: ["langfuse"]
  failure_callback: ["langfuse"]

environment_variables:
  LANGFUSE_PUBLIC_KEY: os.environ/LANGFUSE_PUBLIC_KEY
  LANGFUSE_SECRET_KEY: os.environ/LANGFUSE_SECRET_KEY
  LANGFUSE_HOST: https://cloud.langfuse.com

litellm_settings:
  success_callback: ["langsmith"]

environment_variables:
  LANGCHAIN_API_KEY: os.environ/LANGCHAIN_API_KEY
  LANGCHAIN_PROJECT: my-litellm-project

# custom_callback.py
from litellm.integrations.custom_logger import CustomLogger

class MyCustomLogger(CustomLogger):
    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
        print(f"Model: {kwargs.get('model')}")
        print(f"Cost: ${kwargs.get('response_cost', 0):.6f}")
        print(f"Latency: {(end_time - start_time).total_seconds():.2f}s")
        print(f"Tokens: {response_obj.usage.total_tokens}")

    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
        print(f"FAILED - Model: {kwargs.get('model')}")
        print(f"Error: {kwargs.get('exception')}")

litellm_settings:
  success_callback: ["prometheus"]
  failure_callback: ["prometheus"]

# Request counts
litellm_requests_total
litellm_requests_failed_total

# Latency
litellm_request_duration_seconds_bucket
litellm_request_duration_seconds_sum

# Cost
litellm_spend_total

# Tokens
litellm_tokens_total
litellm_input_tokens_total
litellm_output_tokens_total

litellm_settings:
  cache: true
  cache_params:
    type: redis
    host: redis-host
    port: 6379
    password: os.environ/REDIS_PASSWORD
    ttl: 3600  # 1-hour cache

    # Semantic caching (same response for similar questions)
    supported_call_types:
      - acompletion
      - completion

litellm_settings:
  cache: true
  cache_params:
    type: local
    ttl: 600  # 10-minute cache

# Use cache
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What is 2+2?"}],
    extra_body={
        "metadata": {
            "cache": {
                "no-cache": False,  # Use cache
                "ttl": 3600         # TTL for this request
            }
        }
    }
)

# Skip cache
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What is the latest news?"}],
    extra_body={
        "metadata": {
            "cache": {
                "no-cache": True  # Always get fresh response
            }
        }
    }
)

# litellm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: litellm-proxy
  labels:
    app: litellm
spec:
  replicas: 3
  selector:
    matchLabels:
      app: litellm
  template:
    metadata:
      labels:
        app: litellm
    spec:
      containers:
        - name: litellm
          image: ghcr.io/berriai/litellm:main-latest
          ports:
            - containerPort: 4000
          args: ["--config", "/app/config.yaml", "--port", "4000"]
          volumeMounts:
            - name: config
              mountPath: /app/config.yaml
              subPath: config.yaml
          env:
            - name: OPENAI_API_KEY
              valueFrom:
                secretKeyRef:
                  name: litellm-secrets
                  key: openai-api-key
            - name: ANTHROPIC_API_KEY
              valueFrom:
                secretKeyRef:
                  name: litellm-secrets
                  key: anthropic-api-key
            - name: DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: litellm-secrets
                  key: database-url
          resources:
            requests:
              cpu: "500m"
              memory: "512Mi"
            limits:
              cpu: "2000m"
              memory: "2Gi"
          livenessProbe:
            httpGet:
              path: /health/liveliness
              port: 4000
            initialDelaySeconds: 30
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /health/readiness
              port: 4000
            initialDelaySeconds: 10
            periodSeconds: 5
      volumes:
        - name: config
          configMap:
            name: litellm-config
---
apiVersion: v1
kind: Service
metadata:
  name: litellm-service
spec:
  selector:
    app: litellm
  ports:
    - port: 80
      targetPort: 4000
  type: ClusterIP
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: litellm-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: litellm-proxy
  minReplicas: 2
  maxReplicas: 10
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70

# Install with Helm
helm repo add litellm https://litellm.github.io/litellm-helm
helm repo update

helm install litellm litellm/litellm-helm \
  --set masterKey=sk-your-master-key \
  --set replicaCount=3 \
  --set database.useExisting=true \
  --set database.url=postgresql://user:pass@host:5432/litellm \
  --namespace litellm \
  --create-namespace

Production Deployment Checklist
================================
[ ] PostgreSQL DB configured and connectivity verified
[ ] Master Key set to a strong random value
[ ] All API keys managed via env vars / secrets
[ ] HTTPS/TLS configured (Ingress or LoadBalancer)
[ ] 2+ replicas configured
[ ] HPA (Horizontal Pod Autoscaler) configured
[ ] Liveness/Readiness Probes configured
[ ] Prometheus + Grafana monitoring set up
[ ] Slack/PagerDuty alerting configured
[ ] Redis cache set up (optional)
[ ] Budget and rate limits configured
[ ] Guardrails configured (if needed)
[ ] Logging (Langfuse/LangSmith) set up
[ ] Backup and recovery strategy documented
[ ] Load testing completed

Decision Guide
================================

"I want to get started quickly"
  -> OpenRouter (sign up and use immediately)

"Data security is critical"
  -> LiteLLM (self-hosted, full control)

"I need team/org management"
  -> LiteLLM or Portkey

"I also use local models"
  -> LiteLLM (native Ollama/vLLM support)

"I need enterprise features"
  -> LiteLLM Enterprise or Portkey Enterprise

# production-config.yaml
model_list:
  # === Premium Tier ===
  - model_name: premium-chat
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY
      rpm: 200
      tpm: 400000
    model_info:
      access_groups: ["premium"]

  - model_name: premium-chat
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY
      rpm: 150
    model_info:
      access_groups: ["premium"]

  # === Basic Tier ===
  - model_name: basic-chat
    litellm_params:
      model: openai/gpt-4o-mini
      api_key: os.environ/OPENAI_API_KEY
      rpm: 500
    model_info:
      access_groups: ["basic", "premium"]

  - model_name: basic-chat
    litellm_params:
      model: gemini/gemini-2.0-flash
      api_key: os.environ/GEMINI_API_KEY
      rpm: 1000
    model_info:
      access_groups: ["basic", "premium"]

  # === Coding Dedicated ===
  - model_name: code-model
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY
    model_info:
      access_groups: ["premium"]

  # === Embedding ===
  - model_name: embedding
    litellm_params:
      model: openai/text-embedding-3-small
      api_key: os.environ/OPENAI_API_KEY

# === Router Settings ===
router_settings:
  routing_strategy: latency-based-routing
  allowed_fails: 3
  cooldown_time: 60
  num_retries: 3
  timeout: 120
  retry_after: 5

# === LiteLLM Settings ===
litellm_settings:
  drop_params: true
  set_verbose: false
  num_retries: 3
  request_timeout: 120

  # Fallbacks
  fallbacks:
    - model_name: premium-chat
      fallback_models: ["basic-chat"]
    - model_name: basic-chat
      fallback_models: ["premium-chat"]

  context_window_fallbacks:
    - model_name: basic-chat
      fallback_models: ["premium-chat"]

  # Caching
  cache: true
  cache_params:
    type: redis
    host: os.environ/REDIS_HOST
    port: 6379
    password: os.environ/REDIS_PASSWORD
    ttl: 3600

  # Callbacks
  success_callback: ["langfuse", "prometheus"]
  failure_callback: ["langfuse", "prometheus"]

  # Guardrails
  guardrails:
    - guardrail_name: pii-masking
      litellm_params:
        guardrail: presidio
        mode: during_call

# === General Settings ===
general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
  max_budget: 10000
  budget_duration: 1m

  alerting:
    - slack
  alerting_threshold: 300

  global_max_parallel_requests: 200

# === Environment Variables ===
environment_variables:
  LANGFUSE_PUBLIC_KEY: os.environ/LANGFUSE_PUBLIC_KEY
  LANGFUSE_SECRET_KEY: os.environ/LANGFUSE_SECRET_KEY
  SLACK_WEBHOOK_URL: os.environ/SLACK_WEBHOOK_URL