Split View: LiteLLM 완전 가이드 2025: 100+ LLM을 하나의 API로 통합하는 프록시 서버

# 문제: 각 제공자마다 다른 API 형식
# OpenAI
import openai
response = openai.ChatCompletion.create(model="gpt-4", messages=[...])

# Anthropic
import anthropic
response = anthropic.messages.create(model="claude-3-opus", messages=[...])

# Google
import google.generativeai as genai
response = genai.GenerativeModel("gemini-pro").generate_content(...)

┌──────────────────────────────────────────────────┐
│                  Your Application                │
│              (OpenAI SDK Format)                 │
└─────────────────────┬────────────────────────────┘
                      │
                      ▼
┌──────────────────────────────────────────────────┐
│              LiteLLM Proxy Server                │
│  ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │
│  │  Cost    │ │  Load    │ │  Virtual Keys    │ │
│  │  Tracking│ │  Balance │ │  + Budget Mgmt   │ │
│  └──────────┘ └──────────┘ └──────────────────┘ │
└───┬──────────────┬──────────────┬────────────────┘
    │              │              │
    ▼              ▼              ▼
┌────────┐  ┌──────────┐  ┌───────────┐
│ OpenAI │  │ Anthropic│  │  Google   │
│ GPT-4o │  │ Claude   │  │  Gemini   │
└────────┘  └──────────┘  └───────────┘
    ▲              ▲              ▲
    │              │              │
┌────────┐  ┌──────────┐  ┌───────────┐
│  AWS   │  │  Azure   │  │  Ollama   │
│Bedrock │  │ OpenAI   │  │  (Local)  │
└────────┘  └──────────┘  └───────────┘

# pip install litellm
from litellm import completion

# OpenAI 호출
response = completion(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Hello!"}]
)

# Anthropic 호출 (같은 형식!)
response = completion(
    model="anthropic/claude-sonnet-4-20250514",
    messages=[{"role": "user", "content": "Hello!"}]
)

# Google Gemini 호출 (같은 형식!)
response = completion(
    model="gemini/gemini-2.0-flash",
    messages=[{"role": "user", "content": "Hello!"}]
)

# 프록시 서버 시작
litellm --config config.yaml --port 4000

# 어떤 언어든 OpenAI SDK 형식으로 호출
import openai

client = openai.OpenAI(
    api_key="sk-litellm-your-virtual-key",
    base_url="http://localhost:4000"
)

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "Hello!"}]
)

from litellm import completion
import os

# 환경 변수로 API 키 설정
os.environ["OPENAI_API_KEY"] = "sk-..."
os.environ["ANTHROPIC_API_KEY"] = "sk-ant-..."
os.environ["GEMINI_API_KEY"] = "AI..."

# 동일한 함수, 다른 모델
models = [
    "gpt-4o",
    "anthropic/claude-sonnet-4-20250514",
    "gemini/gemini-2.0-flash",
    "groq/llama-3.1-70b-versatile",
    "mistral/mistral-large-latest",
]

for model in models:
    response = completion(
        model=model,
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Explain quantum computing in 2 sentences."}
        ],
        temperature=0.7,
        max_tokens=200,
    )
    print(f"[{model}]: {response.choices[0].message.content}")

from litellm import completion

response = completion(
    model="anthropic/claude-sonnet-4-20250514",
    messages=[{"role": "user", "content": "Write a short poem about coding."}],
    stream=True,
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

from litellm import embedding

# OpenAI Embedding
response = embedding(
    model="text-embedding-3-small",
    input=["Hello world", "How are you?"]
)

# Cohere Embedding (같은 형식!)
response = embedding(
    model="cohere/embed-english-v3.0",
    input=["Hello world", "How are you?"]
)

print(f"Embedding dimension: {len(response.data[0]['embedding'])}")

from litellm import image_generation

response = image_generation(
    model="dall-e-3",
    prompt="A futuristic city powered by AI, digital art style",
    n=1,
    size="1024x1024",
)

print(f"Image URL: {response.data[0]['url']}")

import asyncio
from litellm import acompletion

async def generate_responses():
    tasks = [
        acompletion(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": f"Tell me fact #{i} about space."}]
        )
        for i in range(5)
    ]
    responses = await asyncio.gather(*tasks)
    for i, resp in enumerate(responses):
        print(f"Fact #{i}: {resp.choices[0].message.content[:100]}")

asyncio.run(generate_responses())

model_list:
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY

  - model_name: claude-sonnet
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY

  - model_name: gemini-flash
    litellm_params:
      model: gemini/gemini-2.0-flash
      api_key: os.environ/GEMINI_API_KEY

  - model_name: llama-local
    litellm_params:
      model: ollama/llama3.1:70b
      api_base: http://localhost:11434

litellm_settings:
  drop_params: true
  set_verbose: false
  num_retries: 3
  request_timeout: 120

general_settings:
  master_key: sk-master-key-1234
  database_url: os.environ/DATABASE_URL

# Docker Compose 파일
cat > docker-compose.yaml << 'EOF'
version: "3.9"
services:
  litellm:
    image: ghcr.io/berriai/litellm:main-latest
    ports:
      - "4000:4000"
    volumes:
      - ./config.yaml:/app/config.yaml
    command: ["--config", "/app/config.yaml", "--port", "4000"]
    environment:
      - OPENAI_API_KEY
      - ANTHROPIC_API_KEY
      - GEMINI_API_KEY
      - DATABASE_URL=postgresql://user:pass@db:5432/litellm
    depends_on:
      - db

  db:
    image: postgres:16
    environment:
      POSTGRES_USER: user
      POSTGRES_PASSWORD: pass
      POSTGRES_DB: litellm
    volumes:
      - pgdata:/var/lib/postgresql/data

volumes:
  pgdata:
EOF

# 시작
docker compose up -d

# 헬스 체크
curl http://localhost:4000/health

# Chat Completion 호출
curl -X POST http://localhost:4000/v1/chat/completions \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "model": "gpt-4o",
    "messages": [{"role": "user", "content": "Hello!"}]
  }'

# 모델 목록 조회
curl http://localhost:4000/v1/models \
  -H "Authorization: Bearer sk-master-key-1234"

┌────────────────────┬──────────────────────────────────────┐
│ 제공자              │ 지원 모델 (예시)                      │
├────────────────────┼──────────────────────────────────────┤
│ OpenAI             │ gpt-4o, gpt-4o-mini, o1, o3-mini    │
│ Anthropic          │ claude-opus-4, claude-sonnet, haiku  │
│ Google (Gemini)    │ gemini-2.0-flash, gemini-1.5-pro    │
│ AWS Bedrock        │ Claude, Titan, Llama via Bedrock     │
│ Azure OpenAI       │ gpt-4o (Azure 호스팅)                │
│ Mistral AI         │ mistral-large, mistral-small         │
│ Groq               │ llama-3.1-70b, mixtral-8x7b         │
│ Together AI        │ llama-3.1, CodeLlama, Qwen          │
│ Ollama (로컬)      │ llama3.1, codellama, mistral 등     │
│ vLLM (자체 호스팅) │ 모든 HuggingFace 모델               │
│ Cohere             │ command-r-plus, embed 모델           │
│ Deepseek           │ deepseek-chat, deepseek-coder        │
│ Fireworks AI       │ llama, mixtral 등                    │
│ Perplexity         │ pplx-70b-online 등                   │
└────────────────────┴──────────────────────────────────────┘

model_list:
  - model_name: bedrock-claude
    litellm_params:
      model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0
      aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
      aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
      aws_region_name: us-east-1

  - model_name: bedrock-llama
    litellm_params:
      model: bedrock/meta.llama3-1-70b-instruct-v1:0
      aws_region_name: us-west-2

model_list:
  - model_name: azure-gpt4
    litellm_params:
      model: azure/gpt-4o-deployment-name
      api_base: https://your-resource.openai.azure.com/
      api_key: os.environ/AZURE_API_KEY
      api_version: "2024-06-01"

model_list:
  - model_name: local-llama
    litellm_params:
      model: ollama/llama3.1:70b
      api_base: http://localhost:11434
      stream: true

  - model_name: local-codellama
    litellm_params:
      model: ollama/codellama:34b
      api_base: http://localhost:11434

router_settings:
  routing_strategy: cost-based-routing

model_list:
  - model_name: general-chat
    litellm_params:
      model: gpt-4o-mini
      # Input: $0.15/1M tokens, Output: $0.60/1M tokens

  - model_name: general-chat
    litellm_params:
      model: anthropic/claude-3-haiku-20240307
      # Input: $0.25/1M tokens, Output: $1.25/1M tokens

  - model_name: general-chat
    litellm_params:
      model: gemini/gemini-2.0-flash
      # Input: $0.075/1M tokens, Output: $0.30/1M tokens

router_settings:
  routing_strategy: latency-based-routing
  routing_strategy_args:
    ttl: 60  # 60초마다 레이턴시 재측정

router_settings:
  routing_strategy: usage-based-routing

model_list:
  - model_name: fast-model
    litellm_params:
      model: gpt-4o-mini
      rpm: 500  # 분당 최대 500 요청

  - model_name: fast-model
    litellm_params:
      model: gemini/gemini-2.0-flash
      rpm: 1000  # 분당 최대 1000 요청

# 요청 시 metadata로 라우팅 제어
response = client.chat.completions.create(
    model="general-chat",
    messages=[{"role": "user", "content": "Complex analysis..."}],
    extra_body={
        "metadata": {
            "tags": ["high-quality", "long-context"]
        }
    }
)

model_list:
  # 같은 모델의 여러 API 키로 분산
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: sk-key-1
      rpm: 100

  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: sk-key-2
      rpm: 100

  # 다른 제공자 간 분산
  - model_name: fast-model
    litellm_params:
      model: openai/gpt-4o-mini
      rpm: 500

  - model_name: fast-model
    litellm_params:
      model: anthropic/claude-3-haiku-20240307
      rpm: 300

router_settings:
  routing_strategy: simple-shuffle  # round-robin 방식
  allowed_fails: 3  # 3번 실패 시 해당 배포 비활성화
  cooldown_time: 60  # 60초 후 재활성화

litellm_settings:
  fallbacks:
    - model_name: gpt-4o
      fallback_models:
        - anthropic/claude-sonnet-4-20250514
        - gemini/gemini-1.5-pro

    - model_name: claude-sonnet
      fallback_models:
        - gpt-4o
        - gemini/gemini-1.5-pro

  # 콘텐츠 정책 위반 시 다른 모델로
  content_policy_fallbacks:
    - model_name: gpt-4o
      fallback_models:
        - anthropic/claude-sonnet-4-20250514

  # 컨텍스트 윈도우 초과 시 더 큰 모델로
  context_window_fallbacks:
    - model_name: gpt-4o-mini
      fallback_models:
        - gpt-4o
        - anthropic/claude-sonnet-4-20250514

사용자 요청: "gpt-4o"로 Chat Completion
    │
    ▼
[1] gpt-4o 호출 시도
    │
    ├── 성공 → 응답 반환
    │
    └── 실패 (429 Rate Limit / 500 Error)
        │
        ▼
[2] claude-sonnet-4 호출 시도 (1차 폴백)
    │
    ├── 성공 → 응답 반환
    │
    └── 실패
        │
        ▼
[3] gemini-1.5-pro 호출 시도 (2차 폴백)
    │
    ├── 성공 → 응답 반환
    │
    └── 실패 → 에러 반환

# 마스터 키로 새 가상 키 생성
curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "key_alias": "frontend-team-key",
    "duration": "30d",
    "max_budget": 100.0,
    "models": ["gpt-4o", "claude-sonnet"],
    "max_parallel_requests": 10,
    "tpm_limit": 100000,
    "rpm_limit": 100,
    "metadata": {
      "team": "frontend",
      "environment": "production"
    }
  }'

# 팀 생성
curl -X POST http://localhost:4000/team/new \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "team_alias": "ml-engineering",
    "max_budget": 500.0,
    "models": ["gpt-4o", "claude-sonnet", "gemini-flash"],
    "members_with_roles": [
      {"role": "admin", "user_id": "user-alice@company.com"},
      {"role": "user", "user_id": "user-bob@company.com"}
    ],
    "metadata": {
      "department": "engineering",
      "cost_center": "ENG-001"
    }
  }'

# 팀에 키 할당
curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "team_id": "team-ml-engineering-id",
    "key_alias": "ml-team-prod-key",
    "max_budget": 200.0
  }'

# config.yaml에서 모델 접근 그룹 설정
model_list:
  - model_name: premium-model
    litellm_params:
      model: openai/gpt-4o
    model_info:
      access_groups: ["premium-tier"]

  - model_name: basic-model
    litellm_params:
      model: openai/gpt-4o-mini
    model_info:
      access_groups: ["basic-tier", "premium-tier"]

general_settings:
  max_budget: 10000.0  # 전체 월 예산 $10,000
  budget_duration: 1m   # 월간 리셋

# 키별 예산 업데이트
curl -X POST http://localhost:4000/key/update \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "key": "sk-virtual-key-xxx",
    "max_budget": 50.0,
    "budget_duration": "1m",
    "soft_budget": 40.0
  }'

# 전체 비용 조회
curl http://localhost:4000/spend/logs \
  -H "Authorization: Bearer sk-master-key-1234" \
  -G \
  -d "start_date=2025-01-01" \
  -d "end_date=2025-01-31"

# 모델별 비용 조회
curl http://localhost:4000/spend/tags \
  -H "Authorization: Bearer sk-master-key-1234" \
  -G \
  -d "group_by=model"

# 팀별 비용 조회
curl http://localhost:4000/spend/tags \
  -H "Authorization: Bearer sk-master-key-1234" \
  -G \
  -d "group_by=team"

general_settings:
  alerting:
    - slack
  alerting_threshold: 300  # 예산의 80% 소진 시 알림
  alert_types:
    - budget_alerts
    - spend_reports
    - failed_tracking

environment_variables:
  SLACK_WEBHOOK_URL: os.environ/SLACK_WEBHOOK_URL

┌─────────────────────────────────────────────────────┐
│           LiteLLM Cost Dashboard (January 2025)     │
├─────────────────────────────────────────────────────┤
│                                                     │
│  Total Spend: $3,245.67 / $10,000.00 budget        │
│  ████████████████░░░░░░░░░░ 32.5%                  │
│                                                     │
│  By Model:                                          │
│  ├─ gpt-4o:          $1,823.45 (56.2%)             │
│  ├─ claude-sonnet:   $891.23  (27.5%)              │
│  ├─ gemini-flash:    $234.56  (7.2%)               │
│  └─ gpt-4o-mini:    $296.43  (9.1%)               │
│                                                     │
│  By Team:                                           │
│  ├─ ML Engineering:  $1,456.78                      │
│  ├─ Frontend:        $987.65                        │
│  └─ Data Science:    $801.24                        │
│                                                     │
│  Daily Trend: ▁▂▃▅▆▇█▇▆▅▃▂▁▂▃▅▆                   │
└─────────────────────────────────────────────────────┘

curl -X POST http://localhost:4000/key/generate \
  -H "Authorization: Bearer sk-master-key-1234" \
  -H "Content-Type: application/json" \
  -d '{
    "key_alias": "rate-limited-key",
    "rpm_limit": 60,
    "tpm_limit": 100000,
    "max_parallel_requests": 5
  }'

model_list:
  - model_name: gpt-4o
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY
      rpm: 200    # 이 배포의 분당 최대 요청
      tpm: 400000 # 이 배포의 분당 최대 토큰

general_settings:
  global_max_parallel_requests: 100  # 전체 동시 요청 제한

litellm_settings:
  guardrails:
    - guardrail_name: pii-masking
      litellm_params:
        guardrail: presidio
        mode: during_call  # 요청 전에 PII 마스킹
        output_parse_pii: true  # 응답에서도 PII 마스킹

# PII가 포함된 요청
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{
        "role": "user",
        "content": "My email is john@example.com and phone is 010-1234-5678"
    }],
    extra_body={
        "metadata": {
            "guardrails": ["pii-masking"]
        }
    }
)
# LLM에는 "My email is [EMAIL] and phone is [PHONE]"로 전송됨

litellm_settings:
  guardrails:
    - guardrail_name: content-filter
      litellm_params:
        guardrail: lakera
        mode: pre_call  # 요청 전에 필터링
        api_key: os.environ/LAKERA_API_KEY

litellm_settings:
  guardrails:
    - guardrail_name: prompt-injection
      litellm_params:
        guardrail: lakera
        mode: pre_call
        prompt_injection: true

# custom_guardrail.py
from litellm.proxy.guardrails.guardrail_hooks import GuardrailHook

class CustomContentFilter(GuardrailHook):
    def __init__(self):
        self.blocked_words = ["harmful", "dangerous", "illegal"]

    async def pre_call_hook(self, data, call_type):
        user_message = data.get("messages", [])[-1].get("content", "")
        for word in self.blocked_words:
            if word.lower() in user_message.lower():
                raise ValueError(f"Blocked content detected: {word}")
        return data

    async def post_call_hook(self, data, response, call_type):
        # 응답 후처리 로직
        return response

litellm_settings:
  success_callback: ["langfuse"]
  failure_callback: ["langfuse"]

environment_variables:
  LANGFUSE_PUBLIC_KEY: os.environ/LANGFUSE_PUBLIC_KEY
  LANGFUSE_SECRET_KEY: os.environ/LANGFUSE_SECRET_KEY
  LANGFUSE_HOST: https://cloud.langfuse.com

litellm_settings:
  success_callback: ["langsmith"]

environment_variables:
  LANGCHAIN_API_KEY: os.environ/LANGCHAIN_API_KEY
  LANGCHAIN_PROJECT: my-litellm-project

# custom_callback.py
from litellm.integrations.custom_logger import CustomLogger

class MyCustomLogger(CustomLogger):
    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
        print(f"Model: {kwargs.get('model')}")
        print(f"Cost: ${kwargs.get('response_cost', 0):.6f}")
        print(f"Latency: {(end_time - start_time).total_seconds():.2f}s")
        print(f"Tokens: {response_obj.usage.total_tokens}")

    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
        print(f"FAILED - Model: {kwargs.get('model')}")
        print(f"Error: {kwargs.get('exception')}")

litellm_settings:
  success_callback: ["prometheus"]
  failure_callback: ["prometheus"]

# 요청 수
litellm_requests_total
litellm_requests_failed_total

# 레이턴시
litellm_request_duration_seconds_bucket
litellm_request_duration_seconds_sum

# 비용
litellm_spend_total

# 토큰
litellm_tokens_total
litellm_input_tokens_total
litellm_output_tokens_total

litellm_settings:
  cache: true
  cache_params:
    type: redis
    host: redis-host
    port: 6379
    password: os.environ/REDIS_PASSWORD
    ttl: 3600  # 1시간 캐시

    # 의미적 캐싱 (유사한 질문에 같은 응답)
    supported_call_types:
      - acompletion
      - completion

litellm_settings:
  cache: true
  cache_params:
    type: local
    ttl: 600  # 10분 캐시

# 캐시 사용
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What is 2+2?"}],
    extra_body={
        "metadata": {
            "cache": {
                "no-cache": False,  # 캐시 사용
                "ttl": 3600         # 이 요청의 TTL
            }
        }
    }
)

# 캐시 무시
response = client.chat.completions.create(
    model="gpt-4o",
    messages=[{"role": "user", "content": "What is the latest news?"}],
    extra_body={
        "metadata": {
            "cache": {
                "no-cache": True  # 항상 새로운 응답
            }
        }
    }
)

# litellm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: litellm-proxy
  labels:
    app: litellm
spec:
  replicas: 3
  selector:
    matchLabels:
      app: litellm
  template:
    metadata:
      labels:
        app: litellm
    spec:
      containers:
        - name: litellm
          image: ghcr.io/berriai/litellm:main-latest
          ports:
            - containerPort: 4000
          args: ["--config", "/app/config.yaml", "--port", "4000"]
          volumeMounts:
            - name: config
              mountPath: /app/config.yaml
              subPath: config.yaml
          env:
            - name: OPENAI_API_KEY
              valueFrom:
                secretKeyRef:
                  name: litellm-secrets
                  key: openai-api-key
            - name: ANTHROPIC_API_KEY
              valueFrom:
                secretKeyRef:
                  name: litellm-secrets
                  key: anthropic-api-key
            - name: DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: litellm-secrets
                  key: database-url
          resources:
            requests:
              cpu: "500m"
              memory: "512Mi"
            limits:
              cpu: "2000m"
              memory: "2Gi"
          livenessProbe:
            httpGet:
              path: /health/liveliness
              port: 4000
            initialDelaySeconds: 30
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /health/readiness
              port: 4000
            initialDelaySeconds: 10
            periodSeconds: 5
      volumes:
        - name: config
          configMap:
            name: litellm-config
---
apiVersion: v1
kind: Service
metadata:
  name: litellm-service
spec:
  selector:
    app: litellm
  ports:
    - port: 80
      targetPort: 4000
  type: ClusterIP
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: litellm-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: litellm-proxy
  minReplicas: 2
  maxReplicas: 10
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70

# Helm으로 설치
helm repo add litellm https://litellm.github.io/litellm-helm
helm repo update

helm install litellm litellm/litellm-helm \
  --set masterKey=sk-your-master-key \
  --set replicaCount=3 \
  --set database.useExisting=true \
  --set database.url=postgresql://user:pass@host:5432/litellm \
  --namespace litellm \
  --create-namespace

Production Deployment Checklist
================================
[ ] PostgreSQL DB 설정 및 연결 확인
[ ] Master Key를 강력한 랜덤 값으로 설정
[ ] 모든 API 키를 환경 변수/시크릿으로 관리
[ ] HTTPS/TLS 설정 (Ingress 또는 LoadBalancer)
[ ] 레플리카 2개 이상 설정
[ ] HPA (Horizontal Pod Autoscaler) 설정
[ ] Liveness/Readiness Probe 설정
[ ] Prometheus + Grafana 모니터링 설정
[ ] Slack/PagerDuty 알림 설정
[ ] Redis 캐시 설정 (선택사항)
[ ] 예산 및 레이트 리밋 설정
[ ] Guardrails 설정 (필요 시)
[ ] 로깅 (Langfuse/LangSmith) 설정
[ ] 백업 및 복구 전략 수립
[ ] 로드 테스트 완료

┌─────────────────────────────────────────────────┐
│ 상황별 추천 솔루션                                │
├─────────────────────────────────────────────────┤
│                                                 │
│ "빠르게 시작하고 싶다"                           │
│  → OpenRouter (가입 후 즉시 사용)                │
│                                                 │
│ "데이터 보안이 중요하다"                          │
│  → LiteLLM (셀프 호스팅, 완전 제어)              │
│                                                 │
│ "팀/조직 관리가 필요하다"                         │
│  → LiteLLM 또는 Portkey                         │
│                                                 │
│ "로컬 모델도 사용한다"                            │
│  → LiteLLM (Ollama/vLLM 네이티브 지원)           │
│                                                 │
│ "엔터프라이즈 기능이 필요하다"                    │
│  → LiteLLM Enterprise 또는 Portkey Enterprise   │
│                                                 │
└─────────────────────────────────────────────────┘

# production-config.yaml
model_list:
  # === 프리미엄 티어 ===
  - model_name: premium-chat
    litellm_params:
      model: openai/gpt-4o
      api_key: os.environ/OPENAI_API_KEY
      rpm: 200
      tpm: 400000
    model_info:
      access_groups: ["premium"]

  - model_name: premium-chat
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY
      rpm: 150
    model_info:
      access_groups: ["premium"]

  # === 기본 티어 ===
  - model_name: basic-chat
    litellm_params:
      model: openai/gpt-4o-mini
      api_key: os.environ/OPENAI_API_KEY
      rpm: 500
    model_info:
      access_groups: ["basic", "premium"]

  - model_name: basic-chat
    litellm_params:
      model: gemini/gemini-2.0-flash
      api_key: os.environ/GEMINI_API_KEY
      rpm: 1000
    model_info:
      access_groups: ["basic", "premium"]

  # === 코딩 전용 ===
  - model_name: code-model
    litellm_params:
      model: anthropic/claude-sonnet-4-20250514
      api_key: os.environ/ANTHROPIC_API_KEY
    model_info:
      access_groups: ["premium"]

  # === 임베딩 ===
  - model_name: embedding
    litellm_params:
      model: openai/text-embedding-3-small
      api_key: os.environ/OPENAI_API_KEY

# === 라우터 설정 ===
router_settings:
  routing_strategy: latency-based-routing
  allowed_fails: 3
  cooldown_time: 60
  num_retries: 3
  timeout: 120
  retry_after: 5

# === LiteLLM 설정 ===
litellm_settings:
  drop_params: true
  set_verbose: false
  num_retries: 3
  request_timeout: 120

  # 폴백
  fallbacks:
    - model_name: premium-chat
      fallback_models: ["basic-chat"]
    - model_name: basic-chat
      fallback_models: ["premium-chat"]

  context_window_fallbacks:
    - model_name: basic-chat
      fallback_models: ["premium-chat"]

  # 캐싱
  cache: true
  cache_params:
    type: redis
    host: os.environ/REDIS_HOST
    port: 6379
    password: os.environ/REDIS_PASSWORD
    ttl: 3600

  # 콜백
  success_callback: ["langfuse", "prometheus"]
  failure_callback: ["langfuse", "prometheus"]

  # Guardrails
  guardrails:
    - guardrail_name: pii-masking
      litellm_params:
        guardrail: presidio
        mode: during_call

# === 일반 설정 ===
general_settings:
  master_key: os.environ/LITELLM_MASTER_KEY
  database_url: os.environ/DATABASE_URL
  max_budget: 10000
  budget_duration: 1m

  alerting:
    - slack
  alerting_threshold: 300

  global_max_parallel_requests: 200

# === 환경 변수 ===
environment_variables:
  LANGFUSE_PUBLIC_KEY: os.environ/LANGFUSE_PUBLIC_KEY
  LANGFUSE_SECRET_KEY: os.environ/LANGFUSE_SECRET_KEY
  SLACK_WEBHOOK_URL: os.environ/SLACK_WEBHOOK_URL