LLM 추론 서빙 프레임워크 비교: TensorRT-LLM vs vLLM vs SGLang 프로덕션 배포 전략

# Static Batching vs Continuous Batching 처리량 차이 시뮬레이션
import numpy as np

def simulate_static_batching(requests, batch_size=8):
    """Static Batching: 배치 내 최장 요청이 끝날 때까지 대기"""
    total_time = 0
    for i in range(0, len(requests), batch_size):
        batch = requests[i:i + batch_size]
        max_tokens = max(r["output_tokens"] for r in batch)
        total_time += max_tokens * 0.01  # 토큰당 10ms 가정
    return total_time

def simulate_continuous_batching(requests, batch_size=8):
    """Continuous Batching: 완료된 슬롯에 즉시 새 요청 삽입"""
    total_time = 0
    active_slots = []
    queue = list(requests)

    while queue or active_slots:
        # 빈 슬롯에 새 요청 삽입
        while len(active_slots) < batch_size and queue:
            active_slots.append(queue.pop(0))

        # 1스텝 진행
        total_time += 0.01
        for slot in active_slots:
            slot["remaining"] = slot.get("remaining", slot["output_tokens"]) - 1

        # 완료된 요청 제거
        active_slots = [s for s in active_slots if s["remaining"] > 0]

    return total_time

# 100개 요청, 출력 길이 10~500 토큰 랜덤
requests = [{"output_tokens": np.random.randint(10, 500)} for _ in range(100)]
static_time = simulate_static_batching(requests)
continuous_time = simulate_continuous_batching(
    [dict(r) for r in requests]
)

print(f"Static Batching 총 시간: {static_time:.1f}s")
print(f"Continuous Batching 총 시간: {continuous_time:.1f}s")
print(f"처리량 향상: {static_time / continuous_time:.1f}x")
# 출력 예시:
# Static Batching 총 시간: 62.5s
# Continuous Batching 총 시간: 27.3s
# 처리량 향상: 2.3x

# 1. TensorRT-LLM 설치 (Docker 권장)
docker pull nvcr.io/nvidia/tritonserver:24.12-trtllm-python-py3

# 2. Hugging Face 모델을 TRT 엔진으로 변환
# Llama-3-70B, FP8 양자화, Tensor Parallelism 4-way
python convert_checkpoint.py \
    --model_dir /models/Llama-3-70B \
    --output_dir /engines/llama-70b-ckpt \
    --dtype float16 \
    --tp_size 4

trtllm-build \
    --checkpoint_dir /engines/llama-70b-ckpt \
    --output_dir /engines/llama-70b-engine \
    --gemm_plugin float16 \
    --gpt_attention_plugin float16 \
    --max_batch_size 64 \
    --max_input_len 4096 \
    --max_seq_len 8192 \
    --use_paged_context_fmha enable \
    --use_fp8_context_fmha enable \
    --workers 4

# 3. Triton Inference Server로 서빙
tritonserver \
    --model-repository=/engines/triton-repo \
    --http-port=8000 \
    --grpc-port=8001 \
    --metrics-port=8002

# TensorRT-LLM Python API로 직접 추론
import tensorrt_llm
from tensorrt_llm import LLM, SamplingParams

# 빌드된 엔진 로드
llm = LLM(
    model="/engines/llama-70b-engine",
    tensor_parallel_size=4,
    dtype="float16",
    kv_cache_config={
        "enable_block_reuse": True,
        "free_gpu_memory_fraction": 0.9,
    },
)

# 배치 추론
prompts = [
    "Explain the concept of attention mechanism in transformers",
    "Write a Python function to implement binary search",
    "What are the key differences between TCP and UDP?",
]

sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
)

outputs = llm.generate(prompts, sampling_params=sampling_params)

for output in outputs:
    print(f"Prompt: {output.prompt[:50]}...")
    print(f"Generated: {output.outputs[0].text[:100]}...")
    print(f"Tokens/sec: {output.outputs[0].token_ids.__len__() / output.metrics.generation_time:.1f}")
    print()

# TensorRT-LLM Speculative Decoding 설정
from tensorrt_llm import LLM, SamplingParams

llm = LLM(
    model="/engines/llama-70b-engine",
    speculative_model="/engines/llama-8b-draft-engine",
    speculative_config={
        "num_draft_tokens": 5,
        "acceptance_method": "typical_acceptance",
    },
    tensor_parallel_size=4,
)

# Speculative Decoding은 투명하게 적용됨
params = SamplingParams(temperature=0.0, max_tokens=1024)
output = llm.generate(["Explain quantum computing"], params)
# 내부적으로 draft model이 5토큰씩 생성하고 target model이 검증

# vLLM 서버 실행 및 OpenAI 호환 API 호출
# 1. 서버 실행
# vllm serve meta-llama/Llama-3.1-70B-Instruct \
#     --tensor-parallel-size 4 \
#     --max-model-len 8192 \
#     --gpu-memory-utilization 0.92 \
#     --enable-prefix-caching \
#     --enable-chunked-prefill \
#     --max-num-seqs 256 \
#     --port 8000

# 2. Python 클라이언트에서 OpenAI 호환 API로 호출
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed",
)

# Chat Completions API
response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-70B-Instruct",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain how PagedAttention works in vLLM."},
    ],
    temperature=0.7,
    max_tokens=512,
    stream=True,
)

for chunk in response:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)

# 3. 배치 추론 (오프라인)
from vllm import LLM, SamplingParams

llm = LLM(
    model="meta-llama/Llama-3.1-70B-Instruct",
    tensor_parallel_size=4,
    max_model_len=8192,
    gpu_memory_utilization=0.92,
    enable_prefix_caching=True,
)

sampling = SamplingParams(temperature=0.0, max_tokens=256)
prompts = [f"Question {i}: What is {topic}?"
           for i, topic in enumerate(["ML", "DL", "NLP", "CV", "RL"])]

outputs = llm.generate(prompts, sampling)
for out in outputs:
    print(f"[{out.request_id}] {out.outputs[0].text[:80]}...")

# vllm-production-stack-values.yaml
# vLLM Production Stack Helm Chart 설정
servingEngineSpec:
  runtimeClassName: nvidia
  modelSpec:
    - name: 'llama-70b'
      repository: 'vllm/vllm-openai'
      tag: 'latest'
      modelURL: 'meta-llama/Llama-3.1-70B-Instruct'
      replicaCount: 2
      requestCPU: 8
      requestMemory: '64Gi'
      requestGPU: 4
      gpuType: 'nvidia.com/gpu'
      tensorParallelSize: 4
      maxModelLen: 8192
      extraArgs:
        - '--enable-prefix-caching'
        - '--enable-chunked-prefill'
        - '--gpu-memory-utilization=0.92'
        - '--max-num-seqs=256'

      hpa:
        enabled: true
        minReplicas: 2
        maxReplicas: 8
        targetValue: '70' # GPU 활용률 70% 타겟

routerSpec:
  repository: 'vllm/production-stack-router'
  tag: 'latest'
  replicaCount: 2
  requestCPU: 4
  requestMemory: '8Gi'
  routingStrategy: 'prefix-aware' # Prefix Cache 친화적 라우팅

# Prometheus 메트릭 수집 설정
monitoring:
  prometheus:
    enabled: true
    serviceMonitor:
      enabled: true
      interval: '15s'
  grafana:
    enabled: true
    dashboards:
      - name: 'vllm-serving'
        url: 'https://grafana.com/grafana/dashboards/vllm'

# Helm으로 vLLM Production Stack 배포
helm repo add vllm https://vllm-project.github.io/production-stack
helm repo update

helm install vllm-serving vllm/vllm-stack \
    -f vllm-production-stack-values.yaml \
    --namespace llm-serving \
    --create-namespace

# 배포 상태 확인
kubectl get pods -n llm-serving
kubectl logs -f deploy/vllm-serving-llama-70b -n llm-serving

# SGLang 서버 실행 및 프론트엔드 사용
# 1. 서버 실행
# python -m sglang.launch_server \
#     --model-path meta-llama/Llama-3.1-70B-Instruct \
#     --tp 4 \
#     --mem-fraction-static 0.88 \
#     --chunked-prefill-size 8192 \
#     --enable-torch-compile \
#     --port 30000

# 2. SGLang 프론트엔드 (Python DSL)
import sglang as sgl

@sgl.function
def multi_turn_qa(s, system_prompt, questions):
    s += sgl.system(system_prompt)
    answers = []
    for q in questions:
        s += sgl.user(q)
        s += sgl.assistant(sgl.gen("answer", max_tokens=256, temperature=0.7))
        answers.append(s["answer"])
    return answers

# RadixAttention이 자동으로 시스템 프롬프트의 KV Cache를 재사용
runtime = sgl.Runtime(
    model_path="meta-llama/Llama-3.1-70B-Instruct",
    tp_size=4,
)
sgl.set_default_backend(runtime)

system = "You are an expert system architect. Provide concise technical answers."
questions_batch = [
    ["What is CQRS?", "How does event sourcing work?"],
    ["What is CQRS?", "When should I avoid CQRS?"],
    ["What is CQRS?", "Compare CQRS with traditional CRUD"],
]

# 3개 요청 모두 "What is CQRS?"에 대한 KV Cache를 자동 공유
import concurrent.futures
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
    futures = [
        executor.submit(multi_turn_qa, system, qs)
        for qs in questions_batch
    ]
    results = [f.result() for f in futures]

for i, r in enumerate(results):
    print(f"Batch {i}: {len(r)} answers generated")

runtime.shutdown()

# SGLang의 구조화 출력 (JSON Schema 기반)
import sglang as sgl
from pydantic import BaseModel
from typing import List

class CodeReview(BaseModel):
    file_name: str
    severity: str  # "critical", "warning", "info"
    line_number: int
    issue: str
    suggestion: str

class ReviewResult(BaseModel):
    reviews: List[CodeReview]
    overall_score: int  # 1-10
    summary: str

@sgl.function
def structured_code_review(s, code_snippet):
    s += sgl.system(
        "You are a senior code reviewer. Analyze the given code and provide "
        "structured feedback in JSON format."
    )
    s += sgl.user(f"Review this code:\n```python\n{code_snippet}\n```")
    s += sgl.assistant(
        sgl.gen(
            "review",
            max_tokens=1024,
            temperature=0.0,
            regex=ReviewResult.model_json_schema(),  # JSON Schema 제약
        )
    )

# Compressed FSM이 JSON Schema를 상태 기계로 컴파일
# 디코딩 시 유효한 JSON만 생성되도록 보장
result = structured_code_review(
    code_snippet="""
def process_data(data):
    result = []
    for i in range(len(data)):
        if data[i] > 0:
            result.append(data[i] * 2)
    return result
"""
)

import json
review = json.loads(result["review"])
print(f"Overall Score: {review['overall_score']}/10")
print(f"Issues Found: {len(review['reviews'])}")
for r in review["reviews"]:
    print(f"  [{r['severity']}] Line {r['line_number']}: {r['issue']}")

# Kubernetes GPU 노드 스케줄링 - Pod Affinity 및 Topology 설정
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llm-serving-70b
  namespace: llm-serving
spec:
  replicas: 2
  selector:
    matchLabels:
      app: llm-serving
      model: llama-70b
  template:
    metadata:
      labels:
        app: llm-serving
        model: llama-70b
    spec:
      # GPU 노드에만 스케줄링
      nodeSelector:
        nvidia.com/gpu.product: 'NVIDIA-H100-80GB-HBM3'
      tolerations:
        - key: 'nvidia.com/gpu'
          operator: 'Exists'
          effect: 'NoSchedule'
      # TP=4 이므로 같은 노드의 4개 GPU를 사용
      # NVLink 연결된 GPU를 보장하기 위한 토폴로지 설정
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: llm-serving
      containers:
        - name: vllm-engine
          image: vllm/vllm-openai:latest
          args:
            - '--model=meta-llama/Llama-3.1-70B-Instruct'
            - '--tensor-parallel-size=4'
            - '--max-model-len=8192'
            - '--gpu-memory-utilization=0.92'
            - '--enable-prefix-caching'
            - '--port=8000'
          resources:
            limits:
              nvidia.com/gpu: 4
              memory: '128Gi'
              cpu: '16'
            requests:
              nvidia.com/gpu: 4
              memory: '96Gi'
              cpu: '12'
          ports:
            - containerPort: 8000
              name: http
            - containerPort: 8002
              name: metrics
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 120
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 180
            periodSeconds: 30
            failureThreshold: 3

# KEDA ScaledObject - GPU 메트릭 기반 오토스케일링
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: llm-serving-scaler
  namespace: llm-serving
spec:
  scaleTargetRef:
    name: llm-serving-70b
  minReplicaCount: 2
  maxReplicaCount: 8
  cooldownPeriod: 300
  pollingInterval: 15
  triggers:
    # GPU 활용률 기반 스케일링
    - type: prometheus
      metadata:
        serverAddress: http://prometheus.monitoring:9090
        metricName: gpu_utilization
        query: |
          avg(DCGM_FI_DEV_GPU_UTIL{
            namespace="llm-serving",
            pod=~"llm-serving-70b.*"
          })
        threshold: '75'
    # 대기열 깊이 기반 스케일링
    - type: prometheus
      metadata:
        serverAddress: http://prometheus.monitoring:9090
        metricName: pending_requests
        query: |
          sum(vllm:num_requests_waiting{
            namespace="llm-serving"
          })
        threshold: '50'

# Prometheus 커스텀 메트릭 수집 스크립트
import requests
import time
from prometheus_client import Gauge, start_http_server

# vLLM / SGLang 메트릭 수집
TTFT_P50 = Gauge("llm_ttft_p50_ms", "Time to First Token p50 in ms")
TTFT_P99 = Gauge("llm_ttft_p99_ms", "Time to First Token p99 in ms")
THROUGHPUT = Gauge("llm_throughput_rps", "Requests per second")
GPU_KV_CACHE_USAGE = Gauge("llm_gpu_kv_cache_usage", "KV Cache usage ratio")
ACTIVE_REQUESTS = Gauge("llm_active_requests", "Number of active requests")
PENDING_REQUESTS = Gauge("llm_pending_requests", "Number of pending requests")

def collect_vllm_metrics(base_url="http://localhost:8000"):
    """vLLM /metrics 엔드포인트에서 메트릭 수집"""
    try:
        resp = requests.get(f"{base_url}/metrics", timeout=5)
        lines = resp.text.strip().split("\n")
        for line in lines:
            if line.startswith("#"):
                continue
            if "vllm:time_to_first_token_seconds" in line and "p50" in line:
                TTFT_P50.set(float(line.split()[-1]) * 1000)
            elif "vllm:time_to_first_token_seconds" in line and "p99" in line:
                TTFT_P99.set(float(line.split()[-1]) * 1000)
            elif "vllm:num_requests_running" in line:
                ACTIVE_REQUESTS.set(float(line.split()[-1]))
            elif "vllm:num_requests_waiting" in line:
                PENDING_REQUESTS.set(float(line.split()[-1]))
            elif "vllm:gpu_cache_usage_perc" in line:
                GPU_KV_CACHE_USAGE.set(float(line.split()[-1]))
    except Exception as e:
        print(f"Metric collection failed: {e}")

if __name__ == "__main__":
    start_http_server(9090)
    while True:
        collect_vllm_metrics()
        time.sleep(15)

# OOM 발생 시 복구 체크리스트
# 1. 현재 GPU 메모리 상태 확인
nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu \
    --format=csv,noheader,nounits

# 2. vLLM 프로세스 상태 확인
curl -s http://localhost:8000/metrics | \
    grep -E "vllm:(num_requests|gpu_cache|cpu_cache)"

# 3. 안전한 설정으로 재시작
vllm serve meta-llama/Llama-3.1-70B-Instruct \
    --tensor-parallel-size 4 \
    --gpu-memory-utilization 0.90 \
    --max-num-seqs 128 \
    --max-model-len 4096 \
    --enable-prefix-caching \
    --enable-chunked-prefill

# 4. Kubernetes에서는 리소스 요청/제한 재설정 후 롤링 업데이트
kubectl set env deploy/llm-serving-70b \
    VLLM_GPU_MEMORY_UTILIZATION=0.90 \
    -n llm-serving
kubectl rollout restart deploy/llm-serving-70b -n llm-serving
kubectl rollout status deploy/llm-serving-70b -n llm-serving