┌─────────────────────────────────────────────────┐
│              Traditional KV Cache                │
│  ┌──────────────────────────────────┐            │
│  │ Seq 1: [used][used][used][waste][waste][waste]│
│  │ Seq 2: [used][waste][waste][waste][waste]     │
│  │ Seq 3: [used][used][waste][waste][waste]      │
│  └──────────────────────────────────┘            │
│              → 60~80% Memory Waste               │
├─────────────────────────────────────────────────┤
│              PagedAttention KV Cache             │
│  Physical Blocks: [B0][B1][B2][B3][B4][B5]...   │
│  Block Table:                                    │
│    Seq 1 → [B0, B3, B5]  (logical → physical)   │
│    Seq 2 → [B1, B4]                             │
│    Seq 3 → [B2, B6]                             │
│              → < 4% Memory Waste                 │
└─────────────────────────────────────────────────┘

Static Batching:
Step 1: [Seq1, Seq2, Seq3, Seq4]  ← Seq2 완료되어도 대기
Step 2: [Seq1, Seq2, Seq3, Seq4]
Step 3: [Seq1, ___, Seq3, Seq4]  ← Seq2 종료 후에도 슬롯 낭비
...
Step N: 모든 시퀀스 완료 후 다음 배치 시작

Continuous Batching:
Step 1: [Seq1, Seq2, Seq3, Seq4]
Step 2: [Seq1, Seq5, Seq3, Seq4]  ← Seq2 완료 즉시 Seq5 투입
Step 3: [Seq1, Seq5, Seq6, Seq4]  ← Seq3 완료 즉시 Seq6 투입
→ GPU 유휴 시간 최소화, Throughput 극대화

# 기본 설치 (CUDA 12.x)
pip install vllm

# 특정 버전 설치
pip install vllm==0.16.0

# CUDA 11.8 환경
pip install vllm --extra-index-url https://download.pytorch.org/whl/cu118

conda create -n vllm python=3.11 -y
conda activate vllm
pip install vllm

# 공식 Docker 이미지 (NVIDIA GPU)
docker run --runtime nvidia --gpus all \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --env "HUGGING_FACE_HUB_TOKEN=<hf_token>" \
  -p 8000:8000 \
  --ipc=host \
  vllm/vllm-openai:latest \
  --model meta-llama/Llama-3.1-8B-Instruct

# ROCm (AMD GPU)
docker run --device /dev/kfd --device /dev/dri \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  vllm/vllm-openai:latest-rocm \
  --model meta-llama/Llama-3.1-8B-Instruct

# vllm serve 명령 (권장)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000

# Python 모듈 직접 실행 (레거시 방식)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000

# YAML 설정 파일로 구동
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --config config.yaml

# vLLM 서버 설정 파일
host: '0.0.0.0'
port: 8000
tensor_parallel_size: 2
gpu_memory_utilization: 0.90
max_model_len: 8192
dtype: 'auto'
enforce_eager: false
enable_prefix_caching: true

from vllm import LLM, SamplingParams

# 모델 로드
llm = LLM(
    model="meta-llama/Llama-3.1-8B-Instruct",
    tensor_parallel_size=1,
    gpu_memory_utilization=0.90,
)

# 샘플링 파라미터 설정
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
)

# 프롬프트 목록
prompts = [
    "Explain PagedAttention in simple terms.",
    "What is continuous batching?",
    "Compare vLLM and TensorRT-LLM.",
]

# 배치 추론 실행
outputs = llm.generate(prompts, sampling_params)

for output in outputs:
    prompt = output.prompt
    generated = output.outputs[0].text
    print(f"Prompt: {prompt!r}")
    print(f"Output: {generated!r}\n")

# 서버 시작
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --served-model-name llama-3.1-8b \
  --api-key my-secret-key

# curl로 Chat Completion 호출
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer my-secret-key" \
  -d '{
    "model": "llama-3.1-8b",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "PagedAttention이란 무엇인가요?"}
    ],
    "temperature": 0.7,
    "max_tokens": 512
  }'

# OpenAI SDK로 호출
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="my-secret-key",
)

response = client.chat.completions.create(
    model="llama-3.1-8b",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "vLLM의 장점을 설명해주세요."},
    ],
    temperature=0.7,
    max_tokens=512,
)

print(response.choices[0].message.content)

# 4-GPU Tensor Parallelism
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4

# 2-GPU Tensor + 2-way Pipeline (총 4 GPU)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 2 \
  --pipeline-parallel-size 2

# Ray 분산 백엔드 사용 (멀티 노드)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 8 \
  --distributed-executor-backend ray

# 메모리 최적화 설정
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --gpu-memory-utilization 0.95 \
  --max-num-seqs 128 \
  --max-model-len 4096 \
  --enable-prefix-caching \
  --enable-chunked-prefill

# Eager 모드 (디버깅/호환성)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enforce-eager \
  --gpu-memory-utilization 0.85

# AWQ 양자화 모델
vllm serve TheBloke/Llama-2-7B-AWQ \
  --quantization awq

# GPTQ 양자화 모델
vllm serve TheBloke/Llama-2-7B-GPTQ \
  --quantization gptq

# FP8 양자화 (H100 이상)
vllm serve neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 \
  --quantization fp8

# BitsAndBytes 4-bit (GPU 메모리 절약)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization bitsandbytes \
  --load-format bitsandbytes

# LoRA 어댑터 서빙
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-lora \
  --max-loras 4 \
  --max-lora-rank 64 \
  --lora-modules \
    adapter1=/path/to/lora1 \
    adapter2=/path/to/lora2

# 별도 draft 모델 사용
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --speculative-model meta-llama/Llama-3.2-1B-Instruct \
  --num-speculative-tokens 5 \
  --tensor-parallel-size 4

# N-gram 기반 speculative decoding (추가 모델 불필요)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --speculative-model "[ngram]" \
  --num-speculative-tokens 5 \
  --ngram-prompt-lookup-max 4

# 기본 Chat Completion
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "서울의 인구는?"}
    ],
    "temperature": 0.3,
    "top_p": 0.9,
    "max_tokens": 256,
    "frequency_penalty": 0.5,
    "seed": 42
  }'

# Structured Output (JSON mode)
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "서울, 부산, 대구의 인구를 JSON으로 알려줘"}
    ],
    "response_format": {
      "type": "json_schema",
      "json_schema": {
        "name": "city_population",
        "schema": {
          "type": "object",
          "properties": {
            "cities": {
              "type": "array",
              "items": {
                "type": "object",
                "properties": {
                  "name": {"type": "string"},
                  "population": {"type": "integer"}
                },
                "required": ["name", "population"]
              }
            }
          },
          "required": ["cities"]
        }
      }
    },
    "temperature": 0.1,
    "max_tokens": 512
  }'

# logprobs 반환
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "1+1=?"}
    ],
    "logprobs": true,
    "top_logprobs": 5,
    "max_tokens": 10
  }'

import requests
import json

url = "http://localhost:8000/v1/chat/completions"
headers = {"Content-Type": "application/json"}

payload = {
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
        {"role": "system", "content": "You are a helpful Korean assistant."},
        {"role": "user", "content": "양자 컴퓨팅이란?"},
    ],
    "temperature": 0.7,
    "top_p": 0.9,
    "top_k": 50,
    "max_tokens": 1024,
    "repetition_penalty": 1.1,
    "stop": ["\n\n\n"],
}

response = requests.post(url, headers=headers, json=payload)
result = response.json()
print(result["choices"][0]["message"]["content"])

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")

# Streaming 응답
stream = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[
        {"role": "user", "content": "Python으로 퀵소트를 구현해줘"},
    ],
    temperature=0.2,
    max_tokens=2048,
    stream=True,
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)
print()

# 멀티 GPU + 로깅 + HF 토큰 설정
export CUDA_VISIBLE_DEVICES=0,1,2,3
export HF_TOKEN="hf_xxxxxxxxxxxx"
export VLLM_LOGGING_LEVEL="DEBUG"
export VLLM_WORKER_MULTIPROC_METHOD="spawn"

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.90

# Docker에서 환경변수 전달
docker run --runtime nvidia --gpus all \
  -e CUDA_VISIBLE_DEVICES=0,1 \
  -e HF_TOKEN="hf_xxxxxxxxxxxx" \
  -e VLLM_LOGGING_LEVEL="INFO" \
  -e VLLM_WORKER_MULTIPROC_METHOD="spawn" \
  -p 8000:8000 \
  --ipc=host \
  vllm/vllm-openai:latest \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --tensor-parallel-size 2

# TP=4 (4 GPU에 모델 분산)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.90

# PP=2, TP=2 (총 4 GPU, 2×2 구성)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 2 \
  --pipeline-parallel-size 2

# 마스터 노드
ray start --head --port=6379

# 워커 노드
ray start --address=<master-ip>:6379

# vLLM 실행 (마스터에서)
vllm serve meta-llama/Llama-3.1-405B-Instruct \
  --tensor-parallel-size 8 \
  --pipeline-parallel-size 2 \
  --distributed-executor-backend ray

# AWQ 사전 양자화 모델 사용
vllm serve TheBloke/Llama-2-13B-chat-AWQ \
  --quantization awq \
  --max-model-len 4096

# Marlin 커널로 더 빠르게 (SM 80+ GPU)
vllm serve TheBloke/Llama-2-13B-chat-AWQ \
  --quantization awq_marlin

# GPTQ 모델 (ExLlamaV2 커널 자동 사용)
vllm serve TheBloke/Llama-2-13B-chat-GPTQ \
  --quantization gptq

# Marlin 커널 사용
vllm serve TheBloke/Llama-2-13B-chat-GPTQ \
  --quantization gptq_marlin

# 사전 양자화 FP8 모델
vllm serve neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 \
  --quantization fp8

# 동적 FP8 양자화 (사전 양자화 불필요)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization fp8 \
  --kv-cache-dtype fp8

vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization bitsandbytes \
  --load-format bitsandbytes \
  --enforce-eager  # BnB는 Eager 모드 필요

# LoRA 어댑터 활성화
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-lora \
  --max-loras 4 \
  --max-lora-rank 64 \
  --lora-modules \
    korean-chat=/path/to/korean-lora \
    code-assist=/path/to/code-lora

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")

# 특정 LoRA 어댑터 사용
response = client.chat.completions.create(
    model="korean-chat",  # LoRA 어댑터 이름
    messages=[{"role": "user", "content": "안녕하세요!"}],
    temperature=0.7,
    max_tokens=256,
)

# v1에서는 기본 활성화, v0에서는 명시 필요
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-prefix-caching

vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-chunked-prefill \
  --max-num-batched-tokens 2048

# JSON Schema 기반 구조화 출력
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "서울의 날씨 정보를 JSON으로 제공해주세요"}
    ],
    "response_format": {
      "type": "json_schema",
      "json_schema": {
        "name": "weather_info",
        "schema": {
          "type": "object",
          "properties": {
            "city": {"type": "string"},
            "temperature_celsius": {"type": "number"},
            "condition": {"type": "string"},
            "humidity_percent": {"type": "integer"}
          },
          "required": ["city", "temperature_celsius", "condition"]
        }
      }
    }
  }'

# Regex 기반 출력 (Completion API)
curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "prompt": "Generate a valid email address:",
    "extra_body": {
      "guided_regex": "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"
    },
    "max_tokens": 50
  }'

# docker-compose.yaml
version: '3.8'

services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - '8000:8000'
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - VLLM_LOGGING_LEVEL=INFO
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
    ipc: host
    command: >
      --model meta-llama/Llama-3.1-8B-Instruct
      --host 0.0.0.0
      --port 8000
      --tensor-parallel-size 2
      --gpu-memory-utilization 0.90
      --max-model-len 8192
      --enable-prefix-caching
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:8000/health']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s

# Docker Compose로 실행
HF_TOKEN=hf_xxxx docker compose up -d

# 로그 확인
docker compose logs -f vllm

# vllm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-llama3
  namespace: ai-serving
  labels:
    app: vllm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
        - name: vllm
          image: vllm/vllm-openai:latest
          ports:
            - containerPort: 8000
              name: http
          args:
            - '--model'
            - 'meta-llama/Llama-3.1-8B-Instruct'
            - '--host'
            - '0.0.0.0'
            - '--port'
            - '8000'
            - '--tensor-parallel-size'
            - '2'
            - '--gpu-memory-utilization'
            - '0.90'
            - '--max-model-len'
            - '8192'
          env:
            - name: HF_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-secret
                  key: token
            - name: VLLM_WORKER_MULTIPROC_METHOD
              value: 'spawn'
          resources:
            limits:
              nvidia.com/gpu: '2'
            requests:
              nvidia.com/gpu: '2'
              memory: '32Gi'
              cpu: '8'
          volumeMounts:
            - name: shm
              mountPath: /dev/shm
            - name: model-cache
              mountPath: /root/.cache/huggingface
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 120
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 180
            periodSeconds: 30
      volumes:
        - name: shm
          emptyDir:
            medium: Memory
            sizeLimit: 2Gi
        - name: model-cache
          persistentVolumeClaim:
            claimName: model-cache-pvc
      nodeSelector:
        nvidia.com/gpu.product: 'NVIDIA-A100-SXM4-80GB'
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-service
  namespace: ai-serving
spec:
  selector:
    app: vllm
  ports:
    - port: 8000
      targetPort: 8000
      name: http
  type: ClusterIP
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: vllm-hpa
  namespace: ai-serving
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: vllm-llama3
  minReplicas: 1
  maxReplicas: 4
  metrics:
    - type: Pods
      pods:
        metric:
          name: vllm_num_requests_running
        target:
          type: AverageValue
          averageValue: '50'

# Homebrew
brew install ollama

# 또는 공식 설치 스크립트
curl -fsSL https://ollama.com/install.sh | sh

# 공식 설치 스크립트 (권장)
curl -fsSL https://ollama.com/install.sh | sh

# 또는 수동 설치
curl -L https://ollama.com/download/ollama-linux-amd64 -o /usr/local/bin/ollama
chmod +x /usr/local/bin/ollama

# CPU only
docker run -d -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama

# NVIDIA GPU
docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama

# AMD GPU (ROCm)
docker run -d --device /dev/kfd --device /dev/dri \
  -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama:rocm

# 서버 시작 (백그라운드에서 자동 시작되지 않는 경우)
ollama serve

# 모델 다운로드 및 대화
ollama run llama3.1

# 특정 태그(크기/양자화) 지정
ollama run llama3.1:8b
ollama run llama3.1:70b-instruct-q4_K_M
ollama run qwen2.5:32b-instruct-q5_K_M

# 모델만 다운로드 (실행하지 않음)
ollama pull llama3.1:8b

# 한줄 프롬프트
ollama run llama3.1 "PagedAttention이란 무엇인가요?"

# 기본 시작 (localhost:11434)
ollama serve

# 환경변수로 바인드 주소 변경
OLLAMA_HOST=0.0.0.0:11434 ollama serve

# 디버그 모드
OLLAMA_DEBUG=1 ollama serve

# 대화형 모드
ollama run llama3.1

# 한줄 프롬프트
ollama run llama3.1 "Explain quantum computing"

# JSON 형식 출력
ollama run llama3.1 "List 3 Korean cities" --format json

# 멀티모달 (이미지 입력)
ollama run llama3.2-vision "What's in this image? /path/to/image.png"

# verbose 모드 (성능 통계 표시)
ollama run llama3.1 --verbose

# 시스템 프롬프트와 함께
ollama run llama3.1 --system "You are a Korean translator."

# Modelfile 기반 생성
ollama create my-model -f ./Modelfile

# GGUF 파일로부터 생성
ollama create my-model -f ./Modelfile-from-gguf

# 양자화 변환
ollama create my-model-q4 --quantize q4_K_M -f ./Modelfile

# 전체 정보
ollama show llama3.1

# Modelfile 출력
ollama show llama3.1 --modelfile

# 파라미터 확인
ollama show llama3.1 --parameters

# 시스템 프롬프트 확인
ollama show llama3.1 --system

# 템플릿 확인
ollama show llama3.1 --template

$ ollama ps
NAME              ID            SIZE     PROCESSOR    UNTIL
llama3.1:8b       a]f2e33d4e25  6.7 GB   100% GPU     4 minutes from now
qwen2.5:7b        845dbda0ea48  4.7 GB   100% GPU     3 minutes from now

# 기본 생성
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "Why is the sky blue?",
  "stream": false
}'

# 스트리밍 (기본값)
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "Write a haiku about coding",
  "options": {
    "temperature": 0.7,
    "num_predict": 100
  }
}'

# JSON 형식 출력
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "List 3 programming languages as JSON",
  "format": "json",
  "stream": false
}'

curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [
    {"role": "system", "content": "You are a helpful Korean assistant."},
    {"role": "user", "content": "서울의 명소를 추천해주세요."}
  ],
  "stream": false,
  "options": {
    "temperature": 0.8,
    "top_p": 0.9,
    "num_ctx": 4096,
    "num_predict": 512
  }
}'

# 단일 텍스트 임베딩
curl http://localhost:11434/api/embed -d '{
  "model": "nomic-embed-text",
  "input": "Hello, world!"
}'

# 복수 텍스트 임베딩
curl http://localhost:11434/api/embed -d '{
  "model": "nomic-embed-text",
  "input": ["Hello world", "Goodbye world"]
}'

# OpenAI 형식 Chat Completion
curl http://localhost:11434/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llama3.1",
    "messages": [
      {"role": "user", "content": "Hello!"}
    ],
    "temperature": 0.7,
    "max_tokens": 256
  }'

# 모델 목록
curl http://localhost:11434/v1/models

import requests

# Generate API
response = requests.post("http://localhost:11434/api/generate", json={
    "model": "llama3.1",
    "prompt": "Explain Docker in Korean",
    "stream": False,
    "options": {
        "temperature": 0.7,
        "num_predict": 512,
    },
})
print(response.json()["response"])

# Chat API
response = requests.post("http://localhost:11434/api/chat", json={
    "model": "llama3.1",
    "messages": [
        {"role": "user", "content": "쿠버네티스란?"},
    ],
    "stream": False,
})
print(response.json()["message"]["content"])

# OpenAI SDK로 Ollama 사용
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama",  # Ollama는 API key 불필요, 임의 값 입력
)

response = client.chat.completions.create(
    model="llama3.1",
    messages=[
        {"role": "user", "content": "Python의 GIL을 설명해주세요."},
    ],
    temperature=0.7,
    max_tokens=512,
)
print(response.choices[0].message.content)

# 기본 모델 지정 (필수)
FROM llama3.1:8b

# 파라미터 설정
PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER top_k 40
PARAMETER num_ctx 4096
PARAMETER num_predict 512
PARAMETER repeat_penalty 1.1
PARAMETER stop "<|eot_id|>"
PARAMETER stop "<|end_of_text|>"

# 시스템 프롬프트
SYSTEM """
당신은 친절한 한국어 AI 어시스턴트입니다.
정확하고 간결한 답변을 제공하며, 필요할 때 예시를 들어 설명합니다.
"""

# 대화 템플릿 (Jinja2 또는 Go template)
TEMPLATE """
{{- if .System }}<|start_header_id|>system<|end_header_id|>
{{ .System }}<|eot_id|>{{ end }}
{{- range .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|>
{{ .Content }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
"""

# LoRA 어댑터 적용 (선택)
ADAPTER /path/to/lora-adapter.gguf

# 라이선스 정보 (선택)
LICENSE """
Apache 2.0
"""

curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [
    {"role": "user", "content": "Hello"}
  ],
  "options": {
    "temperature": 0.3,
    "top_p": 0.9,
    "top_k": 50,
    "num_ctx": 8192,
    "num_predict": 1024,
    "repeat_penalty": 1.2,
    "seed": 42,
    "stop": ["<|eot_id|>"]
  }
}'

# 환경변수 설정
launchctl setenv OLLAMA_HOST "0.0.0.0:11434"
launchctl setenv OLLAMA_MODELS "/Volumes/ExternalSSD/ollama/models"
launchctl setenv OLLAMA_FLASH_ATTENTION "1"
launchctl setenv OLLAMA_KV_CACHE_TYPE "q8_0"
launchctl setenv OLLAMA_NUM_PARALLEL "4"
launchctl setenv OLLAMA_KEEP_ALIVE "-1"

# Ollama 재시작
brew services restart ollama

# systemd 서비스 오버라이드 생성
sudo systemctl edit ollama

# 에디터에서 다음 내용 추가:
[Service]
Environment="OLLAMA_HOST=0.0.0.0:11434"
Environment="OLLAMA_MODELS=/data/ollama/models"
Environment="OLLAMA_FLASH_ATTENTION=1"
Environment="OLLAMA_KV_CACHE_TYPE=q8_0"
Environment="OLLAMA_NUM_PARALLEL=4"
Environment="OLLAMA_KEEP_ALIVE=-1"
Environment="OLLAMA_MAX_LOADED_MODELS=3"
Environment="CUDA_VISIBLE_DEVICES=0,1"

# 서비스 재시작
sudo systemctl daemon-reload
sudo systemctl restart ollama

docker run -d --gpus=all \
  -e OLLAMA_HOST=0.0.0.0:11434 \
  -e OLLAMA_FLASH_ATTENTION=1 \
  -e OLLAMA_KV_CACHE_TYPE=q8_0 \
  -e OLLAMA_NUM_PARALLEL=4 \
  -e OLLAMA_KEEP_ALIVE=-1 \
  -v /data/ollama:/root/.ollama \
  -p 11434:11434 \
  --name ollama \
  ollama/ollama

FROM llama3.1:8b

PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER num_ctx 4096
PARAMETER num_predict 1024
PARAMETER repeat_penalty 1.15
PARAMETER stop "<|eot_id|>"
PARAMETER stop "<|end_of_text|>"

SYSTEM """
당신은 대한민국의 문화와 역사에 정통한 한국어 AI 어시스턴트입니다.
항상 정확하고 친절하게 한국어로 응답하며, 필요시 영어 기술 용어를 병기합니다.
답변은 구조적으로 정리하여 제공합니다.
"""

MESSAGE user 안녕하세요, 자기소개 해주세요.
MESSAGE assistant 안녕하세요! 저는 한국어에 특화된 AI 어시스턴트입니다. 한국의 문화, 역사, 기술 등 다양한 주제에 대해 도움을 드릴 수 있습니다. 무엇이든 물어보세요!

# 모델 생성
ollama create korean-assistant -f ./Modelfile-korean

# 실행
ollama run korean-assistant "서울 3대 궁궐에 대해 알려줘"

FROM qwen2.5-coder:7b

PARAMETER temperature 0.2
PARAMETER top_p 0.85
PARAMETER num_ctx 8192
PARAMETER num_predict 2048

SYSTEM """
You are an expert code reviewer. Analyze code for:
1. Bugs and potential issues
2. Performance improvements
3. Security vulnerabilities
4. Code style and best practices

Provide specific, actionable feedback with corrected code examples.
"""

# NVIDIA 드라이버 확인
nvidia-smi

# 특정 GPU만 사용
CUDA_VISIBLE_DEVICES=0 ollama serve

# 멀티 GPU
CUDA_VISIBLE_DEVICES=0,1 ollama serve

# ROCm 드라이버 확인
rocm-smi

# 특정 GPU 지정
ROCR_VISIBLE_DEVICES=0 ollama serve

# GPU 사용 확인 (ollama ps에서 Processor 컬럼)
ollama ps
# NAME           ID            SIZE    PROCESSOR     UNTIL
# llama3.1:8b    a]f2e33d4e25  6.7 GB  100% GPU      4 minutes from now

# docker-compose.yaml
version: '3.8'

services:
  ollama:
    image: ollama/ollama
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - '11434:11434'
    volumes:
      - ollama_data:/root/.ollama
    environment:
      - OLLAMA_HOST=0.0.0.0:11434
      - OLLAMA_FLASH_ATTENTION=1
      - OLLAMA_KV_CACHE_TYPE=q8_0
      - OLLAMA_NUM_PARALLEL=4
      - OLLAMA_KEEP_ALIVE=24h
    restart: unless-stopped
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:11434/api/version']
      interval: 30s
      timeout: 5s
      retries: 3

  # 모델 초기화 (선택)
  ollama-init:
    image: curlimages/curl:latest
    depends_on:
      ollama:
        condition: service_healthy
    entrypoint: >
      sh -c "
        curl -s http://ollama:11434/api/pull -d '{\"name\": \"llama3.1:8b\"}' &&
        curl -s http://ollama:11434/api/pull -d '{\"name\": \"nomic-embed-text\"}'
      "

volumes:
  ollama_data:

# LLaVA 모델 실행
ollama run llava "What's in this image? /path/to/photo.jpg"

# Llama 3.2 Vision
ollama run llama3.2-vision "이 이미지를 한국어로 설명해주세요. /path/to/image.png"

import requests
import base64

# 이미지를 base64로 인코딩
with open("image.jpg", "rb") as f:
    image_base64 = base64.b64encode(f.read()).decode("utf-8")

response = requests.post("http://localhost:11434/api/chat", json={
    "model": "llava",
    "messages": [
        {
            "role": "user",
            "content": "이 이미지에 무엇이 있나요?",
            "images": [image_base64],
        }
    ],
    "stream": False,
})
print(response.json()["message"]["content"])

from openai import OpenAI
import json

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get current weather for a city",
            "parameters": {
                "type": "object",
                "properties": {
                    "city": {"type": "string", "description": "City name"},
                    "unit": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"],
                    },
                },
                "required": ["city"],
            },
        },
    }
]

response = client.chat.completions.create(
    model="llama3.1",
    messages=[
        {"role": "user", "content": "서울의 현재 날씨는?"}
    ],
    tools=tools,
    tool_choice="auto",
)

message = response.choices[0].message
if message.tool_calls:
    for tool_call in message.tool_calls:
        print(f"Function: {tool_call.function.name}")
        print(f"Arguments: {tool_call.function.arguments}")

# 설치 후 즉시 사용
ollama run llama3.1

# VS Code + Continue 확장 연동
# settings.json에 Ollama 엔드포인트 설정

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.90 \
  --max-num-seqs 256 \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --api-key ${API_KEY}

# 작은 모델 + 고양자화
ollama run phi3:3.8b-mini-instruct-4k-q4_0

# 또는 Qwen 0.5B
ollama run qwen2.5:0.5b

from vllm import LLM, SamplingParams

llm = LLM(
    model="meta-llama/Llama-3.1-8B-Instruct",
    tensor_parallel_size=2,
    gpu_memory_utilization=0.95,
)

# 수천 개의 프롬프트를 한 번에 처리
prompts = load_prompts_from_file("prompts.jsonl")  # 10,000+ prompts
sampling_params = SamplingParams(temperature=0.0, max_tokens=512)

outputs = llm.generate(prompts, sampling_params)
save_outputs(outputs, "results.jsonl")

# Ollama 기반 RAG (개발/소규모)
from langchain_ollama import OllamaLLM, OllamaEmbeddings

llm = OllamaLLM(model="llama3.1")
embeddings = OllamaEmbeddings(model="nomic-embed-text")

# vLLM 기반 RAG (프로덕션)
from langchain_openai import ChatOpenAI, OpenAIEmbeddings

llm = ChatOpenAI(
    base_url="http://vllm-server:8000/v1",
    api_key="token",
    model="meta-llama/Llama-3.1-8B-Instruct",
)

from openai import OpenAI
import uuid

client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")

# 커스텀 request_id 전달
xid = str(uuid.uuid4())

response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "Hello"}],
    extra_headers={"X-Request-ID": xid},
)

print(f"XID: {xid}")
print(f"Response ID: {response.id}")

import requests
import uuid

xid = str(uuid.uuid4())

response = requests.post(
    "http://localhost:11434/api/chat",
    headers={"X-Request-ID": xid},
    json={
        "model": "llama3.1",
        "messages": [{"role": "user", "content": "Hello"}],
        "stream": False,
    },
)

# 로깅에 xid 포함
import logging
logger = logging.getLogger(__name__)
logger.info(f"[xid={xid}] Response: {response.status_code}")

upstream vllm_backend {
    server vllm-server:8000;
}

server {
    listen 80;

    location /v1/ {
        # X-Request-ID가 없으면 자동 생성
        set $request_id $http_x_request_id;
        if ($request_id = "") {
            set $request_id $request_id;
        }

        proxy_pass http://vllm_backend;
        proxy_set_header X-Request-ID $request_id;
        proxy_set_header Host $host;

        # 응답 헤더에 X-Request-ID 추가
        add_header X-Request-ID $request_id always;

        # 액세스 로그에 request_id 포함
        access_log /var/log/nginx/vllm_access.log combined_with_xid;
    }
}

# 로그 포맷 정의
log_format combined_with_xid '$remote_addr - $remote_user [$time_local] '
    '"$request" $status $body_bytes_sent '
    '"$http_referer" "$http_user_agent" '
    'xid="$http_x_request_id"';

# vLLM + OpenTelemetry 분산 추적
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# Tracer 초기화
provider = TracerProvider()
exporter = OTLPSpanExporter(endpoint="http://jaeger:4317")
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)
tracer = trace.get_tracer(__name__)

# LLM 호출을 Span으로 래핑
def call_llm(prompt: str, xid: str) -> str:
    with tracer.start_as_current_span("llm_inference") as span:
        span.set_attribute("xid", xid)
        span.set_attribute("model", "llama-3.1-8b")
        span.set_attribute("prompt_length", len(prompt))

        response = client.chat.completions.create(
            model="meta-llama/Llama-3.1-8B-Instruct",
            messages=[{"role": "user", "content": prompt}],
            extra_headers={"X-Request-ID": xid},
        )

        result = response.choices[0].message.content
        span.set_attribute("response_length", len(result))
        span.set_attribute("tokens_used", response.usage.total_tokens)

        return result

import logging
import uuid
from contextvars import ContextVar

# Context Variable로 xid 관리
request_xid: ContextVar[str] = ContextVar("request_xid", default="")

class XIDFilter(logging.Filter):
    def filter(self, record):
        record.xid = request_xid.get("")
        return True

# 로거 설정
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(
    "%(asctime)s [%(levelname)s] [xid=%(xid)s] %(message)s"
))
handler.addFilter(XIDFilter())

logger = logging.getLogger("llm_service")
logger.addHandler(handler)
logger.setLevel(logging.INFO)

# 사용
async def handle_request(prompt: str):
    xid = str(uuid.uuid4())
    request_xid.set(xid)

    logger.info(f"Received prompt: {prompt[:50]}...")

    response = await call_llm(prompt, xid)

    logger.info(f"Generated {len(response)} chars")
    return {"xid": xid, "response": response}

package main

import (
    "context"
    "fmt"
    "log/slog"
    "net/http"

    "github.com/google/uuid"
)

type contextKey string
const xidKey contextKey = "xid"

// XID 미들웨어
func xidMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        xid := r.Header.Get("X-Request-ID")
        if xid == "" {
            xid = uuid.New().String()
        }

        ctx := context.WithValue(r.Context(), xidKey, xid)
        w.Header().Set("X-Request-ID", xid)

        slog.Info("request received",
            "xid", xid,
            "method", r.Method,
            "path", r.URL.Path,
        )

        next.ServeHTTP(w, r.WithContext(ctx))
    })
}

// Ollama 호출 함수
func callOllama(ctx context.Context, prompt string) (string, error) {
    xid := ctx.Value(xidKey).(string)

    slog.Info("calling ollama",
        "xid", xid,
        "prompt_len", len(prompt),
    )

    // ... Ollama API 호출 로직 ...

    slog.Info("ollama response received",
        "xid", xid,
        "response_len", len(response),
    )

    return response, nil
}