┌─────────────────────────────────────────────────┐
│              Traditional KV Cache                │
│  ┌──────────────────────────────────┐            │
│  │ Seq 1: [used][used][used][waste][waste][waste]│
│  │ Seq 2: [used][waste][waste][waste][waste]     │
│  │ Seq 3: [used][used][waste][waste][waste]      │
│  └──────────────────────────────────┘            │
│              → 60~80% Memory Waste               │
├─────────────────────────────────────────────────┤
│              PagedAttention KV Cache             │
│  Physical Blocks: [B0][B1][B2][B3][B4][B5]...   │
│  Block Table:                                    │
│    Seq 1 → [B0, B3, B5]  (logical → physical)   │
│    Seq 2 → [B1, B4]                             │
│    Seq 3 → [B2, B6]                             │
│              → < 4% Memory Waste                 │
└─────────────────────────────────────────────────┘

Static Batching:
Step 1: [Seq1, Seq2, Seq3, Seq4]  ← Waits even if Seq2 completes
Step 2: [Seq1, Seq2, Seq3, Seq4]
Step 3: [Seq1, ___, Seq3, Seq4]  ← Slot wasted after Seq2 ends
...
Step N: Next batch starts after all sequences complete

Continuous Batching:
Step 1: [Seq1, Seq2, Seq3, Seq4]
Step 2: [Seq1, Seq5, Seq3, Seq4]  ← Seq5 inserted immediately after Seq2 completes
Step 3: [Seq1, Seq5, Seq6, Seq4]  ← Seq6 inserted immediately after Seq3 completes
→ Minimizes GPU idle time, maximizes throughput

# Basic installation (CUDA 12.x)
pip install vllm

# Specific version installation
pip install vllm==0.16.0

# CUDA 11.8 environment
pip install vllm --extra-index-url https://download.pytorch.org/whl/cu118

conda create -n vllm python=3.11 -y
conda activate vllm
pip install vllm

# Official Docker image (NVIDIA GPU)
docker run --runtime nvidia --gpus all \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  --env "HUGGING_FACE_HUB_TOKEN=<hf_token>" \
  -p 8000:8000 \
  --ipc=host \
  vllm/vllm-openai:latest \
  --model meta-llama/Llama-3.1-8B-Instruct

# ROCm (AMD GPU)
docker run --device /dev/kfd --device /dev/dri \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  -p 8000:8000 \
  vllm/vllm-openai:latest-rocm \
  --model meta-llama/Llama-3.1-8B-Instruct

# vllm serve command (recommended)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000

# Direct Python module execution (legacy)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --host 0.0.0.0 \
  --port 8000

# Start with YAML config file
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --config config.yaml

# vLLM server configuration file
host: '0.0.0.0'
port: 8000
tensor_parallel_size: 2
gpu_memory_utilization: 0.90
max_model_len: 8192
dtype: 'auto'
enforce_eager: false
enable_prefix_caching: true

from vllm import LLM, SamplingParams

# Load model
llm = LLM(
    model="meta-llama/Llama-3.1-8B-Instruct",
    tensor_parallel_size=1,
    gpu_memory_utilization=0.90,
)

# Set sampling parameters
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
)

# Prompt list
prompts = [
    "Explain PagedAttention in simple terms.",
    "What is continuous batching?",
    "Compare vLLM and TensorRT-LLM.",
]

# Run batch inference
outputs = llm.generate(prompts, sampling_params)

for output in outputs:
    prompt = output.prompt
    generated = output.outputs[0].text
    print(f"Prompt: {prompt!r}")
    print(f"Output: {generated!r}\n")

# Start server
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --served-model-name llama-3.1-8b \
  --api-key my-secret-key

# Call Chat Completion with curl
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer my-secret-key" \
  -d '{
    "model": "llama-3.1-8b",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "What is PagedAttention?"}
    ],
    "temperature": 0.7,
    "max_tokens": 512
  }'

# Call with OpenAI SDK
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="my-secret-key",
)

response = client.chat.completions.create(
    model="llama-3.1-8b",
    messages=[
        {"role": "system", "content": "You are a helpful assistant."},
        {"role": "user", "content": "Explain the advantages of vLLM."},
    ],
    temperature=0.7,
    max_tokens=512,
)

print(response.choices[0].message.content)

# 4-GPU Tensor Parallelism
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4

# 2-GPU Tensor + 2-way Pipeline (4 GPUs total)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 2 \
  --pipeline-parallel-size 2

# Ray distributed backend (multi-node)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 8 \
  --distributed-executor-backend ray

# Memory optimization settings
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --gpu-memory-utilization 0.95 \
  --max-num-seqs 128 \
  --max-model-len 4096 \
  --enable-prefix-caching \
  --enable-chunked-prefill

# Eager mode (debugging/compatibility)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enforce-eager \
  --gpu-memory-utilization 0.85

# AWQ quantized model
vllm serve TheBloke/Llama-2-7B-AWQ \
  --quantization awq

# GPTQ quantized model
vllm serve TheBloke/Llama-2-7B-GPTQ \
  --quantization gptq

# FP8 quantization (H100 and above)
vllm serve neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 \
  --quantization fp8

# BitsAndBytes 4-bit (GPU memory saving)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization bitsandbytes \
  --load-format bitsandbytes

# LoRA adapter serving
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-lora \
  --max-loras 4 \
  --max-lora-rank 64 \
  --lora-modules \
    adapter1=/path/to/lora1 \
    adapter2=/path/to/lora2

# Using a separate draft model
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --speculative-model meta-llama/Llama-3.2-1B-Instruct \
  --num-speculative-tokens 5 \
  --tensor-parallel-size 4

# N-gram based speculative decoding (no additional model needed)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --speculative-model "[ngram]" \
  --num-speculative-tokens 5 \
  --ngram-prompt-lookup-max 4

# Basic Chat Completion
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "What is the population of Seoul?"}
    ],
    "temperature": 0.3,
    "top_p": 0.9,
    "max_tokens": 256,
    "frequency_penalty": 0.5,
    "seed": 42
  }'

# Structured Output (JSON mode)
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "Give me the population of Seoul, Busan, and Daegu in JSON"}
    ],
    "response_format": {
      "type": "json_schema",
      "json_schema": {
        "name": "city_population",
        "schema": {
          "type": "object",
          "properties": {
            "cities": {
              "type": "array",
              "items": {
                "type": "object",
                "properties": {
                  "name": {"type": "string"},
                  "population": {"type": "integer"}
                },
                "required": ["name", "population"]
              }
            }
          },
          "required": ["cities"]
        }
      }
    },
    "temperature": 0.1,
    "max_tokens": 512
  }'

# Returning logprobs
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "1+1=?"}
    ],
    "logprobs": true,
    "top_logprobs": 5,
    "max_tokens": 10
  }'

import requests
import json

url = "http://localhost:8000/v1/chat/completions"
headers = {"Content-Type": "application/json"}

payload = {
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
        {"role": "system", "content": "You are a helpful Korean assistant."},
        {"role": "user", "content": "What is quantum computing?"},
    ],
    "temperature": 0.7,
    "top_p": 0.9,
    "top_k": 50,
    "max_tokens": 1024,
    "repetition_penalty": 1.1,
    "stop": ["\n\n\n"],
}

response = requests.post(url, headers=headers, json=payload)
result = response.json()
print(result["choices"][0]["message"]["content"])

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")

# Streaming response
stream = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[
        {"role": "user", "content": "Implement quicksort in Python"},
    ],
    temperature=0.2,
    max_tokens=2048,
    stream=True,
)

for chunk in stream:
    if chunk.choices[0].delta.content:
        print(chunk.choices[0].delta.content, end="", flush=True)
print()

# Multi-GPU + logging + HF token setup
export CUDA_VISIBLE_DEVICES=0,1,2,3
export HF_TOKEN="hf_xxxxxxxxxxxx"
export VLLM_LOGGING_LEVEL="DEBUG"
export VLLM_WORKER_MULTIPROC_METHOD="spawn"

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.90

# Passing environment variables in Docker
docker run --runtime nvidia --gpus all \
  -e CUDA_VISIBLE_DEVICES=0,1 \
  -e HF_TOKEN="hf_xxxxxxxxxxxx" \
  -e VLLM_LOGGING_LEVEL="INFO" \
  -e VLLM_WORKER_MULTIPROC_METHOD="spawn" \
  -p 8000:8000 \
  --ipc=host \
  vllm/vllm-openai:latest \
  --model meta-llama/Llama-3.1-8B-Instruct \
  --tensor-parallel-size 2

# TP=4 (distribute model across 4 GPUs)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.90

# PP=2, TP=2 (4 GPUs total, 2x2 configuration)
vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 2 \
  --pipeline-parallel-size 2

# Master node
ray start --head --port=6379

# Worker node
ray start --address=<master-ip>:6379

# Run vLLM (from master)
vllm serve meta-llama/Llama-3.1-405B-Instruct \
  --tensor-parallel-size 8 \
  --pipeline-parallel-size 2 \
  --distributed-executor-backend ray

# Using pre-quantized AWQ model
vllm serve TheBloke/Llama-2-13B-chat-AWQ \
  --quantization awq \
  --max-model-len 4096

# Faster with Marlin kernel (SM 80+ GPU)
vllm serve TheBloke/Llama-2-13B-chat-AWQ \
  --quantization awq_marlin

# GPTQ model (ExLlamaV2 kernel auto-used)
vllm serve TheBloke/Llama-2-13B-chat-GPTQ \
  --quantization gptq

# Using Marlin kernel
vllm serve TheBloke/Llama-2-13B-chat-GPTQ \
  --quantization gptq_marlin

# Pre-quantized FP8 model
vllm serve neuralmagic/Meta-Llama-3.1-8B-Instruct-FP8 \
  --quantization fp8

# Dynamic FP8 quantization (no pre-quantization needed)
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization fp8 \
  --kv-cache-dtype fp8

vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --quantization bitsandbytes \
  --load-format bitsandbytes \
  --enforce-eager  # BnB requires Eager mode

# Enable LoRA adapters
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-lora \
  --max-loras 4 \
  --max-lora-rank 64 \
  --lora-modules \
    korean-chat=/path/to/korean-lora \
    code-assist=/path/to/code-lora

from openai import OpenAI

client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")

# Use a specific LoRA adapter
response = client.chat.completions.create(
    model="korean-chat",  # LoRA adapter name
    messages=[{"role": "user", "content": "Hello!"}],
    temperature=0.7,
    max_tokens=256,
)

# Enabled by default in v1, requires explicit flag in v0
vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-prefix-caching

vllm serve meta-llama/Llama-3.1-8B-Instruct \
  --enable-chunked-prefill \
  --max-num-batched-tokens 2048

# JSON Schema-based structured output
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "messages": [
      {"role": "user", "content": "Please provide Seoul weather info in JSON"}
    ],
    "response_format": {
      "type": "json_schema",
      "json_schema": {
        "name": "weather_info",
        "schema": {
          "type": "object",
          "properties": {
            "city": {"type": "string"},
            "temperature_celsius": {"type": "number"},
            "condition": {"type": "string"},
            "humidity_percent": {"type": "integer"}
          },
          "required": ["city", "temperature_celsius", "condition"]
        }
      }
    }
  }'

# Regex-based output (Completion API)
curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-3.1-8B-Instruct",
    "prompt": "Generate a valid email address:",
    "extra_body": {
      "guided_regex": "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}"
    },
    "max_tokens": 50
  }'

# docker-compose.yaml
version: '3.8'

services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - '8000:8000'
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface
    environment:
      - HF_TOKEN=${HF_TOKEN}
      - VLLM_LOGGING_LEVEL=INFO
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
    ipc: host
    command: >
      --model meta-llama/Llama-3.1-8B-Instruct
      --host 0.0.0.0
      --port 8000
      --tensor-parallel-size 2
      --gpu-memory-utilization 0.90
      --max-model-len 8192
      --enable-prefix-caching
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:8000/health']
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s

# Run with Docker Compose
HF_TOKEN=hf_xxxx docker compose up -d

# Check logs
docker compose logs -f vllm

# vllm-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-llama3
  namespace: ai-serving
  labels:
    app: vllm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: vllm
  template:
    metadata:
      labels:
        app: vllm
    spec:
      containers:
        - name: vllm
          image: vllm/vllm-openai:latest
          ports:
            - containerPort: 8000
              name: http
          args:
            - '--model'
            - 'meta-llama/Llama-3.1-8B-Instruct'
            - '--host'
            - '0.0.0.0'
            - '--port'
            - '8000'
            - '--tensor-parallel-size'
            - '2'
            - '--gpu-memory-utilization'
            - '0.90'
            - '--max-model-len'
            - '8192'
          env:
            - name: HF_TOKEN
              valueFrom:
                secretKeyRef:
                  name: hf-secret
                  key: token
            - name: VLLM_WORKER_MULTIPROC_METHOD
              value: 'spawn'
          resources:
            limits:
              nvidia.com/gpu: '2'
            requests:
              nvidia.com/gpu: '2'
              memory: '32Gi'
              cpu: '8'
          volumeMounts:
            - name: shm
              mountPath: /dev/shm
            - name: model-cache
              mountPath: /root/.cache/huggingface
          readinessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 120
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /health
              port: 8000
            initialDelaySeconds: 180
            periodSeconds: 30
      volumes:
        - name: shm
          emptyDir:
            medium: Memory
            sizeLimit: 2Gi
        - name: model-cache
          persistentVolumeClaim:
            claimName: model-cache-pvc
      nodeSelector:
        nvidia.com/gpu.product: 'NVIDIA-A100-SXM4-80GB'
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-service
  namespace: ai-serving
spec:
  selector:
    app: vllm
  ports:
    - port: 8000
      targetPort: 8000
      name: http
  type: ClusterIP
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: vllm-hpa
  namespace: ai-serving
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: vllm-llama3
  minReplicas: 1
  maxReplicas: 4
  metrics:
    - type: Pods
      pods:
        metric:
          name: vllm_num_requests_running
        target:
          type: AverageValue
          averageValue: '50'

# Homebrew
brew install ollama

# Or official install script
curl -fsSL https://ollama.com/install.sh | sh

# Official install script (recommended)
curl -fsSL https://ollama.com/install.sh | sh

# Or manual installation
curl -L https://ollama.com/download/ollama-linux-amd64 -o /usr/local/bin/ollama
chmod +x /usr/local/bin/ollama

# CPU only
docker run -d -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama

# NVIDIA GPU
docker run -d --gpus=all -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama

# AMD GPU (ROCm)
docker run -d --device /dev/kfd --device /dev/dri \
  -v ollama:/root/.ollama -p 11434:11434 \
  --name ollama ollama/ollama:rocm

# Start server (if not auto-started in background)
ollama serve

# Download model and start chatting
ollama run llama3.1

# Specify a tag (size/quantization)
ollama run llama3.1:8b
ollama run llama3.1:70b-instruct-q4_K_M
ollama run qwen2.5:32b-instruct-q5_K_M

# Download model only (without running)
ollama pull llama3.1:8b

# One-line prompt
ollama run llama3.1 "What is PagedAttention?"

# Default start (localhost:11434)
ollama serve

# Change bind address via environment variable
OLLAMA_HOST=0.0.0.0:11434 ollama serve

# Debug mode
OLLAMA_DEBUG=1 ollama serve

# Interactive mode
ollama run llama3.1

# One-line prompt
ollama run llama3.1 "Explain quantum computing"

# JSON format output
ollama run llama3.1 "List 3 Korean cities" --format json

# Multimodal (image input)
ollama run llama3.2-vision "What's in this image? /path/to/image.png"

# Verbose mode (display performance stats)
ollama run llama3.1 --verbose

# With system prompt
ollama run llama3.1 --system "You are a Korean translator."

# Create from Modelfile
ollama create my-model -f ./Modelfile

# Create from GGUF file
ollama create my-model -f ./Modelfile-from-gguf

# Quantization conversion
ollama create my-model-q4 --quantize q4_K_M -f ./Modelfile

# Full info
ollama show llama3.1

# Output Modelfile
ollama show llama3.1 --modelfile

# Check parameters
ollama show llama3.1 --parameters

# Check system prompt
ollama show llama3.1 --system

# Check template
ollama show llama3.1 --template

$ ollama ps
NAME              ID            SIZE     PROCESSOR    UNTIL
llama3.1:8b       a]f2e33d4e25  6.7 GB   100% GPU     4 minutes from now
qwen2.5:7b        845dbda0ea48  4.7 GB   100% GPU     3 minutes from now

# Basic generation
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "Why is the sky blue?",
  "stream": false
}'

# Streaming (default)
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "Write a haiku about coding",
  "options": {
    "temperature": 0.7,
    "num_predict": 100
  }
}'

# JSON format output
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.1",
  "prompt": "List 3 programming languages as JSON",
  "format": "json",
  "stream": false
}'

curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [
    {"role": "system", "content": "You are a helpful Korean assistant."},
    {"role": "user", "content": "Recommend tourist spots in Seoul."}
  ],
  "stream": false,
  "options": {
    "temperature": 0.8,
    "top_p": 0.9,
    "num_ctx": 4096,
    "num_predict": 512
  }
}'

# Single text embedding
curl http://localhost:11434/api/embed -d '{
  "model": "nomic-embed-text",
  "input": "Hello, world!"
}'

# Multiple text embeddings
curl http://localhost:11434/api/embed -d '{
  "model": "nomic-embed-text",
  "input": ["Hello world", "Goodbye world"]
}'

# OpenAI format Chat Completion
curl http://localhost:11434/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llama3.1",
    "messages": [
      {"role": "user", "content": "Hello!"}
    ],
    "temperature": 0.7,
    "max_tokens": 256
  }'

# Model list
curl http://localhost:11434/v1/models

import requests

# Generate API
response = requests.post("http://localhost:11434/api/generate", json={
    "model": "llama3.1",
    "prompt": "Explain Docker in Korean",
    "stream": False,
    "options": {
        "temperature": 0.7,
        "num_predict": 512,
    },
})
print(response.json()["response"])

# Chat API
response = requests.post("http://localhost:11434/api/chat", json={
    "model": "llama3.1",
    "messages": [
        {"role": "user", "content": "What is Kubernetes?"},
    ],
    "stream": False,
})
print(response.json()["message"]["content"])

# Using Ollama with OpenAI SDK
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama",  # Ollama doesn't require an API key, any value works
)

response = client.chat.completions.create(
    model="llama3.1",
    messages=[
        {"role": "user", "content": "Explain Python's GIL."},
    ],
    temperature=0.7,
    max_tokens=512,
)
print(response.choices[0].message.content)

# Specify base model (required)
FROM llama3.1:8b

# Parameter settings
PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER top_k 40
PARAMETER num_ctx 4096
PARAMETER num_predict 512
PARAMETER repeat_penalty 1.1
PARAMETER stop "<|eot_id|>"
PARAMETER stop "<|end_of_text|>"

# System prompt
SYSTEM """
You are a friendly Korean AI assistant.
You provide accurate and concise answers, explaining with examples when needed.
"""

# Chat template (Jinja2 or Go template)
TEMPLATE """
{{- if .System }}<|start_header_id|>system<|end_header_id|>
{{ .System }}<|eot_id|>{{ end }}
{{- range .Messages }}<|start_header_id|>{{ .Role }}<|end_header_id|>
{{ .Content }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
"""

# LoRA adapter (optional)
ADAPTER /path/to/lora-adapter.gguf

# License info (optional)
LICENSE """
Apache 2.0
"""

curl http://localhost:11434/api/chat -d '{
  "model": "llama3.1",
  "messages": [
    {"role": "user", "content": "Hello"}
  ],
  "options": {
    "temperature": 0.3,
    "top_p": 0.9,
    "top_k": 50,
    "num_ctx": 8192,
    "num_predict": 1024,
    "repeat_penalty": 1.2,
    "seed": 42,
    "stop": ["<|eot_id|>"]
  }
}'

# Set environment variables
launchctl setenv OLLAMA_HOST "0.0.0.0:11434"
launchctl setenv OLLAMA_MODELS "/Volumes/ExternalSSD/ollama/models"
launchctl setenv OLLAMA_FLASH_ATTENTION "1"
launchctl setenv OLLAMA_KV_CACHE_TYPE "q8_0"
launchctl setenv OLLAMA_NUM_PARALLEL "4"
launchctl setenv OLLAMA_KEEP_ALIVE "-1"

# Restart Ollama
brew services restart ollama

# Create systemd service override
sudo systemctl edit ollama

# Add the following in the editor:
[Service]
Environment="OLLAMA_HOST=0.0.0.0:11434"
Environment="OLLAMA_MODELS=/data/ollama/models"
Environment="OLLAMA_FLASH_ATTENTION=1"
Environment="OLLAMA_KV_CACHE_TYPE=q8_0"
Environment="OLLAMA_NUM_PARALLEL=4"
Environment="OLLAMA_KEEP_ALIVE=-1"
Environment="OLLAMA_MAX_LOADED_MODELS=3"
Environment="CUDA_VISIBLE_DEVICES=0,1"

# Restart service
sudo systemctl daemon-reload
sudo systemctl restart ollama

docker run -d --gpus=all \
  -e OLLAMA_HOST=0.0.0.0:11434 \
  -e OLLAMA_FLASH_ATTENTION=1 \
  -e OLLAMA_KV_CACHE_TYPE=q8_0 \
  -e OLLAMA_NUM_PARALLEL=4 \
  -e OLLAMA_KEEP_ALIVE=-1 \
  -v /data/ollama:/root/.ollama \
  -p 11434:11434 \
  --name ollama \
  ollama/ollama

FROM llama3.1:8b

PARAMETER temperature 0.7
PARAMETER top_p 0.9
PARAMETER num_ctx 4096
PARAMETER num_predict 1024
PARAMETER repeat_penalty 1.15
PARAMETER stop "<|eot_id|>"
PARAMETER stop "<|end_of_text|>"

SYSTEM """
You are a Korean AI assistant well-versed in Korean culture and history.
You always respond accurately and kindly in Korean, using English technical terms alongside when needed.
You provide answers in a structured format.
"""

MESSAGE user Hello, please introduce yourself.
MESSAGE assistant Hello! I'm an AI assistant specialized in Korean. I can help with various topics including Korean culture, history, technology, and more. Feel free to ask me anything!

# Create model
ollama create korean-assistant -f ./Modelfile-korean

# Run
ollama run korean-assistant "Tell me about the three grand palaces of Seoul"

FROM qwen2.5-coder:7b

PARAMETER temperature 0.2
PARAMETER top_p 0.85
PARAMETER num_ctx 8192
PARAMETER num_predict 2048

SYSTEM """
You are an expert code reviewer. Analyze code for:
1. Bugs and potential issues
2. Performance improvements
3. Security vulnerabilities
4. Code style and best practices

Provide specific, actionable feedback with corrected code examples.
"""

# Check NVIDIA driver
nvidia-smi

# Use specific GPU only
CUDA_VISIBLE_DEVICES=0 ollama serve

# Multi-GPU
CUDA_VISIBLE_DEVICES=0,1 ollama serve

# Check ROCm driver
rocm-smi

# Specify GPU
ROCR_VISIBLE_DEVICES=0 ollama serve

# Check GPU usage (Processor column in ollama ps)
ollama ps
# NAME           ID            SIZE    PROCESSOR     UNTIL
# llama3.1:8b    a]f2e33d4e25  6.7 GB  100% GPU      4 minutes from now

# docker-compose.yaml
version: '3.8'

services:
  ollama:
    image: ollama/ollama
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    ports:
      - '11434:11434'
    volumes:
      - ollama_data:/root/.ollama
    environment:
      - OLLAMA_HOST=0.0.0.0:11434
      - OLLAMA_FLASH_ATTENTION=1
      - OLLAMA_KV_CACHE_TYPE=q8_0
      - OLLAMA_NUM_PARALLEL=4
      - OLLAMA_KEEP_ALIVE=24h
    restart: unless-stopped
    healthcheck:
      test: ['CMD', 'curl', '-f', 'http://localhost:11434/api/version']
      interval: 30s
      timeout: 5s
      retries: 3

  # Model initialization (optional)
  ollama-init:
    image: curlimages/curl:latest
    depends_on:
      ollama:
        condition: service_healthy
    entrypoint: >
      sh -c "
        curl -s http://ollama:11434/api/pull -d '{\"name\": \"llama3.1:8b\"}' &&
        curl -s http://ollama:11434/api/pull -d '{\"name\": \"nomic-embed-text\"}'
      "

volumes:
  ollama_data:

# Run LLaVA model
ollama run llava "What's in this image? /path/to/photo.jpg"

# Llama 3.2 Vision
ollama run llama3.2-vision "Describe this image in Korean. /path/to/image.png"

import requests
import base64

# Encode image to base64
with open("image.jpg", "rb") as f:
    image_base64 = base64.b64encode(f.read()).decode("utf-8")

response = requests.post("http://localhost:11434/api/chat", json={
    "model": "llava",
    "messages": [
        {
            "role": "user",
            "content": "What's in this image?",
            "images": [image_base64],
        }
    ],
    "stream": False,
})
print(response.json()["message"]["content"])

from openai import OpenAI
import json

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get current weather for a city",
            "parameters": {
                "type": "object",
                "properties": {
                    "city": {"type": "string", "description": "City name"},
                    "unit": {
                        "type": "string",
                        "enum": ["celsius", "fahrenheit"],
                    },
                },
                "required": ["city"],
            },
        },
    }
]

response = client.chat.completions.create(
    model="llama3.1",
    messages=[
        {"role": "user", "content": "What's the current weather in Seoul?"}
    ],
    tools=tools,
    tool_choice="auto",
)

message = response.choices[0].message
if message.tool_calls:
    for tool_call in message.tool_calls:
        print(f"Function: {tool_call.function.name}")
        print(f"Arguments: {tool_call.function.arguments}")

# Install and use immediately
ollama run llama3.1

# VS Code + Continue extension integration
# Set Ollama endpoint in settings.json

vllm serve meta-llama/Llama-3.1-70B-Instruct \
  --tensor-parallel-size 4 \
  --gpu-memory-utilization 0.90 \
  --max-num-seqs 256 \
  --enable-prefix-caching \
  --enable-chunked-prefill \
  --api-key ${API_KEY}

# Small model + high quantization
ollama run phi3:3.8b-mini-instruct-4k-q4_0

# Or Qwen 0.5B
ollama run qwen2.5:0.5b

from vllm import LLM, SamplingParams

llm = LLM(
    model="meta-llama/Llama-3.1-8B-Instruct",
    tensor_parallel_size=2,
    gpu_memory_utilization=0.95,
)

# Process thousands of prompts at once
prompts = load_prompts_from_file("prompts.jsonl")  # 10,000+ prompts
sampling_params = SamplingParams(temperature=0.0, max_tokens=512)

outputs = llm.generate(prompts, sampling_params)
save_outputs(outputs, "results.jsonl")

# Ollama-based RAG (development/small-scale)
from langchain_ollama import OllamaLLM, OllamaEmbeddings

llm = OllamaLLM(model="llama3.1")
embeddings = OllamaEmbeddings(model="nomic-embed-text")

# vLLM-based RAG (production)
from langchain_openai import ChatOpenAI, OpenAIEmbeddings

llm = ChatOpenAI(
    base_url="http://vllm-server:8000/v1",
    api_key="token",
    model="meta-llama/Llama-3.1-8B-Instruct",
)

from openai import OpenAI
import uuid

client = OpenAI(base_url="http://localhost:8000/v1", api_key="token")

# Pass custom request_id
xid = str(uuid.uuid4())

response = client.chat.completions.create(
    model="meta-llama/Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "Hello"}],
    extra_headers={"X-Request-ID": xid},
)

print(f"XID: {xid}")
print(f"Response ID: {response.id}")

import requests
import uuid

xid = str(uuid.uuid4())

response = requests.post(
    "http://localhost:11434/api/chat",
    headers={"X-Request-ID": xid},
    json={
        "model": "llama3.1",
        "messages": [{"role": "user", "content": "Hello"}],
        "stream": False,
    },
)

# Include xid in logging
import logging
logger = logging.getLogger(__name__)
logger.info(f"[xid={xid}] Response: {response.status_code}")

upstream vllm_backend {
    server vllm-server:8000;
}

server {
    listen 80;

    location /v1/ {
        # Auto-generate X-Request-ID if missing
        set $request_id $http_x_request_id;
        if ($request_id = "") {
            set $request_id $request_id;
        }

        proxy_pass http://vllm_backend;
        proxy_set_header X-Request-ID $request_id;
        proxy_set_header Host $host;

        # Add X-Request-ID to response headers
        add_header X-Request-ID $request_id always;

        # Include request_id in access log
        access_log /var/log/nginx/vllm_access.log combined_with_xid;
    }
}

# Log format definition
log_format combined_with_xid '$remote_addr - $remote_user [$time_local] '
    '"$request" $status $body_bytes_sent '
    '"$http_referer" "$http_user_agent" '
    'xid="$http_x_request_id"';

# vLLM + OpenTelemetry distributed tracing
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

# Initialize Tracer
provider = TracerProvider()
exporter = OTLPSpanExporter(endpoint="http://jaeger:4317")
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)
tracer = trace.get_tracer(__name__)

# Wrap LLM call as a Span
def call_llm(prompt: str, xid: str) -> str:
    with tracer.start_as_current_span("llm_inference") as span:
        span.set_attribute("xid", xid)
        span.set_attribute("model", "llama-3.1-8b")
        span.set_attribute("prompt_length", len(prompt))

        response = client.chat.completions.create(
            model="meta-llama/Llama-3.1-8B-Instruct",
            messages=[{"role": "user", "content": prompt}],
            extra_headers={"X-Request-ID": xid},
        )

        result = response.choices[0].message.content
        span.set_attribute("response_length", len(result))
        span.set_attribute("tokens_used", response.usage.total_tokens)

        return result

import logging
import uuid
from contextvars import ContextVar

# Manage xid with Context Variable
request_xid: ContextVar[str] = ContextVar("request_xid", default="")

class XIDFilter(logging.Filter):
    def filter(self, record):
        record.xid = request_xid.get("")
        return True

# Logger setup
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(
    "%(asctime)s [%(levelname)s] [xid=%(xid)s] %(message)s"
))
handler.addFilter(XIDFilter())

logger = logging.getLogger("llm_service")
logger.addHandler(handler)
logger.setLevel(logging.INFO)

# Usage
async def handle_request(prompt: str):
    xid = str(uuid.uuid4())
    request_xid.set(xid)

    logger.info(f"Received prompt: {prompt[:50]}...")

    response = await call_llm(prompt, xid)

    logger.info(f"Generated {len(response)} chars")
    return {"xid": xid, "response": response}

package main

import (
    "context"
    "fmt"
    "log/slog"
    "net/http"

    "github.com/google/uuid"
)

type contextKey string
const xidKey contextKey = "xid"

// XID Middleware
func xidMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        xid := r.Header.Get("X-Request-ID")
        if xid == "" {
            xid = uuid.New().String()
        }

        ctx := context.WithValue(r.Context(), xidKey, xid)
        w.Header().Set("X-Request-ID", xid)

        slog.Info("request received",
            "xid", xid,
            "method", r.Method,
            "path", r.URL.Path,
        )

        next.ServeHTTP(w, r.WithContext(ctx))
    })
}

// Ollama call function
func callOllama(ctx context.Context, prompt string) (string, error) {
    xid := ctx.Value(xidKey).(string)

    slog.Info("calling ollama",
        "xid", xid,
        "prompt_len", len(prompt),
    )

    // ... Ollama API call logic ...

    slog.Info("ollama response received",
        "xid", xid,
        "response_len", len(response),
    )

    return response, nil
}