Split View: Unsloth로 LLM 파인튜닝 완전 가이드 2025: QLoRA, 4bit 양자화, 2배 빠른 학습

Full Fine-tuning (update all parameters)
+------------------------+
|   W (d x d)            |  <- Update entire weights
|   e.g.: 4096 x 4096   |     = 16M parameters
|   = 64MB (FP16)        |
+------------------------+

LoRA (Low-Rank Adaptation)
+------------------------+
|   W0 (frozen) + B * A  |
|   W0: 4096 x 4096     |  <- Frozen (no updates)
|   B: 4096 x 16         |  <- Trainable (65K params)
|   A: 16 x 4096         |  <- Trainable (65K params)
|   = 0.25MB (FP16)      |     Total 130K params
+------------------------+

QLoRA (Quantized LoRA)
+------------------------+
|   W0 (4bit) + B * A   |
|   W0: 4096 x 4096     |  <- 4-bit quantized (8MB)
|   B: 4096 x 16         |  <- FP16 trainable
|   A: 16 x 4096         |  <- FP16 trainable
|   = 8.25MB total        |
+------------------------+

W_new = W_old + delta_W

delta_W = B * A
where:
  B is a d x r matrix (d=model dimension, r=LoRA rank)
  A is a r x d matrix
  r << d (e.g., r=16, d=4096)

# Full Fine-tuning parameters
d = 4096
full_params = d * d  # = 16,777,216 (16.7M)

# LoRA parameters
r = 16
lora_params = d * r + r * d  # = 131,072 (131K)

# Savings ratio
savings = 1 - (lora_params / full_params)
print(f"Parameter savings: {savings:.2%}")  # 99.22%

# NF4 quantization values (based on normal distribution quantiles)
nf4_values = [
    -1.0, -0.6962, -0.5251, -0.3949,
    -0.2844, -0.1848, -0.0911, 0.0,
    0.0796, 0.1609, 0.2461, 0.3379,
    0.4407, 0.5626, 0.7230, 1.0,
]

# Install Unsloth on Colab (T4 GPU)
# Runtime -> Change runtime type -> Select T4 GPU

# 1. Install Unsloth
!pip install "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"
!pip install --no-deps "xformers<0.0.27" "trl<0.9.0" peft accelerate bitsandbytes

# 2. Verify GPU
import torch
print(f"GPU: {torch.cuda.get_device_name(0)}")
print(f"VRAM: {torch.cuda.get_device_properties(0).total_mem / 1024**3:.1f} GB")

# Create Conda environment
conda create -n unsloth python=3.11
conda activate unsloth

# Install PyTorch (CUDA 12.1)
conda install pytorch torchvision torchaudio pytorch-cuda=12.1 -c pytorch -c nvidia

# Install Unsloth
pip install "unsloth[cu121] @ git+https://github.com/unslothai/unsloth.git"
pip install --no-deps trl peft accelerate bitsandbytes

# Verify installation
python -c "from unsloth import FastLanguageModel; print('Unsloth OK')"

FROM nvidia/cuda:12.1.0-devel-ubuntu22.04

RUN apt-get update && apt-get install -y python3.11 python3-pip git

RUN pip install torch --index-url https://download.pytorch.org/whl/cu121
RUN pip install "unsloth[cu121] @ git+https://github.com/unslothai/unsloth.git"
RUN pip install --no-deps trl peft accelerate bitsandbytes

WORKDIR /workspace
CMD ["python3"]

from unsloth import FastLanguageModel
import torch

# Load model and tokenizer
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/Meta-Llama-3.1-8B-bnb-4bit",  # Pre-quantized 4bit model
    max_seq_length=2048,    # Maximum sequence length
    dtype=None,             # Auto-detect (A100: bfloat16, others: float16)
    load_in_4bit=True,      # Load with 4bit quantization
)

# Check GPU memory
gpu_stats = torch.cuda.get_device_properties(0)
start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
max_memory = round(gpu_stats.total_mem / 1024 / 1024 / 1024, 3)
print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
print(f"{start_gpu_memory} GB of memory reserved.")

# Add LoRA adapter
model = FastLanguageModel.get_peft_model(
    model,
    r=16,                          # LoRA rank (8, 16, 32, 64)
    target_modules=[               # Modules to apply LoRA
        "q_proj", "k_proj", "v_proj", "o_proj",  # Attention
        "gate_proj", "up_proj", "down_proj",       # MLP
    ],
    lora_alpha=16,                 # LoRA alpha (usually same as r)
    lora_dropout=0,                # 0 is optimal for Unsloth
    bias="none",                   # No bias training
    use_gradient_checkpointing="unsloth",  # Unsloth-optimized checkpointing
    random_state=3407,
    use_rslora=False,              # Rank-Stabilized LoRA (experimental)
    loftq_config=None,             # LoftQ configuration
)

# Check trainable parameters
def print_trainable_parameters(model):
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    total = sum(p.numel() for p in model.parameters())
    print(f"Trainable: {trainable:,} / Total: {total:,} = {trainable/total:.2%}")

print_trainable_parameters(model)
# Trainable: 41,943,040 / Total: 8,030,261,248 = 0.52%

# Alpaca prompt template
alpaca_prompt = """Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
{}

### Input:
{}

### Response:
{}"""

# Dataset formatting function
EOS_TOKEN = tokenizer.eos_token

def formatting_prompts_func(examples):
    instructions = examples["instruction"]
    inputs = examples["input"]
    outputs = examples["output"]
    texts = []
    for instruction, input_text, output in zip(instructions, inputs, outputs):
        text = alpaca_prompt.format(instruction, input_text, output) + EOS_TOKEN
        texts.append(text)
    return {"text": texts}

from datasets import load_dataset

# Load KoAlpaca dataset
dataset = load_dataset("beomi/KoAlpaca-v1.1a", split="train")

# Format conversion
def format_koalpaca(examples):
    texts = []
    for instruction, output in zip(examples["instruction"], examples["output"]):
        text = alpaca_prompt.format(instruction, "", output) + EOS_TOKEN
        texts.append(text)
    return {"text": texts}

dataset = dataset.map(format_koalpaca, batched=True)

# OpenAI Messages format (using Llama 3 chat template)
def format_openai_messages(examples):
    texts = []
    for messages in examples["messages"]:
        text = tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=False,
        )
        texts.append(text)
    return {"text": texts}

# Analyze sequence length distribution
def analyze_sequence_lengths(dataset, tokenizer):
    lengths = []
    for item in dataset:
        tokens = tokenizer.encode(item["text"])
        lengths.append(len(tokens))

    import numpy as np
    print(f"Mean length: {np.mean(lengths):.0f}")
    print(f"Median: {np.median(lengths):.0f}")
    print(f"95th percentile: {np.percentile(lengths, 95):.0f}")
    print(f"99th percentile: {np.percentile(lengths, 99):.0f}")
    print(f"Max length: {max(lengths)}")

    recommended = int(np.percentile(lengths, 95))
    print(f"\nRecommended max_seq_length: {recommended}")
    return lengths

analyze_sequence_lengths(dataset, tokenizer)

from trl import SFTTrainer
from transformers import TrainingArguments
from unsloth import is_bfloat16_supported

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    dataset_text_field="text",
    max_seq_length=2048,
    dataset_num_proc=2,
    packing=False,
    args=TrainingArguments(
        # === Basic ===
        output_dir="./outputs",
        num_train_epochs=3,

        # === Batch & Memory ===
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,  # Effective batch = 2 * 4 = 8

        # === Learning Rate ===
        learning_rate=2e-4,             # QLoRA recommended LR
        lr_scheduler_type="cosine",
        warmup_steps=5,

        # === Precision ===
        fp16=not is_bfloat16_supported(),
        bf16=is_bfloat16_supported(),

        # === Logging ===
        logging_steps=1,
        logging_dir="./logs",
        report_to="wandb",

        # === Saving ===
        save_strategy="steps",
        save_steps=100,
        save_total_limit=3,

        # === Optimization ===
        optim="adamw_8bit",
        weight_decay=0.01,
        max_grad_norm=0.3,
        seed=3407,
    ),
)

# Two ways to achieve effective batch size of 8

# Method 1: Large batch (needs more VRAM)
per_device_train_batch_size = 8
gradient_accumulation_steps = 1
# Effective batch = 8 * 1 = 8, VRAM: ~12GB

# Method 2: Small batch + Gradient Accumulation (less VRAM)
per_device_train_batch_size = 2
gradient_accumulation_steps = 4
# Effective batch = 2 * 4 = 8, VRAM: ~6GB
# Note: Training slightly slower

# Start training
trainer_stats = trainer.train()

# Print results
print(f"Training time: {trainer_stats.metrics['train_runtime']:.2f}s")
print(f"Final Loss: {trainer_stats.metrics['train_loss']:.4f}")

# Check GPU memory usage
used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
print(f"Peak VRAM usage: {used_memory} GB")

# Unsloth-optimized Gradient Checkpointing
model = FastLanguageModel.get_peft_model(
    model,
    r=16,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj",
                     "gate_proj", "up_proj", "down_proj"],
    use_gradient_checkpointing="unsloth",  # Key! 30% VRAM savings
)

# Gradient checkpointing options:
# "unsloth": Unsloth-optimized version (faster and more memory efficient)
# True: Standard PyTorch gradient checkpointing
# False: Disabled (fastest but uses most memory)

# Pack short sequences together for better GPU utilization
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    packing=True,           # Enable sequence packing
    max_seq_length=2048,    # Total packed length
)

# Packing effect:
# Packing OFF: [tokentokenPAD PAD PAD PAD] [tokenPAD PAD PAD PAD PAD]
# Packing ON:  [tokentokentokenSEPtokentokentoken] -> Better GPU utilization

# Save LoRA adapter only (small size)
model.save_pretrained("lora_adapter")
tokenizer.save_pretrained("lora_adapter")

# Check saved files
import os
for f in os.listdir("lora_adapter"):
    size = os.path.getsize(f"lora_adapter/{f}") / 1024 / 1024
    print(f"  {f}: {size:.1f} MB")

# adapter_config.json: 0.0 MB
# adapter_model.safetensors: 160.0 MB  <- LoRA weights
# tokenizer.json: 17.1 MB

# Merge LoRA adapter with base model
merged_model = model.merge_and_unload()

# Save merged model
merged_model.save_pretrained("merged_model")
tokenizer.save_pretrained("merged_model")

# Unsloth's built-in GGUF conversion
# Supports various quantization levels

# Q4_K_M: Most common (quality/size balance)
model.save_pretrained_gguf(
    "model_gguf",
    tokenizer,
    quantization_method="q4_k_m",
)

# Q5_K_M: Higher quality
model.save_pretrained_gguf(
    "model_q5",
    tokenizer,
    quantization_method="q5_k_m",
)

# Q8_0: Highest quality (larger size)
model.save_pretrained_gguf(
    "model_q8",
    tokenizer,
    quantization_method="q8_0",
)

# F16: No quantization (largest)
model.save_pretrained_gguf(
    "model_f16",
    tokenizer,
    quantization_method="f16",
)

# GPTQ quantization (for GPU inference)
from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig

quantize_config = BaseQuantizeConfig(
    bits=4,
    group_size=128,
    desc_act=False,
)

# Prepare calibration data
calibration_data = [
    tokenizer(text, return_tensors="pt")
    for text in calibration_texts[:128]
]

# Run GPTQ quantization
gptq_model = AutoGPTQForCausalLM.from_pretrained(
    "merged_model",
    quantize_config=quantize_config,
)
gptq_model.quantize(calibration_data)
gptq_model.save_quantized("model_gptq")

# Upload model to Hugging Face Hub

# Upload LoRA adapter only
model.push_to_hub(
    "my-org/llama3-8b-korean-lora",
    token="hf_xxxxx",
    private=True,
)
tokenizer.push_to_hub(
    "my-org/llama3-8b-korean-lora",
    token="hf_xxxxx",
    private=True,
)

# Upload GGUF file
model.push_to_hub_gguf(
    "my-org/llama3-8b-korean-gguf",
    tokenizer,
    quantization_method="q4_k_m",
    token="hf_xxxxx",
)

# Switch to inference mode
FastLanguageModel.for_inference(model)

# Single prompt inference
def generate_response(instruction, input_text=""):
    prompt = alpaca_prompt.format(instruction, input_text, "")
    inputs = tokenizer([prompt], return_tensors="pt").to("cuda")

    outputs = model.generate(
        **inputs,
        max_new_tokens=512,
        temperature=0.7,
        top_p=0.9,
        repetition_penalty=1.15,
        do_sample=True,
    )

    response = tokenizer.batch_decode(outputs)[0]
    response = response.split("### Response:\n")[-1]
    response = response.replace(tokenizer.eos_token, "").strip()
    return response

# Test
test_questions = [
    "Explain the traditional holidays of Korea.",
    "Explain how decorators work in Python.",
    "Give me tips for healthy eating habits.",
]

for q in test_questions:
    print(f"Q: {q}")
    print(f"A: {generate_response(q)}")
    print("-" * 80)

# Generation parameter effects
generation_configs = {
    "Factual answers": {
        "temperature": 0.1,
        "top_p": 0.9,
        "repetition_penalty": 1.0,
    },
    "Creative answers": {
        "temperature": 0.8,
        "top_p": 0.95,
        "repetition_penalty": 1.15,
    },
    "Balanced answers": {
        "temperature": 0.5,
        "top_p": 0.9,
        "repetition_penalty": 1.1,
    },
}

# Benchmark evaluation with lm-eval-harness
pip install lm-eval

# Korean benchmark evaluation
lm_eval --model hf \
    --model_args pretrained=./merged_model \
    --tasks kobest_boolq,kobest_copa,kobest_hellaswag,kobest_sentineg,kobest_wic \
    --batch_size 4 \
    --output_path ./eval_results

# Run from Python
from lm_eval import evaluator

results = evaluator.simple_evaluate(
    model="hf",
    model_args="pretrained=./merged_model",
    tasks=["kobest_boolq", "kobest_copa", "kobest_hellaswag"],
    batch_size=4,
)

for task, metrics in results["results"].items():
    print(f"{task}: acc={metrics.get('acc', 'N/A')}")

# deepspeed_config.json
"""
{
    "zero_optimization": {
        "stage": 2,
        "offload_optimizer": {
            "device": "cpu",
            "pin_memory": true
        },
        "allgather_partitions": true,
        "reduce_scatter": true
    },
    "bf16": {
        "enabled": true
    },
    "train_batch_size": "auto",
    "train_micro_batch_size_per_gpu": "auto"
}
"""

# Execute
# deepspeed --num_gpus 4 train.py --deepspeed deepspeed_config.json

from trl import DPOTrainer, DPOConfig
from unsloth import FastLanguageModel, PatchDPOTrainer

# Apply DPO patch
PatchDPOTrainer()

# Prepare DPO dataset
dpo_dataset = load_dataset("argilla/ultrafeedback-binarized-preferences", split="train")

# Configure DPO Trainer
dpo_trainer = DPOTrainer(
    model=model,
    ref_model=None,           # None in Unsloth (auto-handled)
    args=DPOConfig(
        output_dir="./dpo_output",
        per_device_train_batch_size=2,
        gradient_accumulation_steps=4,
        learning_rate=5e-7,       # DPO uses lower LR
        num_train_epochs=1,
        beta=0.1,                 # DPO beta (KL divergence weight)
        fp16=not is_bfloat16_supported(),
        bf16=is_bfloat16_supported(),
        logging_steps=1,
    ),
    train_dataset=dpo_dataset,
    tokenizer=tokenizer,
)

dpo_trainer.train()

# Continued Pre-training with domain-specific text
from trl import SFTTrainer

# Domain text data (medical, legal, financial, etc.)
domain_dataset = load_dataset("my-org/medical-korean-corpus", split="train")

# Use lower learning rate for Continued Pre-training
trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=domain_dataset,
    dataset_text_field="text",
    max_seq_length=4096,     # Long documents
    packing=True,            # Use packing for efficiency
    args=TrainingArguments(
        output_dir="./cpt_output",
        learning_rate=2e-5,  # Very low learning rate
        num_train_epochs=1,  # 1 epoch is sufficient
        per_device_train_batch_size=1,
        gradient_accumulation_steps=8,
        optim="adamw_8bit",
        warmup_ratio=0.1,
    ),
)

trainer.train()

# Symptom: CUDA out of memory
# RuntimeError: CUDA out of memory.

# Solutions in order:
# 1. Reduce batch_size
per_device_train_batch_size = 1  # Minimum

# 2. Increase gradient_accumulation_steps
gradient_accumulation_steps = 8

# 3. Reduce max_seq_length
max_seq_length = 1024  # 2048 -> 1024

# 4. Reduce LoRA rank
r = 8  # 16 -> 8

# 5. Verify gradient checkpointing
use_gradient_checkpointing = "unsloth"

# 6. Clear cache
torch.cuda.empty_cache()
import gc
gc.collect()

# Symptom: loss diverges to NaN
# Cause: Learning rate too high or data issues

# Solutions:
# 1. Lower learning rate
learning_rate = 1e-5  # 2e-4 -> 1e-5

# 2. Set max_grad_norm
max_grad_norm = 0.3  # Gradient clipping

# 3. Validate data
def check_data_issues(dataset, tokenizer):
    """Check for data problems"""
    issues = []
    for i, item in enumerate(dataset):
        text = item["text"]
        if not text.strip():
            issues.append(f"[{i}] Empty text")
        tokens = tokenizer.encode(text)
        if len(tokens) > 4096:
            issues.append(f"[{i}] Text too long: {len(tokens)} tokens")
        if not any(c.isalnum() for c in text):
            issues.append(f"[{i}] No valid text content")
    return issues

# Symptom: Existing knowledge disappears after fine-tuning
# Solutions:

# 1. Use lower learning rate
learning_rate = 5e-5

# 2. Fewer epochs (1-3)
num_train_epochs = 1

# 3. Mix general knowledge into training data
# Original data 80% + general knowledge data 20%

# 4. Lower LoRA rank (limits change magnitude)
r = 8

# 5. Stronger regularization
weight_decay = 0.1

# Overfitting indicators:
# 1. Train loss decreases but eval loss increases
# 2. Model output nearly memorizes training data
# 3. Performance degrades on new prompts

# Solutions:
# 1. Increase data volume
# 2. Regularization (dropout, weight_decay)
# 3. Early stopping
from transformers import EarlyStoppingCallback

trainer = SFTTrainer(
    callbacks=[EarlyStoppingCallback(early_stopping_patience=3)],
    args=TrainingArguments(
        evaluation_strategy="steps",
        eval_steps=50,
        load_best_model_at_end=True,
    ),
)