Split View: LLM 파인튜닝 실전 — LoRA, QLoRA, PEFT로 나만의 모델 만들기

Original W (4096 x 4096) = 16M parameters [frozen]

LoRA:
A (r x 4096) + B (4096 x r) = r x 8192 parameters
With r=8: 65,536 parameters (0.4%)

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM, AutoTokenizer

# 1. Load the base model
model_name = "meta-llama/Llama-3.1-8B"
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained(model_name)

# 2. LoRA configuration
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,                          # Rank
    lora_alpha=32,                 # Scaling (typically 2x the rank)
    lora_dropout=0.05,             # Dropout
    target_modules=[               # Modules to apply LoRA to
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    bias="none"
)

# 3. Create PEFT model
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# trainable params: 41,943,040 || all params: 8,072,204,288 || trainable%: 0.5194

# Check all Linear layers in the model
for name, module in model.named_modules():
    if isinstance(module, torch.nn.Linear):
        print(name, module.in_features, module.out_features)

# Common choices:
# - Attention only: ["q_proj", "v_proj"] — Minimum VRAM
# - Full attention: ["q_proj", "k_proj", "v_proj", "o_proj"] — Recommended
# - Including MLP: above + ["gate_proj", "up_proj", "down_proj"] — Maximum performance

from transformers import BitsAndBytesConfig
import torch

# 4-bit quantization configuration
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",           # NormalFloat4
    bnb_4bit_compute_dtype=torch.bfloat16, # Computation in bf16
    bnb_4bit_use_double_quant=True,        # Double Quantization
)

# Load the quantized model
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B",
    quantization_config=bnb_config,
    device_map="auto"
)

# Apply LoRA (QLoRA = 4-bit quantized model + LoRA)
model = get_peft_model(model, lora_config)

from datasets import load_dataset

# Alpaca-style dataset
dataset = load_dataset("json", data_files="train_data.json")

# Data example
# {
#   "instruction": "Please summarize the following text.",
#   "input": "Kubernetes is a platform for containerized workloads and services...",
#   "output": "Kubernetes is a container orchestration platform."
# }

# Apply prompt template
def format_instruction(sample):
    if sample["input"]:
        text = f"""### Instruction:
{sample["instruction"]}

### Input:
{sample["input"]}

### Response:
{sample["output"]}"""
    else:
        text = f"""### Instruction:
{sample["instruction"]}

### Response:
{sample["output"]}"""
    return {"text": text}

dataset = dataset.map(format_instruction)

from trl import SFTTrainer, SFTConfig

training_args = SFTConfig(
    output_dir="./output",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,    # Effective batch = 4 x 4 = 16
    learning_rate=2e-4,
    lr_scheduler_type="cosine",
    warmup_ratio=0.03,
    max_seq_length=2048,
    bf16=True,
    logging_steps=10,
    save_strategy="epoch",
    optim="paged_adamw_8bit",         # Paged optimizer for QLoRA
    gradient_checkpointing=True,      # Additional VRAM savings
)

trainer = SFTTrainer(
    model=model,
    tokenizer=tokenizer,
    train_dataset=dataset["train"],
    args=training_args,
)

trainer.train()

# Save only the LoRA adapter (tens of MB)
model.save_pretrained("./lora-adapter")

# Load the adapter later
from peft import PeftModel
base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Llama-3.1-8B")
model = PeftModel.from_pretrained(base_model, "./lora-adapter")

# Merge adapter with the base model (for deployment)
merged_model = model.merge_and_unload()
merged_model.save_pretrained("./merged-model")

# Characteristics by rank
# r=4:   Fewest parameters, suitable for simple domain adaptation
# r=8:   Typical starting point
# r=16:  Good balance (recommended)
# r=32:  Complex tasks, requires more VRAM
# r=64+: Performance close to full fine-tuning, proportionally less efficient

# Empirically, r=16 + alpha=32 works well in most cases

# LoRA/QLoRA learning rates are set higher than full FT
# Full FT: 1e-5 ~ 5e-5
# LoRA:    1e-4 ~ 3e-4
# QLoRA:   2e-4 (typical)

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],
    use_dora=True,   # Enable DoRA
)

from peft import PeftModel

# Load multiple adapters on the base model
model = PeftModel.from_pretrained(base_model, "./adapter-korean")
model.load_adapter("./adapter-code", adapter_name="code")

# Switch adapters
model.set_adapter("code")

# Or combine adapter weights
model.add_weighted_adapter(
    adapters=["default", "code"],
    weights=[0.7, 0.3],
    adapter_name="merged"
)