HuggingFaceエコシステム完全ガイド：Transformers、Datasets、PEFT、Accelerateをマスターする

pip install transformers datasets tokenizers peft accelerate trl diffusers evaluate huggingface_hub

from huggingface_hub import login

# 方法1: トークンを直接指定
login(token="hf_your_token_here")

# 方法2: 環境変数（推奨）
import os
os.environ["HUGGINGFACE_TOKEN"] = "hf_your_token_here"

# 方法3: CLI
# huggingface-cli login

from transformers import pipeline

# テキスト分類
classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
result = classifier("This library is incredibly useful!")
print(result)
# [{'label': 'POSITIVE', 'score': 0.9997}]

# テキスト生成
generator = pipeline(
    "text-generation",
    model="meta-llama/Meta-Llama-3-8B-Instruct",
    torch_dtype="auto",
    device_map="auto"
)
output = generator(
    "The most famous landmarks in New York are",
    max_new_tokens=100,
    do_sample=True,
    temperature=0.7
)
print(output[0]["generated_text"])

# 抽出型QA
qa = pipeline("question-answering", model="deepset/roberta-base-squad2")
result = qa(
    question="Where is HuggingFace headquartered?",
    context="HuggingFace is headquartered in New York and also has an office in Paris."
)
print(result)
# {'score': 0.99, 'start': 30, 'end': 38, 'answer': 'New York'}

# 翻訳
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr")
result = translator("Hello, I am an AI researcher.")
print(result)

# 要約
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
long_text = """HuggingFace was founded in 2016 as an AI company..."""
summary = summarizer(long_text, max_length=100, min_length=30)
print(summary)

# 固有表現認識
ner = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english", aggregation_strategy="simple")
result = ner("Apple is looking at buying a U.K. startup for $1 billion.")
print(result)

from transformers import AutoTokenizer, AutoModel
import torch

# トークナイザーの読み込み
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# テキストのエンコード
text = "HuggingFace is a really useful library."
inputs = tokenizer(text, return_tensors="pt")
print(inputs)
# {'input_ids': tensor([[...]]), 'attention_mask': tensor([[...]])}

# トークンのデコード
decoded = tokenizer.decode(inputs["input_ids"][0])
print(decoded)

# 特殊トークンの確認
print(f"BOS: {tokenizer.bos_token}")
print(f"EOS: {tokenizer.eos_token}")
print(f"PAD: {tokenizer.pad_token}")
print(f"UNK: {tokenizer.unk_token}")
print(f"Vocab size: {tokenizer.vocab_size}")

# バッチエンコード
texts = ["First sentence.", "Second sentence, which is a little longer."]
batch_inputs = tokenizer(
    texts,
    padding=True,
    truncation=True,
    max_length=128,
    return_tensors="pt"
)
print(batch_inputs["input_ids"].shape)  # [2, 128]

# モデルの読み込み
model = AutoModel.from_pretrained("bert-base-uncased")
model.eval()

with torch.no_grad():
    outputs = model(**batch_inputs)
    print(outputs.last_hidden_state.shape)  # [batch, seq_len, hidden_dim]
    print(outputs.pooler_output.shape)      # [batch, hidden_dim]

from transformers import (
    AutoModelForSequenceClassification,
    AutoModelForCausalLM,
    AutoModelForSeq2SeqLM,
    AutoModelForTokenClassification,
    AutoModelForQuestionAnswering,
    AutoModelForMaskedLM,
)
import torch

# シーケンス分類（感情分析、テキスト分類）
clf_model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=2
)

# 因果言語モデル（GPTスタイルのテキスト生成）
causal_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    torch_dtype=torch.bfloat16,
    device_map="auto",
    attn_implementation="flash_attention_2"
)

# Seq2Seq（翻訳、要約）
seq2seq_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large-cnn")

# トークン分類（NER、品詞タグ付け）
ner_model = AutoModelForTokenClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=9
)

# 抽出型QA
qa_model = AutoModelForQuestionAnswering.from_pretrained("deepset/roberta-large-squad2")

# マスク言語モデル（BERTスタイルのマスク予測）
mlm_model = AutoModelForMaskedLM.from_pretrained("bert-base-uncased")

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

def load_model(model_name: str):
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype=torch.bfloat16,
        device_map="auto"
    )
    return tokenizer, model

def generate_text(
    model,
    tokenizer,
    prompt: str,
    max_new_tokens: int = 200,
    temperature: float = 0.7,
    top_p: float = 0.9,
    do_sample: bool = True
) -> str:
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=temperature,
            top_p=top_p,
            do_sample=do_sample,
            repetition_penalty=1.1,
            eos_token_id=tokenizer.eos_token_id,
            pad_token_id=tokenizer.pad_token_id,
        )

    # 新しく生成されたトークンのみ返す（入力プロンプトを除く）
    generated_ids = outputs[0][inputs["input_ids"].shape[1]:]
    return tokenizer.decode(generated_ids, skip_special_tokens=True)

# 使用例
tokenizer, model = load_model("Qwen/Qwen2.5-7B-Instruct")

# チャットテンプレートの適用
messages = [
    {"role": "system", "content": "You are a helpful AI assistant."},
    {"role": "user", "content": "Write a Python function to compute Fibonacci numbers."}
]

prompt = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)

response = generate_text(model, tokenizer, prompt)
print(response)

from datasets import load_dataset

# Hubから読み込み
dataset = load_dataset("glue", "sst2")
print(dataset)
# DatasetDict({train: Dataset({features: [...], num_rows: ...})})

# 特定のスプリットを読み込み
train_ds = load_dataset("glue", "sst2", split="train")
val_ds = load_dataset("glue", "sst2", split="validation")

# ローカルファイルから読み込み
local_ds = load_dataset("json", data_files={"train": "train.jsonl", "test": "test.jsonl"})
csv_ds = load_dataset("csv", data_files="data.csv")
text_ds = load_dataset("text", data_files="corpus.txt")

# パーセンテージベースのスプリット
split_ds = load_dataset("glue", "sst2", split="train[:80%]")
val_split = load_dataset("glue", "sst2", split="train[80%:]")

# データセットの検査
print(train_ds.features)
print(train_ds.column_names)
print(train_ds.num_rows)
print(train_ds[0])    # 最初のサンプル
print(train_ds[:5])   # 最初の5サンプル

from datasets import load_dataset
from transformers import AutoTokenizer

dataset = load_dataset("glue", "sst2")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

def tokenize_function(examples):
    return tokenizer(
        examples["sentence"],
        padding="max_length",
        truncation=True,
        max_length=128
    )

# マルチプロセスによるバッチ処理
tokenized_ds = dataset.map(
    tokenize_function,
    batched=True,
    num_proc=4,
    remove_columns=["sentence", "idx"]
)

# filter: 条件を満たすサンプルのみ保持
long_samples = dataset["train"].filter(
    lambda x: len(x["sentence"].split()) > 10
)
print(f"After filter: {len(long_samples)} samples")

# select: インデックスベースの選択
small_ds = dataset["train"].select(range(1000))

# ソート
sorted_ds = dataset["train"].sort("label", reverse=True)

# シャッフル
shuffled_ds = dataset["train"].shuffle(seed=42)

# カラムの追加
def add_text_length(example):
    example["text_length"] = len(example["sentence"].split())
    return example

ds_with_length = dataset["train"].map(add_text_length)

# データセットの保存/読み込み
tokenized_ds.save_to_disk("./tokenized_sst2")

from datasets import load_from_disk
loaded_ds = load_from_disk("./tokenized_sst2")

from datasets import Dataset, DatasetDict, Features, Value, ClassLabel
import pandas as pd

# pandas DataFrameからDatasetを作成
df = pd.DataFrame({
    "text": ["Positive sentence.", "Negative sentence.", "Neutral sentence."],
    "label": [1, 0, 2]
})
custom_ds = Dataset.from_pandas(df)

# 辞書からDatasetを作成
data_dict = {
    "text": ["sample 1", "sample 2"],
    "score": [0.8, 0.3]
}
ds_from_dict = Dataset.from_dict(data_dict)

# Hubにプッシュ
from huggingface_hub import login
login()

custom_ds.push_to_hub("your-username/my-custom-dataset")

from datasets import load_dataset

# ストリーミングモード（完全なデータセットをメモリに読み込まない）
streaming_ds = load_dataset(
    "HuggingFaceFW/fineweb",
    "sample-10BT",
    split="train",
    streaming=True
)

# イテレータとして反復処理
for i, example in enumerate(streaming_ds):
    if i >= 5:
        break
    print(example["text"][:100])

# mapとfilterもストリーミングで動作
filtered_stream = streaming_ds.filter(lambda x: len(x["text"]) > 500)
mapped_stream = filtered_stream.map(lambda x: {"text_len": len(x["text"])})

from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders, processors
from tokenizers.models import BPE, WordPiece, Unigram

# BPEトークナイザーの学習
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)

trainer = trainers.BpeTrainer(
    vocab_size=30000,
    min_frequency=2,
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
)

files = ["corpus.txt"]
tokenizer.train(files, trainer)

# 保存と読み込み
tokenizer.save("my_tokenizer.json")
loaded_tokenizer = Tokenizer.load("my_tokenizer.json")

# エンコード
output = tokenizer.encode("Hello, HuggingFace!")
print(output.tokens)
print(output.ids)

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# ベースモデルの読み込み
model_name = "meta-llama/Meta-Llama-3-8B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

# LoRA設定
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,                        # ランク（低いほどパラメータが少ない）
    lora_alpha=32,               # スケーリング係数
    target_modules=[             # LoRAを適用するレイヤー
        "q_proj", "v_proj",
        "k_proj", "o_proj",
        "gate_proj", "up_proj",
        "down_proj"
    ],
    lora_dropout=0.05,
    bias="none",
    inference_mode=False
)

# LoRAを適用
peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 41,943,040 || all params: 8,072,925,184 || trainable%: 0.5196

from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch

# 4ビット量子化設定
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

# 量子化モデルの読み込み
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    quantization_config=bnb_config,
    device_map="auto"
)

# kbitトレーニングの準備（勾配チェックポインティングを有効化）
model = prepare_model_for_kbit_training(model)

# LoRAを適用
lora_config = LoraConfig(
    r=64,
    lora_alpha=128,
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.1,
    bias="none",
    task_type="CAUSAL_LM"
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

from trl import SFTTrainer, SFTConfig
from peft import LoraConfig
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from datasets import load_dataset
import torch

# 1. データセット
dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft[:10%]")

# 2. モデル設定
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    quantization_config=bnb_config,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
tokenizer.pad_token = tokenizer.eos_token

# 3. LoRA設定
peft_config = LoraConfig(
    r=32,
    lora_alpha=64,
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"]
)

# 4. トレーニング設定
sft_config = SFTConfig(
    output_dir="./qlora-llama3",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    bf16=True,
    logging_steps=10,
    save_steps=100,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    max_seq_length=2048,
    report_to="wandb"
)

# 5. 学習
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    args=sft_config,
    processing_class=tokenizer,
)

trainer.train()
trainer.save_model("./qlora-llama3-final")

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# LoRAアダプターのみ保存
peft_model.save_pretrained("./lora-adapter")
tokenizer.save_pretrained("./lora-adapter")

# アダプターをHubにアップロード
peft_model.push_to_hub("your-username/llama3-lora")

# アダプターの読み込み（ベースモデル + アダプター）
base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
peft_model = PeftModel.from_pretrained(base_model, "./lora-adapter")

# アダプターをベースモデルにマージ（推論速度を最適化）
merged_model = peft_model.merge_and_unload()
merged_model.save_pretrained("./merged-llama3")
tokenizer.save_pretrained("./merged-llama3")

# インタラクティブ設定
accelerate config

# 非インタラクティブ設定
accelerate config --config_file ./accelerate_config.yaml

# accelerate_config.yaml
compute_environment: LOCAL_MACHINE
distributed_type: MULTI_GPU
num_machines: 1
num_processes: 4
gpu_ids: all
mixed_precision: bf16

from accelerate import Accelerator
from transformers import AutoModelForSequenceClassification, AutoTokenizer, AdamW
from datasets import load_dataset
from torch.utils.data import DataLoader

def training_function():
    accelerator = Accelerator(
        mixed_precision="bf16",
        gradient_accumulation_steps=4,
        log_with="wandb"
    )

    model_name = "bert-base-uncased"
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

    dataset = load_dataset("glue", "sst2", split="train")

    def tokenize(examples):
        return tokenizer(examples["sentence"], truncation=True, max_length=128)

    tokenized = dataset.map(tokenize, batched=True)
    tokenized.set_format("torch", columns=["input_ids", "attention_mask", "label"])

    dataloader = DataLoader(tokenized, batch_size=16, shuffle=True)
    optimizer = AdamW(model.parameters(), lr=2e-5)

    # Accelerateですべてを準備
    model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader)

    for epoch in range(3):
        model.train()
        for batch in dataloader:
            with accelerator.accumulate(model):
                outputs = model(**batch)
                loss = outputs.loss
                accelerator.backward(loss)
                accelerator.clip_grad_norm_(model.parameters(), 1.0)
                optimizer.step()
                optimizer.zero_grad()

        accelerator.print(f"Epoch {epoch} completed")

    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained("./output", save_function=accelerator.save)

# 実行: accelerate launch train.py
training_function()

from trl import SFTTrainer, SFTConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-7B")
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B")

dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft[:5%]")

training_args = SFTConfig(
    output_dir="./sft-qwen",
    max_seq_length=2048,
    num_train_epochs=1,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=2,
    learning_rate=2e-5,
    bf16=True,
    save_strategy="epoch",
    logging_steps=10,
    dataset_text_field="messages",
)

trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    processing_class=tokenizer,
)

trainer.train()

from trl import DPOTrainer, DPOConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

model = AutoModelForCausalLM.from_pretrained("your-sft-model")
ref_model = AutoModelForCausalLM.from_pretrained("your-sft-model")  # 凍結
tokenizer = AutoTokenizer.from_pretrained("your-sft-model")

# DPOデータセットには prompt、chosen、rejected カラムが必要
dpo_dataset = load_dataset("HuggingFaceH4/ultrafeedback_binarized", split="train_prefs")

dpo_config = DPOConfig(
    output_dir="./dpo-model",
    num_train_epochs=1,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=5e-7,
    beta=0.1,  # KLペナルティの強度
    bf16=True,
    loss_type="sigmoid",
)

trainer = DPOTrainer(
    model=model,
    ref_model=ref_model,
    args=dpo_config,
    train_dataset=dpo_dataset,
    processing_class=tokenizer,
)

trainer.train()

from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
import torch

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
    use_safetensors=True
)

# より速いスケジューラーを使用
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
pipe = pipe.to("cuda")

# メモリ最適化
pipe.enable_attention_slicing()
pipe.enable_xformers_memory_efficient_attention()

# 画像の生成
image = pipe(
    prompt="A beautiful mountain landscape at sunset, photorealistic, 8k",
    negative_prompt="blurry, low quality, cartoon",
    num_inference_steps=20,
    guidance_scale=7.5,
    width=512,
    height=512,
    generator=torch.Generator("cuda").manual_seed(42)
).images[0]

image.save("landscape.png")

import evaluate

# BLEU（翻訳品質）
bleu = evaluate.load("bleu")
result = bleu.compute(
    predictions=["the cat is on the mat"],
    references=[["the cat is on the mat", "there is a cat on the mat"]]
)
print(f"BLEU: {result['bleu']:.4f}")

# ROUGE（要約品質）
rouge = evaluate.load("rouge")
result = rouge.compute(
    predictions=["This is a very useful library."],
    references=["This is an extremely useful library."]
)
print(result)  # rouge1, rouge2, rougeL, rougeLsum

# BERTScore（意味的類似度）
bertscore = evaluate.load("bertscore")
result = bertscore.compute(
    predictions=["AI is changing the world."],
    references=["Artificial intelligence is transforming the globe."],
    lang="en"
)
print(f"BERTScore F1: {result['f1'][0]:.4f}")

# 精度
accuracy = evaluate.load("accuracy")
result = accuracy.compute(predictions=[0, 1, 1, 0], references=[0, 1, 0, 0])
print(f"Accuracy: {result['accuracy']:.4f}")

from huggingface_hub import HfApi, hf_hub_download, snapshot_download
from huggingface_hub import create_repo, upload_file, upload_folder

api = HfApi()

# 単一ファイルのダウンロード
local_path = hf_hub_download(
    repo_id="meta-llama/Meta-Llama-3-8B",
    filename="config.json"
)

# リポジトリ全体のダウンロード
snapshot_download(
    repo_id="meta-llama/Meta-Llama-3-8B",
    local_dir="./llama3-8b",
    ignore_patterns=["*.bin", "*.pt"]  # safetensorsのみダウンロード
)

# リポジトリの作成
create_repo("your-username/my-model", private=True)
create_repo("your-username/my-dataset", repo_type="dataset")
create_repo("your-username/my-space", repo_type="space", space_sdk="gradio")

# ファイルのアップロード
upload_file(
    path_or_fileobj="./my_model.bin",
    path_in_repo="pytorch_model.bin",
    repo_id="your-username/my-model"
)

# フォルダのアップロード
upload_folder(
    folder_path="./output-model",
    repo_id="your-username/my-model",
    ignore_patterns=["*.log", "__pycache__"]
)

# モデルの検索
models = api.list_models(
    filter="text-generation",
    language="en",
    sort="downloads",
    limit=10
)
for m in models:
    print(m.id, m.downloads)

from optimum.exporters.onnx import main_export

# モデルをONNX形式に変換
main_export(
    model_name_or_path="bert-base-uncased",
    output="./bert-onnx",
    task="text-classification"
)

# 推論にONNXモデルを使用
from optimum.onnxruntime import ORTModelForSequenceClassification
from transformers import AutoTokenizer

ort_model = ORTModelForSequenceClassification.from_pretrained("./bert-onnx")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

inputs = tokenizer("This movie was fantastic!", return_tensors="pt")
outputs = ort_model(**inputs)
print(outputs.logits)

from datasets import load_dataset, concatenate_datasets

# 指示データセットの読み込み
alpaca = load_dataset("tatsu-lab/alpaca", split="train")

def format_instruction_dataset(example):
    """AlpacaフォーマットをChatMLフォーマットに変換"""
    instruction = example["instruction"]
    inp = example.get("input", "").strip()
    output = example["output"]

    if inp:
        user_content = f"{instruction}\n\n{inp}"
    else:
        user_content = instruction

    messages = [
        {"role": "system", "content": "You are a helpful AI assistant."},
        {"role": "user", "content": user_content},
        {"role": "assistant", "content": output}
    ]
    return {"messages": messages}

formatted_dataset = alpaca.map(format_instruction_dataset)
print(formatted_dataset[0])

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, prepare_model_for_kbit_training, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset

MODEL_NAME = "meta-llama/Meta-Llama-3-8B"
OUTPUT_DIR = "./llama3-qlora"
MAX_SEQ_LEN = 2048
NUM_EPOCHS = 2
BATCH_SIZE = 2
GRAD_ACCUM = 8
LR = 2e-4

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"

model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    quantization_config=bnb_config,
    device_map="auto",
    attn_implementation="flash_attention_2"
)
model = prepare_model_for_kbit_training(model)

peft_config = LoraConfig(
    r=64,
    lora_alpha=128,
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)

dataset = load_dataset("tatsu-lab/alpaca", split="train")

def format_sample(example):
    messages = [
        {"role": "system", "content": "You are a helpful AI assistant."},
        {"role": "user", "content": example["instruction"]},
        {"role": "assistant", "content": example["output"]}
    ]
    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
    return {"text": text}

dataset = dataset.map(format_sample)

training_args = SFTConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=NUM_EPOCHS,
    per_device_train_batch_size=BATCH_SIZE,
    gradient_accumulation_steps=GRAD_ACCUM,
    learning_rate=LR,
    bf16=True,
    gradient_checkpointing=True,
    optim="paged_adamw_8bit",
    warmup_ratio=0.05,
    lr_scheduler_type="cosine",
    save_strategy="epoch",
    logging_steps=10,
    dataset_text_field="text",
    max_seq_length=MAX_SEQ_LEN,
    packing=True,
)

trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    args=training_args,
    processing_class=tokenizer,
)

trainer.train()
trainer.model.save_pretrained(f"{OUTPUT_DIR}/adapter")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/adapter")
print("Training complete!")

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
model = PeftModel.from_pretrained(base_model, "./llama3-qlora/adapter")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")

def chat(user_message: str, max_new_tokens: int = 512) -> str:
    messages = [
        {"role": "system", "content": "You are a helpful AI assistant."},
        {"role": "user", "content": user_message}
    ]
    prompt = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True
    )
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            repetition_penalty=1.1
        )

    return tokenizer.decode(
        outputs[0][inputs["input_ids"].shape[1]:],
        skip_special_tokens=True
    )

test_questions = [
    "Write a Python function to check if a number is prime.",
    "Explain the difference between machine learning and deep learning.",
    "What are the main benefits of using HuggingFace?",
]

for q in test_questions:
    print(f"Q: {q}")
    print(f"A: {chat(q)}")
    print("-" * 50)

import gradio as gr
from transformers import pipeline
import torch

pipe = pipeline(
    "text-generation",
    model="./llama3-qlora/merged",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

def respond(message, history, system_prompt, max_tokens, temperature):
    messages = [{"role": "system", "content": system_prompt}]
    for user, assistant in history:
        messages.append({"role": "user", "content": user})
        messages.append({"role": "assistant", "content": assistant})
    messages.append({"role": "user", "content": message})

    output = pipe(
        messages,
        max_new_tokens=max_tokens,
        temperature=temperature,
        do_sample=True
    )
    return output[0]["generated_text"][-1]["content"]

demo = gr.ChatInterface(
    respond,
    additional_inputs=[
        gr.Textbox("You are a helpful AI assistant.", label="System Prompt"),
        gr.Slider(50, 1000, 256, label="Max Tokens"),
        gr.Slider(0.1, 2.0, 0.7, label="Temperature")
    ],
    title="Llama-3 Chatbot",
    description="QLoRAファインチューニング済みLLM"
)

if __name__ == "__main__":
    demo.launch()