HuggingFace 生态系统完全指南：精通 Transformers、Datasets、PEFT 与 Accelerate

pip install transformers datasets tokenizers peft accelerate trl diffusers evaluate huggingface_hub

from huggingface_hub import login

# 方式一：直接传入令牌
login(token="hf_your_token_here")

# 方式二：环境变量（推荐）
import os
os.environ["HUGGINGFACE_TOKEN"] = "hf_your_token_here"

# 方式三：CLI
# huggingface-cli login

from transformers import pipeline

# 文本分类
classifier = pipeline("text-classification", model="distilbert-base-uncased-finetuned-sst-2-english")
result = classifier("This library is incredibly useful!")
print(result)
# [{'label': 'POSITIVE', 'score': 0.9997}]

# 文本生成
generator = pipeline(
    "text-generation",
    model="meta-llama/Meta-Llama-3-8B-Instruct",
    torch_dtype="auto",
    device_map="auto"
)
output = generator(
    "The most famous landmarks in New York are",
    max_new_tokens=100,
    do_sample=True,
    temperature=0.7
)
print(output[0]["generated_text"])

# 抽取式问答
qa = pipeline("question-answering", model="deepset/roberta-base-squad2")
result = qa(
    question="Where is HuggingFace headquartered?",
    context="HuggingFace is headquartered in New York and also has an office in Paris."
)
print(result)
# {'score': 0.99, 'start': 30, 'end': 38, 'answer': 'New York'}

# 翻译
translator = pipeline("translation", model="Helsinki-NLP/opus-mt-en-fr")
result = translator("Hello, I am an AI researcher.")
print(result)

# 摘要
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
long_text = """HuggingFace was founded in 2016 as an AI company..."""
summary = summarizer(long_text, max_length=100, min_length=30)
print(summary)

# 命名实体识别
ner = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english", aggregation_strategy="simple")
result = ner("Apple is looking at buying a U.K. startup for $1 billion.")
print(result)

from transformers import AutoTokenizer, AutoModel
import torch

# 加载分词器
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# 编码文本
text = "HuggingFace is a really useful library."
inputs = tokenizer(text, return_tensors="pt")
print(inputs)
# {'input_ids': tensor([[...]]), 'attention_mask': tensor([[...]])}

# 解码 token
decoded = tokenizer.decode(inputs["input_ids"][0])
print(decoded)

# 查看特殊 token
print(f"BOS: {tokenizer.bos_token}")
print(f"EOS: {tokenizer.eos_token}")
print(f"PAD: {tokenizer.pad_token}")
print(f"UNK: {tokenizer.unk_token}")
print(f"Vocab size: {tokenizer.vocab_size}")

# 批量编码
texts = ["First sentence.", "Second sentence, which is a little longer."]
batch_inputs = tokenizer(
    texts,
    padding=True,
    truncation=True,
    max_length=128,
    return_tensors="pt"
)
print(batch_inputs["input_ids"].shape)  # [2, 128]

# 加载模型
model = AutoModel.from_pretrained("bert-base-uncased")
model.eval()

with torch.no_grad():
    outputs = model(**batch_inputs)
    print(outputs.last_hidden_state.shape)  # [batch, seq_len, hidden_dim]
    print(outputs.pooler_output.shape)      # [batch, hidden_dim]

from transformers import (
    AutoModelForSequenceClassification,
    AutoModelForCausalLM,
    AutoModelForSeq2SeqLM,
    AutoModelForTokenClassification,
    AutoModelForQuestionAnswering,
    AutoModelForMaskedLM,
)
import torch

# 序列分类（情感分析、文本分类）
clf_model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=2
)

# 因果语言模型（GPT 风格文本生成）
causal_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    torch_dtype=torch.bfloat16,
    device_map="auto",
    attn_implementation="flash_attention_2"
)

# Seq2Seq（翻译、摘要）
seq2seq_model = AutoModelForSeq2SeqLM.from_pretrained("facebook/bart-large-cnn")

# Token 分类（NER、词性标注）
ner_model = AutoModelForTokenClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=9
)

# 抽取式问答
qa_model = AutoModelForQuestionAnswering.from_pretrained("deepset/roberta-large-squad2")

# 掩码语言模型（BERT 风格掩码预测）
mlm_model = AutoModelForMaskedLM.from_pretrained("bert-base-uncased")

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

def load_model(model_name: str):
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForCausalLM.from_pretrained(
        model_name,
        torch_dtype=torch.bfloat16,
        device_map="auto"
    )
    return tokenizer, model

def generate_text(
    model,
    tokenizer,
    prompt: str,
    max_new_tokens: int = 200,
    temperature: float = 0.7,
    top_p: float = 0.9,
    do_sample: bool = True
) -> str:
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            temperature=temperature,
            top_p=top_p,
            do_sample=do_sample,
            repetition_penalty=1.1,
            eos_token_id=tokenizer.eos_token_id,
            pad_token_id=tokenizer.pad_token_id,
        )

    # 只返回新生成的 token（排除输入提示部分）
    generated_ids = outputs[0][inputs["input_ids"].shape[1]:]
    return tokenizer.decode(generated_ids, skip_special_tokens=True)

# 使用示例
tokenizer, model = load_model("Qwen/Qwen2.5-7B-Instruct")

# 应用聊天模板
messages = [
    {"role": "system", "content": "You are a helpful AI assistant."},
    {"role": "user", "content": "Write a Python function to compute Fibonacci numbers."}
]

prompt = tokenizer.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True
)

response = generate_text(model, tokenizer, prompt)
print(response)

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")

text = "Hello! Let's learn HuggingFace."

# 基础编码
ids = tokenizer.encode(text)
print(f"Token IDs: {ids}")
print(f"Token count: {len(ids)}")

# 查看 token
tokens = tokenizer.tokenize(text)
print(f"Tokens: {tokens}")

# 包含/排除特殊 token
ids_with_special = tokenizer.encode(text, add_special_tokens=True)
ids_without_special = tokenizer.encode(text, add_special_tokens=False)
print(f"With special tokens: {len(ids_with_special)}")
print(f"Without special tokens: {len(ids_without_special)}")

# 带填充的批量处理
batch = [
    "Short sentence.",
    "This is a much longer sentence that will need padding applied to it."
]
encoded = tokenizer(
    batch,
    padding="max_length",
    max_length=64,
    truncation=True,
    return_tensors="pt",
    return_attention_mask=True
)

print("input_ids shape:", encoded["input_ids"].shape)
print("attention_mask shape:", encoded["attention_mask"].shape)

# Token type ID（BERT 风格的 pair encoding）
bert_tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
pair_encoded = bert_tokenizer(
    "Question: What is AI?",
    "Context: AI stands for artificial intelligence.",
    return_tensors="pt"
)
print("token_type_ids:", pair_encoded.get("token_type_ids"))

# 添加特殊 token
special_tokens = {"additional_special_tokens": ["[DOMAIN]", "[ENTITY]", "[DATE]"]}
num_added = tokenizer.add_special_tokens(special_tokens)
print(f"Added {num_added} special tokens")

from datasets import load_dataset

# 从 Hub 加载
dataset = load_dataset("glue", "sst2")
print(dataset)
# DatasetDict({train: Dataset({features: [...], num_rows: ...})})

# 加载特定 split
train_ds = load_dataset("glue", "sst2", split="train")
val_ds = load_dataset("glue", "sst2", split="validation")

# 从本地文件加载
local_ds = load_dataset("json", data_files={"train": "train.jsonl", "test": "test.jsonl"})
csv_ds = load_dataset("csv", data_files="data.csv")
text_ds = load_dataset("text", data_files="corpus.txt")

# 按百分比切分
split_ds = load_dataset("glue", "sst2", split="train[:80%]")
val_split = load_dataset("glue", "sst2", split="train[80%:]")

# 查看数据集信息
print(train_ds.features)
print(train_ds.column_names)
print(train_ds.num_rows)
print(train_ds[0])    # 第一个样本
print(train_ds[:5])   # 前 5 个样本

from datasets import load_dataset
from transformers import AutoTokenizer

dataset = load_dataset("glue", "sst2")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

def tokenize_function(examples):
    return tokenizer(
        examples["sentence"],
        padding="max_length",
        truncation=True,
        max_length=128
    )

# 使用多进程做批处理
tokenized_ds = dataset.map(
    tokenize_function,
    batched=True,
    num_proc=4,
    remove_columns=["sentence", "idx"]
)

# filter：只保留满足条件的样本
long_samples = dataset["train"].filter(
    lambda x: len(x["sentence"].split()) > 10
)
print(f"After filter: {len(long_samples)} samples")

# select：按索引选择
small_ds = dataset["train"].select(range(1000))

# sort
sorted_ds = dataset["train"].sort("label", reverse=True)

# shuffle
shuffled_ds = dataset["train"].shuffle(seed=42)

# rename_column
renamed_ds = dataset["train"].rename_column("label", "sentiment")

# 添加一列
def add_text_length(example):
    example["text_length"] = len(example["sentence"].split())
    return example

ds_with_length = dataset["train"].map(add_text_length)

# 保存/加载数据集
tokenized_ds.save_to_disk("./tokenized_sst2")

from datasets import load_from_disk
loaded_ds = load_from_disk("./tokenized_sst2")

from datasets import Dataset, DatasetDict, Features, Value, ClassLabel
import pandas as pd

# 从 pandas DataFrame 创建 Dataset
df = pd.DataFrame({
    "text": ["Positive sentence.", "Negative sentence.", "Neutral sentence."],
    "label": [1, 0, 2]
})
custom_ds = Dataset.from_pandas(df)

# 从字典创建 Dataset
data_dict = {
    "text": ["sample 1", "sample 2"],
    "score": [0.8, 0.3]
}
ds_from_dict = Dataset.from_dict(data_dict)

# 定义 Features schema
features = Features({
    "text": Value("string"),
    "label": ClassLabel(names=["negative", "positive", "neutral"]),
    "score": Value("float32")
})

# 上传到 Hub
from huggingface_hub import login
login()

custom_ds.push_to_hub("your-username/my-custom-dataset")

# 附带 train/test split 一起上传
dataset_dict = DatasetDict({
    "train": custom_ds,
    "test": custom_ds
})
dataset_dict.push_to_hub("your-username/my-custom-dataset")

from datasets import load_dataset

# 流式模式（不会把整个数据集加载进内存）
streaming_ds = load_dataset(
    "HuggingFaceFW/fineweb",
    "sample-10BT",
    split="train",
    streaming=True
)

# 以迭代器方式遍历
for i, example in enumerate(streaming_ds):
    if i >= 5:
        break
    print(example["text"][:100])

# map 和 filter 在流式模式下同样可用
filtered_stream = streaming_ds.filter(lambda x: len(x["text"]) > 500)
mapped_stream = filtered_stream.map(lambda x: {"text_len": len(x["text"])})

# 批处理
def process_batch(batch):
    return {"processed": [t.lower() for t in batch["text"]]}

batched_stream = streaming_ds.map(process_batch, batched=True, batch_size=16)

# 转换为 DataLoader
from torch.utils.data import DataLoader

dataloader = DataLoader(
    list(streaming_ds.take(1000)),  # 只加载前 1000 条
    batch_size=8,
    shuffle=True
)

from transformers import (
    DataCollatorWithPadding,
    DataCollatorForSeq2Seq,
    DataCollatorForLanguageModeling,
    AutoTokenizer
)

tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

# 动态填充（在每个 batch 内填充到最大长度）
data_collator = DataCollatorWithPadding(tokenizer=tokenizer)

# Seq2Seq collator（自动生成 decoder 输入）
t5_tokenizer = AutoTokenizer.from_pretrained("t5-base")
seq2seq_collator = DataCollatorForSeq2Seq(
    tokenizer=t5_tokenizer,
    padding=True,
    return_tensors="pt"
)

# 掩码语言模型 collator
mlm_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer,
    mlm=True,
    mlm_probability=0.15  # 屏蔽 15% 的 token
)

# 因果语言模型 collator
clm_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer,
    mlm=False
)

from tokenizers import Tokenizer, models, trainers, pre_tokenizers, decoders, processors
from tokenizers.models import BPE, WordPiece, Unigram

# 训练一个 BPE 分词器
tokenizer = Tokenizer(BPE(unk_token="[UNK]"))
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)

trainer = trainers.BpeTrainer(
    vocab_size=30000,
    min_frequency=2,
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
)

files = ["corpus.txt"]
tokenizer.train(files, trainer)

# 保存与加载
tokenizer.save("my_tokenizer.json")
loaded_tokenizer = Tokenizer.load("my_tokenizer.json")

# 编码
output = tokenizer.encode("Hello, HuggingFace!")
print(output.tokens)
print(output.ids)

# 批量编码
batch_output = tokenizer.encode_batch(["sentence 1", "sentence 2"])

from tokenizers import Tokenizer
from tokenizers.models import WordPiece
from tokenizers.trainers import WordPieceTrainer
from tokenizers.pre_tokenizers import Whitespace
from tokenizers.processors import TemplateProcessing

tokenizer = Tokenizer(WordPiece(unk_token="[UNK]"))
tokenizer.pre_tokenizer = Whitespace()

trainer = WordPieceTrainer(
    vocab_size=30522,
    special_tokens=["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]"]
)

tokenizer.train(["corpus.txt"], trainer)

# 添加 BERT 风格的后处理
tokenizer.post_processor = TemplateProcessing(
    single="[CLS] $A [SEP]",
    pair="[CLS] $A [SEP] $B:1 [SEP]:1",
    special_tokens=[
        ("[CLS]", tokenizer.token_to_id("[CLS]")),
        ("[SEP]", tokenizer.token_to_id("[SEP]")),
    ],
)

# 与 HuggingFace transformers 集成
from transformers import PreTrainedTokenizerFast
fast_tokenizer = PreTrainedTokenizerFast(
    tokenizer_object=tokenizer,
    unk_token="[UNK]",
    pad_token="[PAD]",
    cls_token="[CLS]",
    sep_token="[SEP]",
    mask_token="[MASK]"
)

# 上传到 Hub
fast_tokenizer.push_to_hub("your-username/my-tokenizer")

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# 加载基座模型
model_name = "meta-llama/Meta-Llama-3-8B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

# LoRA 配置
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,                        # 秩（越低参数越少）
    lora_alpha=32,               # 缩放因子
    target_modules=[             # 应用 LoRA 的层
        "q_proj", "v_proj",
        "k_proj", "o_proj",
        "gate_proj", "up_proj",
        "down_proj"
    ],
    lora_dropout=0.05,
    bias="none",
    inference_mode=False
)

# 应用 LoRA
peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 41,943,040 || all params: 8,072,925,184 || trainable%: 0.5196

from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model
from transformers import AutoModelForCausalLM, BitsAndBytesConfig
import torch

# 4 位量化配置
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

# 加载量化后的模型
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    quantization_config=bnb_config,
    device_map="auto"
)

# 为 kbit 训练做准备（启用梯度检查点）
model = prepare_model_for_kbit_training(model)

# 应用 LoRA
lora_config = LoraConfig(
    r=64,
    lora_alpha=128,
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"],
    lora_dropout=0.1,
    bias="none",
    task_type="CAUSAL_LM"
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()

from trl import SFTTrainer, SFTConfig
from peft import LoraConfig
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from datasets import load_dataset
import torch

# 1. 数据准备
dataset = load_dataset("tatsu-lab/alpaca", split="train")

def format_conversation(sample):
    system = "You are a helpful AI assistant."
    instruction = sample["instruction"]
    inp = sample.get("input", "")
    output = sample["output"]

    if inp:
        user_msg = f"{instruction}\n\nInput: {inp}"
    else:
        user_msg = instruction

    return {
        "text": f"<|system|>\n{system}\n<|user|>\n{user_msg}\n<|assistant|>\n{output}"
    }

dataset = dataset.map(format_conversation)

# 2. 模型设置
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    quantization_config=bnb_config,
    device_map="auto"
)
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
tokenizer.pad_token = tokenizer.eos_token

# 3. LoRA 配置
peft_config = LoraConfig(
    r=32,
    lora_alpha=64,
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
    target_modules=["q_proj", "v_proj", "k_proj", "o_proj"]
)

# 4. 训练配置
sft_config = SFTConfig(
    output_dir="./qlora-llama3",
    num_train_epochs=3,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    fp16=False,
    bf16=True,
    logging_steps=10,
    save_steps=100,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    dataset_text_field="text",
    max_seq_length=2048,
    report_to="wandb"
)

# 5. 训练
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    args=sft_config,
    processing_class=tokenizer,
)

trainer.train()
trainer.save_model("./qlora-llama3-final")

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

# 只保存 LoRA 适配器
peft_model.save_pretrained("./lora-adapter")
tokenizer.save_pretrained("./lora-adapter")

# 上传适配器到 Hub
peft_model.push_to_hub("your-username/llama3-lora")

# 加载适配器（基座模型 + 适配器）
base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
peft_model = PeftModel.from_pretrained(base_model, "./lora-adapter")

# 将适配器合并进基座模型（优化推理速度）
merged_model = peft_model.merge_and_unload()
merged_model.save_pretrained("./merged-llama3")
tokenizer.save_pretrained("./merged-llama3")

# 交互式设置
accelerate config

# 非交互式设置
accelerate config --config_file ./accelerate_config.yaml

# accelerate_config.yaml
compute_environment: LOCAL_MACHINE
distributed_type: MULTI_GPU
num_machines: 1
num_processes: 4
gpu_ids: all
mixed_precision: bf16

from accelerate import Accelerator
from transformers import AutoModelForSequenceClassification, AutoTokenizer, AdamW
from datasets import load_dataset
from torch.utils.data import DataLoader

def training_function():
    # 初始化 Accelerator
    accelerator = Accelerator(
        mixed_precision="bf16",
        gradient_accumulation_steps=4,
        log_with="wandb"
    )

    # 模型与分词器
    model_name = "bert-base-uncased"
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

    # 数据集
    dataset = load_dataset("glue", "sst2", split="train")

    def tokenize(examples):
        return tokenizer(examples["sentence"], truncation=True, max_length=128)

    tokenized = dataset.map(tokenize, batched=True)
    tokenized.set_format("torch", columns=["input_ids", "attention_mask", "label"])

    dataloader = DataLoader(tokenized, batch_size=16, shuffle=True)

    # 优化器
    optimizer = AdamW(model.parameters(), lr=2e-5)

    # 用 Accelerate 一次性准备好所有对象
    model, optimizer, dataloader = accelerator.prepare(model, optimizer, dataloader)

    for epoch in range(3):
        model.train()
        for batch in dataloader:
            with accelerator.accumulate(model):
                outputs = model(**batch)
                loss = outputs.loss
                accelerator.backward(loss)
                accelerator.clip_grad_norm_(model.parameters(), 1.0)
                optimizer.step()
                optimizer.zero_grad()

        accelerator.print(f"Epoch {epoch} completed")

    # 保存
    accelerator.wait_for_everyone()
    unwrapped_model = accelerator.unwrap_model(model)
    unwrapped_model.save_pretrained("./output", save_function=accelerator.save)

# 运行方式: accelerate launch train.py
training_function()

from accelerate import Accelerator
from accelerate.utils import DeepSpeedPlugin

ds_plugin = DeepSpeedPlugin(
    hf_ds_config={
        "zero_optimization": {
            "stage": 2,
            "allgather_partitions": True,
            "allgather_bucket_size": 2e8,
            "overlap_comm": True,
            "reduce_scatter": True,
            "reduce_bucket_size": 2e8,
            "contiguous_gradients": True
        },
        "bf16": {"enabled": True},
        "optimizer": {
            "type": "AdamW",
            "params": {
                "lr": "auto",
                "weight_decay": "auto"
            }
        }
    }
)

accelerator = Accelerator(deepspeed_plugin=ds_plugin)

from accelerate import Accelerator

accelerator = Accelerator(
    mixed_precision="bf16",
    gradient_accumulation_steps=8  # 每 8 步才真正更新一次
)

for batch in dataloader:
    with accelerator.accumulate(model):
        output = model(**batch)
        loss = output.loss / accelerator.gradient_accumulation_steps
        accelerator.backward(loss)

        if accelerator.sync_gradients:
            accelerator.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

from trl import SFTTrainer, SFTConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-7B")
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-7B")

dataset = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft[:5%]")

training_args = SFTConfig(
    output_dir="./sft-qwen",
    max_seq_length=2048,
    num_train_epochs=1,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=2,
    learning_rate=2e-5,
    bf16=True,
    save_strategy="epoch",
    logging_steps=10,
    dataset_text_field="messages",
)

trainer = SFTTrainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    processing_class=tokenizer,
)

trainer.train()

from trl import DPOTrainer, DPOConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
from datasets import load_dataset

model = AutoModelForCausalLM.from_pretrained("your-sft-model")
ref_model = AutoModelForCausalLM.from_pretrained("your-sft-model")  # frozen
tokenizer = AutoTokenizer.from_pretrained("your-sft-model")

# DPO 数据集需要: prompt、chosen、rejected 三列
dpo_dataset = load_dataset("HuggingFaceH4/ultrafeedback_binarized", split="train_prefs")

dpo_config = DPOConfig(
    output_dir="./dpo-model",
    num_train_epochs=1,
    per_device_train_batch_size=2,
    gradient_accumulation_steps=4,
    learning_rate=5e-7,
    beta=0.1,  # KL 惩罚强度
    bf16=True,
    loss_type="sigmoid",
)

trainer = DPOTrainer(
    model=model,
    ref_model=ref_model,
    args=dpo_config,
    train_dataset=dpo_dataset,
    processing_class=tokenizer,
)

trainer.train()

from trl import RewardTrainer, RewardConfig
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from datasets import load_dataset

model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels=1  # 标量奖励输出
)
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

dataset = load_dataset("HuggingFaceH4/ultrafeedback_binarized", split="train_prefs")

reward_config = RewardConfig(
    output_dir="./reward-model",
    num_train_epochs=1,
    per_device_train_batch_size=8,
    learning_rate=1e-5,
    max_length=512,
)

trainer = RewardTrainer(
    model=model,
    args=reward_config,
    train_dataset=dataset,
    processing_class=tokenizer,
)

trainer.train()

from diffusers import StableDiffusionPipeline, DPMSolverMultistepScheduler
import torch

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
    use_safetensors=True
)

# 使用更快的调度器
pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config)
pipe = pipe.to("cuda")

# 内存优化
pipe.enable_attention_slicing()
pipe.enable_xformers_memory_efficient_attention()

# 生成图像
image = pipe(
    prompt="A beautiful mountain landscape at sunset, photorealistic, 8k",
    negative_prompt="blurry, low quality, cartoon",
    num_inference_steps=20,
    guidance_scale=7.5,
    width=512,
    height=512,
    generator=torch.Generator("cuda").manual_seed(42)
).images[0]

image.save("landscape.png")

# SDXL
from diffusers import StableDiffusionXLPipeline

xl_pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    torch_dtype=torch.float16,
    variant="fp16"
).to("cuda")

image = xl_pipe(
    prompt="A majestic mountain landscape, 4k photo",
    num_inference_steps=30,
    guidance_scale=5.0
).images[0]

from diffusers import StableDiffusionPipeline
import torch

pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16
).to("cuda")

pipe.load_lora_weights("path/to/lora/weights", weight_name="lora.safetensors")
pipe.fuse_lora(lora_scale=0.8)

image = pipe(
    "a photo of sks person in a fantasy world",
    num_inference_steps=30
).images[0]

import evaluate

# BLEU（翻译质量）
bleu = evaluate.load("bleu")
result = bleu.compute(
    predictions=["the cat is on the mat"],
    references=[["the cat is on the mat", "there is a cat on the mat"]]
)
print(f"BLEU: {result['bleu']:.4f}")

# ROUGE（摘要质量）
rouge = evaluate.load("rouge")
result = rouge.compute(
    predictions=["This is a very useful library."],
    references=["This is an extremely useful library."]
)
print(result)  # rouge1, rouge2, rougeL, rougeLsum

# BERTScore（语义相似度）
bertscore = evaluate.load("bertscore")
result = bertscore.compute(
    predictions=["AI is changing the world."],
    references=["Artificial intelligence is transforming the globe."],
    lang="en"
)
print(f"BERTScore F1: {result['f1'][0]:.4f}")

# Accuracy
accuracy = evaluate.load("accuracy")
result = accuracy.compute(predictions=[0, 1, 1, 0], references=[0, 1, 0, 0])
print(f"Accuracy: {result['accuracy']:.4f}")

# Perplexity
perplexity = evaluate.load("perplexity", module_type="metric")

from transformers import Trainer, TrainingArguments
import evaluate
import numpy as np

accuracy = evaluate.load("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return accuracy.compute(predictions=predictions, references=labels)

training_args = TrainingArguments(
    output_dir="./model-output",
    evaluation_strategy="epoch",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=32,
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    compute_metrics=compute_metrics,
)

from huggingface_hub import HfApi, hf_hub_download, snapshot_download
from huggingface_hub import create_repo, upload_file, upload_folder

api = HfApi()

# 下载单个文件
local_path = hf_hub_download(
    repo_id="meta-llama/Meta-Llama-3-8B",
    filename="config.json"
)

# 下载整个仓库
snapshot_download(
    repo_id="meta-llama/Meta-Llama-3-8B",
    local_dir="./llama3-8b",
    ignore_patterns=["*.bin", "*.pt"]  # 只下载 safetensors
)

# 创建仓库
create_repo("your-username/my-model", private=True)
create_repo("your-username/my-dataset", repo_type="dataset")
create_repo("your-username/my-space", repo_type="space", space_sdk="gradio")

# 上传文件
upload_file(
    path_or_fileobj="./my_model.bin",
    path_in_repo="pytorch_model.bin",
    repo_id="your-username/my-model"
)

# 上传文件夹
upload_folder(
    folder_path="./output-model",
    repo_id="your-username/my-model",
    ignore_patterns=["*.log", "__pycache__"]
)

# 查询仓库信息
model_info = api.model_info("meta-llama/Meta-Llama-3-8B")
print(model_info.card_data)
print(model_info.safetensors)

# 搜索模型
models = api.list_models(
    filter="text-generation",
    language="en",
    sort="downloads",
    limit=10
)
for m in models:
    print(m.id, m.downloads)

from huggingface_hub import ModelCard, ModelCardData

card_data = ModelCardData(
    language="en",
    license="apache-2.0",
    library_name="transformers",
    tags=["llama", "instruction-tuned", "fine-tuned"],
    datasets=["HuggingFaceH4/ultrachat_200k"],
    base_model="meta-llama/Meta-Llama-3-8B",
    metrics=[{"type": "accuracy", "value": 0.95}]
)

card = ModelCard.from_template(
    card_data,
    template_str="""
---
{{ card_data }}
---

# Llama-3-8B Instruct

This model is a fine-tuned version of Llama-3-8B for instruction following.

## Usage

```python
from transformers import pipeline
generator = pipeline("text-generation", model="your-username/llama3-instruct")
```

## Training Details

- Base model: meta-llama/Meta-Llama-3-8B
- Method: QLoRA (4-bit)
- Dataset: UltraChat 200k
  """
  )

card.push_to_hub("your-username/llama3-instruct")

from optimum.exporters.onnx import main_export

# 将模型转换为 ONNX 格式
main_export(
    model_name_or_path="bert-base-uncased",
    output="./bert-onnx",
    task="text-classification"
)

# 加载 ONNX 模型并推理
from optimum.onnxruntime import ORTModelForSequenceClassification
from transformers import AutoTokenizer

ort_model = ORTModelForSequenceClassification.from_pretrained("./bert-onnx")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")

inputs = tokenizer("This movie was fantastic!", return_tensors="pt")
outputs = ort_model(**inputs)
print(outputs.logits)

from optimum.bettertransformer import BetterTransformer
from transformers import AutoModelForSequenceClassification
import torch

model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")

# 转换为 BetterTransformer（利用 PyTorch 2.0+ 的优化）
model = BetterTransformer.transform(model)

# 推理（自动使用 Flash Attention 等优化）
model.eval()
with torch.no_grad():
    outputs = model(**inputs)

from datasets import load_dataset, concatenate_datasets

# 常见基准数据集
topic_ds = load_dataset("ag_news", split="train")                       # 主题分类
sts_ds = load_dataset("glue", "stsb", split="train")                    # 语义相似度
nli_ds = load_dataset("glue", "mnli", split="train")                    # 自然语言推理
ner_ds = load_dataset("conll2003", split="train")                       # 命名实体识别
re_ds = load_dataset("docred", split="train")                           # 关系抽取
dp_ds = load_dataset("universal_dependencies", "en_ewt", split="train") # 依存句法分析

# 指令数据集
alpaca = load_dataset("tatsu-lab/alpaca", split="train")

# 对话数据集
chat_ds = load_dataset("HuggingFaceH4/ultrachat_200k", split="train_sft")

def format_instruction_dataset(example):
    """将 Alpaca 格式转换为 ChatML 格式"""
    instruction = example["instruction"]
    inp = example.get("input", "").strip()
    output = example["output"]

    if inp:
        user_content = f"{instruction}\n\n{inp}"
    else:
        user_content = instruction

    messages = [
        {"role": "system", "content": "You are a helpful AI assistant."},
        {"role": "user", "content": user_content},
        {"role": "assistant", "content": output}
    ]
    return {"messages": messages}

formatted_dataset = alpaca.map(format_instruction_dataset)
print(formatted_dataset[0])

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, prepare_model_for_kbit_training, get_peft_model
from trl import SFTTrainer, SFTConfig
from datasets import load_dataset

# ─── 配置 ───────────────────────────────────────────────
MODEL_NAME = "meta-llama/Meta-Llama-3-8B"
OUTPUT_DIR = "./llama3-qlora"
MAX_SEQ_LEN = 2048
NUM_EPOCHS = 2
BATCH_SIZE = 2
GRAD_ACCUM = 8
LR = 2e-4

# ─── 4bit 量化 ─────────────────────────────────────────
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

# ─── 模型/分词器加载 ─────────────────────────────────
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"

model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    quantization_config=bnb_config,
    device_map="auto",
    attn_implementation="flash_attention_2"
)
model = prepare_model_for_kbit_training(model)

# ─── LoRA 配置 ───────────────────────────────────────────
peft_config = LoraConfig(
    r=64,
    lora_alpha=128,
    target_modules=[
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)

# ─── 数据集 ────────────────────────────────────────────
dataset = load_dataset("tatsu-lab/alpaca", split="train")

def format_sample(example):
    messages = [
        {"role": "system", "content": "You are a helpful AI assistant."},
        {"role": "user", "content": example["instruction"]},
        {"role": "assistant", "content": example["output"]}
    ]
    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=False)
    return {"text": text}

dataset = dataset.map(format_sample)

# ─── 训练配置 ───────────────────────────────────────────
training_args = SFTConfig(
    output_dir=OUTPUT_DIR,
    num_train_epochs=NUM_EPOCHS,
    per_device_train_batch_size=BATCH_SIZE,
    gradient_accumulation_steps=GRAD_ACCUM,
    learning_rate=LR,
    bf16=True,
    gradient_checkpointing=True,
    optim="paged_adamw_8bit",
    warmup_ratio=0.05,
    lr_scheduler_type="cosine",
    save_strategy="epoch",
    logging_steps=10,
    dataset_text_field="text",
    max_seq_length=MAX_SEQ_LEN,
    packing=True,
)

# ─── 训练 ────────────────────────────────────────────────
trainer = SFTTrainer(
    model=model,
    train_dataset=dataset,
    peft_config=peft_config,
    args=training_args,
    processing_class=tokenizer,
)

trainer.train()

# ─── 保存 ────────────────────────────────────────────────
trainer.model.save_pretrained(f"{OUTPUT_DIR}/adapter")
tokenizer.save_pretrained(f"{OUTPUT_DIR}/adapter")
print("Training complete!")

from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

base_model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3-8B",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
model = PeftModel.from_pretrained(base_model, "./llama3-qlora/adapter")
tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")

def chat(user_message: str, max_new_tokens: int = 512) -> str:
    messages = [
        {"role": "system", "content": "You are a helpful AI assistant."},
        {"role": "user", "content": user_message}
    ]
    prompt = tokenizer.apply_chat_template(
        messages,
        tokenize=False,
        add_generation_prompt=True
    )
    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=max_new_tokens,
            do_sample=True,
            temperature=0.7,
            top_p=0.9,
            repetition_penalty=1.1
        )

    return tokenizer.decode(
        outputs[0][inputs["input_ids"].shape[1]:],
        skip_special_tokens=True
    )

test_questions = [
    "Write a Python function to check if a number is prime.",
    "Explain the difference between machine learning and deep learning.",
    "What are the main benefits of using HuggingFace?",
]

for q in test_questions:
    print(f"Q: {q}")
    print(f"A: {chat(q)}")
    print("-" * 50)

import gradio as gr
from transformers import pipeline
import torch

pipe = pipeline(
    "text-generation",
    model="./llama3-qlora/merged",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

def respond(message, history, system_prompt, max_tokens, temperature):
    messages = [{"role": "system", "content": system_prompt}]
    for user, assistant in history:
        messages.append({"role": "user", "content": user})
        messages.append({"role": "assistant", "content": assistant})
    messages.append({"role": "user", "content": message})

    output = pipe(
        messages,
        max_new_tokens=max_tokens,
        temperature=temperature,
        do_sample=True
    )
    return output[0]["generated_text"][-1]["content"]

demo = gr.ChatInterface(
    respond,
    additional_inputs=[
        gr.Textbox("You are a helpful AI assistant.", label="System Prompt"),
        gr.Slider(50, 1000, 256, label="Max Tokens"),
        gr.Slider(0.1, 2.0, 0.7, label="Temperature")
    ],
    title="Llama-3 Chatbot",
    description="QLoRA fine-tuned LLM"
)

if __name__ == "__main__":
    demo.launch()