필사 모드: 开源 LLM 生态完全指南：2026 年的模型、工具与部署策略

Multi-Head Attention (MHA):
Q heads: 32    K heads: 32    V heads: 32
KV cache: 2 × 32 × seq_len × d_head

Grouped Query Attention (GQA):
Q heads: 32    K heads: 8    V heads: 8
KV cache: 2 × 8 × seq_len × d_head   (4× 减少！)

Multi-Query Attention (MQA):
Q heads: 32    K heads: 1    V heads: 1
KV cache: 2 × 1 × seq_len × d_head   (32× 减少，有一定品质损失)

# MoE 层的概念示意(简化版)
class MoELayer:
    def __init__(self, num_experts=8, top_k=2, d_model=4096, d_ff=14336):
        self.experts = [FeedForward(d_model, d_ff) for _ in range(num_experts)]
        self.router = nn.Linear(d_model, num_experts)
        self.top_k = top_k  # 每个 token 只激活 top_k 个专家

    def forward(self, x):
        # 路由器选出 top-k 个专家
        logits = self.router(x)
        weights, indices = logits.topk(self.top_k)
        weights = F.softmax(weights, dim=-1)

        # 对被选中专家的输出做加权求和
        output = sum(
            weights[:, i] * self.experts[indices[:, i]](x)
            for i in range(self.top_k)
        )
        return output

KV cache size = 2 × num_layers × num_kv_heads × head_dim × seq_len × dtype_bytes

示例：Llama 3.1 8B，fp16，128K 上下文
= 2 × 32 × 8 × 128 × 131072 × 2 bytes
= 约 17.2 GB(仅 KV 缓存)

# 安装(macOS/Linux)
curl -fsSL https://ollama.com/install.sh | sh

# 交互式运行模型
ollama run llama3.1:8b

# 运行特定量化版本
ollama run llama3.1:70b-instruct-q4_K_M

# 仅下载不运行
ollama pull mistral:7b

# 列出已安装的模型
ollama list

# 作为 API 服务器运行(兼容 OpenAI！)
ollama serve  # 在 localhost:11434 启动

from openai import OpenAI

# 连接本地 Ollama
client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama"  # 必填，但会被忽略
)

response = client.chat.completions.create(
    model="llama3.1:8b",
    messages=[{"role": "user", "content": "简单说明一下 Transformer。"}]
)
print(response.choices[0].message.content)

# 构建
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
cmake -B build -DLLAMA_CUDA=ON  # 支持 CUDA GPU
cmake --build build --config Release -j

# 下载 GGUF 模型(示例)
# 来自 Hugging Face：bartowski/Meta-Llama-3.1-8B-Instruct-GGUF

# 运行推理
./build/bin/llama-cli \
  -m models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  -n 512 \
  --prompt "说明一下 Transformer 中的注意力机制。"

# 作为兼容 OpenAI 的服务器运行
./build/bin/llama-server \
  -m models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  --host 0.0.0.0 \
  --port 8080 \
  -c 4096 \
  -ngl 35  # 加载到 GPU 上的层数

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",          # 自动分散到可用 GPU 上
)

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "什么是 Transformer 架构？"}
]

input_ids = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    return_tensors="pt"
).to(model.device)

with torch.no_grad():
    outputs = model.generate(
        input_ids,
        max_new_tokens=512,
        temperature=0.7,
        do_sample=True,
        pad_token_id=tokenizer.eos_token_id,
    )

response = tokenizer.decode(outputs[0][input_ids.shape[-1]:], skip_special_tokens=True)
print(response)

# 安装
pip install vllm

# 启动服务器(单 GPU)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-8B-Instruct \
  --dtype bfloat16 \
  --max-model-len 32768 \
  --port 8000

# 用张量并行做多 GPU(4 块 GPU)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-70B-Instruct \
  --dtype bfloat16 \
  --tensor-parallel-size 4 \
  --max-model-len 32768 \
  --port 8000

# 应用量化(AWQ)
python -m vllm.entrypoints.openai.api_server \
  --model TheBloke/Llama-3-70B-Instruct-AWQ \
  --quantization awq \
  --dtype float16 \
  --tensor-parallel-size 2

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed"
)

response = client.chat.completions.create(
    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "什么是 PagedAttention？"}],
    max_tokens=256,
    temperature=0.7,
)
print(response.choices[0].message.content)

# 流式输出
with client.chat.completions.stream(
    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "详细说明一下 RAG。"}],
) as stream:
    for text in stream.text_stream:
        print(text, end="", flush=True)

# Dockerfile
FROM vllm/vllm-openai:latest

ENV HUGGING_FACE_HUB_TOKEN=""

CMD ["--model", "meta-llama/Meta-Llama-3.1-8B-Instruct", \
     "--dtype", "bfloat16", \
     "--max-model-len", "16384"]

docker run --gpus all \
  -p 8000:8000 \
  -e HUGGING_FACE_HUB_TOKEN=hf_xxx \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  my-vllm-server

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

# LoRA 配置
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,               # 适配矩阵的秩
    lora_alpha=32,      # 缩放因子
    lora_dropout=0.05,
    target_modules=[    # 要适配的层
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    bias="none",
)

peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 83,886,080 || all params: 8,114,474,240 || trainable%: 1.03%

from transformers import BitsAndBytesConfig
from peft import prepare_model_for_kbit_training

# 4 比特量化配置
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto"
)

# 为 k 比特训练做准备(将层归一化转为 fp32 等)
model = prepare_model_for_kbit_training(model)

# 在量化后的模型上应用 LoRA
peft_model = get_peft_model(model, lora_config)

from trl import SFTTrainer, SFTConfig
from datasets import load_dataset

dataset = load_dataset("your-org/your-dataset", split="train")

def format_example(example):
    return {
        "text": f"<|user|>\n{example['instruction']}\n<|assistant|>\n{example['output']}"
    }

dataset = dataset.map(format_example)

trainer = SFTTrainer(
    model=peft_model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=SFTConfig(
        output_dir="./checkpoints",
        num_train_epochs=3,
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        warmup_ratio=0.03,
        learning_rate=2e-4,
        fp16=True,
        logging_steps=10,
        save_steps=100,
        dataset_text_field="text",
        max_seq_length=2048,
    ),
)

trainer.train()
trainer.model.save_pretrained("./fine-tuned-model")

from peft import PeftModel

# 加载基础模型
base_model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="cpu"
)

# 加载并合并 LoRA 权重
model = PeftModel.from_pretrained(base_model, "./fine-tuned-model")
merged_model = model.merge_and_unload()

# 保存合并后的模型(标准 HF 格式，可配合 vLLM 等使用)
merged_model.save_pretrained("./merged-model")
tokenizer.save_pretrained("./merged-model")

from huggingface_hub import list_models, model_info

# 搜索模型
models = list_models(
    filter="text-generation",
    sort="downloads",
    direction=-1,
    limit=10
)
for m in models:
    print(m.id, m.downloads)

# 查看模型信息
info = model_info("meta-llama/Meta-Llama-3.1-8B-Instruct")
print(info.tags)
print(info.cardData)

Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
│                            │
└─ 模型名                    └─ 量化方式

常见的量化后缀：
Q4_K_M  - 4 比特，品质/速度的均衡选择(多数场景下推荐)
Q5_K_M  - 5 比特，品质更高，占用更多内存
Q6_K    - 6 比特，接近无损，内存占用较高
Q8_0    - 8 比特，几乎无损
Q2_K    - 2 比特，品质很低，内存占用极小
IQ4_XS  - 4 比特"重要性量化"，品质优于 Q4_K_M

# 通过 API 调用 Space
from gradio_client import Client

client = Client("meta-llama/Llama-3.1-8B-Instruct")
result = client.predict(
    message="说明一下 Transformer 的注意力机制",
    api_name="/chat"
)
print(result)

from transformers import AutoModelForCausalLM, GPTQConfig

# 一次性量化(需要校准数据)
quantization_config = GPTQConfig(
    bits=4,
    dataset="c4",
    block_size=128
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3.1-8B-Instruct",
    quantization_config=quantization_config,
    device_map="auto"
)

model.save_pretrained("./llama-3.1-8b-gptq")

# 安装
pip install autoawq

# 量化
python -c "
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = 'meta-llama/Meta-Llama-3.1-8B-Instruct'
quant_path = './llama-3.1-8b-awq'

model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)

quant_config = {'zero_point': True, 'q_group_size': 128, 'w_bit': 4, 'version': 'GEMM'}
model.quantize(tokenizer, quant_config=quant_config)

model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)
print('量化完成')
"

# 已在上面的 QLoRA 小节中展示
# 使用 load_in_4bit=True 的 BitsAndBytesConfig
# NF4 量化最适合 QLoRA 微调

                 ┌──────────────────────┐
                 │      你的任务是？      │
                 └──────────┬───────────┘
                            │
          ┌─────────────────┼─────────────────┐
          ▼                 ▼                 ▼
      代码生成            多语言处理          通用对话
          │                │                  │
    Qwen2.5-Coder     Qwen2.5 / Llama    Llama 3.1 8B
    DeepSeek-Coder    / Mistral NeMo     Mistral 7B
    CodeGemma                            Gemma 2 9B

          │
          ▼
      是否需要
      本地运行？
          │
    ┌─────┴─────┐
    │是         │否
    ▼           ▼
 内存够吗？   使用 vLLM
    │       + Llama 70B
  ┌─┴──┐    或 405B
  │    │
 <=8GB >8GB
  │    │
 8B   70B
 Q4   Q4_K_M

def route_request(request: dict) -> str:
    """根据需求把请求路由到开源或闭源模型。"""

    # 敏感数据始终使用开源
    if request.get("contains_pii") or request.get("confidential"):
        return "local_llama_70b"

    # 大规模的简单任务使用开源
    if request.get("task_type") in ["classification", "extraction", "summarization"]:
        if request.get("volume") == "high":
            return "local_llama_8b"

    # 复杂推理或多模态使用闭源
    if request.get("requires_vision") or request.get("complexity") == "high":
        return "gpt_4o"

    # 默认：为控制成本使用本地模型
    return "local_llama_70b"