Open-Source LLM Landscape Guide: Models, Tools, and Deployment in 2026

Multi-Head Attention (MHA):
Q heads: 32    K heads: 32    V heads: 32
KV cache: 2 × 32 × seq_len × d_head

Grouped Query Attention (GQA):
Q heads: 32    K heads: 8    V heads: 8
KV cache: 2 × 8 × seq_len × d_head   (4× reduction!)

Multi-Query Attention (MQA):
Q heads: 32    K heads: 1    V heads: 1
KV cache: 2 × 1 × seq_len × d_head   (32× reduction, some quality loss)

# Simplified MoE layer concept
class MoELayer:
    def __init__(self, num_experts=8, top_k=2, d_model=4096, d_ff=14336):
        self.experts = [FeedForward(d_model, d_ff) for _ in range(num_experts)]
        self.router = nn.Linear(d_model, num_experts)
        self.top_k = top_k  # Only activate top_k experts per token

    def forward(self, x):
        # Router selects top-k experts
        logits = self.router(x)
        weights, indices = logits.topk(self.top_k)
        weights = F.softmax(weights, dim=-1)

        # Weighted sum of selected expert outputs
        output = sum(
            weights[:, i] * self.experts[indices[:, i]](x)
            for i in range(self.top_k)
        )
        return output

KV cache size = 2 × num_layers × num_kv_heads × head_dim × seq_len × dtype_bytes

Example: Llama 3.1 8B, fp16, 128K context
= 2 × 32 × 8 × 128 × 131072 × 2 bytes
= ~17.2 GB just for KV cache

# Install (macOS/Linux)
curl -fsSL https://ollama.com/install.sh | sh

# Run a model interactively
ollama run llama3.1:8b

# Run a specific quantization
ollama run llama3.1:70b-instruct-q4_K_M

# Pull without running
ollama pull mistral:7b

# List installed models
ollama list

# Run as API server (OpenAI-compatible!)
ollama serve  # starts on localhost:11434

from openai import OpenAI

# Point at local Ollama
client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama"  # required but ignored
)

response = client.chat.completions.create(
    model="llama3.1:8b",
    messages=[{"role": "user", "content": "Explain transformers briefly."}]
)
print(response.choices[0].message.content)

# Build
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp
cmake -B build -DLLAMA_CUDA=ON  # CUDA GPU support
cmake --build build --config Release -j

# Download a GGUF model (example)
# From Hugging Face: bartowski/Meta-Llama-3.1-8B-Instruct-GGUF

# Run inference
./build/bin/llama-cli \
  -m models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  -n 512 \
  --prompt "Explain the attention mechanism in transformers."

# Run as OpenAI-compatible server
./build/bin/llama-server \
  -m models/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf \
  --host 0.0.0.0 \
  --port 8080 \
  -c 4096 \
  -ngl 35  # layers on GPU

from transformers import AutoTokenizer, AutoModelForCausalLM
import torch

model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"

tokenizer = AutoTokenizer.from_pretrained(model_id)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",          # auto-splits across available GPUs
)

messages = [
    {"role": "system", "content": "You are a helpful assistant."},
    {"role": "user", "content": "What is the transformer architecture?"}
]

input_ids = tokenizer.apply_chat_template(
    messages,
    add_generation_prompt=True,
    return_tensors="pt"
).to(model.device)

with torch.no_grad():
    outputs = model.generate(
        input_ids,
        max_new_tokens=512,
        temperature=0.7,
        do_sample=True,
        pad_token_id=tokenizer.eos_token_id,
    )

response = tokenizer.decode(outputs[0][input_ids.shape[-1]:], skip_special_tokens=True)
print(response)

# Install
pip install vllm

# Start server (single GPU)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-8B-Instruct \
  --dtype bfloat16 \
  --max-model-len 32768 \
  --port 8000

# Multi-GPU with tensor parallelism (4 GPUs)
python -m vllm.entrypoints.openai.api_server \
  --model meta-llama/Meta-Llama-3.1-70B-Instruct \
  --dtype bfloat16 \
  --tensor-parallel-size 4 \
  --max-model-len 32768 \
  --port 8000

# With quantization (AWQ)
python -m vllm.entrypoints.openai.api_server \
  --model TheBloke/Llama-3-70B-Instruct-AWQ \
  --quantization awq \
  --dtype float16 \
  --tensor-parallel-size 2

from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:8000/v1",
    api_key="not-needed"
)

response = client.chat.completions.create(
    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "What is PagedAttention?"}],
    max_tokens=256,
    temperature=0.7,
)
print(response.choices[0].message.content)

# Streaming
with client.chat.completions.stream(
    model="meta-llama/Meta-Llama-3.1-8B-Instruct",
    messages=[{"role": "user", "content": "Explain RAG in detail."}],
) as stream:
    for text in stream.text_stream:
        print(text, end="", flush=True)

# Dockerfile
FROM vllm/vllm-openai:latest

ENV HUGGING_FACE_HUB_TOKEN=""

CMD ["--model", "meta-llama/Meta-Llama-3.1-8B-Instruct", \
     "--dtype", "bfloat16", \
     "--max-model-len", "16384"]

docker run --gpus all \
  -p 8000:8000 \
  -e HUGGING_FACE_HUB_TOKEN=hf_xxx \
  -v ~/.cache/huggingface:/root/.cache/huggingface \
  my-vllm-server

from peft import LoraConfig, get_peft_model, TaskType
from transformers import AutoModelForCausalLM, AutoTokenizer
import torch

model_id = "meta-llama/Meta-Llama-3.1-8B-Instruct"
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto"
)

# LoRA configuration
lora_config = LoraConfig(
    task_type=TaskType.CAUSAL_LM,
    r=16,               # Rank of adaptation matrices
    lora_alpha=32,      # Scaling factor
    lora_dropout=0.05,
    target_modules=[    # Which layers to adapt
        "q_proj", "k_proj", "v_proj", "o_proj",
        "gate_proj", "up_proj", "down_proj"
    ],
    bias="none",
)

peft_model = get_peft_model(model, lora_config)
peft_model.print_trainable_parameters()
# trainable params: 83,886,080 || all params: 8,114,474,240 || trainable%: 1.03%

from transformers import BitsAndBytesConfig
from peft import prepare_model_for_kbit_training

# 4-bit quantization config
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16
)

model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto"
)

# Prepare for k-bit training (casts layer norms to fp32, etc.)
model = prepare_model_for_kbit_training(model)

# Apply LoRA on top of quantized model
peft_model = get_peft_model(model, lora_config)

from trl import SFTTrainer, SFTConfig
from datasets import load_dataset

dataset = load_dataset("your-org/your-dataset", split="train")

def format_example(example):
    return {
        "text": f"<|user|>\n{example['instruction']}\n<|assistant|>\n{example['output']}"
    }

dataset = dataset.map(format_example)

trainer = SFTTrainer(
    model=peft_model,
    tokenizer=tokenizer,
    train_dataset=dataset,
    args=SFTConfig(
        output_dir="./checkpoints",
        num_train_epochs=3,
        per_device_train_batch_size=4,
        gradient_accumulation_steps=4,
        warmup_ratio=0.03,
        learning_rate=2e-4,
        fp16=True,
        logging_steps=10,
        save_steps=100,
        dataset_text_field="text",
        max_seq_length=2048,
    ),
)

trainer.train()
trainer.model.save_pretrained("./fine-tuned-model")

from peft import PeftModel

# Load base model
base_model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="cpu"
)

# Load and merge LoRA weights
model = PeftModel.from_pretrained(base_model, "./fine-tuned-model")
merged_model = model.merge_and_unload()

# Save merged model (standard HF format, works with vLLM etc.)
merged_model.save_pretrained("./merged-model")
tokenizer.save_pretrained("./merged-model")

from huggingface_hub import list_models, model_info

# Search for models
models = list_models(
    filter="text-generation",
    sort="downloads",
    direction=-1,
    limit=10
)
for m in models:
    print(m.id, m.downloads)

# Get model info
info = model_info("meta-llama/Meta-Llama-3.1-8B-Instruct")
print(info.tags)
print(info.cardData)

Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
│                            │
└─ Model name                └─ Quantization method

Common quantization suffixes:
Q4_K_M  - 4-bit, medium quality/speed tradeoff (recommended for most use cases)
Q5_K_M  - 5-bit, better quality, more memory
Q6_K    - 6-bit, near-lossless, high memory
Q8_0    - 8-bit, virtually lossless
Q2_K    - 2-bit, very low quality, tiny memory footprint
IQ4_XS  - 4-bit with "importance quantization", better quality than Q4_K_M

# Call a Space via API
from gradio_client import Client

client = Client("meta-llama/Llama-3.1-8B-Instruct")
result = client.predict(
    message="Explain the transformer attention mechanism",
    api_name="/chat"
)
print(result)

from transformers import AutoModelForCausalLM, GPTQConfig

# One-time quantization (requires calibration data)
quantization_config = GPTQConfig(
    bits=4,
    dataset="c4",
    block_size=128
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Meta-Llama-3.1-8B-Instruct",
    quantization_config=quantization_config,
    device_map="auto"
)

model.save_pretrained("./llama-3.1-8b-gptq")

# Install
pip install autoawq

# Quantize
python -c "
from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = 'meta-llama/Meta-Llama-3.1-8B-Instruct'
quant_path = './llama-3.1-8b-awq'

model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)

quant_config = {'zero_point': True, 'q_group_size': 128, 'w_bit': 4, 'version': 'GEMM'}
model.quantize(tokenizer, quant_config=quant_config)

model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)
print('Quantization complete')
"

# Already shown in the QLoRA section above
# BitsAndBytesConfig with load_in_4bit=True
# NF4 quantization is best for QLoRA fine-tuning

                 ┌──────────────────────┐
                 │  What is your task?  │
                 └──────────┬───────────┘
                            │
          ┌─────────────────┼─────────────────┐
          ▼                 ▼                 ▼
    Code generation    Multilingual       General chat
          │                │                  │
    Qwen2.5-Coder     Qwen2.5 / Llama    Llama 3.1 8B
    DeepSeek-Coder    / Mistral NeMo     Mistral 7B
    CodeGemma                            Gemma 2 9B

          │
          ▼
   Do you need to
   run locally?
          │
    ┌─────┴─────┐
    │Yes        │No
    ▼           ▼
 Fit in RAM?  Use vLLM
    │         + Llama 70B
  ┌─┴──┐      or 405B
  │    │
 <=8GB >8GB
  │    │
 8B   70B
 Q4   Q4_K_M

def route_request(request: dict) -> str:
    """Route to open-source or proprietary based on requirements."""

    # Always use open-source for sensitive data
    if request.get("contains_pii") or request.get("confidential"):
        return "local_llama_70b"

    # Use open-source for high-volume simple tasks
    if request.get("task_type") in ["classification", "extraction", "summarization"]:
        if request.get("volume") == "high":
            return "local_llama_8b"

    # Use proprietary for complex reasoning or multimodal
    if request.get("requires_vision") or request.get("complexity") == "high":
        return "gpt_4o"

    # Default: local model for cost control
    return "local_llama_70b"