LLM Evaluation Production Guide: From MMLU Benchmarks to Custom Evaluation Pipelines

# Example: How HumanEval pass@k is calculated
import numpy as np
from typing import List

def estimate_pass_at_k(
    num_samples: int,
    num_correct: int,
    k: int
) -> float:
    """
    Estimates pass@k using the unbiased estimator from Chen et al. (2021).

    This avoids the high-variance naive estimator (sampling k and checking).
    Instead, it computes: 1 - C(n-c, k) / C(n, k)
    where n = num_samples, c = num_correct.
    """
    if num_correct == 0:
        return 0.0
    if num_samples - num_correct < k:
        return 1.0

    # Use logarithms for numerical stability
    log_numerator = sum(
        np.log(num_samples - num_correct - i) for i in range(k)
    )
    log_denominator = sum(
        np.log(num_samples - i) for i in range(k)
    )
    return 1.0 - np.exp(log_numerator - log_denominator)

# Example: model got 120 correct out of 164 problems, k=1
score = estimate_pass_at_k(num_samples=164, num_correct=120, k=1)
print(f"pass@1: {score:.4f}")  # pass@1: 0.7317

+------------------+     +-------------------+     +------------------+
|   Eval Dataset   |     |   Model Registry  |     |  Eval Config     |
|  (versioned,     |---->|  (model versions, |---->|  (metrics, k,    |
|   immutable)     |     |   endpoints)      |     |   thresholds)    |
+------------------+     +-------------------+     +------------------+
         |                        |                        |
         v                        v                        v
+--------------------------------------------------------------+
|                    Eval Orchestrator                          |
|  - Loads dataset splits                                      |
|  - Dispatches inference requests (async, rate-limited)       |
|  - Collects raw outputs                                      |
|  - Routes to appropriate graders                             |
+--------------------------------------------------------------+
         |                        |                        |
         v                        v                        v
+------------------+  +-------------------+  +------------------+
|  Exact Match     |  |  LLM-as-Judge     |  |  Code Execution  |
|  Grader          |  |  Grader           |  |  Grader          |
+------------------+  +-------------------+  +------------------+
         |                        |                        |
         v                        v                        v
+--------------------------------------------------------------+
|                    Results Aggregator                         |
|  - Per-category scores                                       |
|  - Confidence intervals                                      |
|  - Statistical significance tests                            |
|  - Regression detection                                      |
+--------------------------------------------------------------+
         |                        |
         v                        v
+------------------+     +-------------------+
|  Dashboard /     |     |  CI/CD Gate       |
|  Alerting        |     |  (pass/fail)      |
+------------------+     +-------------------+

import json
import hashlib
from dataclasses import dataclass, field, asdict
from typing import List, Optional
from datetime import datetime

@dataclass
class EvalSample:
    """A single evaluation example."""
    id: str
    input_prompt: str
    expected_output: str
    category: str
    difficulty: str  # "easy", "medium", "hard"
    metadata: dict = field(default_factory=dict)

@dataclass
class EvalDataset:
    """Versioned, immutable evaluation dataset."""
    name: str
    version: str
    created_at: str
    samples: List[EvalSample]
    description: str = ""

    def __post_init__(self):
        self._checksum = self._compute_checksum()

    def _compute_checksum(self) -> str:
        content = json.dumps(
            [asdict(s) for s in self.samples],
            sort_keys=True
        )
        return hashlib.sha256(content.encode()).hexdigest()[:12]

    @property
    def checksum(self) -> str:
        return self._checksum

    def validate_integrity(self, expected_checksum: str) -> bool:
        return self._checksum == expected_checksum

    def split_by_category(self) -> dict:
        categories = {}
        for sample in self.samples:
            categories.setdefault(sample.category, []).append(sample)
        return categories

    def save(self, path: str):
        data = {
            "name": self.name,
            "version": self.version,
            "created_at": self.created_at,
            "description": self.description,
            "checksum": self.checksum,
            "sample_count": len(self.samples),
            "samples": [asdict(s) for s in self.samples],
        }
        with open(path, "w") as f:
            json.dump(data, f, indent=2)

# Example: Create an evaluation dataset for a customer support bot
samples = [
    EvalSample(
        id="cs-001",
        input_prompt="Customer says: 'My order #12345 hasn't arrived.' "
                     "Generate a support response.",
        expected_output="apologize, look up order status, provide ETA",
        category="order_inquiry",
        difficulty="easy",
    ),
    EvalSample(
        id="cs-002",
        input_prompt="Customer says: 'I want a refund AND a replacement.' "
                     "Generate a support response following policy.",
        expected_output="explain policy (refund OR replacement, not both), "
                       "offer alternatives",
        category="refund_policy",
        difficulty="hard",
    ),
]

dataset = EvalDataset(
    name="customer_support_eval",
    version="1.0.0",
    created_at=datetime.now().isoformat(),
    samples=samples,
    description="Evaluation set for customer support chatbot v2",
)
print(f"Dataset: {dataset.name} v{dataset.version}")
print(f"Checksum: {dataset.checksum}")
print(f"Categories: {list(dataset.split_by_category().keys())}")

import re
import asyncio
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, Optional

@dataclass
class GradeResult:
    score: float  # 0.0 to 1.0
    passed: bool
    reasoning: str
    grader_type: str
    metadata: dict = None

class BaseGrader(ABC):
    """Base class for all graders."""

    @abstractmethod
    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        pass

class ExactMatchGrader(BaseGrader):
    """For tasks with deterministic correct answers."""

    def __init__(self, normalize: bool = True):
        self.normalize = normalize

    def _normalize(self, text: str) -> str:
        text = text.strip().lower()
        text = re.sub(r'\s+', ' ', text)
        return text

    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        if self.normalize:
            expected = self._normalize(expected)
            actual = self._normalize(actual)
        match = expected == actual
        return GradeResult(
            score=1.0 if match else 0.0,
            passed=match,
            reasoning=f"Exact match: {match}",
            grader_type="exact_match",
        )

class LLMJudgeGrader(BaseGrader):
    """Uses a strong LLM to judge response quality."""

    JUDGE_PROMPT_TEMPLATE = """You are an expert evaluator. Score the following
response on a scale of 1-5 based on these criteria:
- Accuracy: Does it contain correct information?
- Completeness: Does it address all parts of the question?
- Helpfulness: Is it actionable and useful?

Question: {prompt}
Expected answer should cover: {expected}
Actual response: {actual}

Respond with ONLY a JSON object:
{{"score": <1-5>, "reasoning": "<brief explanation>"}}"""

    def __init__(self, judge_client, judge_model: str = "claude-sonnet-4-20250514"):
        self.client = judge_client
        self.model = judge_model

    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        judge_prompt = self.JUDGE_PROMPT_TEMPLATE.format(
            prompt=prompt, expected=expected, actual=actual
        )
        response = await self.client.messages.create(
            model=self.model,
            max_tokens=256,
            messages=[{"role": "user", "content": judge_prompt}],
        )
        result = json.loads(response.content[0].text)
        normalized_score = result["score"] / 5.0
        return GradeResult(
            score=normalized_score,
            passed=normalized_score >= 0.6,
            reasoning=result["reasoning"],
            grader_type="llm_judge",
            metadata={"judge_model": self.model, "raw_score": result["score"]},
        )

class CodeExecutionGrader(BaseGrader):
    """Grades code generation by running test cases."""

    def __init__(self, timeout_seconds: int = 10):
        self.timeout = timeout_seconds

    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        test_code = f"{actual}\n\n{expected}"
        try:
            proc = await asyncio.create_subprocess_exec(
                "python", "-c", test_code,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            stdout, stderr = await asyncio.wait_for(
                proc.communicate(), timeout=self.timeout
            )
            passed = proc.returncode == 0
            return GradeResult(
                score=1.0 if passed else 0.0,
                passed=passed,
                reasoning=stderr.decode() if not passed else "All tests passed",
                grader_type="code_execution",
            )
        except asyncio.TimeoutError:
            return GradeResult(
                score=0.0,
                passed=False,
                reasoning=f"Execution timed out after {self.timeout}s",
                grader_type="code_execution",
            )

import asyncio
import time
from dataclasses import dataclass, field
from typing import Dict, List

@dataclass
class EvalConfig:
    model_id: str
    model_endpoint: str
    dataset_path: str
    grader_type: str  # "exact_match", "llm_judge", "code_execution"
    max_concurrent: int = 10
    timeout_per_request: int = 30
    temperature: float = 0.0
    max_tokens: int = 1024

@dataclass
class EvalResult:
    sample_id: str
    category: str
    grade: GradeResult
    latency_ms: float
    token_count: int = 0

class EvalOrchestrator:
    """Orchestrates the evaluation pipeline with rate limiting."""

    def __init__(self, config: EvalConfig, client, grader: BaseGrader):
        self.config = config
        self.client = client
        self.grader = grader
        self.semaphore = asyncio.Semaphore(config.max_concurrent)
        self.results: List[EvalResult] = []

    async def evaluate_sample(self, sample: EvalSample) -> EvalResult:
        async with self.semaphore:
            start = time.monotonic()
            response = await self.client.messages.create(
                model=self.config.model_id,
                max_tokens=self.config.max_tokens,
                temperature=self.config.temperature,
                messages=[{"role": "user", "content": sample.input_prompt}],
            )
            latency_ms = (time.monotonic() - start) * 1000
            actual_output = response.content[0].text
            grade = await self.grader.grade(
                sample.input_prompt,
                sample.expected_output,
                actual_output,
            )
            return EvalResult(
                sample_id=sample.id,
                category=sample.category,
                grade=grade,
                latency_ms=latency_ms,
                token_count=response.usage.output_tokens,
            )

    async def run(self, dataset: EvalDataset) -> Dict:
        tasks = [
            self.evaluate_sample(sample) for sample in dataset.samples
        ]
        self.results = await asyncio.gather(*tasks, return_exceptions=True)

        # Filter out exceptions
        valid = [r for r in self.results if isinstance(r, EvalResult)]
        errors = [r for r in self.results if isinstance(r, Exception)]

        # Aggregate by category
        by_category = {}
        for r in valid:
            by_category.setdefault(r.category, []).append(r)

        summary = {
            "model": self.config.model_id,
            "total_samples": len(dataset.samples),
            "completed": len(valid),
            "errors": len(errors),
            "overall_score": (
                sum(r.grade.score for r in valid) / len(valid)
                if valid else 0
            ),
            "overall_pass_rate": (
                sum(1 for r in valid if r.grade.passed) / len(valid)
                if valid else 0
            ),
            "avg_latency_ms": (
                sum(r.latency_ms for r in valid) / len(valid)
                if valid else 0
            ),
            "by_category": {
                cat: {
                    "score": sum(r.grade.score for r in rs) / len(rs),
                    "pass_rate": sum(
                        1 for r in rs if r.grade.passed
                    ) / len(rs),
                    "count": len(rs),
                }
                for cat, rs in by_category.items()
            },
        }
        return summary

import numpy as np
from scipy import stats
from dataclasses import dataclass
from typing import List, Tuple

@dataclass
class ComparisonResult:
    model_a: str
    model_b: str
    mean_a: float
    mean_b: float
    difference: float
    p_value: float
    confidence_interval: Tuple[float, float]
    significant: bool
    effect_size: float  # Cohen's d
    test_used: str
    n_samples: int

def compare_models(
    scores_a: List[float],
    scores_b: List[float],
    model_a_name: str = "Model A",
    model_b_name: str = "Model B",
    alpha: float = 0.05,
) -> ComparisonResult:
    """
    Paired comparison of two models on the same evaluation set.
    Uses paired t-test when assumptions hold, Wilcoxon signed-rank otherwise.
    """
    a = np.array(scores_a)
    b = np.array(scores_b)
    assert len(a) == len(b), "Both models must be evaluated on identical samples"

    differences = a - b
    n = len(differences)

    # Check normality of differences (Shapiro-Wilk test)
    if n >= 20:
        _, normality_p = stats.shapiro(differences)
    else:
        normality_p = 0  # Too few samples, use non-parametric

    if normality_p > 0.05:
        # Paired t-test
        t_stat, p_value = stats.ttest_rel(a, b)
        test_name = "paired_t_test"
    else:
        # Wilcoxon signed-rank test (non-parametric alternative)
        try:
            w_stat, p_value = stats.wilcoxon(differences)
            test_name = "wilcoxon_signed_rank"
        except ValueError:
            # All differences are zero
            p_value = 1.0
            test_name = "wilcoxon_signed_rank (degenerate)"

    # Effect size (Cohen's d for paired samples)
    diff_std = np.std(differences, ddof=1)
    effect_size = np.mean(differences) / diff_std if diff_std > 0 else 0.0

    # Bootstrap confidence interval for the mean difference
    rng = np.random.default_rng(42)
    boot_means = []
    for _ in range(10000):
        boot_sample = rng.choice(differences, size=n, replace=True)
        boot_means.append(np.mean(boot_sample))
    ci_lower = np.percentile(boot_means, 100 * alpha / 2)
    ci_upper = np.percentile(boot_means, 100 * (1 - alpha / 2))

    return ComparisonResult(
        model_a=model_a_name,
        model_b=model_b_name,
        mean_a=float(np.mean(a)),
        mean_b=float(np.mean(b)),
        difference=float(np.mean(differences)),
        p_value=float(p_value),
        confidence_interval=(float(ci_lower), float(ci_upper)),
        significant=p_value < alpha,
        effect_size=float(effect_size),
        test_used=test_name,
        n_samples=n,
    )

# Usage example
np.random.seed(42)
model_a_scores = np.random.binomial(1, 0.82, size=500).astype(float)
model_b_scores = np.random.binomial(1, 0.78, size=500).astype(float)

result = compare_models(
    model_a_scores.tolist(),
    model_b_scores.tolist(),
    "Claude 3.5 Sonnet",
    "GPT-4o",
)

print(f"Comparison: {result.model_a} vs {result.model_b}")
print(f"  Scores: {result.mean_a:.4f} vs {result.mean_b:.4f}")
print(f"  Difference: {result.difference:+.4f}")
print(f"  p-value: {result.p_value:.4f} ({result.test_used})")
print(f"  95% CI: [{result.confidence_interval[0]:+.4f}, "
      f"{result.confidence_interval[1]:+.4f}]")
print(f"  Cohen's d: {result.effect_size:.3f}")
print(f"  Significant: {result.significant}")

from itertools import combinations

def compare_multiple_models(
    model_scores: Dict[str, List[float]],
    alpha: float = 0.05,
) -> List[ComparisonResult]:
    """
    Compare all pairs of models with Bonferroni correction.
    """
    model_names = list(model_scores.keys())
    pairs = list(combinations(model_names, 2))
    n_comparisons = len(pairs)

    # Bonferroni-corrected alpha
    corrected_alpha = alpha / n_comparisons

    results = []
    for name_a, name_b in pairs:
        result = compare_models(
            model_scores[name_a],
            model_scores[name_b],
            name_a,
            name_b,
            alpha=corrected_alpha,
        )
        results.append(result)

    print(f"Bonferroni-corrected alpha: {corrected_alpha:.4f} "
          f"({n_comparisons} comparisons)")
    for r in results:
        sig_marker = "*" if r.significant else ""
        print(f"  {r.model_a} vs {r.model_b}: "
              f"diff={r.difference:+.4f}, "
              f"p={r.p_value:.4f}{sig_marker}")

    return results

# .github/workflows/llm-eval.yml
name: LLM Evaluation Pipeline

on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'model_config/**'
  schedule:
    - cron: '0 6 * * 1' # Weekly Monday 6AM UTC

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: pip install -r requirements-eval.txt

      - name: Run evaluation suite
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          python -m eval.run \
            --dataset eval/datasets/production_v2.json \
            --model claude-sonnet-4-20250514 \
            --output results/latest.json \
            --threshold 0.85

      - name: Compare with baseline
        run: |
          python -m eval.compare \
            --baseline results/baseline.json \
            --current results/latest.json \
            --alpha 0.05

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: results/

      - name: Gate check
        run: |
          python -m eval.gate \
            --results results/latest.json \
            --min-score 0.85 \
            --max-regression 0.02