LLM評価プロダクションガイド：MMLUベンチマークからカスタム評価パイプラインまで

# 例：HumanEval pass@k の計算方法
import numpy as np
from typing import List

def estimate_pass_at_k(
    num_samples: int,
    num_correct: int,
    k: int
) -> float:
    """
    Chen et al. (2021)の不偏推定量を使用してpass@kを推定します。

    高分散のナイーブ推定量（k個をサンプリングして確認）を回避します。
    代わりに以下を計算します: 1 - C(n-c, k) / C(n, k)
    ここで n = num_samples, c = num_correct
    """
    if num_correct == 0:
        return 0.0
    if num_samples - num_correct < k:
        return 1.0

    # 数値安定性のため対数を使用
    log_numerator = sum(
        np.log(num_samples - num_correct - i) for i in range(k)
    )
    log_denominator = sum(
        np.log(num_samples - i) for i in range(k)
    )
    return 1.0 - np.exp(log_numerator - log_denominator)

# 例：164問中120問が正解の場合、k=1
score = estimate_pass_at_k(num_samples=164, num_correct=120, k=1)
print(f"pass@1: {score:.4f}")  # pass@1: 0.7317

+------------------+     +-------------------+     +------------------+
|   評価データセット  |     |  モデルレジストリ    |     |  評価設定          |
|  (バージョン管理,  |---->|  (モデルバージョン,  |---->|  (メトリクス, k,  |
|   不変)           |     |   エンドポイント)    |     |   閾値)           |
+------------------+     +-------------------+     +------------------+
         |                        |                        |
         v                        v                        v
+--------------------------------------------------------------+
|                    評価オーケストレーター                       |
|  - データセットの分割をロード                                  |
|  - 推論リクエストの非同期・レート制限付きディスパッチ              |
|  - 生の出力を収集                                            |
|  - 適切なグレーダーにルーティング                               |
+--------------------------------------------------------------+
         |                        |                        |
         v                        v                        v
+------------------+  +-------------------+  +------------------+
|  完全一致         |  |  LLMジャッジ       |  |  コード実行       |
|  グレーダー       |  |  グレーダー        |  |  グレーダー       |
+------------------+  +-------------------+  +------------------+
         |                        |                        |
         v                        v                        v
+--------------------------------------------------------------+
|                    結果集約エンジン                            |
|  - カテゴリ別スコア                                          |
|  - 信頼区間                                                 |
|  - 統計的有意性検定                                          |
|  - リグレッション検出                                        |
+--------------------------------------------------------------+
         |                        |
         v                        v
+------------------+     +-------------------+
|  ダッシュボード /  |     |  CI/CDゲート       |
|  アラート         |     |  (合格/不合格)      |
+------------------+     +-------------------+

import json
import hashlib
from dataclasses import dataclass, field, asdict
from typing import List, Optional
from datetime import datetime

@dataclass
class EvalSample:
    """単一の評価サンプル"""
    id: str
    input_prompt: str
    expected_output: str
    category: str
    difficulty: str  # "easy", "medium", "hard"
    metadata: dict = field(default_factory=dict)

@dataclass
class EvalDataset:
    """バージョン管理された不変の評価データセット"""
    name: str
    version: str
    created_at: str
    samples: List[EvalSample]
    description: str = ""

    def __post_init__(self):
        self._checksum = self._compute_checksum()

    def _compute_checksum(self) -> str:
        content = json.dumps(
            [asdict(s) for s in self.samples],
            sort_keys=True
        )
        return hashlib.sha256(content.encode()).hexdigest()[:12]

    @property
    def checksum(self) -> str:
        return self._checksum

    def validate_integrity(self, expected_checksum: str) -> bool:
        return self._checksum == expected_checksum

    def split_by_category(self) -> dict:
        categories = {}
        for sample in self.samples:
            categories.setdefault(sample.category, []).append(sample)
        return categories

    def save(self, path: str):
        data = {
            "name": self.name,
            "version": self.version,
            "created_at": self.created_at,
            "description": self.description,
            "checksum": self.checksum,
            "sample_count": len(self.samples),
            "samples": [asdict(s) for s in self.samples],
        }
        with open(path, "w") as f:
            json.dump(data, f, indent=2)

# 例：カスタマーサポートボットの評価データセットを作成
samples = [
    EvalSample(
        id="cs-001",
        input_prompt="顧客：「注文番号#12345がまだ届きません。」"
                     "サポート対応を生成してください。",
        expected_output="謝罪、注文状況の確認、到着予定日の提供",
        category="order_inquiry",
        difficulty="easy",
    ),
    EvalSample(
        id="cs-002",
        input_prompt="顧客：「返金と交換の両方をお願いします。」"
                     "ポリシーに従ったサポート対応を生成してください。",
        expected_output="ポリシーの説明（返金または交換のいずれか一方）、"
                       "代替案の提示",
        category="refund_policy",
        difficulty="hard",
    ),
]

dataset = EvalDataset(
    name="customer_support_eval",
    version="1.0.0",
    created_at=datetime.now().isoformat(),
    samples=samples,
    description="カスタマーサポートチャットボットv2の評価セット",
)
print(f"データセット: {dataset.name} v{dataset.version}")
print(f"チェックサム: {dataset.checksum}")
print(f"カテゴリ: {list(dataset.split_by_category().keys())}")

import re
import asyncio
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, Optional

@dataclass
class GradeResult:
    score: float  # 0.0〜1.0
    passed: bool
    reasoning: str
    grader_type: str
    metadata: dict = None

class BaseGrader(ABC):
    """全グレーダーの基底クラス"""

    @abstractmethod
    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        pass

class ExactMatchGrader(BaseGrader):
    """確定的な正解があるタスク用"""

    def __init__(self, normalize: bool = True):
        self.normalize = normalize

    def _normalize(self, text: str) -> str:
        text = text.strip().lower()
        text = re.sub(r'\s+', ' ', text)
        return text

    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        if self.normalize:
            expected = self._normalize(expected)
            actual = self._normalize(actual)
        match = expected == actual
        return GradeResult(
            score=1.0 if match else 0.0,
            passed=match,
            reasoning=f"完全一致: {match}",
            grader_type="exact_match",
        )

class LLMJudgeGrader(BaseGrader):
    """強力なLLMを使用して回答品質を判定"""

    JUDGE_PROMPT_TEMPLATE = """あなたは専門的な評価者です。以下の基準に基づいて
回答を1-5のスケールで評価してください：
- 正確性：正しい情報が含まれているか？
- 完全性：質問のすべての部分に対応しているか？
- 有用性：実行可能で有用か？

質問: {prompt}
期待される回答の要素: {expected}
実際の回答: {actual}

JSONオブジェクトのみで回答してください：
{{"score": <1-5>, "reasoning": "<簡潔な説明>"}}"""

    def __init__(self, judge_client, judge_model: str = "claude-sonnet-4-20250514"):
        self.client = judge_client
        self.model = judge_model

    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        judge_prompt = self.JUDGE_PROMPT_TEMPLATE.format(
            prompt=prompt, expected=expected, actual=actual
        )
        response = await self.client.messages.create(
            model=self.model,
            max_tokens=256,
            messages=[{"role": "user", "content": judge_prompt}],
        )
        result = json.loads(response.content[0].text)
        normalized_score = result["score"] / 5.0
        return GradeResult(
            score=normalized_score,
            passed=normalized_score >= 0.6,
            reasoning=result["reasoning"],
            grader_type="llm_judge",
            metadata={"judge_model": self.model, "raw_score": result["score"]},
        )

class CodeExecutionGrader(BaseGrader):
    """テストケースを実行してコード生成を評価"""

    def __init__(self, timeout_seconds: int = 10):
        self.timeout = timeout_seconds

    async def grade(
        self, prompt: str, expected: str, actual: str
    ) -> GradeResult:
        test_code = f"{actual}\n\n{expected}"
        try:
            proc = await asyncio.create_subprocess_exec(
                "python", "-c", test_code,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            stdout, stderr = await asyncio.wait_for(
                proc.communicate(), timeout=self.timeout
            )
            passed = proc.returncode == 0
            return GradeResult(
                score=1.0 if passed else 0.0,
                passed=passed,
                reasoning=stderr.decode() if not passed else "全テスト合格",
                grader_type="code_execution",
            )
        except asyncio.TimeoutError:
            return GradeResult(
                score=0.0,
                passed=False,
                reasoning=f"{self.timeout}秒後にタイムアウト",
                grader_type="code_execution",
            )

import asyncio
import time
from dataclasses import dataclass, field
from typing import Dict, List

@dataclass
class EvalConfig:
    model_id: str
    model_endpoint: str
    dataset_path: str
    grader_type: str  # "exact_match", "llm_judge", "code_execution"
    max_concurrent: int = 10
    timeout_per_request: int = 30
    temperature: float = 0.0
    max_tokens: int = 1024

@dataclass
class EvalResult:
    sample_id: str
    category: str
    grade: GradeResult
    latency_ms: float
    token_count: int = 0

class EvalOrchestrator:
    """レート制限付きで評価パイプラインをオーケストレーション"""

    def __init__(self, config: EvalConfig, client, grader: BaseGrader):
        self.config = config
        self.client = client
        self.grader = grader
        self.semaphore = asyncio.Semaphore(config.max_concurrent)
        self.results: List[EvalResult] = []

    async def evaluate_sample(self, sample: EvalSample) -> EvalResult:
        async with self.semaphore:
            start = time.monotonic()
            response = await self.client.messages.create(
                model=self.config.model_id,
                max_tokens=self.config.max_tokens,
                temperature=self.config.temperature,
                messages=[{"role": "user", "content": sample.input_prompt}],
            )
            latency_ms = (time.monotonic() - start) * 1000
            actual_output = response.content[0].text
            grade = await self.grader.grade(
                sample.input_prompt,
                sample.expected_output,
                actual_output,
            )
            return EvalResult(
                sample_id=sample.id,
                category=sample.category,
                grade=grade,
                latency_ms=latency_ms,
                token_count=response.usage.output_tokens,
            )

    async def run(self, dataset: EvalDataset) -> Dict:
        tasks = [
            self.evaluate_sample(sample) for sample in dataset.samples
        ]
        self.results = await asyncio.gather(*tasks, return_exceptions=True)

        # 例外をフィルタリング
        valid = [r for r in self.results if isinstance(r, EvalResult)]
        errors = [r for r in self.results if isinstance(r, Exception)]

        # カテゴリ別に集計
        by_category = {}
        for r in valid:
            by_category.setdefault(r.category, []).append(r)

        summary = {
            "model": self.config.model_id,
            "total_samples": len(dataset.samples),
            "completed": len(valid),
            "errors": len(errors),
            "overall_score": (
                sum(r.grade.score for r in valid) / len(valid)
                if valid else 0
            ),
            "overall_pass_rate": (
                sum(1 for r in valid if r.grade.passed) / len(valid)
                if valid else 0
            ),
            "avg_latency_ms": (
                sum(r.latency_ms for r in valid) / len(valid)
                if valid else 0
            ),
            "by_category": {
                cat: {
                    "score": sum(r.grade.score for r in rs) / len(rs),
                    "pass_rate": sum(
                        1 for r in rs if r.grade.passed
                    ) / len(rs),
                    "count": len(rs),
                }
                for cat, rs in by_category.items()
            },
        }
        return summary

import numpy as np
from scipy import stats
from dataclasses import dataclass
from typing import List, Tuple

@dataclass
class ComparisonResult:
    model_a: str
    model_b: str
    mean_a: float
    mean_b: float
    difference: float
    p_value: float
    confidence_interval: Tuple[float, float]
    significant: bool
    effect_size: float  # Cohen's d
    test_used: str
    n_samples: int

def compare_models(
    scores_a: List[float],
    scores_b: List[float],
    model_a_name: str = "Model A",
    model_b_name: str = "Model B",
    alpha: float = 0.05,
) -> ComparisonResult:
    """
    同一評価セットにおける2モデルの対応比較。
    仮定が成り立つ場合は対応のあるt検定、それ以外はWilcoxonの符号順位検定を使用。
    """
    a = np.array(scores_a)
    b = np.array(scores_b)
    assert len(a) == len(b), "両モデルは同一サンプルで評価される必要があります"

    differences = a - b
    n = len(differences)

    # 差の正規性を検定（Shapiro-Wilk検定）
    if n >= 20:
        _, normality_p = stats.shapiro(differences)
    else:
        normality_p = 0  # サンプル数が少なすぎるため、ノンパラメトリックを使用

    if normality_p > 0.05:
        # 対応のあるt検定
        t_stat, p_value = stats.ttest_rel(a, b)
        test_name = "paired_t_test"
    else:
        # Wilcoxonの符号順位検定（ノンパラメトリック代替法）
        try:
            w_stat, p_value = stats.wilcoxon(differences)
            test_name = "wilcoxon_signed_rank"
        except ValueError:
            # すべての差がゼロ
            p_value = 1.0
            test_name = "wilcoxon_signed_rank (degenerate)"

    # 効果量（対応サンプルのCohen's d）
    diff_std = np.std(differences, ddof=1)
    effect_size = np.mean(differences) / diff_std if diff_std > 0 else 0.0

    # 平均差のブートストラップ信頼区間
    rng = np.random.default_rng(42)
    boot_means = []
    for _ in range(10000):
        boot_sample = rng.choice(differences, size=n, replace=True)
        boot_means.append(np.mean(boot_sample))
    ci_lower = np.percentile(boot_means, 100 * alpha / 2)
    ci_upper = np.percentile(boot_means, 100 * (1 - alpha / 2))

    return ComparisonResult(
        model_a=model_a_name,
        model_b=model_b_name,
        mean_a=float(np.mean(a)),
        mean_b=float(np.mean(b)),
        difference=float(np.mean(differences)),
        p_value=float(p_value),
        confidence_interval=(float(ci_lower), float(ci_upper)),
        significant=p_value < alpha,
        effect_size=float(effect_size),
        test_used=test_name,
        n_samples=n,
    )

# 使用例
np.random.seed(42)
model_a_scores = np.random.binomial(1, 0.82, size=500).astype(float)
model_b_scores = np.random.binomial(1, 0.78, size=500).astype(float)

result = compare_models(
    model_a_scores.tolist(),
    model_b_scores.tolist(),
    "Claude 3.5 Sonnet",
    "GPT-4o",
)

print(f"比較: {result.model_a} vs {result.model_b}")
print(f"  スコア: {result.mean_a:.4f} vs {result.mean_b:.4f}")
print(f"  差分: {result.difference:+.4f}")
print(f"  p値: {result.p_value:.4f} ({result.test_used})")
print(f"  95%信頼区間: [{result.confidence_interval[0]:+.4f}, "
      f"{result.confidence_interval[1]:+.4f}]")
print(f"  Cohen's d: {result.effect_size:.3f}")
print(f"  有意: {result.significant}")

from itertools import combinations

def compare_multiple_models(
    model_scores: Dict[str, List[float]],
    alpha: float = 0.05,
) -> List[ComparisonResult]:
    """
    Bonferroni補正を適用して、全モデルペアを比較。
    """
    model_names = list(model_scores.keys())
    pairs = list(combinations(model_names, 2))
    n_comparisons = len(pairs)

    # Bonferroni補正後のalpha
    corrected_alpha = alpha / n_comparisons

    results = []
    for name_a, name_b in pairs:
        result = compare_models(
            model_scores[name_a],
            model_scores[name_b],
            name_a,
            name_b,
            alpha=corrected_alpha,
        )
        results.append(result)

    print(f"Bonferroni補正後alpha: {corrected_alpha:.4f} "
          f"（{n_comparisons}回の比較）")
    for r in results:
        sig_marker = "*" if r.significant else ""
        print(f"  {r.model_a} vs {r.model_b}: "
              f"差分={r.difference:+.4f}, "
              f"p={r.p_value:.4f}{sig_marker}")

    return results

# .github/workflows/llm-eval.yml
name: LLM Evaluation Pipeline

on:
  pull_request:
    paths:
      - 'prompts/**'
      - 'model_config/**'
  schedule:
    - cron: '0 6 * * 1' # 毎週月曜日 UTC 6:00

jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: pip install -r requirements-eval.txt

      - name: Run evaluation suite
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          python -m eval.run \
            --dataset eval/datasets/production_v2.json \
            --model claude-sonnet-4-20250514 \
            --output results/latest.json \
            --threshold 0.85

      - name: Compare with baseline
        run: |
          python -m eval.compare \
            --baseline results/baseline.json \
            --current results/latest.json \
            --alpha 0.05

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: results/

      - name: Gate check
        run: |
          python -m eval.gate \
            --results results/latest.json \
            --min-score 0.85 \
            --max-regression 0.02