不靠感觉做 LLM 评测 — 样本量、评判者偏差与 CI 回归测试

import re, json

def assert_output_contract(resp, ctx):
    """与模型质量无关、必须永远成立的那些条件"""
    checks = []

    checks.append(("파싱 가능", is_valid_json(resp.text)))
    checks.append(("스키마 준수", validate_schema(resp.text) is True))
    checks.append(("잘리지 않음", resp.finish_reason != "length"))
    checks.append(("금칙 표현 없음", not FORBIDDEN.search(resp.text)))
    checks.append(("문맥 밖 URL 없음", set(urls(resp.text)) <= set(urls(ctx))))
    checks.append(("지연 상한", resp.latency_ms < 4000))
    checks.append(("출력 길이 상한", resp.output_tokens < 800))

    return [name for name, ok in checks if not ok]

def harvest_candidates(logs, judge, sample_rate=0.02):
    """自动挑出值得放进评测集的请求"""
    out = []
    for r in logs:
        reason = None
        if r.user_feedback == "negative":     reason = "사용자 부정 피드백"
        elif r.retry_count > 0:               reason = "재시도 발생"
        elif r.assertion_failures:            reason = "어서션 실패"
        elif r.judge_score is not None and r.judge_score < 3:
            reason = "심사자 저점"
        elif random.random() < sample_rate:   reason = "무작위 표본"

        if reason:
            out.append({"input": r.input, "output": r.output, "reason": reason})
    return out   # 标准答案标签由人来打

def judge_pairwise(judge, question, answer_a, answer_b, rubric):
    """把顺序换过来评两次，只有两次一致才认定胜负"""
    def ask(first, second):
        prompt = (f"{rubric}\n\n질문: {question}\n\n"
                  f"[답변 1]\n{first}\n\n[답변 2]\n{second}\n\n"
                  "먼저 기준별 근거를 쓰고, 마지막 줄에 '승자: 1' 또는 '승자: 2' 또는 '승자: 무승부'.")
        return parse_winner(judge(prompt))

    r1 = ask(answer_a, answer_b)        # A 在前
    r2 = ask(answer_b, answer_a)        # B 在前
    if r1 == "1" and r2 == "2":  return "A"
    if r1 == "2" and r2 == "1":  return "B"
    return "무승부"                      # 换个顺序就翻转，说明判不了

import math

def wilson(successes, n, z=1.96):
    if n == 0:
        return (0.0, 1.0)
    p = successes / n
    denom = 1 + z * z / n
    center = (p + z * z / (2 * n)) / denom
    half = (z / denom) * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))
    return (max(0.0, center - half), min(1.0, center + half))

for n in (20, 60, 200, 1000):
    lo, hi = wilson(round(0.85 * n), n)
    print(f"n={n:4}  85% 관측 → 95% 신뢰구간 [{lo:.3f}, {hi:.3f}]  폭 {hi - lo:.3f}")
# n=  20  85% 관측 → 95% 신뢰구간 [0.640, 0.948]  폭 0.308
# n=  60  85% 관측 → 95% 신뢰구간 [0.739, 0.919]  폭 0.180
# n= 200  85% 관측 → 95% 신뢰구간 [0.794, 0.893]  폭 0.099
# n=1000  85% 관측 → 95% 신뢰구간 [0.827, 0.871]  폭 0.044

Z_A, Z_B = 1.96, 0.8416   # 显著性水平 5%，检验效能 80%

def n_unpaired(p1, p2):
    """用互不相同的样例衡量两个版本时，每组需要的样例数"""
    num = (Z_A + Z_B) ** 2 * (p1 * (1 - p1) + p2 * (1 - p2))
    return math.ceil(num / (p1 - p2) ** 2)

def n_paired(win_ratio, discordant_rate):
    """用同一批样例把两个版本对着比时需要的样例数。
    win_ratio: 意见分歧的样例中新版本获胜的比例"""
    num = (Z_A / 2 + Z_B * math.sqrt(win_ratio * (1 - win_ratio))) ** 2
    need_discordant = math.ceil(num / (win_ratio - 0.5) ** 2)
    return math.ceil(need_discordant / discordant_rate), need_discordant

print(f"짝짓지 않음: 그룹당 {n_unpaired(0.80, 0.85)}개 (총 {2 * n_unpaired(0.80, 0.85)}개)")
total, disc = n_paired(0.70, 0.20)
print(f"짝지은 비교: 총 {total}개 (의견 갈린 {disc}개 필요)")
# 짝짓지 않음: 그룹당 903개 (총 1806개)
# 짝지은 비교: 총 235개 (의견 갈린 47개 필요)

def ci_gate(baseline, candidate, n, threshold=0.03):
    """按各指标的下降阈值判定。不因单个样例不一致就判失败。"""
    failures = []
    for metric in ("accuracy", "assertion_pass", "citation_valid"):
        drop = baseline[metric] - candidate[metric]
        lo, hi = wilson(round(candidate[metric] * n), n)
        if drop > threshold and baseline[metric] > hi:
            failures.append(f"{metric}: {baseline[metric]:.3f} → {candidate[metric]:.3f} "
                            f"(구간 상한 {hi:.3f})")
    return failures