AIベンチマークデータセット完全ガイド: ImageNet、COCO、GLUE、MMLU、HumanEval

# PyTorchでImageNet検証精度を測定する
import torch
import torchvision.transforms as transforms
import torchvision.datasets as datasets
import torchvision.models as models
from torch.utils.data import DataLoader

def evaluate_imagenet(model, val_dir, batch_size=256):
    # 標準前処理（ImageNet検証標準）
    val_transform = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(224),
        transforms.ToTensor(),
        transforms.Normalize(
            mean=[0.485, 0.456, 0.406],
            std=[0.229, 0.224, 0.225]
        )
    ])

    val_dataset = datasets.ImageFolder(val_dir, transform=val_transform)
    val_loader = DataLoader(
        val_dataset,
        batch_size=batch_size,
        shuffle=False,
        num_workers=8,
        pin_memory=True
    )

    model.eval()
    top1_correct = 0
    top5_correct = 0
    total = 0

    with torch.no_grad():
        for images, labels in val_loader:
            images = images.cuda()
            labels = labels.cuda()

            outputs = model(images)
            _, predicted = outputs.topk(5, 1, True, True)
            predicted = predicted.t()
            correct = predicted.eq(labels.view(1, -1).expand_as(predicted))

            top1_correct += correct[:1].reshape(-1).float().sum(0)
            top5_correct += correct[:5].reshape(-1).float().sum(0)
            total += labels.size(0)

    top1_acc = top1_correct / total * 100
    top5_acc = top5_correct / total * 100
    print(f"Top-1 精度: {top1_acc:.2f}%")
    print(f"Top-5 精度: {top5_acc:.2f}%")
    return top1_acc, top5_acc

# 例: ResNet-50を評価
model = models.resnet50(pretrained=True).cuda()
evaluate_imagenet(model, '/path/to/imagenet/val')

from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import json

def evaluate_coco_detection(annotation_file, result_file):
    # COCOグランドトゥルースを読み込む
    coco_gt = COCO(annotation_file)

    # 予測を読み込む
    coco_dt = coco_gt.loadRes(result_file)

    # バウンディングボックス評価
    coco_eval = COCOeval(coco_gt, coco_dt, 'bbox')
    coco_eval.evaluate()
    coco_eval.accumulate()
    coco_eval.summarize()

    stats = coco_eval.stats
    print(f"\n=== COCO 検出結果 ===")
    print(f"AP @ IoU=0.50:0.95 (COCO主要指標): {stats[0]:.3f}")
    print(f"AP @ IoU=0.50 (PASCAL VOCスタイル): {stats[1]:.3f}")
    print(f"AP @ IoU=0.75 (厳格): {stats[2]:.3f}")
    print(f"AP small (面積 < 32^2): {stats[3]:.3f}")
    print(f"AP medium: {stats[4]:.3f}")
    print(f"AP large: {stats[5]:.3f}")
    print(f"AR (画像あたり最大1件): {stats[6]:.3f}")
    print(f"AR (画像あたり最大10件): {stats[7]:.3f}")
    print(f"AR (画像あたり最大100件): {stats[8]:.3f}")
    return stats

# COCOアノテーションを探索する
coco = COCO('instances_val2017.json')
cat_ids = coco.getCatIds(catNms=['person', 'car', 'dog'])
img_ids = coco.getImgIds(catIds=cat_ids[:1])

img = coco.loadImgs(img_ids[0])[0]
ann_ids = coco.getAnnIds(imgIds=img['id'])
anns = coco.loadAnns(ann_ids)
print(f"画像: {img['file_name']}, アノテーション数: {len(anns)}")
for ann in anns[:3]:
    cat = coco.loadCats(ann['category_id'])[0]
    print(f"  カテゴリ: {cat['name']}, 面積: {ann['area']:.0f}px^2")

import numpy as np

def compute_miou(pred_mask, gt_mask, num_classes=150):
    """mIoUを計算する。"""
    iou_list = []

    for cls in range(num_classes):
        pred_cls = (pred_mask == cls)
        gt_cls = (gt_mask == cls)

        intersection = np.logical_and(pred_cls, gt_cls).sum()
        union = np.logical_or(pred_cls, gt_cls).sum()

        if union == 0:
            continue  # 画像にクラスが存在しない場合はスキップ

        iou = intersection / union
        iou_list.append(iou)

    return np.mean(iou_list) if iou_list else 0.0

# mmsegmentationで評価
from mmseg.apis import inference_segmentor, init_segmentor

config_file = 'configs/segformer/segformer_mit-b5_8xb2-160k_ade20k-512x512.py'
checkpoint_file = 'segformer_mit-b5_8x2_512x512_160k_ade20k_20220617_203542-745f14da.pth'

model = init_segmentor(config_file, checkpoint_file, device='cuda:0')
result = inference_segmentor(model, 'test_image.jpg')

import torch
import torchvision
import torchvision.transforms as transforms
from torch.utils.data import DataLoader

def evaluate_cifar10(model, batch_size=128):
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
    ])

    testset = torchvision.datasets.CIFAR10(
        root='./data', train=False, download=True, transform=transform
    )
    testloader = DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)

    model.eval()
    correct = 0
    total = 0

    with torch.no_grad():
        for images, labels in testloader:
            outputs = model(images)
            _, predicted = torch.max(outputs, 1)
            total += labels.size(0)
            correct += (predicted == labels).sum().item()

    accuracy = 100 * correct / total
    print(f"CIFAR-10 精度: {accuracy:.2f}%")
    return accuracy

from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import numpy as np
from sklearn.metrics import matthews_corrcoef

def evaluate_glue_cola(model_name="bert-base-uncased"):
    """CoLA（文法的受容性）を評価する。"""
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(
        model_name, num_labels=2
    )

    dataset = load_dataset("glue", "cola")
    val_data = dataset["validation"]

    predictions = []
    labels = []

    model.eval()
    import torch

    for item in val_data:
        inputs = tokenizer(
            item['sentence'],
            return_tensors='pt',
            padding=True,
            truncation=True,
            max_length=128
        )

        with torch.no_grad():
            outputs = model(**inputs)
            pred = outputs.logits.argmax(dim=-1).item()

        predictions.append(pred)
        labels.append(item['label'])

    mcc = matthews_corrcoef(labels, predictions)
    print(f"CoLA Matthews相関係数: {mcc:.4f}")
    return mcc

def evaluate_glue_sst2(model_name="textattack/bert-base-uncased-SST-2"):
    """SST-2（感情分類）を評価する。"""
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(model_name)

    dataset = load_dataset("glue", "sst2")
    val_data = dataset["validation"]

    correct = 0
    total = len(val_data)

    model.eval()
    import torch

    for item in val_data:
        inputs = tokenizer(
            item['sentence'],
            return_tensors='pt',
            truncation=True,
            max_length=128
        )
        with torch.no_grad():
            outputs = model(**inputs)
            pred = outputs.logits.argmax(dim=-1).item()
        if pred == item['label']:
            correct += 1

    acc = correct / total
    print(f"SST-2 精度: {acc:.4f}")
    return acc

from datasets import load_dataset
from transformers import pipeline

def evaluate_squad(model_name="deepset/roberta-base-squad2"):
    """SQuAD 2.0を評価する。"""
    qa_pipeline = pipeline("question-answering", model=model_name)
    dataset = load_dataset("squad_v2", split="validation")

    em_scores = []
    f1_scores = []
    no_answer_correct = 0
    no_answer_total = 0

    for item in dataset.select(range(200)):
        context = item['context']
        question = item['question']
        answers = item['answers']

        result = qa_pipeline(question=question, context=context)
        predicted = result['answer'].lower().strip()

        has_answer = len(answers['text']) > 0

        if not has_answer:
            no_answer_total += 1
            if result['score'] < 0.1:
                no_answer_correct += 1
            em_scores.append(0)
            f1_scores.append(0)
        else:
            gold_answers = [a.lower().strip() for a in answers['text']]
            em = max(int(predicted == gold) for gold in gold_answers)
            em_scores.append(em)

            best_f1 = 0
            for gold in gold_answers:
                pred_tokens = set(predicted.split())
                gold_tokens = set(gold.split())
                common = pred_tokens & gold_tokens
                if len(common) == 0:
                    f1 = 0
                else:
                    precision = len(common) / len(pred_tokens)
                    recall = len(common) / len(gold_tokens)
                    f1 = 2 * precision * recall / (precision + recall)
                best_f1 = max(best_f1, f1)
            f1_scores.append(best_f1)

    print(f"SQuAD 2.0 結果（200サンプル）:")
    print(f"  EM: {sum(em_scores)/len(em_scores)*100:.1f}%")
    print(f"  F1: {sum(f1_scores)/len(f1_scores)*100:.1f}%")
    if no_answer_total > 0:
        print(f"  答えなし精度: {no_answer_correct/no_answer_total*100:.1f}%")

import sacrebleu

def compute_bleu(predictions, references):
    """BLEUスコアを計算する。"""
    bleu = sacrebleu.corpus_bleu(predictions, [references])
    print(f"BLEU: {bleu.score:.2f}")
    print(f"BP: {bleu.bp:.3f}")
    print(f"比率: {bleu.sys_len/bleu.ref_len:.3f}")
    return bleu.score

from transformers import MarianMTModel, MarianTokenizer

def evaluate_translation(src_texts, tgt_texts, model_name="Helsinki-NLP/opus-mt-en-de"):
    tokenizer = MarianTokenizer.from_pretrained(model_name)
    model = MarianMTModel.from_pretrained(model_name)

    predictions = []
    for text in src_texts[:100]:
        inputs = tokenizer([text], return_tensors="pt", padding=True, truncation=True)
        translated = model.generate(**inputs, max_length=512)
        pred = tokenizer.decode(translated[0], skip_special_tokens=True)
        predictions.append(pred)

    bleu_score = compute_bleu(predictions, tgt_texts[:100])
    return bleu_score

from datasets import load_dataset

def evaluate_mmlu(model_fn, subjects=None, num_few_shot=5):
    """MMLU評価関数。"""
    if subjects is None:
        subjects = ['abstract_algebra', 'anatomy', 'astronomy', 'college_mathematics']

    results = {}

    for subject in subjects:
        dataset = load_dataset("lukaemon/mmlu", subject)
        test_data = dataset['test']
        dev_data = dataset['dev']

        correct = 0
        total = 0

        # few-shotプロンプトを構築する
        few_shot_examples = ""
        for i, item in enumerate(dev_data.select(range(num_few_shot))):
            few_shot_examples += f"Q: {item['input']}\n"
            few_shot_examples += f"(A) {item['A']}  (B) {item['B']}  (C) {item['C']}  (D) {item['D']}\n"
            few_shot_examples += f"Answer: {item['target']}\n\n"

        for item in test_data:
            prompt = few_shot_examples
            prompt += f"Q: {item['input']}\n"
            prompt += f"(A) {item['A']}  (B) {item['B']}  (C) {item['C']}  (D) {item['D']}\n"
            prompt += "Answer:"

            response = model_fn(prompt)
            pred = response.strip()[0] if response.strip() else 'A'

            if pred == item['target']:
                correct += 1
            total += 1

        accuracy = correct / total
        results[subject] = accuracy
        print(f"{subject}: {accuracy:.3f} ({correct}/{total})")

    overall = sum(results.values()) / len(results)
    print(f"\n全体平均: {overall:.3f}")
    return results

from lm_eval import evaluator

# lm-evaluation-harnessでBIG-Benchを評価する
results = evaluator.simple_evaluate(
    model="hf",
    model_args="pretrained=meta-llama/Llama-3.2-3B-Instruct",
    tasks=["bigbench_causal_judgment", "bigbench_date_understanding"],
    num_fewshot=3,
    batch_size="auto"
)
print(results['results'])

from datasets import load_dataset
import torch
from transformers import AutoTokenizer, AutoModelForMultipleChoice

def evaluate_hellaswag(model_name="microsoft/deberta-v2-xxlarge"):
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForMultipleChoice.from_pretrained(model_name)

    dataset = load_dataset("hellaswag", split="validation")

    correct = 0
    total = min(500, len(dataset))

    for item in dataset.select(range(total)):
        context = item['ctx']
        endings = item['endings']
        label = int(item['label'])

        choices = [context + " " + ending for ending in endings]

        encoding = tokenizer(
            [context] * 4,
            choices,
            return_tensors='pt',
            padding=True,
            truncation=True,
            max_length=256
        )

        with torch.no_grad():
            outputs = model(**{k: v.unsqueeze(0) for k, v in encoding.items()})
            logits = outputs.logits
            predicted = logits.argmax(dim=-1).item()

        if predicted == label:
            correct += 1

    accuracy = correct / total
    print(f"HellaSwag 精度: {accuracy:.4f}")
    return accuracy

from datasets import load_dataset
from transformers import pipeline

def evaluate_truthfulqa(model_name="gpt2-xl"):
    """TruthfulQA MC1（単一正解）評価。"""
    dataset = load_dataset("truthful_qa", "multiple_choice")
    val_data = dataset["validation"]

    generator = pipeline("text-generation", model=model_name)
    correct = 0
    total = min(100, len(val_data))

    for item in val_data.select(range(total)):
        question = item['question']
        choices = item['mc1_targets']['choices']
        labels = item['mc1_targets']['labels']
        correct_idx = labels.index(1)

        prompt = f"Q: {question}\n選択肢:\n"
        for i, choice in enumerate(choices):
            letter = chr(65 + i)
            prompt += f"{letter}. {choice}\n"
        prompt += "答え:"

        response = generator(prompt, max_new_tokens=5, do_sample=False)
        generated = response[0]['generated_text'][len(prompt):].strip()
        pred_letter = generated[0] if generated else 'A'
        pred_idx = ord(pred_letter) - 65

        if pred_idx == correct_idx:
            correct += 1

    accuracy = correct / total
    print(f"TruthfulQA MC1 精度: {accuracy:.4f}")
    return accuracy

from datasets import load_dataset
import re

def extract_number(text):
    """テキストから最終的な数値の答えを抽出する。"""
    numbers = re.findall(r'-?\d+\.?\d*', text)
    return numbers[-1] if numbers else None

def evaluate_gsm8k_chain_of_thought(model_fn, num_shot=8):
    """チェーン・オブ・ソートプロンプティングでGSM8Kを評価する。"""
    dataset = load_dataset("gsm8k", "main")
    test_data = dataset['test']

    few_shot_prompt = """Q: Rogerはテニスボールを5個持っています。テニスボールのカンを2缶買いました。
各カンには3個のテニスボールが入っています。今、テニスボールは何個ありますか？
A: Rogerは最初5個持っていました。2缶 x 3個 = 6個。5 + 6 = 11個。答えは11。

Q: 食堂にはリンゴが23個ありました。昼食に20個使い、さらに6個買いました。今何個ありますか？
A: 最初23個。20個使用: 23 - 20 = 3。6個購入: 3 + 6 = 9。答えは9。

"""

    correct = 0
    total = min(200, len(test_data))

    for item in test_data.select(range(total)):
        question = item['question']
        gold_answer = item['answer'].split('####')[-1].strip()

        prompt = few_shot_prompt + f"Q: {question}\nA:"
        response = model_fn(prompt, max_tokens=256)

        pred = extract_number(response)
        gold = extract_number(gold_answer)

        if pred and gold and abs(float(pred) - float(gold)) < 0.01:
            correct += 1

    accuracy = correct / total
    print(f"GSM8K 精度（チェーン・オブ・ソート）: {accuracy:.4f}")
    return accuracy

from datasets import load_dataset
import subprocess
import tempfile
import os

def evaluate_humaneval(model_fn, k=1, n=10, temperature=0.8):
    """HumanEval pass@k評価。"""
    dataset = load_dataset("openai_humaneval")
    test_data = dataset['test']

    task_results = {}

    for item in test_data.select(range(20)):
        task_id = item['task_id']
        prompt = item['prompt']
        tests = item['test']
        entry_point = item['entry_point']

        passes = 0

        for attempt in range(n):
            code = model_fn(prompt, temperature=temperature)

            full_code = prompt + code + "\n" + tests + f"\ncheck({entry_point})"

            with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f:
                f.write(full_code)
                tmp_path = f.name

            try:
                result = subprocess.run(
                    ['python', tmp_path],
                    timeout=10,
                    capture_output=True,
                    text=True
                )
                if result.returncode == 0:
                    passes += 1
            except subprocess.TimeoutExpired:
                pass
            finally:
                os.unlink(tmp_path)

        task_results[task_id] = passes / n

    pass_at_1 = sum(task_results.values()) / len(task_results)
    print(f"pass@1: {pass_at_1:.4f}")
    return pass_at_1

# 主要モデルのHumanEval性能（2025年）
humaneval_scores = {
    "GPT-3 (175B)": 0.0,
    "Codex (12B)": 0.288,
    "GPT-4": 0.870,
    "Claude 3.5 Sonnet": 0.900,
    "DeepSeek-Coder-33B": 0.823,
    "Llama 3.1 70B": 0.803,
}

from datasets import load_dataset

def explore_mbpp():
    """MBPPデータセットを探索する。"""
    dataset = load_dataset("mbpp")
    test_data = dataset['test']

    print("MBPPサンプル問題:")
    for item in test_data.select(range(3)):
        print(f"\nタスクID: {item['task_id']}")
        print(f"問題: {item['text']}")
        print(f"テストケース: {item['test_list'][:2]}")
        print(f"参考コード:\n{item['code']}")
        print("-" * 50)

from openai import OpenAI

def mt_bench_judge(question, answer, reference_answer=None):
    """GPT-4を使ってMT-Benchの回答を評価する。"""
    client = OpenAI()

    system_prompt = """あなたはAIの回答を評価する有能なアシスタントです。
精度、関連性、完全性、明確さに基づいて1〜10のスケールで回答を評価してください。
出力フォーマット: スコア: X/10\n根拠: [簡潔な説明]"""

    user_prompt = f"""質問: {question}

AIの回答: {answer}

{f'参考回答: {reference_answer}' if reference_answer else ''}

この回答を評価してください。"""

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt}
        ],
        temperature=0.0
    )

    judge_response = response.choices[0].message.content
    print(f"評価:\n{judge_response}")
    return judge_response

# 公式MT-Bench使用方法（FastChat）:
# git clone https://github.com/lm-sys/FastChat
# python -m fastchat.llm_judge.gen_model_answer --model-path your-model
# python -m fastchat.llm_judge.gen_judgment --judge-model gpt-4
# python -m fastchat.llm_judge.show_result

# HELM評価を実行する
pip install crfm-helm

helm-run \
    --conf src/helm/benchmark/presentation/run_specs_lite.conf \
    --local \
    --max-eval-instances 1000 \
    --num-train-trials 1

# 結果を表示する
helm-summarize --suite v1
helm-server

from huggingface_hub import HfApi

def fetch_leaderboard_data():
    """Open LLM Leaderboardデータを取得する。"""
    api = HfApi()

    dataset_info = api.dataset_info("open-llm-leaderboard/results")
    print(f"最終更新: {dataset_info.lastModified}")

    files = api.list_repo_files(
        repo_id="open-llm-leaderboard/results",
        repo_type="dataset"
    )

    for f in list(files)[:5]:
        print(f"ファイル: {f}")

from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import torch

def evaluate_klue_nli(model_name="klue/roberta-large"):
    """KLUE-NLIを評価する。"""
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    model = AutoModelForSequenceClassification.from_pretrained(
        model_name, num_labels=3
    )

    dataset = load_dataset("klue", "nli")
    val_data = dataset['validation']

    correct = 0
    total = min(500, len(val_data))

    model.eval()
    for item in val_data.select(range(total)):
        premise = item['premise']
        hypothesis = item['hypothesis']
        gold_label = item['label']

        inputs = tokenizer(
            premise, hypothesis,
            return_tensors='pt',
            truncation=True,
            max_length=512
        )

        with torch.no_grad():
            outputs = model(**inputs)
            pred = outputs.logits.argmax(dim=-1).item()

        if pred == gold_label:
            correct += 1

    accuracy = correct / total
    print(f"KLUE-NLI 精度: {accuracy:.4f}")
    return accuracy

from datasets import load_dataset

def evaluate_kmmlu_sample():
    """KMMLUデータセットを探索する。"""
    dataset = load_dataset("HAERAE-HUB/KMMLU")
    test_data = dataset['test']

    print(f"合計問題数: {len(test_data)}")

    subjects = set(test_data['subject'])
    print(f"科目数: {len(subjects)}")
    print(f"サンプル科目: {list(subjects)[:10]}")

    item = test_data[0]
    print(f"\n科目: {item['subject']}")
    print(f"問題: {item['question']}")
    print(f"A: {item['A']}")
    print(f"B: {item['B']}")
    print(f"C: {item['C']}")
    print(f"D: {item['D']}")
    print(f"答え: {item['answer']}")

from transformers import BlipProcessor, BlipForQuestionAnswering
import torch
from PIL import Image

def evaluate_vqa_blip():
    """BLIPでVQAを評価する。"""
    processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base")
    model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base")
    model.eval()

    questions = [
        ("車は何色ですか？", "test_car.jpg"),
        ("画像に何人いますか？", "test_crowd.jpg"),
        ("雨が降っていますか？", "test_outdoor.jpg")
    ]

    for question, image_path in questions:
        try:
            image = Image.open(image_path).convert('RGB')
            inputs = processor(image, question, return_tensors="pt")

            with torch.no_grad():
                out = model.generate(**inputs, max_length=20)

            answer = processor.decode(out[0], skip_special_tokens=True)
            print(f"Q: {question}")
            print(f"A: {answer}\n")
        except FileNotFoundError:
            print(f"画像が見つかりません: {image_path}")

from datasets import load_dataset
import torch

def explore_mmmu():
    """MMMUデータセットを探索する。"""
    dataset = load_dataset("MMMU/MMMU", "Accounting")
    print(f"会計学検証セット: {len(dataset['validation'])}問")

    item = dataset['validation'][0]
    print(f"\n問題: {item['question']}")
    print(f"選択肢A: {item['option_A']}")
    print(f"選択肢B: {item['option_B']}")
    print(f"答え: {item['answer']}")

    if item['image_1']:
        print("画像あり")

# インストール
git clone https://github.com/EleutherAI/lm-evaluation-harness
cd lm-evaluation-harness
pip install -e .

# GPT-2でMMLUを評価する
lm_eval --model hf \
    --model_args pretrained=gpt2 \
    --tasks mmlu \
    --num_fewshot 5 \
    --batch_size 8 \
    --output_path results/gpt2_mmlu

# 複数タスクを同時に評価する
lm_eval --model hf \
    --model_args pretrained=meta-llama/Llama-3.2-3B-Instruct \
    --tasks mmlu,arc_challenge,hellaswag,truthfulqa_mc1,gsm8k \
    --num_fewshot 5 \
    --batch_size 4 \
    --output_path results/llama3.2_3b

# 4ビット量子化で実行する
lm_eval --model hf \
    --model_args pretrained=meta-llama/Meta-Llama-3-8B,load_in_4bit=True \
    --tasks mmlu \
    --num_fewshot 5 \
    --batch_size 1

import lm_eval
from lm_eval import evaluator
import json
import os

def run_comprehensive_evaluation(model_path, output_dir="./results"):
    """包括的なLM-Evaluation-Harness評価を実行する。"""
    os.makedirs(output_dir, exist_ok=True)

    task_groups = {
        "knowledge": ["mmlu", "arc_challenge", "arc_easy"],
        "reasoning": ["hellaswag", "winogrande", "piqa"],
        "truthfulness": ["truthfulqa_mc1"],
        "math": ["gsm8k"],
        "coding": ["humaneval"],
    }

    all_results = {}

    fewshot_map = {
        "mmlu": 5, "arc_challenge": 25, "arc_easy": 25,
        "hellaswag": 10, "winogrande": 5, "piqa": 0,
        "truthfulqa_mc1": 0, "gsm8k": 5, "humaneval": 0
    }

    for group, tasks in task_groups.items():
        print(f"\n=== {group.upper()} を評価中 ===")

        results = evaluator.simple_evaluate(
            model="hf",
            model_args=f"pretrained={model_path}",
            tasks=tasks,
            num_fewshot=fewshot_map.get(tasks[0], 0),
            batch_size="auto",
            device="cuda" if __import__("torch").cuda.is_available() else "cpu",
        )

        all_results[group] = results['results']

        for task, metrics in results['results'].items():
            if 'acc,none' in metrics:
                print(f"  {task}: {metrics['acc,none']*100:.1f}%")
            elif 'exact_match,strict-match' in metrics:
                print(f"  {task}: {metrics['exact_match,strict-match']*100:.1f}%")

    with open(f"{output_dir}/evaluation_results.json", "w", encoding="utf-8") as f:
        json.dump(all_results, f, ensure_ascii=False, indent=2)

    print(f"\n結果を保存しました: {output_dir}/evaluation_results.json")
    return all_results

# custom_task.py
from lm_eval.api.task import Task
from lm_eval.api.instance import Instance

class CustomQATask(Task):
    """lm-evaluation-harnessのカスタムQ&Aタスク。"""
    VERSION = 1.0
    DATASET_PATH = "your-org/your-qa-dataset"
    DATASET_NAME = None

    def has_training_docs(self):
        return False

    def has_validation_docs(self):
        return True

    def has_test_docs(self):
        return True

    def validation_docs(self):
        return self.dataset["validation"]

    def test_docs(self):
        return self.dataset["test"]

    def doc_to_text(self, doc):
        return f"質問: {doc['question']}\n答え:"

    def doc_to_target(self, doc):
        return " " + doc['answer']

    def construct_requests(self, doc, ctx):
        return [Instance(
            request_type="generate_until",
            doc=doc,
            arguments=(ctx, {"until": ["\n", "質問:"]}),
            idx=0
        )]

    def process_results(self, doc, results):
        gold = doc['answer'].lower().strip()
        pred = results[0].lower().strip()
        return {"exact_match": int(gold == pred)}

    def aggregation(self):
        return {"exact_match": "mean"}

    def higher_is_better(self):
        return {"exact_match": True}