みんなのためのAI 第3回 — 損失0.0017なのに正解率7.5%、犯人はパディング1マスだった

[手書き数字の画像] + "is it even or odd?"  ->  "odd"
[手書き数字の画像] + "how many holes?"      ->  "zero"

class VQA(nn.Module):
    def __init__(self):
        super().__init__()
        self.cnn = nn.Sequential(
            nn.Conv2d(1, 32, 3, 2, 1), nn.GELU(),    # 28 -> 14
            nn.Conv2d(32, 64, 3, 2, 1), nn.GELU(),   # 14 -> 7
            nn.Conv2d(64, 128, 3, 2, 1), nn.GELU(),  # 7 -> 4
        )
        self.img_proj = nn.Linear(128, 192)
        self.emb = nn.Embedding(V, 192)
        self.pos = nn.Embedding(64, 192)
        layer = nn.TransformerEncoderLayer(192, 6, 768, batch_first=True, norm_first=True)
        self.tr = nn.TransformerEncoder(layer, 3)
        self.head = nn.Linear(192, V)

[画像トークン16個][質問トークン24個][回答トークン6個]

T = h.shape[1]
A0 = 16 + q.shape[1]            # 回答が始まる位置
m = torch.zeros(T, T, device=h.device)
m[:, A0:] = float("-inf")       # まず回答区間を誰にも見せない
idx = torch.arange(T, device=h.device)
m[idx.unsqueeze(0) <= idx.unsqueeze(1)] = 0   # 自分自身と過去は許可
m[:A0, A0:] = float("-inf")     # 前の区間が回答を先読みするのだけ再遮断

[  603.7s] SUMMARY {"name": "03-image-text-to-text", "steps": 82120,
                    "final_loss": 0.0017, "vqa_acc": 0.075, "params": 1475870}

Q: how many holes?         (正解 zero) -> A: zeroe
Q: is it even or odd?      (正解 even) -> A: evene
Q: what digit is this?     (正解 one)  -> A: oneeo
Q: is it bigger than four? (正解 no)   -> A: nonrn
Q: is it even or odd?      (正解 odd)  -> A: oddee

def tok(s, L):
    ids = [enc[c] for c in s][:L]
    return ids + [0] * (L - len(ids))     # 0 = PAD

a_full = torch.tensor([tok(s, AL - 1) + [1] for s in ans])   # 1 = EOS

tok("even", 5)  ->  [e, v, e, n, PAD]
+ [EOS]         ->  [e, v, e, n, PAD, EOS]

loss = F.cross_entropy(..., ignore_index=0)   # PAD は無視

def tok_ans(s, L):
    """回答用: EOS を答えの「すぐ後ろ」に付け、その後をパディングする。"""
    ids = [enc[c] for c in s][:L - 1] + [1]
    return ids + [0] * (L - len(ids))

tok_ans("even", 6)  ->  [e, v, e, n, EOS, PAD]

[  603.1s] SUMMARY {"name": "03-image-text-to-text", "steps": 82030,
                    "final_loss": 0.00075, "vqa_acc": 0.995, "params": 1475870}

Q: is it even or odd?      (正解 odd)  -> A: odd
Q: how many holes?         (正解 zero) -> A: zero
Q: is it bigger than four? (正解 no)   -> A: no
Q: what digit is this?     (正解 nine) -> A: nine