필사 모드: 三行配置文件让 4 张 GPU 死了一周 — containerd drop-in 合并的真相

$ kubectl get nodes -o custom-columns="NODE:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu"
NODE     GPU
nuc1     0
nuc2     0
omen     0
omen2    0

$ ssh omen 'grep -c "runtimes.nvidia" /etc/containerd/conf.d/99-nvidia.toml'
6

$ ssh omen 'containerd config dump | grep -n "runtimes"'
111:      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
113:        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
128:          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]

containerd: failed to load TOML: /etc/containerd/config.toml: (300, 2): duplicated tables
Job for containerd.service failed because the control process exited with error code.

$ ssh omen 'ls /etc/containerd/conf.d/'
99-nvidia.toml
zz-labhub-registry.toml

# LabHub: 信任 Harbor(HTTP) registry
[plugins."io.containerd.grpc.v1.cri".registry]
  config_path = "/etc/containerd/certs.d"

$ # 给 zz 文件加上 version = 2 的副本，拿去 dump
$ containerd --config /tmp/t2.toml config dump 2>/dev/null | grep -c "runtimes.nvidia"
0

$ for f in 99-nvidia zz-labhub-registry; do
    printf 'version = 2\nimports = ["/etc/containerd/conf.d/%s.toml"]\n' $f > /tmp/t.toml
    echo -n "只 import $f → "
    containerd --config /tmp/t.toml config dump 2>/dev/null | grep -c "runtimes.nvidia"
  done
只 import 99-nvidia → 6
只 import zz-labhub-registry → 0

$ # 两个都要
$ printf 'version = 2\nimports = ["/etc/containerd/conf.d/*.toml"]\n' > /tmp/t3.toml
$ containerd --config /tmp/t3.toml config dump 2>/dev/null | grep -c "runtimes.nvidia"
0

读取顺序           CRI 插件配置
─────────────────  ────────────────────────────────────
config.toml        runc、sandbox_image、cgroup 设置 …
99-nvidia.toml     runc + nvidia ×3、certs.d …          ← 把上面的整块替换掉
zz-registry.toml   只有一个 config_path                  ← 又一次整块替换
─────────────────  ────────────────────────────────────
最终               config_path + 其余全部默认值

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  runtime_type = "io.containerd.runc.v2"

sandbox_image = "registry.k8s.io/pause:3.8"

def strip_nvidia(lines):
    """把 nvidia 表和它的所有子表全部扒掉。"""
    out, cur, dropped = [], "", 0
    for line in lines:
        h = header_of(line)
        if h is not None:
            cur = h
        if cur == NV or cur.startswith(NV + "."):
            dropped += 1
            continue
        out.append(line)
    return out, dropped

rc, out, err = dump(candidate)          # containerd --config <候选> config dump
if rc != 0 or out.count("runtimes.nvidia") == 0:
    print("候选配置里没有抓到 nvidia 运行时。原文件保持不动。")
    return 1
if "/etc/containerd/certs.d" not in out:
    print("registry 配置会消失。原文件保持不动。")
    return 1
# 从这里开始才真的动手

$ ssh omen2 'systemctl is-active containerd'
active

$ kubectl describe node omen2 | grep -A1 "Ready "
  Ready   False   KubeletNotReady   container runtime is down

level=warning msg="failed to load plugin io.containerd.grpc.v1.cri"
  error="failed to create CRI service: failed to create cni conf monitor for default:
         failed to create fsnotify watcher: too many open files"
level=info msg="containerd successfully booted in 0.417476s"

$ sudo sh -c 'printf "fs.inotify.max_user_instances = 8192\nfs.inotify.max_user_watches = 1048576\n" \
    > /etc/sysctl.d/99-inotify.conf && sysctl -p /etc/sysctl.d/99-inotify.conf'
$ sudo systemctl restart containerd

$ kubectl get nodes -o custom-columns="NODE:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu"
NODE     GPU
nuc1     1
nuc2     1
omen     1
omen2    1

$ kubectl -n gpu-operator logs gputest
GPU 0: NVIDIA GeForce RTX 4070 Laptop GPU (UUID: GPU-53be3556-a942-531c-a9a5-20e92af45279)