Kubernetes ImagePullBackOff 与 ErrImagePull 完全解剖 — 一个原因字符串就能收尾

kubectl get pod -n analytics

NAME                        READY   STATUS             RESTARTS   AGE
ingest-6b8c94f7d5-4tzqr     0/1     ImagePullBackOff   0          3m12s
ingest-6b8c94f7d5-9wdhm     0/1     ErrImagePull       0          3m12s

kubectl rollout restart deployment/ingest -n analytics

kubectl describe pod ingest-6b8c94f7d5-4tzqr -n analytics | tail -12

Events:
  Type     Reason     Age                   From               Message
  ----     ------     ----                  ----               -------
  Normal   Scheduled  3m24s                 default-scheduler  Successfully assigned analytics/ingest-6b8c94f7d5-4tzqr to ip-10-0-2-77
  Normal   Pulling    2m1s (x4 over 3m23s)  kubelet            Pulling image "ghcr.io/example/ingest:2.7.0"
  Warning  Failed     2m1s (x4 over 3m22s)  kubelet            Failed to pull image "ghcr.io/example/ingest:2.7.0": rpc error: code = NotFound desc = failed to pull and unpack image "ghcr.io/example/ingest:2.7.0": failed to resolve reference "ghcr.io/example/ingest:2.7.0": ghcr.io/example/ingest:2.7.0: not found
  Warning  Failed     2m1s (x4 over 3m22s)  kubelet            Error: ErrImagePull
  Normal   BackOff    97s (x6 over 3m22s)   kubelet            Back-off pulling image "ghcr.io/example/ingest:2.7.0"
  Warning  Failed     97s (x6 over 3m22s)   kubelet            Error: ImagePullBackOff

kubectl get events -A --field-selector reason=Failed --sort-by=.lastTimestamp | tail -5

NAMESPACE   LAST SEEN   TYPE      REASON   OBJECT                          MESSAGE
analytics   41s         Warning   Failed   pod/ingest-6b8c94f7d5-4tzqr     Failed to pull image "ghcr.io/example/ingest:2.7.0": ... not found
payments    2m8s        Warning   Failed   pod/checkout-5d7f8b9c4-x2klm    Failed to pull image "registry.example.com/checkout:1.14.2": ... 401 Unauthorized
media       6m11s       Warning   Failed   pod/transcode-79c5d6f8b-vn4pq   Failed to pull image "redis:7.2": ... toomanyrequests: You have reached your pull rate limit.

crane ls ghcr.io/example/ingest | tail -5

2.6.4
2.6.5
2.7.0-rc1
2.7.1

crane manifest ghcr.io/example/ingest:2.7.0
Error: fetching manifest ghcr.io/example/ingest:2.7.0: GET https://ghcr.io/v2/example/ingest/manifests/2.7.0: MANIFEST_UNKNOWN

kubectl describe node ip-10-0-2-77 | grep -A8 "Conditions:"

Conditions:
  Type             Status  LastTransitionTime                Reason                       Message
  ----             ------  ------------------                ------                       -------
  MemoryPressure   False   Sat, 25 Jul 2026 22:10:44 +0900   KubeletHasSufficientMemory   kubelet has sufficient memory available
  DiskPressure     True    Sun, 26 Jul 2026 08:52:03 +0900   KubeletHasDiskPressure       kubelet has disk pressure
  PIDPressure      False   Sat, 25 Jul 2026 22:10:44 +0900   KubeletHasSufficientPID      kubelet has sufficient PID available
  Ready            True    Sat, 25 Jul 2026 22:10:54 +0900   KubeletReady                 kubelet is posting ready status

kubectl create secret docker-registry regcred \
  --docker-server=registry.example.com \
  --docker-username=deploy-bot \
  --docker-password="$(cat ~/.registry-token)" \
  --namespace=payments

kubectl get secret regcred -n payments \
  -o jsonpath='{.data.\.dockerconfigjson}' | base64 -d | jq .

{
  "auths": {
    "registry.example.com": {
      "username": "deploy-bot",
      "password": "glpat-xxxxxxxxxxxx",
      "auth": "ZGVwbG95LWJvdDpnbHBhdC14eHh4eHh4eHh4eHg="
    }
  }
}

kubectl patch serviceaccount default -n payments \
  -p '{"imagePullSecrets": [{"name": "regcred"}]}'

serviceaccount/default patched

kubectl rollout restart deployment/checkout -n payments

kubectl get pod -n payments -l app=checkout \
  -o jsonpath='{.items[0].spec.imagePullSecrets}'

[{"name":"regcred"}]

kubectl get pod -n payments -l app=checkout \
  -o jsonpath='{.items[0].spec.serviceAccountName}'

checkout-sa

kubectl describe pod ingest-6b8c94f7d5-4tzqr -n analytics | grep "Failed to pull"

  Warning  Failed  8s  kubelet  Failed to pull image "ghcr.io/example/ingest:2.7.1": no match for platform in manifest: not found

kubectl logs ingest-6b8c94f7d5-4tzqr -n analytics --previous

exec /usr/local/bin/ingest: exec format error

docker buildx imagetools inspect ghcr.io/example/ingest:2.7.1

Name:      ghcr.io/example/ingest:2.7.1
MediaType: application/vnd.oci.image.index.v1+json
Digest:    sha256:2b1e7f43c9d8a0b6e2f14c7d9a3b58e0c1f26d4a8b7e930f5c2a1d6b4e83f97c

Manifests:
  Name:      ghcr.io/example/ingest:2.7.1@sha256:8f3c1d...
  MediaType: application/vnd.oci.image.manifest.v1+json
  Platform:  linux/arm64

kubectl get nodes -o custom-columns=NAME:.metadata.name,ARCH:.status.nodeInfo.architecture

NAME              ARCH
ip-10-0-1-14      amd64
ip-10-0-2-77      amd64
ip-10-0-3-91      amd64

docker buildx create --use --name multiarch
docker buildx build \
  --platform linux/amd64,linux/arm64 \
  -t ghcr.io/example/ingest:2.7.1 \
  --push .

spec:
  nodeSelector:
    kubernetes.io/arch: amd64

kubectl describe pod transcode-79c5d6f8b-vn4pq -n media | grep "Failed to pull"

  Warning  Failed  15s  kubelet  Failed to pull image "redis:7.2": failed to pull and unpack image "docker.io/library/redis:7.2": failed to resolve reference "docker.io/library/redis:7.2": unexpected status from HEAD request to https://registry-1.docker.io/v2/library/redis/manifests/7.2: 429 Too Many Requests

TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:ratelimitpreview/test:pull" | jq -r .token)
curl -s --head -H "Authorization: Bearer $TOKEN" \
  https://registry-1.docker.io/v2/ratelimitpreview/test/manifests/latest | grep -i ratelimit

ratelimit-limit: 100;w=21600
ratelimit-remaining: 7;w=21600

# /etc/containerd/certs.d/docker.io/hosts.toml
server = "https://registry-1.docker.io"

[host."https://registry-mirror.example.com/v2"]
  capabilities = ["pull", "resolve"]
  skip_verify = false

kubectl get pods -n analytics -l app=ingest \
  -o custom-columns=POD:.metadata.name,NODE:.spec.nodeName,IMAGEID:.status.containerStatuses[0].imageID

POD                       NODE           IMAGEID
ingest-6b8c94f7d5-4tzqr   ip-10-0-1-14   ghcr.io/example/ingest@sha256:8f3c1d...
ingest-6b8c94f7d5-9wdhm   ip-10-0-2-77   ghcr.io/example/ingest@sha256:2b1e7f...

spec:
  containers:
    - name: ingest
      image: ghcr.io/example/ingest@sha256:8f3c1d5b2e7a94c0f13d68b5a2e9c47f0d81b3a65e2c9f7048d1b6a35c8e29f4
      imagePullPolicy: IfNotPresent

skopeo inspect \
  --creds "deploy-bot:${REGISTRY_TOKEN}" \
  docker://registry.example.com/checkout:1.14.2 \
  --format '{{.Digest}} {{.Architecture}}'

sha256:9c1f0f4d2a8b73e5c16f9d20a4b8e7c35f1d69a20c8b4e73f5a1d92c6b8e40f7 amd64

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: image-pull-alerts
  namespace: monitoring
spec:
  groups:
    - name: image-pull
      rules:
        - alert: ImagePullFailing
          expr: kube_pod_container_status_waiting_reason{reason=~"ImagePullBackOff|ErrImagePull"} == 1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} cannot pull its image"