필사 모드: Kubernetes CrashLoopBackOff 원인별 진단과 해결 — 로그가 비어 있을 때 무엇을 봐야 하나

kubectl get pod -n payments

NAME                        READY   STATUS             RESTARTS        AGE
checkout-7d9f6c5b4d-2xk9p   0/1     CrashLoopBackOff   6 (2m41s ago)   11m
checkout-7d9f6c5b4d-lm4vz   0/1     CrashLoopBackOff   6 (2m38s ago)   11m
checkout-7d9f6c5b4d-q8w2n   0/1     CrashLoopBackOff   6 (2m44s ago)   11m

kubectl logs checkout-7d9f6c5b4d-2xk9p -n payments

Error from server (BadRequest): container "checkout" in pod "checkout-7d9f6c5b4d-2xk9p" is waiting to start: CrashLoopBackOff

kubectl delete pod checkout-7d9f6c5b4d-2xk9p -n payments

kubectl get pods -A --sort-by='.status.containerStatuses[0].restartCount' | tail -20

NAMESPACE   NAME                          READY   STATUS    RESTARTS         AGE
search      indexer-6c4d9f7b8-h2klp       1/1     Running   17 (43m ago)     6d
payments    ledger-5f8b7c6d9-nm3xt        1/1     Running   23 (12m ago)     9d

kubectl describe pod checkout-7d9f6c5b4d-2xk9p -n payments

Containers:
  checkout:
    Container ID:   containerd://3f2a91c4e88b7d5641a0c2f9e37b1d80
    Image:          registry.example.com/checkout:1.14.2
    Image ID:       registry.example.com/checkout@sha256:9c1f...
    Port:           8080/TCP
    State:          Waiting
      Reason:       CrashLoopBackOff
    Last State:     Terminated
      Reason:       Error
      Exit Code:    1
      Started:      Sun, 26 Jul 2026 09:41:12 +0900
      Finished:     Sun, 26 Jul 2026 09:41:13 +0900
    Ready:          False
    Restart Count:  6
Events:
  Type     Reason     Age                    From     Message
  ----     ------     ----                   ----     -------
  Normal   Pulled     3m12s (x5 over 11m)    kubelet  Container image "registry.example.com/checkout:1.14.2" already present on machine
  Normal   Created    3m12s (x5 over 11m)    kubelet  Created container checkout
  Normal   Started    3m11s (x5 over 11m)    kubelet  Started container checkout
  Warning  BackOff    2m41s (x24 over 10m)   kubelet  Back-off restarting failed container checkout

kubectl logs checkout-7d9f6c5b4d-2xk9p -n payments --previous

2026-07-26T00:41:12.881Z INFO  starting checkout 1.14.2
2026-07-26T00:41:13.104Z ERROR config: required key PAYMENT_SIGNING_KEY is not set
2026-07-26T00:41:13.105Z FATAL exiting with status 1

kubectl get pod checkout-7d9f6c5b4d-2xk9p -n payments \
  -o jsonpath='{range .status.containerStatuses[*]}{.name}{"\t"}{.restartCount}{"\t"}{.lastState.terminated.reason}{"\t"}{.lastState.terminated.exitCode}{"\n"}{end}'

checkout    6    Error    1
istio-proxy 0    <none>   <none>

kubectl get secret payment-keys -n payments -o jsonpath='{.data}' | tr ',' '\n'

{"PAYMENT_API_URL":"aHR0cHM6...","PAYMENT_WEBHOOK_SECRET":"czNjcjN0"}

env:
  - name: PAYMENT_SIGNING_KEY
    valueFrom:
      secretKeyRef:
        name: payment-keys
        key: PAYMENT_SIGNING_KEY
        # optional을 생략하면 기본값 false — 키가 없으면 컨테이너를 시작하지 않는다

kubectl describe pod checkout-7d9f6c5b4d-2xk9p -n payments | grep -A3 "Warning  Failed"

  Warning  Failed     9s (x3 over 25s)  kubelet  Error: couldn't find key PAYMENT_SIGNING_KEY in Secret payments/payment-keys

kubectl get events -n payments --field-selector involvedObject.name=checkout-7d9f6c5b4d-2xk9p --sort-by=.lastTimestamp

LAST SEEN   TYPE      REASON      OBJECT                          MESSAGE
4m12s       Warning   Unhealthy   pod/checkout-7d9f6c5b4d-2xk9p   Liveness probe failed: Get "http://10.42.3.17:8080/healthz": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
4m12s       Normal    Killing     pod/checkout-7d9f6c5b4d-2xk9p   Container checkout failed liveness probe, will be restarted

kubectl describe pod migrate-runner-0 -n payments | grep -A2 "Last State"

    Last State:     Terminated
      Reason:       StartError
      Message:      failed to create containerd task: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: exec: "/app/entrypoint.sh": permission denied

kubectl describe pod ledger-0 -n payments | tail -8

Events:
  Type     Reason       Age                 From               Message
  ----     ------       ----                ----               -------
  Warning  FailedMount  2m15s (x9 over 12m) kubelet            MountVolume.SetUp failed for volume "ledger-data" : rpc error: code = Internal desc = volume attachment is being deleted
  Warning  FailedMount  38s                 kubelet            Unable to attach or mount volumes: unmounted volumes=[ledger-data], unattached volumes=[ledger-data kube-api-access-x9k2m]: timed out waiting for the condition

kubectl get pod ledger-0 -n payments

NAME       READY   STATUS                  RESTARTS      AGE
ledger-0   0/1     Init:CrashLoopBackOff   4 (48s ago)   3m

kubectl logs ledger-0 -n payments -c wait-for-db --previous

waiting for postgres.payments.svc.cluster.local:5432 ...
timeout after 30s

apiVersion: v1
kind: Pod
metadata:
  name: checkout-shell
  namespace: payments
spec:
  restartPolicy: Never
  containers:
    - name: checkout
      image: registry.example.com/checkout:1.14.2
      command: ['sh', '-c', 'sleep infinity']
      envFrom:
        - secretRef:
            name: payment-keys
      volumeMounts:
        - name: config
          mountPath: /etc/checkout
  volumes:
    - name: config
      configMap:
        name: checkout-config

kubectl apply -f checkout-shell.yaml
kubectl exec -it checkout-shell -n payments -- sh

/ # env | grep PAYMENT
PAYMENT_API_URL=https://api.example.com
PAYMENT_WEBHOOK_SECRET=s3cr3t
/ # /app/checkout
ERROR config: required key PAYMENT_SIGNING_KEY is not set

kubectl debug checkout-7d9f6c5b4d-2xk9p -n payments \
  --copy-to=checkout-debug \
  --container=checkout \
  -- sleep infinity

kubectl exec -it checkout-debug -n payments -- sh

kubectl debug -it checkout-7d9f6c5b4d-2xk9p -n payments \
  --image=busybox:1.36 \
  --target=checkout \
  -- sh

Defaulting debug container name to debugger-7v2mp.
/ # ls /proc
1  14  self ...
/ # cat /proc/1/cmdline
/app/checkout
/ # wget -qO- http://localhost:8080/healthz
wget: can't connect to remote host: Connection refused

/ # ls /proc/1/root/etc/checkout
application.yaml  logging.yaml

kubectl apply -f deploy/checkout.yaml
kubectl rollout status deployment/checkout -n payments --timeout=180s

Waiting for deployment "checkout" rollout to finish: 0 of 3 updated replicas are available...
error: deployment "checkout" exceeded its progress deadline

apiVersion: apps/v1
kind: Deployment
metadata:
  name: checkout
  namespace: payments
spec:
  replicas: 3
  progressDeadlineSeconds: 180
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: pod-restart-alerts
  namespace: monitoring
spec:
  groups:
    - name: pod-health
      rules:
        - alert: PodRestartingRepeatedly
          expr: increase(kube_pod_container_status_restarts_total[1h]) > 3
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} restarted more than 3 times in an hour"
        - alert: PodInCrashLoop
          expr: kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff"} == 1
          for: 5m
          labels:
            severity: critical