필사 모드: Kubernetes OOMKilled(137) 메모리 문제 해결 — 한도를 올리기 전에 확인할 것들

kubectl get pod -n search

NAME                      READY   STATUS             RESTARTS       AGE
indexer-6c4d9f7b8-h2klp   0/1     CrashLoopBackOff   4 (94s ago)    22m

kubectl logs indexer-6c4d9f7b8-h2klp -n search --previous | tail -3

2026-07-26T00:38:41.220Z INFO  segment merge started shard=7 docs=1284000
2026-07-26T00:38:44.902Z INFO  segment merge progress shard=7 pct=61

kubectl describe pod indexer-6c4d9f7b8-h2klp -n search | grep -A6 "Last State"

    Last State:     Terminated
      Reason:       OOMKilled
      Exit Code:    137
      Started:      Sun, 26 Jul 2026 09:36:02 +0900
      Finished:     Sun, 26 Jul 2026 09:38:45 +0900
    Restart Count:  4

kubectl get pods -A -o custom-columns='NS:.metadata.namespace,POD:.metadata.name,REASON:.status.containerStatuses[*].lastState.terminated.reason,CODE:.status.containerStatuses[*].lastState.terminated.exitCode' \
  | grep -v '<none>' | head -5

NS          POD                            REASON      CODE
search      indexer-6c4d9f7b8-h2klp        OOMKilled   137
payments    ledger-5f8b7c6d9-nm3xt         Error       137
media       transcode-79c5d6f8b-vn4pq      Error       143

kubectl exec -it indexer-6c4d9f7b8-h2klp -n search -- cat /sys/fs/cgroup/memory.events

low 0
high 0
max 3421
oom 2
oom_kill 2

kubectl debug node/ip-10-0-1-14 -it --image=busybox -- sh

/ # dmesg -T | grep -i "out of memory" | tail -3
[Sun Jul 26 09:38:45 2026] Memory cgroup out of memory: Killed process 21847 (java) total-vm:9421884kB, anon-rss:2088412kB, file-rss:2104kB, shmem-rss:0kB, UID:1000 pgtables:5120kB oom_score_adj:939

kubectl get events -A --field-selector reason=Evicted --sort-by=.lastTimestamp | tail -3

NAMESPACE   LAST SEEN   TYPE      REASON    OBJECT                        MESSAGE
media       4m12s       Warning   Evicted   pod/transcode-79c5d6f8b-9klm  The node was low on resource: memory. Threshold quantity: 100Mi, available: 84Mi. Container transcode was using 3187Mi, request is 512Mi.

kubectl get pods -A -o custom-columns='NS:.metadata.namespace,POD:.metadata.name,QOS:.status.qosClass' | grep BestEffort | head -3

NS          POD                             QOS
default     debug-shell                     BestEffort
media       thumbnailer-5c8d7f9b6-p4wnt     BestEffort

kubectl exec -it indexer-6c4d9f7b8-h2klp -n search -- \
  java -XX:+PrintFlagsFinal -version 2>/dev/null | grep -E "MaxHeapSize|MaxRAMPercentage"

   size_t MaxHeapSize     = 536870912    {product} {ergonomic}
 double MaxRAMPercentage  = 25.000000    {product} {default}

kubectl exec -it indexer-6c4d9f7b8-h2klp -n search -- jcmd 1 VM.native_memory summary

Total: reserved=4183MB, committed=1971MB
-                 Java Heap (reserved=512MB, committed=512MB)
-                     Class (reserved=1084MB, committed=68MB)
-                    Thread (reserved=418MB, committed=418MB)
-                      Code (reserved=250MB, committed=42MB)
-                        GC (reserved=76MB, committed=76MB)
-                  Internal (reserved=812MB, committed=804MB)

env:
  - name: JAVA_TOOL_OPTIONS
    value: >-
      -XX:MaxRAMPercentage=70.0
      -XX:MaxMetaspaceSize=256m
      -XX:MaxDirectMemorySize=256m
      -XX:+ExitOnOutOfMemoryError
      -XX:+HeapDumpOnOutOfMemoryError
      -XX:HeapDumpPath=/dumps
resources:
  requests:
    memory: 2Gi
  limits:
    memory: 2Gi

kubectl exec -it api-gateway-7d8f9c6b5-x2mnp -n edge -- \
  node -e "console.log(require('v8').getHeapStatistics().heap_size_limit / 1048576)"

4144.0

env:
  - name: NODE_OPTIONS
    value: '--max-old-space-size=768'
resources:
  requests:
    memory: 1Gi
  limits:
    memory: 1Gi

env:
  - name: MALLOC_ARENA_MAX
    value: '2'

command:
  - gunicorn
  - --workers=4
  - --max-requests=2000
  - --max-requests-jitter=200
  - app:application

kubectl exec -it api-gateway-7d8f9c6b5-x2mnp -n edge -- cat /sys/fs/cgroup/memory.events | grep oom_kill

oom_kill 3

kubectl exec -it indexer-6c4d9f7b8-h2klp -n search -- sh -c \
  'cat /sys/fs/cgroup/memory.max; cat /sys/fs/cgroup/memory.current; grep -E "^(anon|file|slab) " /sys/fs/cgroup/memory.stat'

2147483648
2091483136
anon 1183842304
file 874512384
slab 33124352

kubectl top pod indexer-6c4d9f7b8-h2klp -n search

NAME                      CPU(cores)   MEMORY(bytes)
indexer-6c4d9f7b8-h2klp   410m         1216Mi

volumes:
  - name: scratch
    emptyDir:
      medium: Memory
      sizeLimit: 256Mi

resources:
  requests:
    cpu: 500m
    memory: 2Gi
  limits:
    memory: 2Gi

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: indexer
  namespace: search
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: indexer
  updatePolicy:
    updateMode: 'Off'

kubectl get vpa indexer -n search \
  -o jsonpath='{.status.recommendation.containerRecommendations[0]}'

{"containerName":"indexer","lowerBound":{"cpu":"320m","memory":"1300Mi"},"target":{"cpu":"480m","memory":"1730Mi"},"uncappedTarget":{"cpu":"480m","memory":"1730Mi"},"upperBound":{"cpu":"1100m","memory":"2600Mi"}}

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: memory-alerts
  namespace: monitoring
spec:
  groups:
    - name: memory
      rules:
        - alert: ContainerOOMKilled
          expr: increase(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) > 0
          labels:
            severity: critical
          annotations:
            summary: "{{ $labels.namespace }}/{{ $labels.pod }} was OOMKilled"
        - alert: ContainerMemoryNearLimit
          expr: |
            container_memory_working_set_bytes{container!="",container!="POD"}
              / on(namespace,pod,container) container_spec_memory_limit_bytes{container!=""} > 0.9
          for: 15m
          labels:
            severity: warning
        - alert: ContainerMemoryGrowsMonotonically
          expr: |
            deriv(container_memory_working_set_bytes{container!="",container!="POD"}[6h]) > 0
              and increase(container_memory_working_set_bytes{container!="",container!="POD"}[6h]) > 200e6
          for: 1h
          labels:
            severity: warning

volumeMounts:
  - name: dumps
    mountPath: /dumps
volumes:
  - name: dumps
    persistentVolumeClaim:
      claimName: heap-dumps