Prometheus 운영 총정리 — 메트릭 수집, PromQL, 알림, 대시보드, 모범사례

http_requests_total{method="GET", handler="/api/users", status="200"}

graph TB
    subgraph Targets
        A[Application /metrics]
        B[Node Exporter]
        C[cAdvisor]
        D[Custom Exporter]
    end

    subgraph "Prometheus Server"
        E[Retrieval<br/>스크레이프 엔진]
        F[TSDB<br/>시계열 데이터베이스]
        G[HTTP Server<br/>PromQL API]
    end

    H[Service Discovery<br/>Kubernetes / Consul / DNS]
    I[Pushgateway<br/>단기 작업용]
    J[Alertmanager<br/>알림 라우팅/그룹핑]
    K[Grafana<br/>대시보드]

    H --> E
    A --> E
    B --> E
    C --> E
    D --> E
    I --> E
    E --> F
    F --> G
    G --> K
    G --> J
    J --> L[Slack / PagerDuty / Email]

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.53.0
    container_name: prometheus
    ports:
      - '9090:9090'
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules/:/etc/prometheus/rules/
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'
    restart: unless-stopped

  alertmanager:
    image: prom/alertmanager:v0.27.0
    container_name: alertmanager
    ports:
      - '9093:9093'
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
    restart: unless-stopped

  grafana:
    image: grafana/grafana:11.1.0
    container_name: grafana
    ports:
      - '3000:3000'
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning/:/etc/grafana/provisioning/
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:v1.8.1
    container_name: node-exporter
    ports:
      - '9100:9100'
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

# prometheus/prometheus.yml
global:
  scrape_interval: 15s # 기본 스크레이프 간격
  evaluation_interval: 15s # 알림 규칙 평가 간격
  scrape_timeout: 10s # 스크레이프 타임아웃

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  # Prometheus 자기 자신 모니터링
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  # 애플리케이션 (relabel_configs 활용)
  - job_name: 'app'
    metrics_path: /metrics
    scheme: http
    static_configs:
      - targets: ['app:8080']
        labels:
          env: production
          team: backend

  # Kubernetes 서비스 디스커버리 (참고)
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod

# 5분간 초당 요청 비율 (알림에 적합)
sum(rate(http_requests_total[5m])) by (service)

# 순간 변화율 (대시보드 시각화에 적합)
sum(irate(http_requests_total[$__rate_interval])) by (service)

# 95번째 백분위수 응답 시간
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
)

# 50번째 백분위수 (중간값)
histogram_quantile(0.50,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
)

# 서비스별 에러율 상위 5개
topk(5,
  sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
  /
  sum(rate(http_requests_total[5m])) by (service)
)

# 네임스페이스별 메모리 사용량
sum(container_memory_usage_bytes{container!=""}) by (namespace)

# CPU 사용률이 80% 초과인 노드
node_cpu_seconds_total{mode="idle"} < 0.2

# predict_linear으로 디스크 용량 예측 (4시간 뒤)
predict_linear(
  node_filesystem_avail_bytes{mountpoint="/"}[6h], 4 * 3600
) < 0

# prometheus/rules/alerts.yml
groups:
  - name: instance-alerts
    rules:
      - alert: InstanceDown
        expr: up == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: '인스턴스 {{ $labels.instance }}가 다운되었습니다'
          description: '{{ $labels.job }} 작업의 {{ $labels.instance }}가 3분 이상 응답하지 않습니다.'

      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)
          > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.service }} 에러율 {{ $value | humanizePercentage }} 초과'
          description: '서비스 {{ $labels.service }}의 5xx 에러율이 5%를 초과했습니다.'

      - alert: HighMemoryUsage
        expr: |
          (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: '노드 메모리 사용률 90% 초과'

      - alert: DiskSpaceRunningOut
        expr: |
          predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 24*3600) < 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: '24시간 내 디스크 공간이 부족할 것으로 예측됩니다'

      - alert: HighLatencyP95
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.service }} P95 응답 시간 1초 초과'

# alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'

route:
  receiver: 'default-slack'
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 1h
    - match:
        severity: warning
      receiver: 'slack-warnings'
      repeat_interval: 4h

receivers:
  - name: 'default-slack'
    slack_configs:
      - channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        send_resolved: true

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_SERVICE_KEY'
        severity: critical

  - name: 'slack-warnings'
    slack_configs:
      - channel: '#alerts-warning'
        send_resolved: true

inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'instance']

# grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true
    jsonData:
      timeInterval: '15s'
      httpMethod: POST

{
  "title": "Service Error Rate",
  "type": "timeseries",
  "datasource": "Prometheus",
  "targets": [
    {
      "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
      "legendFormat": "{{ service }}"
    }
  ],
  "fieldConfig": {
    "defaults": {
      "unit": "percentunit",
      "thresholds": {
        "steps": [
          { "value": 0, "color": "green" },
          { "value": 0.01, "color": "yellow" },
          { "value": 0.05, "color": "red" }
        ]
      }
    }
  }
}

# 글로벌 Prometheus의 scrape_configs
scrape_configs:
  - job_name: 'federate'
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{job=~".+"}' # 모든 job 메트릭
        - '{__name__=~"job:.*"}' # Recording Rule 결과만
    static_configs:
      - targets:
          - 'prometheus-team-a:9090'
          - 'prometheus-team-b:9090'

# relabel_configs로 불필요한 레이블 제거
relabel_configs:
  - action: labeldrop
    regex: '(pod_template_hash|controller_revision_hash)'

# metric_relabel_configs로 고카디널리티 메트릭 제거
metric_relabel_configs:
  - source_labels: [__name__]
    regex: '(go_gc_.*|go_memstats_.*)'
    action: drop

# 시계열 수가 가장 많은 메트릭 상위 10개
topk(10, count by (__name__) ({__name__=~".+"}))

# 특정 메트릭의 카디널리티 확인
count(http_requests_total) by (service, method, status)