Split View: Prometheus 운영 총정리 — 메트릭 수집, PromQL, 알림, 대시보드, 모범사례

http_requests_total{method="GET", handler="/api/users", status="200"}

graph TB
    subgraph Targets
        A[Application /metrics]
        B[Node Exporter]
        C[cAdvisor]
        D[Custom Exporter]
    end

    subgraph "Prometheus Server"
        E[Retrieval<br/>Scrape Engine]
        F[TSDB<br/>Time Series Database]
        G[HTTP Server<br/>PromQL API]
    end

    H[Service Discovery<br/>Kubernetes / Consul / DNS]
    I[Pushgateway<br/>For Short-lived Jobs]
    J[Alertmanager<br/>Alert Routing/Grouping]
    K[Grafana<br/>Dashboards]

    H --> E
    A --> E
    B --> E
    C --> E
    D --> E
    I --> E
    E --> F
    F --> G
    G --> K
    G --> J
    J --> L[Slack / PagerDuty / Email]

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.53.0
    container_name: prometheus
    ports:
      - '9090:9090'
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules/:/etc/prometheus/rules/
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'
    restart: unless-stopped

  alertmanager:
    image: prom/alertmanager:v0.27.0
    container_name: alertmanager
    ports:
      - '9093:9093'
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
    restart: unless-stopped

  grafana:
    image: grafana/grafana:11.1.0
    container_name: grafana
    ports:
      - '3000:3000'
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning/:/etc/grafana/provisioning/
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:v1.8.1
    container_name: node-exporter
    ports:
      - '9100:9100'
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

# prometheus/prometheus.yml
global:
  scrape_interval: 15s # Default scrape interval
  evaluation_interval: 15s # Alert rule evaluation interval
  scrape_timeout: 10s # Scrape timeout

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  # Prometheus self-monitoring
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  # Application (using relabel_configs)
  - job_name: 'app'
    metrics_path: /metrics
    scheme: http
    static_configs:
      - targets: ['app:8080']
        labels:
          env: production
          team: backend

  # Kubernetes Service Discovery (reference)
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod

# Per-second request rate over 5 minutes (suitable for alerts)
sum(rate(http_requests_total[5m])) by (service)

# Instantaneous rate of change (suitable for dashboard visualization)
sum(irate(http_requests_total[$__rate_interval])) by (service)

# 95th percentile response time
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
)

# 50th percentile (median)
histogram_quantile(0.50,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
)

# Top 5 services by error rate
topk(5,
  sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
  /
  sum(rate(http_requests_total[5m])) by (service)
)

# Memory usage by namespace
sum(container_memory_usage_bytes{container!=""}) by (namespace)

# Nodes with CPU usage exceeding 80%
node_cpu_seconds_total{mode="idle"} < 0.2

# Predict disk space 4 hours from now using predict_linear
predict_linear(
  node_filesystem_avail_bytes{mountpoint="/"}[6h], 4 * 3600
) < 0

# prometheus/rules/alerts.yml
groups:
  - name: instance-alerts
    rules:
      - alert: InstanceDown
        expr: up == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: 'Instance {{ $labels.instance }} is down'
          description: '{{ $labels.instance }} of job {{ $labels.job }} has been unreachable for more than 3 minutes.'

      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)
          > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.service }} error rate exceeds {{ $value | humanizePercentage }}'
          description: 'Service {{ $labels.service }} has a 5xx error rate exceeding 5%.'

      - alert: HighMemoryUsage
        expr: |
          (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: 'Node memory usage exceeds 90%'

      - alert: DiskSpaceRunningOut
        expr: |
          predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 24*3600) < 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: 'Disk space predicted to run out within 24 hours'

      - alert: HighLatencyP95
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.service }} P95 latency exceeds 1 second'

# alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'

route:
  receiver: 'default-slack'
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 1h
    - match:
        severity: warning
      receiver: 'slack-warnings'
      repeat_interval: 4h

receivers:
  - name: 'default-slack'
    slack_configs:
      - channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        send_resolved: true

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_SERVICE_KEY'
        severity: critical

  - name: 'slack-warnings'
    slack_configs:
      - channel: '#alerts-warning'
        send_resolved: true

inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'instance']

# grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true
    jsonData:
      timeInterval: '15s'
      httpMethod: POST

{
  "title": "Service Error Rate",
  "type": "timeseries",
  "datasource": "Prometheus",
  "targets": [
    {
      "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
      "legendFormat": "{{ service }}"
    }
  ],
  "fieldConfig": {
    "defaults": {
      "unit": "percentunit",
      "thresholds": {
        "steps": [
          { "value": 0, "color": "green" },
          { "value": 0.01, "color": "yellow" },
          { "value": 0.05, "color": "red" }
        ]
      }
    }
  }
}

# Global Prometheus scrape_configs
scrape_configs:
  - job_name: 'federate'
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{job=~".+"}' # All job metrics
        - '{__name__=~"job:.*"}' # Only recording rule results
    static_configs:
      - targets:
          - 'prometheus-team-a:9090'
          - 'prometheus-team-b:9090'

# Drop unnecessary labels with relabel_configs
relabel_configs:
  - action: labeldrop
    regex: '(pod_template_hash|controller_revision_hash)'

# Drop high-cardinality metrics with metric_relabel_configs
metric_relabel_configs:
  - source_labels: [__name__]
    regex: '(go_gc_.*|go_memstats_.*)'
    action: drop

# Top 10 metrics by time series count
topk(10, count by (__name__) ({__name__=~".+"}))

# Check cardinality for a specific metric
count(http_requests_total) by (service, method, status)