Prometheus運用完全ガイド — メトリクス収集・PromQL・アラート・ダッシュボード・ベストプラクティス

http_requests_total{method="GET", handler="/api/users", status="200"}

graph TB
    subgraph Targets
        A[Application /metrics]
        B[Node Exporter]
        C[cAdvisor]
        D[Custom Exporter]
    end

    subgraph "Prometheus Server"
        E[Retrieval<br/>スクレイプエンジン]
        F[TSDB<br/>時系列データベース]
        G[HTTP Server<br/>PromQL API]
    end

    H[Service Discovery<br/>Kubernetes / Consul / DNS]
    I[Pushgateway<br/>短期ジョブ用]
    J[Alertmanager<br/>アラートルーティング/グルーピング]
    K[Grafana<br/>ダッシュボード]

    H --> E
    A --> E
    B --> E
    C --> E
    D --> E
    I --> E
    E --> F
    F --> G
    G --> K
    G --> J
    J --> L[Slack / PagerDuty / Email]

# docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.53.0
    container_name: prometheus
    ports:
      - '9090:9090'
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/rules/:/etc/prometheus/rules/
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'
    restart: unless-stopped

  alertmanager:
    image: prom/alertmanager:v0.27.0
    container_name: alertmanager
    ports:
      - '9093:9093'
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml
    restart: unless-stopped

  grafana:
    image: grafana/grafana:11.1.0
    container_name: grafana
    ports:
      - '3000:3000'
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=admin
    volumes:
      - grafana_data:/var/lib/grafana
      - ./grafana/provisioning/:/etc/grafana/provisioning/
    restart: unless-stopped

  node-exporter:
    image: prom/node-exporter:v1.8.1
    container_name: node-exporter
    ports:
      - '9100:9100'
    restart: unless-stopped

volumes:
  prometheus_data:
  grafana_data:

# prometheus/prometheus.yml
global:
  scrape_interval: 15s # デフォルトスクレイプ間隔
  evaluation_interval: 15s # アラートルール評価間隔
  scrape_timeout: 10s # スクレイプタイムアウト

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - /etc/prometheus/rules/*.yml

scrape_configs:
  # Prometheus自身のモニタリング
  - job_name: 'prometheus'
    static_configs:
      - targets: ['localhost:9090']

  # Node Exporter
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  # アプリケーション（relabel_configsの活用）
  - job_name: 'app'
    metrics_path: /metrics
    scheme: http
    static_configs:
      - targets: ['app:8080']
        labels:
          env: production
          team: backend

  # Kubernetesサービスディスカバリ（参考）
  - job_name: 'kubernetes-pods'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: pod

# 5分間の秒間リクエストレート（アラートに適切）
sum(rate(http_requests_total[5m])) by (service)

# 瞬間変化率（ダッシュボードの可視化に適切）
sum(irate(http_requests_total[$__rate_interval])) by (service)

# 95パーセンタイルレスポンスタイム
histogram_quantile(0.95,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
)

# 50パーセンタイル（中央値）
histogram_quantile(0.50,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le)
)

# サービス別エラー率トップ5
topk(5,
  sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
  /
  sum(rate(http_requests_total[5m])) by (service)
)

# ネームスペース別メモリ使用量
sum(container_memory_usage_bytes{container!=""}) by (namespace)

# CPU使用率が80%を超えるノード
node_cpu_seconds_total{mode="idle"} < 0.2

# predict_linearでディスク容量を予測（4時間後）
predict_linear(
  node_filesystem_avail_bytes{mountpoint="/"}[6h], 4 * 3600
) < 0

# prometheus/rules/alerts.yml
groups:
  - name: instance-alerts
    rules:
      - alert: InstanceDown
        expr: up == 0
        for: 3m
        labels:
          severity: critical
        annotations:
          summary: 'インスタンス {{ $labels.instance }} がダウンしました'
          description: '{{ $labels.job }} ジョブの {{ $labels.instance }} が3分以上応答していません。'

      - alert: HighErrorRate
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)
          > 0.05
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.service }} エラー率 {{ $value | humanizePercentage }} 超過'
          description: 'サービス {{ $labels.service }} の5xxエラー率が5%を超過しました。'

      - alert: HighMemoryUsage
        expr: |
          (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 0.9
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: 'ノードメモリ使用率90%超過'

      - alert: DiskSpaceRunningOut
        expr: |
          predict_linear(node_filesystem_avail_bytes{mountpoint="/"}[6h], 24*3600) < 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: '24時間以内にディスク容量が不足すると予測されます'

      - alert: HighLatencyP95
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          ) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: '{{ $labels.service }} P95レスポンスタイム1秒超過'

# alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  slack_api_url: 'https://hooks.slack.com/services/YOUR/SLACK/WEBHOOK'

route:
  receiver: 'default-slack'
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 1h
    - match:
        severity: warning
      receiver: 'slack-warnings'
      repeat_interval: 4h

receivers:
  - name: 'default-slack'
    slack_configs:
      - channel: '#alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}'
        send_resolved: true

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_SERVICE_KEY'
        severity: critical

  - name: 'slack-warnings'
    slack_configs:
      - channel: '#alerts-warning'
        send_resolved: true

inhibit_rules:
  - source_match:
      severity: critical
    target_match:
      severity: warning
    equal: ['alertname', 'instance']

# grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true
    jsonData:
      timeInterval: '15s'
      httpMethod: POST

{
  "title": "Service Error Rate",
  "type": "timeseries",
  "datasource": "Prometheus",
  "targets": [
    {
      "expr": "sum(rate(http_requests_total{status=~\"5..\"}[5m])) by (service) / sum(rate(http_requests_total[5m])) by (service)",
      "legendFormat": "{{ service }}"
    }
  ],
  "fieldConfig": {
    "defaults": {
      "unit": "percentunit",
      "thresholds": {
        "steps": [
          { "value": 0, "color": "green" },
          { "value": 0.01, "color": "yellow" },
          { "value": 0.05, "color": "red" }
        ]
      }
    }
  }
}

# グローバルPrometheusのscrape_configs
scrape_configs:
  - job_name: 'federate'
    honor_labels: true
    metrics_path: '/federate'
    params:
      'match[]':
        - '{job=~".+"}' # すべてのjobメトリクス
        - '{__name__=~"job:.*"}' # Recording Ruleの結果のみ
    static_configs:
      - targets:
          - 'prometheus-team-a:9090'
          - 'prometheus-team-b:9090'

# relabel_configsで不要なラベルを削除
relabel_configs:
  - action: labeldrop
    regex: '(pod_template_hash|controller_revision_hash)'

# metric_relabel_configsで高カーディナリティメトリクスを削除
metric_relabel_configs:
  - source_labels: [__name__]
    regex: '(go_gc_.*|go_memstats_.*)'
    action: drop

# 時系列数が最も多いメトリクストップ10
topk(10, count by (__name__) ({__name__=~".+"}))

# 特定メトリクスのカーディナリティ確認
count(http_requests_total) by (service, method, status)