読まれるダッシュボードと鳴らす価値のあるアラート — 問いの定義、変数構成、SLO、そしてアラート疲れ

# provisioning/datasources/prometheus.yaml
apiVersion: 1
datasources:
  - name: Prometheus-prod
    uid: prom-prod
    type: prometheus
    access: proxy
    url: http://prometheus.observability.svc:9090
    jsonData:
      httpMethod: POST
      timeInterval: 15s          # スクレイプ間隔。rate間隔計算の基準になる
      prometheusType: Prometheus
      prometheusVersion: 3.13.0
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: tempo-prod
    isDefault: true

  - name: Prometheus-staging
    uid: prom-staging
    type: prometheus
    access: proxy
    url: http://prometheus.staging.svc:9090
    jsonData:
      httpMethod: POST
      timeInterval: 15s

{
  "templating": {
    "list": [
      {
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus",
        "current": { "text": "Prometheus-prod", "value": "prom-prod" }
      },
      {
        "name": "namespace",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": "label_values(kube_namespace_status_phase, namespace)",
        "refresh": 1,
        "sort": 1
      },
      {
        "name": "service",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": "label_values(http_requests_total{namespace=\"$namespace\"}, service)",
        "refresh": 2,
        "includeAll": true,
        "multi": true
      },
      {
        "name": "route",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": "label_values(http_requests_total{namespace=\"$namespace\", service=~\"$service\"}, route)",
        "refresh": 2,
        "includeAll": true,
        "multi": true
      }
    ]
  }
}

# 複数選択の変数は正規表現で受ける。等号で受けると値がひとつのときしか動かない
sum by (route) (
  rate(http_requests_total{namespace="$namespace", service=~"$service", route=~"$route"}[$__rate_interval])
)

# ダッシュボードのパネルではrate間隔変数を使う。
# 固定の[5m]を使うと、時間範囲を広く見るときは解像度が無駄になり、
# 狭く見るときはサンプルが足りずグラフに穴が空く
histogram_quantile(0.99,
  sum by (le, route) (
    rate(http_request_duration_seconds_bucket{namespace="$namespace", service=~"$service"}[$__rate_interval])
  )
)

# 上位10ルートのみ。残りが気になれば変数で絞って見る
topk(10,
  sum by (route) (
    rate(http_requests_total{service=~"$service", status_class="5xx"}[$__rate_interval])
  )
)

# デプロイ時刻を注釈として重ねて描くと「いつからか」という問いが即座に解決する
changes(kube_deployment_status_observed_generation{namespace="$namespace"}[$__rate_interval]) > 0

# 原因ベース — こういうルールが40個ほど積み上がるとアラート疲れが始まる
- alert: HighCPU
  expr: rate(process_cpu_seconds_total[5m]) > 0.8
  for: 5m
- alert: HighMemory
  expr: process_resident_memory_bytes / container_spec_memory_limit_bytes > 0.9
  for: 5m
- alert: ManyGoroutines
  expr: go_goroutines > 10000
  for: 5m
- alert: DBConnectionsHigh
  expr: pg_stat_activity_count > 80
  for: 5m

# 症状ベース — ユーザーが経験することに掛ける
groups:
  - name: symptom_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx"}[5m]))
            /
          sum by (service) (rate(http_requests_total[5m]))
            > 0.02
        for: 5m
        labels:
          severity: page
        annotations:
          summary: '5xx比率が2%を超えました'
          runbook_url: https://wiki.internal/runbook/high-error-rate

      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum by (le, service) (rate(http_request_duration_seconds_bucket[5m]))
          ) > 1.5
        for: 10m
        labels:
          severity: page
        annotations:
          summary: 'p99レイテンシが1.5秒を超えました'

      - alert: RequestsStopped
        expr: |
          sum by (service) (rate(http_requests_total[5m])) == 0
            and
          sum by (service) (rate(http_requests_total[5m] offset 1h)) > 1
        for: 5m
        labels:
          severity: page
        annotations:
          summary: '普段トラフィックがあるはずのサービスにリクエストが来ていません'

# 問題のあるルール — 5分平均がしきい値を超えることは事実上ない
avg_over_time(http_request_duration_seconds_sum[5m])
  / avg_over_time(http_request_duration_seconds_count[5m]) > 5

# なぜ鳴らないか: 平均はテールを隠す。p99が8秒でも平均は0.2秒かもしれない。
# このルールはサービス全体が軒並み8秒かかるときにしか発火しない。

# 過去30日間に一度も発火していないルールを見つける
# ALERTS_FOR_STATEはアクティブなアラートにしか存在しないため、ルール一覧と突き合わせる必要がある
count by (alertname) (max_over_time(ALERTS[30d]))

# ルールの評価自体が失敗していないかも確認する
increase(prometheus_rule_evaluation_failures_total[1h]) > 0

# tests/alerts_test.yml
rule_files:
  - ../rules/symptom_alerts.yml

evaluation_interval: 30s

tests:
  - interval: 15s
    input_series:
      - series: 'http_requests_total{service="checkout", status_class="2xx"}'
        values: '0+90x60'
      - series: 'http_requests_total{service="checkout", status_class="5xx"}'
        values: '0+10x60'
    alert_rule_test:
      - eval_time: 10m
        alertname: HighErrorRate
        exp_alerts:
          - exp_labels:
              service: checkout
              severity: page
            exp_annotations:
              summary: '5xx比率が2%を超えました'
              runbook_url: https://wiki.internal/runbook/high-error-rate

# rules/slo.yml
groups:
  - name: slo_sli
    interval: 30s
    rules:
      # 良いリクエストの割合 — ヘルスチェックは除外する
      - record: service:sli_availability:ratio_rate5m
        expr: |
          sum by (service) (
            rate(http_requests_total{status_class!="5xx", route!~"/healthz|/readyz"}[5m])
          )
          /
          sum by (service) (
            rate(http_requests_total{route!~"/healthz|/readyz"}[5m])
          )

      # 各ウィンドウのエラー比率 — バーンレートアラートが参照する
      - record: service:http_error_ratio:rate5m
        expr: |
          1 - service:sli_availability:ratio_rate5m

      - record: service:http_error_ratio:rate1h
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[1h]))
          /
          sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[1h]))

      - record: service:http_error_ratio:rate6h
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[6h]))
          /
          sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[6h]))

      - record: service:http_error_ratio:rate30m
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[30m]))
          /
          sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[30m]))

      # 残エラーバジェット比率 — ダッシュボードの中核パネル
      - record: service:error_budget_remaining:ratio30d
        expr: |
          1 - (
            (
              sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[30d]))
              /
              sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[30d]))
            )
            / 0.001
          )

groups:
  - name: slo_burn_rate
    rules:
      - alert: ErrorBudgetBurnFast
        expr: |
          service:http_error_ratio:rate1h > (14.4 * 0.001)
            and
          service:http_error_ratio:rate5m > (14.4 * 0.001)
        for: 2m
        labels:
          severity: page
          slo: availability
        annotations:
          summary: 'エラーバジェットを14.4倍の速度で消費中です。約2日で全量消費します'
          runbook_url: https://wiki.internal/runbook/slo-burn

      - alert: ErrorBudgetBurnSlow
        expr: |
          service:http_error_ratio:rate6h > (6 * 0.001)
            and
          service:http_error_ratio:rate30m > (6 * 0.001)
        for: 15m
        labels:
          severity: ticket
          slo: availability
        annotations:
          summary: 'エラーバジェットを6倍の速度で消費中です'

# alertmanager.yml
route:
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: slack-default
  routes:
    - matchers: [severity = page]
      receiver: pagerduty
      group_wait: 10s
      repeat_interval: 1h
      continue: true
    - matchers: [severity = ticket]
      receiver: jira
      repeat_interval: 24h

inhibit_rules:
  # クラスタ全体の障害中は個別サービスのアラートを抑制する
  - source_matchers: [alertname = ClusterUnreachable]
    target_matchers: [severity =~ 'page|ticket']
    equal: [cluster]
  # 同じサービスでpageが立っていればticketは抑制する
  - source_matchers: [severity = page]
    target_matchers: [severity = ticket]
    equal: [service]

receivers:
  - name: pagerduty
    pagerduty_configs:
      - routing_key_file: /etc/alertmanager/pd_key
  - name: jira
    webhook_configs:
      - url: http://alert-to-jira.internal/hook
  - name: slack-default
    slack_configs:
      - channel: '#alerts'
        send_resolved: true

# provisioning/dashboards/dashboards.yaml
apiVersion: 1
providers:
  - name: gitops
    orgId: 1
    folder: Services
    type: file
    disableDeletion: true
    updateIntervalSeconds: 60
    allowUiUpdates: false        # UI編集を禁止する。変更はリポジトリを経由する
    options:
      path: /etc/grafana/dashboards
      foldersFromFilesStructure: true

# provisioning/alerting/rules.yaml — Grafana管理のアラートをファイルとして定義する
apiVersion: 1
groups:
  - orgId: 1
    name: checkout_slo
    folder: Alerts
    interval: 1m
    rules:
      - uid: checkout-burn-fast
        title: Checkout error budget burn (fast)
        condition: THRESHOLD
        data:
          - refId: BURN
            relativeTimeRange:
              from: 3600
              to: 0
            datasourceUid: prom-prod
            model:
              expr: service:http_error_ratio:rate1h{service="checkout"}
              instant: true
              refId: BURN
          - refId: THRESHOLD
            datasourceUid: __expr__
            model:
              type: threshold
              expression: BURN
              conditions:
                - evaluator:
                    type: gt
                    params: [0.0144]
              refId: THRESHOLD
        for: 2m
        noDataState: NoData
        execErrState: Alerting
        labels:
          severity: page
          slo: availability
        annotations:
          summary: Checkoutのエラーバジェットが14.4倍の速度で消費されています
          runbook_url: https://wiki.internal/runbook/slo-burn

- alert: ScrapeTargetDown
  expr: up{job=~"checkout.*|payment.*"} == 0
  for: 5m
  labels:
    severity: page
  annotations:
    summary: 'スクレイプ対象が5分間応答していません'