SLI/SLO/Error Budget基盤の信頼性エンジニアリング実践ガイド

# error-budget-policy.yaml
error_budget_policy:
  service: payment-api
  slo_window: 30d
  slo_target: 99.9%

  thresholds:
    - level: green
      budget_remaining: '>50%'
      actions:
        - 通常の機能開発を継続
        - 週次SLOレビューミーティング

    - level: yellow
      budget_remaining: '25%-50%'
      actions:
        - 新機能リリース前に信頼性レビュー必須
        - 技術的負債の解消作業を並行実施
        - 日次SLOレビューミーティング

    - level: orange
      budget_remaining: '10%-25%'
      actions:
        - 新機能リリースを停止
        - エンジニアリング時間の50%以上を信頼性作業に投入
        - 障害原因分析（Postmortem）を即時実行

    - level: red
      budget_remaining: '<10%'
      actions:
        - 全機能開発を凍結
        - 全エンジニアリングリソースを信頼性復旧に集中
        - 経営陣へのエスカレーション
        - SLO未達の根本原因が解消されるまで凍結を維持

# multi-window-burn-rate-alerts.yaml
# Google SRE Workbook推奨アラート設定
groups:
  - name: slo-burn-rate-alerts
    rules:
      # ページアラート: 2%の予算を1時間以内に消費（Burn Rate 14.4）
      - alert: SLOBurnRateHigh_Page
        expr: |
          (
            sli:error_ratio:rate1h > (14.4 * 0.001)
            and
            sli:error_ratio:rate5m > (14.4 * 0.001)
          )
        for: 2m
        labels:
          severity: page
        annotations:
          summary: 'Error budget burn rate is very high'
          description: 'Service is consuming error budget 14.4x faster than normal. 2% of 30-day budget will be consumed in 1 hour.'

      # ページアラート: 5%の予算を6時間以内に消費（Burn Rate 6）
      - alert: SLOBurnRateMedium_Page
        expr: |
          (
            sli:error_ratio:rate6h > (6 * 0.001)
            and
            sli:error_ratio:rate30m > (6 * 0.001)
          )
        for: 5m
        labels:
          severity: page
        annotations:
          summary: 'Error budget burn rate is elevated'
          description: 'Service is consuming error budget 6x faster than normal. 5% of 30-day budget will be consumed in 6 hours.'

      # チケットアラート: 10%の予算を3日以内に消費（Burn Rate 1）
      - alert: SLOBurnRateSlow_Ticket
        expr: |
          (
            sli:error_ratio:rate3d > (1 * 0.001)
            and
            sli:error_ratio:rate6h > (1 * 0.001)
          )
        for: 30m
        labels:
          severity: ticket
        annotations:
          summary: 'Error budget burn rate indicates slow degradation'
          description: 'Service is steadily consuming error budget. 10% of 30-day budget will be consumed in 3 days.'

# sli-recording-rules.yaml
groups:
  - name: sli-recording-rules
    interval: 30s
    rules:
      # 全リクエスト中のエラー比率（様々なウィンドウ）
      - record: sli:error_ratio:rate5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m]))

      - record: sli:error_ratio:rate30m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[30m]))
          /
          sum(rate(http_requests_total[30m]))

      - record: sli:error_ratio:rate1h
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[1h]))
          /
          sum(rate(http_requests_total[1h]))

      - record: sli:error_ratio:rate6h
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[6h]))
          /
          sum(rate(http_requests_total[6h]))

      - record: sli:error_ratio:rate3d
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[3d]))
          /
          sum(rate(http_requests_total[3d]))

      # レイテンシSLI: 200ms以内の応答比率
      - record: sli:latency_good_ratio:rate5m
        expr: |
          sum(rate(http_request_duration_seconds_bucket{le="0.2"}[5m]))
          /
          sum(rate(http_request_duration_seconds_count[5m]))

      - record: sli:latency_good_ratio:rate1h
        expr: |
          sum(rate(http_request_duration_seconds_bucket{le="0.2"}[1h]))
          /
          sum(rate(http_request_duration_seconds_count[1h]))

# Error Budget残存率（30日ウィンドウ、SLO 99.9%）
# 結果が1なら100%残存、0なら完全消費
(
  1 - (
    sum(increase(http_requests_total{status=~"5.."}[30d]))
    /
    sum(increase(http_requests_total[30d]))
  )
  - 0.999
) / 0.001

#!/usr/bin/env python3
"""SLOレポート生成スクリプト"""

import requests
from datetime import datetime, timedelta

PROMETHEUS_URL = "http://prometheus:9090"
SLO_CONFIGS = [
    {
        "name": "payment-api-availability",
        "sli_query": 'sum(rate(http_requests_total{job="payment-api",status!~"5.."}[30d])) / sum(rate(http_requests_total{job="payment-api"}[30d]))',
        "slo_target": 0.999,
        "window_days": 30,
    },
    {
        "name": "payment-api-latency",
        "sli_query": 'sum(rate(http_request_duration_seconds_bucket{job="payment-api",le="0.2"}[30d])) / sum(rate(http_request_duration_seconds_count{job="payment-api"}[30d]))',
        "slo_target": 0.995,
        "window_days": 30,
    },
]


def query_prometheus(query: str) -> float:
    """Prometheus instant queryの実行"""
    resp = requests.get(
        f"{PROMETHEUS_URL}/api/v1/query",
        params={"query": query},
    )
    result = resp.json()["data"]["result"]
    if not result:
        return 0.0
    return float(result[0]["value"][1])


def generate_report():
    """SLOレポートの生成"""
    print(f"=== SLO Report ({datetime.now().strftime('%Y-%m-%d %H:%M')}) ===\n")

    for config in SLO_CONFIGS:
        current_sli = query_prometheus(config["sli_query"])
        slo_target = config["slo_target"]
        error_budget_total = 1 - slo_target
        error_budget_consumed = max(0, slo_target - current_sli)
        error_budget_remaining = max(
            0, (error_budget_total - error_budget_consumed) / error_budget_total
        )

        status = "OK" if current_sli >= slo_target else "VIOLATED"
        budget_pct = error_budget_remaining * 100

        print(f"Service: {config['name']}")
        print(f"  SLI (current): {current_sli:.4%}")
        print(f"  SLO (target):  {slo_target:.4%}")
        print(f"  Status:        {status}")
        print(f"  Error Budget:  {budget_pct:.1f}% remaining")

        if budget_pct < 25:
            print(f"  WARNING: Error budget below 25%!")
        print()


if __name__ == "__main__":
    generate_report()

# grafana-slo-dashboard.json（パネル構成の概要）
panels:
  - title: 'Current Availability SLI'
    type: stat
    targets:
      - expr: '1 - sli:error_ratio:rate30d'
        legendFormat: 'Availability'
    thresholds:
      - value: 0.999
        color: green
      - value: 0.995
        color: yellow
      - value: 0.99
        color: red

  - title: 'Error Budget Remaining'
    type: gauge
    targets:
      - expr: |
          clamp_min(
            (0.001 - sli:error_ratio:rate30d) / 0.001,
            0
          )
        legendFormat: 'Budget Remaining'
    min: 0
    max: 1
    thresholds:
      - value: 0.5
        color: green
      - value: 0.25
        color: yellow
      - value: 0
        color: red

  - title: 'Burn Rate (1h window)'
    type: timeseries
    targets:
      - expr: 'sli:error_ratio:rate1h / 0.001'
        legendFormat: 'Burn Rate (1h)'
    overrides:
      - matcher: 'Burn Rate Threshold'
        properties:
          - id: custom.lineStyle
            value: dash