SLI/SLO/Error Budget 기반 신뢰성 엔지니어링 실전 가이드

# error-budget-policy.yaml
error_budget_policy:
  service: payment-api
  slo_window: 30d
  slo_target: 99.9%

  thresholds:
    - level: green
      budget_remaining: '>50%'
      actions:
        - 정상 기능 개발 진행
        - 주간 SLO 리뷰 미팅

    - level: yellow
      budget_remaining: '25%-50%'
      actions:
        - 신규 기능 릴리스 전 신뢰성 리뷰 필수
        - 기술 부채 해소 작업 병행
        - 일간 SLO 리뷰 미팅

    - level: orange
      budget_remaining: '10%-25%'
      actions:
        - 신규 기능 릴리스 중단
        - 엔지니어링 시간 50% 이상 신뢰성 작업에 투입
        - 장애 원인 분석(Postmortem) 즉시 실행

    - level: red
      budget_remaining: '<10%'
      actions:
        - 모든 기능 개발 동결
        - 전체 엔지니어링 리소스 신뢰성 복구에 집중
        - 경영진 에스컬레이션
        - SLO 미달 근본 원인 해소 완료 시까지 동결 유지

# multi-window-burn-rate-alerts.yaml
# Google SRE Workbook 권장 알럿 설정
groups:
  - name: slo-burn-rate-alerts
    rules:
      # 페이지 알럿: 2% 예산을 1시간 내 소진 (Burn Rate 14.4)
      - alert: SLOBurnRateHigh_Page
        expr: |
          (
            sli:error_ratio:rate1h > (14.4 * 0.001)
            and
            sli:error_ratio:rate5m > (14.4 * 0.001)
          )
        for: 2m
        labels:
          severity: page
        annotations:
          summary: 'Error budget burn rate is very high'
          description: 'Service is consuming error budget 14.4x faster than normal. 2% of 30-day budget will be consumed in 1 hour.'

      # 페이지 알럿: 5% 예산을 6시간 내 소진 (Burn Rate 6)
      - alert: SLOBurnRateMedium_Page
        expr: |
          (
            sli:error_ratio:rate6h > (6 * 0.001)
            and
            sli:error_ratio:rate30m > (6 * 0.001)
          )
        for: 5m
        labels:
          severity: page
        annotations:
          summary: 'Error budget burn rate is elevated'
          description: 'Service is consuming error budget 6x faster than normal. 5% of 30-day budget will be consumed in 6 hours.'

      # 티켓 알럿: 10% 예산을 3일 내 소진 (Burn Rate 1)
      - alert: SLOBurnRateSlow_Ticket
        expr: |
          (
            sli:error_ratio:rate3d > (1 * 0.001)
            and
            sli:error_ratio:rate6h > (1 * 0.001)
          )
        for: 30m
        labels:
          severity: ticket
        annotations:
          summary: 'Error budget burn rate indicates slow degradation'
          description: 'Service is steadily consuming error budget. 10% of 30-day budget will be consumed in 3 days.'

# sli-recording-rules.yaml
groups:
  - name: sli-recording-rules
    interval: 30s
    rules:
      # 전체 요청 중 에러 비율 (다양한 윈도우)
      - record: sli:error_ratio:rate5m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m]))
          /
          sum(rate(http_requests_total[5m]))

      - record: sli:error_ratio:rate30m
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[30m]))
          /
          sum(rate(http_requests_total[30m]))

      - record: sli:error_ratio:rate1h
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[1h]))
          /
          sum(rate(http_requests_total[1h]))

      - record: sli:error_ratio:rate6h
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[6h]))
          /
          sum(rate(http_requests_total[6h]))

      - record: sli:error_ratio:rate3d
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[3d]))
          /
          sum(rate(http_requests_total[3d]))

      # 지연시간 SLI: 200ms 이내 응답 비율
      - record: sli:latency_good_ratio:rate5m
        expr: |
          sum(rate(http_request_duration_seconds_bucket{le="0.2"}[5m]))
          /
          sum(rate(http_request_duration_seconds_count[5m]))

      - record: sli:latency_good_ratio:rate1h
        expr: |
          sum(rate(http_request_duration_seconds_bucket{le="0.2"}[1h]))
          /
          sum(rate(http_request_duration_seconds_count[1h]))

# Error Budget 잔여율 (30일 윈도우, SLO 99.9%)
# 결과가 1이면 100% 남음, 0이면 완전 소진
(
  1 - (
    sum(increase(http_requests_total{status=~"5.."}[30d]))
    /
    sum(increase(http_requests_total[30d]))
  )
  - 0.999
) / 0.001

#!/usr/bin/env python3
"""SLO 보고서 생성 스크립트"""

import requests
from datetime import datetime, timedelta

PROMETHEUS_URL = "http://prometheus:9090"
SLO_CONFIGS = [
    {
        "name": "payment-api-availability",
        "sli_query": 'sum(rate(http_requests_total{job="payment-api",status!~"5.."}[30d])) / sum(rate(http_requests_total{job="payment-api"}[30d]))',
        "slo_target": 0.999,
        "window_days": 30,
    },
    {
        "name": "payment-api-latency",
        "sli_query": 'sum(rate(http_request_duration_seconds_bucket{job="payment-api",le="0.2"}[30d])) / sum(rate(http_request_duration_seconds_count{job="payment-api"}[30d]))',
        "slo_target": 0.995,
        "window_days": 30,
    },
]


def query_prometheus(query: str) -> float:
    """Prometheus instant query 실행"""
    resp = requests.get(
        f"{PROMETHEUS_URL}/api/v1/query",
        params={"query": query},
    )
    result = resp.json()["data"]["result"]
    if not result:
        return 0.0
    return float(result[0]["value"][1])


def generate_report():
    """SLO 보고서 생성"""
    print(f"=== SLO Report ({datetime.now().strftime('%Y-%m-%d %H:%M')}) ===\n")

    for config in SLO_CONFIGS:
        current_sli = query_prometheus(config["sli_query"])
        slo_target = config["slo_target"]
        error_budget_total = 1 - slo_target
        error_budget_consumed = max(0, slo_target - current_sli)
        error_budget_remaining = max(
            0, (error_budget_total - error_budget_consumed) / error_budget_total
        )

        status = "OK" if current_sli >= slo_target else "VIOLATED"
        budget_pct = error_budget_remaining * 100

        print(f"Service: {config['name']}")
        print(f"  SLI (current): {current_sli:.4%}")
        print(f"  SLO (target):  {slo_target:.4%}")
        print(f"  Status:        {status}")
        print(f"  Error Budget:  {budget_pct:.1f}% remaining")

        if budget_pct < 25:
            print(f"  WARNING: Error budget below 25%!")
        print()


if __name__ == "__main__":
    generate_report()

# grafana-slo-dashboard.json (패널 구성 요약)
panels:
  - title: 'Current Availability SLI'
    type: stat
    targets:
      - expr: '1 - sli:error_ratio:rate30d'
        legendFormat: 'Availability'
    thresholds:
      - value: 0.999
        color: green
      - value: 0.995
        color: yellow
      - value: 0.99
        color: red

  - title: 'Error Budget Remaining'
    type: gauge
    targets:
      - expr: |
          clamp_min(
            (0.001 - sli:error_ratio:rate30d) / 0.001,
            0
          )
        legendFormat: 'Budget Remaining'
    min: 0
    max: 1
    thresholds:
      - value: 0.5
        color: green
      - value: 0.25
        color: yellow
      - value: 0
        color: red

  - title: 'Burn Rate (1h window)'
    type: timeseries
    targets:
      - expr: 'sli:error_ratio:rate1h / 0.001'
        legendFormat: 'Burn Rate (1h)'
    overrides:
      - matcher: 'Burn Rate Threshold'
        properties:
          - id: custom.lineStyle
            value: dash