Grafana OnCall과 인시던트 관리 자동화: PagerDuty 통합부터 Runbook 자동화까지

┌─────────────────────────────────────────────────────────────────────────┐
│                     인시던트 관리 자동화 아키텍처                          │
├─────────────────────────────────────────────────────────────────────────┤
│                                                                         │
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐                   │
│  │ Prometheus   │  │ Grafana      │  │ 외부 모니터링  │                   │
│  │ Alertmanager │  │ Alerting     │  │ (Datadog 등)  │                   │
│  └──────┬───────┘  └──────┬───────┘  └──────┬───────┘                   │
│         │                 │                  │                           │
│         └─────────────────┼──────────────────┘                          │
│                           ▼                                             │
│              ┌────────────────────────┐                                  │
│              │   Grafana OnCall/IRM   │                                  │
│              │  ┌──────────────────┐  │                                  │
│              │  │  Integration     │  │  Webhook / API 수신              │
│              │  │  Layer           │  │                                  │
│              │  └────────┬─────────┘  │                                  │
│              │           ▼            │                                  │
│              │  ┌──────────────────┐  │                                  │
│              │  │  Route Engine    │  │  라벨 기반 라우팅                 │
│              │  └────────┬─────────┘  │                                  │
│              │           ▼            │                                  │
│              │  ┌──────────────────┐  │                                  │
│              │  │  Escalation      │  │  에스컬레이션 체인 실행           │
│              │  │  Engine          │  │                                  │
│              │  └────────┬─────────┘  │                                  │
│              │           ▼            │                                  │
│              │  ┌──────────────────┐  │                                  │
│              │  │  Schedule        │  │  온콜 스케줄 조회                 │
│              │  │  Manager         │  │                                  │
│              │  └──────────────────┘  │                                  │
│              └────────────────────────┘                                  │
│                       │          │                                       │
│          ┌────────────┘          └────────────┐                          │
│          ▼                                    ▼                          │
│  ┌──────────────────┐              ┌──────────────────┐                  │
│  │  Notification     │              │  Outgoing         │                 │
│  │  Channels         │              │  Webhooks         │                 │
│  │                   │              │                   │                 │
│  │  - Slack          │              │  - Runbook 실행   │                 │
│  │  - MS Teams       │              │  - Jira 티켓 생성 │                 │
│  │  - Phone Call     │              │  - PagerDuty 연동 │                 │
│  │  - SMS            │              │  - 자동 복구 스크립트│                │
│  │  - Email          │              │                   │                 │
│  └──────────────────┘              └──────────────────┘                  │
│                                                                         │
└─────────────────────────────────────────────────────────────────────────┘

# docker-compose.yml
version: '3.8'

services:
  engine:
    image: grafana/oncall:latest
    restart: always
    ports:
      - '8080:8080'
    command: >
      sh -c "uwsgi --ini uwsgi.ini"
    environment:
      BASE_URL: http://localhost:8080
      SECRET_KEY: ${ONCALL_SECRET_KEY:-my-secret-key-change-in-production}
      RABBITMQ_USERNAME: rabbitmq
      RABBITMQ_PASSWORD: rabbitmq
      RABBITMQ_HOST: rabbitmq
      RABBITMQ_PORT: 5672
      RABBITMQ_DEFAULT_VHOST: /
      MYSQL_DB_NAME: oncall
      MYSQL_USER: root
      MYSQL_PASSWORD: oncall
      MYSQL_HOST: mysql
      MYSQL_PORT: 3306
      REDIS_URI: redis://redis:6379/0
      DJANGO_SETTINGS_MODULE: settings.hobby
      CELERY_WORKER_QUEUE: default,critical,long,slack,telegram,webhook,retry,celery,grafana
      GRAFANA_API_URL: http://grafana:3000
    depends_on:
      mysql:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      redis:
        condition: service_healthy

  celery:
    image: grafana/oncall:latest
    restart: always
    command: >
      sh -c "./celery_with_exporter.sh"
    environment:
      BASE_URL: http://localhost:8080
      SECRET_KEY: ${ONCALL_SECRET_KEY:-my-secret-key-change-in-production}
      RABBITMQ_USERNAME: rabbitmq
      RABBITMQ_PASSWORD: rabbitmq
      RABBITMQ_HOST: rabbitmq
      RABBITMQ_PORT: 5672
      MYSQL_DB_NAME: oncall
      MYSQL_USER: root
      MYSQL_PASSWORD: oncall
      MYSQL_HOST: mysql
      MYSQL_PORT: 3306
      REDIS_URI: redis://redis:6379/0
      DJANGO_SETTINGS_MODULE: settings.hobby
      CELERY_WORKER_QUEUE: default,critical,long,slack,telegram,webhook,retry,celery,grafana
    depends_on:
      mysql:
        condition: service_healthy
      rabbitmq:
        condition: service_healthy
      redis:
        condition: service_healthy

  mysql:
    image: mysql:8.0
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: oncall
      MYSQL_DATABASE: oncall
    volumes:
      - oncall-mysql:/var/lib/mysql
    healthcheck:
      test: ['CMD', 'mysqladmin', 'ping', '-h', 'localhost']
      interval: 10s
      timeout: 5s
      retries: 5

  redis:
    image: redis:7-alpine
    restart: always
    healthcheck:
      test: ['CMD', 'redis-cli', 'ping']
      interval: 10s
      timeout: 5s
      retries: 5

  rabbitmq:
    image: rabbitmq:3.12-management-alpine
    restart: always
    environment:
      RABBITMQ_DEFAULT_USER: rabbitmq
      RABBITMQ_DEFAULT_PASS: rabbitmq
    healthcheck:
      test: ['CMD', 'rabbitmq-diagnostics', 'check_running']
      interval: 10s
      timeout: 5s
      retries: 5

  grafana:
    image: grafana/grafana:latest
    restart: always
    ports:
      - '3000:3000'
    environment:
      GF_SECURITY_ADMIN_USER: admin
      GF_SECURITY_ADMIN_PASSWORD: admin
      GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS: grafana-oncall-app
      GF_INSTALL_PLUGINS: grafana-oncall-app
    volumes:
      - grafana-data:/var/lib/grafana

volumes:
  oncall-mysql:
  grafana-data:

# 1. Docker Compose 시작
docker-compose up -d

# 2. DB 마이그레이션 실행
docker-compose exec engine python manage.py migrate

# 3. Grafana OnCall 플러그인 활성화 확인
# 브라우저에서 http://localhost:3000 접속
# Grafana 좌측 메뉴 -> Alerts & IRM -> OnCall

# 4. OnCall API 토큰 생성 (Terraform/API 연동용)
curl -X POST http://localhost:3000/api/plugins/grafana-oncall-app/resources/api/v1/api_token \
  -H "Authorization: Bearer <grafana-admin-api-key>" \
  -H "Content-Type: application/json"

# 5. 헬스체크
curl http://localhost:8080/health/

# Helm 저장소 추가
helm repo add grafana https://grafana.github.io/helm-charts
helm repo update

# OnCall 설치 (기본 설정)
helm install oncall grafana/oncall \
  --namespace oncall \
  --create-namespace \
  --set base_url=oncall.example.com \
  --set grafana."grafana\.ini".server.domain=grafana.example.com \
  --set ingress.enabled=true \
  --set ingress.annotations."kubernetes\.io/ingress\.class"=nginx \
  --set celery.workers=4 \
  --set engine.replicaCount=2

# terraform/oncall-schedules.tf

terraform {
  required_providers {
    grafana = {
      source  = "grafana/grafana"
      version = ">= 3.0.0"
    }
  }
}

provider "grafana" {
  url                  = var.grafana_url
  auth                 = var.grafana_auth
  oncall_access_token  = var.oncall_access_token
}

# 팀 데이터 소스
data "grafana_oncall_user" "engineer_a" {
  username = "engineer-a"
}

data "grafana_oncall_user" "engineer_b" {
  username = "engineer-b"
}

data "grafana_oncall_user" "engineer_c" {
  username = "engineer-c"
}

data "grafana_oncall_user" "engineer_d" {
  username = "engineer-d"
}

# 프라이머리 온콜 스케줄 - 주간 로테이션
resource "grafana_oncall_schedule" "primary" {
  name      = "Platform Team - Primary"
  type      = "calendar"
  team_id   = var.team_id
  time_zone = "Asia/Seoul"

  shifts = [
    grafana_oncall_on_call_shift.primary_rotation.id,
  ]
}

resource "grafana_oncall_on_call_shift" "primary_rotation" {
  name       = "Primary Weekly Rotation"
  type       = "rolling_users"
  start      = "2026-03-09T00:00:00"
  duration   = 60 * 60 * 24 * 7  # 7일 (초 단위)
  frequency  = "weekly"
  by_day     = ["MO", "TU", "WE", "TH", "FR", "SA", "SU"]
  time_zone  = "Asia/Seoul"

  rolling_users = [
    [data.grafana_oncall_user.engineer_a.id],
    [data.grafana_oncall_user.engineer_b.id],
    [data.grafana_oncall_user.engineer_c.id],
    [data.grafana_oncall_user.engineer_d.id],
  ]
}

# 백업 스케줄 - 시니어 엔지니어
resource "grafana_oncall_schedule" "backup" {
  name      = "Platform Team - Backup"
  type      = "calendar"
  team_id   = var.team_id
  time_zone = "Asia/Seoul"

  shifts = [
    grafana_oncall_on_call_shift.backup_rotation.id,
  ]
}

resource "grafana_oncall_on_call_shift" "backup_rotation" {
  name       = "Backup Bi-Weekly Rotation"
  type       = "rolling_users"
  start      = "2026-03-09T00:00:00"
  duration   = 60 * 60 * 24 * 14  # 14일
  frequency  = "weekly"
  interval   = 2
  time_zone  = "Asia/Seoul"

  rolling_users = [
    [data.grafana_oncall_user.engineer_a.id],
    [data.grafana_oncall_user.engineer_c.id],
  ]
}

# terraform/oncall-escalation.tf

# 심각도별 에스컬레이션 체인

# Critical (P1) - 빠른 에스컬레이션
resource "grafana_oncall_escalation_chain" "critical" {
  name    = "Critical - P1 Incidents"
  team_id = var.team_id
}

resource "grafana_oncall_escalation" "critical_step_1" {
  escalation_chain_id = grafana_oncall_escalation_chain.critical.id
  type                = "notify_on_call_from_schedule"
  notify_on_call_from_schedule = grafana_oncall_schedule.primary.id
  position            = 0
  important           = true
}

resource "grafana_oncall_escalation" "critical_wait_1" {
  escalation_chain_id = grafana_oncall_escalation_chain.critical.id
  type                = "wait"
  duration            = 300  # 5분
  position            = 1
}

resource "grafana_oncall_escalation" "critical_step_2" {
  escalation_chain_id = grafana_oncall_escalation_chain.critical.id
  type                = "notify_on_call_from_schedule"
  notify_on_call_from_schedule = grafana_oncall_schedule.backup.id
  position            = 2
  important           = true
}

resource "grafana_oncall_escalation" "critical_wait_2" {
  escalation_chain_id = grafana_oncall_escalation_chain.critical.id
  type                = "wait"
  duration            = 300  # 5분
  position            = 3
}

resource "grafana_oncall_escalation" "critical_step_3" {
  escalation_chain_id = grafana_oncall_escalation_chain.critical.id
  type                = "notify_whole_channel"
  position            = 4
}

# Warning (P2) - 표준 에스컬레이션
resource "grafana_oncall_escalation_chain" "warning" {
  name    = "Warning - P2 Incidents"
  team_id = var.team_id
}

resource "grafana_oncall_escalation" "warning_step_1" {
  escalation_chain_id = grafana_oncall_escalation_chain.warning.id
  type                = "notify_on_call_from_schedule"
  notify_on_call_from_schedule = grafana_oncall_schedule.primary.id
  position            = 0
  important           = false
}

resource "grafana_oncall_escalation" "warning_wait_1" {
  escalation_chain_id = grafana_oncall_escalation_chain.warning.id
  type                = "wait"
  duration            = 900  # 15분
  position            = 1
}

resource "grafana_oncall_escalation" "warning_step_2" {
  escalation_chain_id = grafana_oncall_escalation_chain.warning.id
  type                = "notify_on_call_from_schedule"
  notify_on_call_from_schedule = grafana_oncall_schedule.backup.id
  position            = 2
  important           = false
}

# Integration 설정 - Alertmanager 연동
resource "grafana_oncall_integration" "alertmanager" {
  name = "Prometheus Alertmanager"
  type = "alertmanager"

  default_route {
    escalation_chain_id = grafana_oncall_escalation_chain.warning.id
  }
}

# 라우팅 규칙 - 심각도에 따라 다른 에스컬레이션 체인 적용
resource "grafana_oncall_route" "critical_route" {
  integration_id      = grafana_oncall_integration.alertmanager.id
  escalation_chain_id = grafana_oncall_escalation_chain.critical.id
  routing_regex       = "\"severity\":\"critical\""
  position            = 0
}

resource "grafana_oncall_route" "warning_route" {
  integration_id      = grafana_oncall_integration.alertmanager.id
  escalation_chain_id = grafana_oncall_escalation_chain.warning.id
  routing_regex       = "\"severity\":\"warning\""
  position            = 1
}

# 1. Slack App 생성
# https://api.slack.com/apps 에서 "Create New App" -> "From an app manifest" 선택

# 2. App Manifest (YAML 형식)
# Slack 앱 생성 시 아래 매니페스트를 사용

# slack-app-manifest.yml
display_information:
  name: Grafana OnCall
  description: On-call management and incident response
  background_color: '#1a1a2e'

features:
  bot_user:
    display_name: Grafana OnCall
    always_online: true
  shortcuts:
    - name: Create Incident
      type: message
      callback_id: incident_create
      description: Create a new incident from this message

oauth_config:
  scopes:
    bot:
      - app_mentions:read
      - channels:history
      - channels:read
      - chat:write
      - commands
      - files:write
      - groups:history
      - groups:read
      - im:history
      - im:read
      - im:write
      - reactions:write
      - team:read
      - usergroups:read
      - usergroups:write
      - users:read
      - users:read.email

settings:
  event_subscriptions:
    request_url: https://oncall.example.com/slack/event_api_endpoint/
    bot_events:
      - app_mention
      - message.im
  interactivity:
    is_enabled: true
    request_url: https://oncall.example.com/slack/interactive_api_endpoint/
  org_deploy_enabled: false
  socket_mode_enabled: false

# Grafana Alerting - PagerDuty Contact Point 설정
# grafana/provisioning/alerting/contactpoints.yml

apiVersion: 1
contactPoints:
  - orgId: 1
    name: PagerDuty-Critical
    receivers:
      - uid: pagerduty-critical
        type: pagerduty
        settings:
          integrationKey: '${PAGERDUTY_INTEGRATION_KEY}'
          severity: critical
          class: 'production-incident'
          component: '{{ .CommonLabels.service }}'
          group: '{{ .CommonLabels.alertname }}'
        disableResolveMessage: false

  - orgId: 1
    name: PagerDuty-Warning
    receivers:
      - uid: pagerduty-warning
        type: pagerduty
        settings:
          integrationKey: '${PAGERDUTY_WARNING_KEY}'
          severity: warning
          class: 'production-warning'
          component: '{{ .CommonLabels.service }}'
          group: '{{ .CommonLabels.alertname }}'
        disableResolveMessage: false

# Notification Policy - 심각도별 라우팅
policies:
  - orgId: 1
    receiver: PagerDuty-Warning
    group_by: ['alertname', 'service']
    group_wait: 30s
    group_interval: 5m
    repeat_interval: 4h
    routes:
      - receiver: PagerDuty-Critical
        matchers:
          - severity = critical
        group_wait: 10s
        group_interval: 1m
        repeat_interval: 1h
        continue: false

# PagerDuty 마이그레이션 도구 사용
# 1. PagerDuty API 키 생성 (Read-only 권한)
# 2. 마이그레이션 스크립트 실행

# PagerDuty 설정 내보내기
pip install pdpyras

# 마이그레이션 Python 스크립트
python3 migrate_pagerduty_to_oncall.py \
  --pagerduty-api-key="${PAGERDUTY_API_KEY}" \
  --oncall-api-url="http://localhost:8080" \
  --oncall-api-token="${ONCALL_API_TOKEN}" \
  --dry-run  # 먼저 시뮬레이션으로 확인

# webhook_sync.py - PagerDuty <-> Grafana OnCall 양방향 동기화
import os
import json
import hmac
import hashlib
from flask import Flask, request, jsonify
import requests

app = Flask(__name__)

ONCALL_API_URL = os.environ["ONCALL_API_URL"]
ONCALL_API_TOKEN = os.environ["ONCALL_API_TOKEN"]
PAGERDUTY_API_KEY = os.environ["PAGERDUTY_API_KEY"]
WEBHOOK_SECRET = os.environ["WEBHOOK_SECRET"]


def verify_signature(payload: bytes, signature: str) -> bool:
    """웹훅 서명 검증"""
    expected = hmac.new(
        WEBHOOK_SECRET.encode(), payload, hashlib.sha256
    ).hexdigest()
    return hmac.compare_digest(expected, signature)


@app.route("/webhook/pagerduty-to-oncall", methods=["POST"])
def pagerduty_to_oncall():
    """PagerDuty 이벤트를 Grafana OnCall로 전달"""
    payload = request.get_json()

    for message in payload.get("messages", []):
        event = message.get("event", "")
        incident = message.get("incident", {})

        if event == "incident.triggered":
            # OnCall에 알림 생성
            oncall_payload = {
                "title": incident.get("title", "PagerDuty Incident"),
                "message": incident.get("description", ""),
                "severity": map_severity(incident.get("urgency", "high")),
                "source_link": incident.get("html_url", ""),
            }

            headers = {
                "Authorization": ONCALL_API_TOKEN,
                "Content-Type": "application/json",
            }

            response = requests.post(
                f"{ONCALL_API_URL}/integrations/v1/webhook/<integration-id>/",
                json=oncall_payload,
                headers=headers,
                timeout=10,
            )
            app.logger.info(
                "Forwarded PagerDuty incident to OnCall: %s", response.status_code
            )

        elif event == "incident.resolved":
            # OnCall에서 해당 알림 해결 처리
            resolve_oncall_alert(incident.get("id"))

    return jsonify({"status": "ok"}), 200


@app.route("/webhook/oncall-to-pagerduty", methods=["POST"])
def oncall_to_pagerduty():
    """Grafana OnCall 이벤트를 PagerDuty로 전달"""
    payload = request.get_json()
    event_type = payload.get("event", {}).get("type", "")
    alert_payload = payload.get("alert_payload", {})

    if event_type == "acknowledge":
        # PagerDuty에서 해당 인시던트 Acknowledge
        pd_event = {
            "routing_key": os.environ["PAGERDUTY_ROUTING_KEY"],
            "event_action": "acknowledge",
            "dedup_key": alert_payload.get("id", ""),
        }
    elif event_type == "resolve":
        pd_event = {
            "routing_key": os.environ["PAGERDUTY_ROUTING_KEY"],
            "event_action": "resolve",
            "dedup_key": alert_payload.get("id", ""),
        }
    else:
        return jsonify({"status": "ignored"}), 200

    response = requests.post(
        "https://events.pagerduty.com/v2/enqueue",
        json=pd_event,
        timeout=10,
    )
    app.logger.info("Forwarded OnCall event to PagerDuty: %s", response.status_code)
    return jsonify({"status": "ok"}), 200


def map_severity(pd_urgency: str) -> str:
    """PagerDuty urgency를 OnCall severity로 매핑"""
    mapping = {"high": "critical", "low": "warning"}
    return mapping.get(pd_urgency, "warning")


def resolve_oncall_alert(pd_incident_id: str):
    """PagerDuty 인시던트 ID로 OnCall 알림 해결"""
    headers = {
        "Authorization": ONCALL_API_TOKEN,
        "Content-Type": "application/json",
    }
    # OnCall API로 해당 알림 검색 및 해결
    response = requests.get(
        f"{ONCALL_API_URL}/api/v1/alert_groups/",
        headers=headers,
        params={"search": pd_incident_id},
        timeout=10,
    )
    if response.status_code == 200:
        for alert_group in response.json().get("results", []):
            requests.post(
                f"{ONCALL_API_URL}/api/v1/alert_groups/{alert_group['id']}/resolve/",
                headers=headers,
                timeout=10,
            )


if __name__ == "__main__":
    app.run(host="0.0.0.0", port=5000)

# runbook_executor.py - Runbook 자동 실행 서버
import os
import json
import subprocess
import logging
from datetime import datetime
from flask import Flask, request, jsonify
import requests

app = Flask(__name__)
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# Runbook 레지스트리 - 알림 유형별 자동화 스크립트 매핑
RUNBOOK_REGISTRY = {
    "HighCPUUsage": {
        "script": "/opt/runbooks/high_cpu_usage.sh",
        "auto_execute": True,
        "severity_threshold": "warning",
        "description": "CPU 사용률 임계치 초과 시 자동 대응",
        "actions": [
            "프로세스별 CPU 사용량 수집",
            "Top 5 프로세스 식별",
            "비정상 프로세스 자동 재시작 (화이트리스트 기반)",
        ],
    },
    "DiskSpaceCritical": {
        "script": "/opt/runbooks/disk_cleanup.sh",
        "auto_execute": True,
        "severity_threshold": "critical",
        "description": "디스크 공간 부족 시 자동 정리",
        "actions": [
            "임시 파일 정리",
            "오래된 로그 압축 및 아카이브",
            "Docker 미사용 이미지 정리",
        ],
    },
    "DatabaseConnectionPoolExhausted": {
        "script": "/opt/runbooks/db_connection_pool.sh",
        "auto_execute": False,  # 수동 승인 필요
        "severity_threshold": "critical",
        "description": "DB 커넥션 풀 고갈 시 대응 절차",
        "actions": [
            "유휴 커넥션 강제 종료",
            "커넥션 풀 크기 동적 확장",
            "슬로우 쿼리 식별 및 킬",
        ],
    },
    "PodCrashLoopBackOff": {
        "script": "/opt/runbooks/pod_crashloop.sh",
        "auto_execute": True,
        "severity_threshold": "warning",
        "description": "Pod CrashLoopBackOff 자동 진단",
        "actions": [
            "Pod 로그 수집",
            "이전 Pod 이벤트 분석",
            "리소스 제한 확인",
            "최근 배포 롤백 여부 판단",
        ],
    },
}

SLACK_WEBHOOK_URL = os.environ.get("SLACK_WEBHOOK_URL", "")
ONCALL_API_URL = os.environ.get("ONCALL_API_URL", "")
ONCALL_API_TOKEN = os.environ.get("ONCALL_API_TOKEN", "")


@app.route("/webhook/runbook", methods=["POST"])
def execute_runbook():
    """OnCall Outgoing Webhook에서 호출 - Runbook 자동 실행"""
    payload = request.get_json()

    alert_name = extract_alert_name(payload)
    severity = extract_severity(payload)
    alert_id = payload.get("alert_group_id", "unknown")

    logger.info("Received alert: %s (severity: %s, id: %s)", alert_name, severity, alert_id)

    runbook = RUNBOOK_REGISTRY.get(alert_name)
    if not runbook:
        logger.warning("No runbook found for alert: %s", alert_name)
        return jsonify({"status": "no_runbook", "alert": alert_name}), 200

    # 자동 실행 가능 여부 확인
    if not runbook["auto_execute"]:
        notify_manual_runbook(alert_name, runbook, payload)
        return jsonify({"status": "manual_required", "alert": alert_name}), 200

    # Runbook 스크립트 실행
    result = run_script(
        runbook["script"],
        env_vars={
            "ALERT_NAME": alert_name,
            "ALERT_ID": alert_id,
            "SEVERITY": severity,
            "PAYLOAD": json.dumps(payload),
        },
    )

    # 실행 결과를 Slack으로 통보
    notify_runbook_result(alert_name, runbook, result, alert_id)

    # 성공 시 OnCall 알림 자동 해결
    if result["returncode"] == 0:
        auto_resolve_alert(alert_id)

    return jsonify({
        "status": "executed",
        "alert": alert_name,
        "success": result["returncode"] == 0,
        "output": result["stdout"][:500],
    }), 200


def extract_alert_name(payload: dict) -> str:
    """페이로드에서 알림 이름 추출"""
    alert_payload = payload.get("alert_payload", {})
    labels = alert_payload.get("labels", {})
    return labels.get("alertname", payload.get("title", "Unknown"))


def extract_severity(payload: dict) -> str:
    """페이로드에서 심각도 추출"""
    alert_payload = payload.get("alert_payload", {})
    labels = alert_payload.get("labels", {})
    return labels.get("severity", "unknown")


def run_script(script_path: str, env_vars: dict, timeout: int = 300) -> dict:
    """Runbook 스크립트를 실행하고 결과를 반환"""
    env = os.environ.copy()
    env.update(env_vars)

    try:
        result = subprocess.run(
            ["/bin/bash", script_path],
            capture_output=True,
            text=True,
            timeout=timeout,
            env=env,
        )
        return {
            "returncode": result.returncode,
            "stdout": result.stdout,
            "stderr": result.stderr,
        }
    except subprocess.TimeoutExpired:
        return {
            "returncode": -1,
            "stdout": "",
            "stderr": f"Script timed out after {timeout}s",
        }
    except Exception as e:
        return {
            "returncode": -1,
            "stdout": "",
            "stderr": str(e),
        }


def notify_runbook_result(alert_name: str, runbook: dict, result: dict, alert_id: str):
    """Runbook 실행 결과를 Slack으로 통보"""
    if not SLACK_WEBHOOK_URL:
        return

    status_emoji = "white_check_mark" if result["returncode"] == 0 else "x"
    status_text = "SUCCESS" if result["returncode"] == 0 else "FAILED"

    slack_message = {
        "text": f"Runbook Execution: {status_text}",
        "blocks": [
            {
                "type": "header",
                "text": {
                    "type": "plain_text",
                    "text": f"Runbook: {alert_name} - {status_text}",
                },
            },
            {
                "type": "section",
                "fields": [
                    {"type": "mrkdwn", "text": f"*Alert ID:*\n{alert_id}"},
                    {"type": "mrkdwn", "text": f"*Description:*\n{runbook['description']}"},
                    {"type": "mrkdwn", "text": f"*Timestamp:*\n{datetime.utcnow().isoformat()}"},
                ],
            },
        ],
    }

    if result["stdout"]:
        slack_message["blocks"].append({
            "type": "section",
            "text": {
                "type": "mrkdwn",
                "text": f"*Output:*\n```{result['stdout'][:1000]}```",
            },
        })

    requests.post(SLACK_WEBHOOK_URL, json=slack_message, timeout=10)


def notify_manual_runbook(alert_name: str, runbook: dict, payload: dict):
    """수동 실행이 필요한 Runbook 안내를 Slack으로 전송"""
    if not SLACK_WEBHOOK_URL:
        return

    actions_text = "\n".join(f"  {i+1}. {a}" for i, a in enumerate(runbook["actions"]))
    slack_message = {
        "text": f"Manual Runbook Required: {alert_name}",
        "blocks": [
            {
                "type": "header",
                "text": {
                    "type": "plain_text",
                    "text": f"Manual Runbook: {alert_name}",
                },
            },
            {
                "type": "section",
                "text": {
                    "type": "mrkdwn",
                    "text": (
                        f"*Description:* {runbook['description']}\n\n"
                        f"*Steps:*\n{actions_text}"
                    ),
                },
            },
        ],
    }
    requests.post(SLACK_WEBHOOK_URL, json=slack_message, timeout=10)


def auto_resolve_alert(alert_id: str):
    """Runbook 성공 시 OnCall 알림 자동 해결"""
    if not ONCALL_API_URL or not ONCALL_API_TOKEN:
        return

    headers = {
        "Authorization": ONCALL_API_TOKEN,
        "Content-Type": "application/json",
    }
    try:
        requests.post(
            f"{ONCALL_API_URL}/api/v1/alert_groups/{alert_id}/resolve/",
            headers=headers,
            timeout=10,
        )
        logger.info("Auto-resolved alert: %s", alert_id)
    except Exception as e:
        logger.error("Failed to auto-resolve alert %s: %s", alert_id, e)


if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8000)

#!/bin/bash
# /opt/runbooks/disk_cleanup.sh
# 디스크 공간 부족 시 자동 정리 Runbook

set -euo pipefail

LOG_FILE="/var/log/runbook/disk_cleanup_$(date +%Y%m%d_%H%M%S).log"
mkdir -p /var/log/runbook

exec > >(tee -a "$LOG_FILE") 2>&1

echo "=== Disk Cleanup Runbook Started ==="
echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "Alert: ${ALERT_NAME:-unknown}"
echo "Severity: ${SEVERITY:-unknown}"
echo ""

# 1. 현재 디스크 사용량 확인
echo "--- Step 1: Current Disk Usage ---"
df -h / /var /tmp 2>/dev/null || df -h /
echo ""

# 2. 대용량 파일 식별
echo "--- Step 2: Top 10 Largest Files in /var ---"
find /var -type f -size +100M -exec ls -lh {} \; 2>/dev/null | sort -k5 -hr | head -10
echo ""

# 3. 임시 파일 정리
echo "--- Step 3: Cleaning Temporary Files ---"
TEMP_CLEANED=$(find /tmp -type f -atime +7 -delete -print 2>/dev/null | wc -l)
echo "Removed ${TEMP_CLEANED} temporary files older than 7 days"
echo ""

# 4. 오래된 로그 파일 압축
echo "--- Step 4: Compressing Old Log Files ---"
LOG_COMPRESSED=0
for logfile in $(find /var/log -name "*.log" -size +50M -mtime +3 2>/dev/null); do
    gzip "$logfile" && LOG_COMPRESSED=$((LOG_COMPRESSED + 1))
done
echo "Compressed ${LOG_COMPRESSED} log files"
echo ""

# 5. Docker 정리 (Docker가 설치된 경우)
if command -v docker &> /dev/null; then
    echo "--- Step 5: Docker Cleanup ---"
    echo "Removing dangling images..."
    docker image prune -f 2>/dev/null || true
    echo "Removing unused volumes..."
    docker volume prune -f 2>/dev/null || true
    echo "Removing stopped containers older than 24h..."
    docker container prune -f --filter "until=24h" 2>/dev/null || true
    echo ""
fi

# 6. systemd journal 정리
if command -v journalctl &> /dev/null; then
    echo "--- Step 6: Journal Cleanup ---"
    journalctl --vacuum-time=7d 2>/dev/null || true
    echo ""
fi

# 7. 정리 후 디스크 사용량 확인
echo "--- Step 7: Disk Usage After Cleanup ---"
df -h / /var /tmp 2>/dev/null || df -h /

# 8. 결과 판단
USAGE_PERCENT=$(df / | tail -1 | awk '{print $5}' | tr -d '%')
if [ "$USAGE_PERCENT" -lt 85 ]; then
    echo ""
    echo "=== Disk Cleanup SUCCESS: Usage is now ${USAGE_PERCENT}% ==="
    exit 0
else
    echo ""
    echo "=== Disk Cleanup PARTIAL: Usage is still ${USAGE_PERCENT}% - Manual intervention needed ==="
    exit 1
fi

# Slack 통합 진단 체크리스트
# 1. OnCall 엔진 로그 확인
docker-compose logs engine | grep -i slack

# 2. Slack 앱 이벤트 구독 URL 검증
# https://api.slack.com/apps -> 앱 선택 -> Event Subscriptions
# Request URL이 https://oncall.example.com/slack/event_api_endpoint/ 인지 확인

# 3. 봇 토큰 유효성 검증
curl -X POST https://slack.com/api/auth.test \
  -H "Authorization: Bearer xoxb-your-bot-token" \
  -H "Content-Type: application/json"

# 4. 채널 접근 권한 확인
curl -X POST https://slack.com/api/conversations.info \
  -H "Authorization: Bearer xoxb-your-bot-token" \
  -H "Content-Type: application/json" \
  -d '{"channel": "C0XXXXXXX"}'

# 5. 사용자 Slack 계정 연동 확인
# Grafana OnCall -> Users -> 해당 사용자 -> Slack 계정 연동 여부 확인