会被读的仪表盘与值得呼人的告警 — 定义问题、组织变量、SLO 与告警疲劳

# provisioning/datasources/prometheus.yaml
apiVersion: 1
datasources:
  - name: Prometheus-prod
    uid: prom-prod
    type: prometheus
    access: proxy
    url: http://prometheus.observability.svc:9090
    jsonData:
      httpMethod: POST
      timeInterval: 15s          # 抓取间隔，是计算 rate 窗口的基准
      prometheusType: Prometheus
      prometheusVersion: 3.13.0
      exemplarTraceIdDestinations:
        - name: trace_id
          datasourceUid: tempo-prod
    isDefault: true

  - name: Prometheus-staging
    uid: prom-staging
    type: prometheus
    access: proxy
    url: http://prometheus.staging.svc:9090
    jsonData:
      httpMethod: POST
      timeInterval: 15s

{
  "templating": {
    "list": [
      {
        "name": "datasource",
        "type": "datasource",
        "query": "prometheus",
        "current": { "text": "Prometheus-prod", "value": "prom-prod" }
      },
      {
        "name": "namespace",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": "label_values(kube_namespace_status_phase, namespace)",
        "refresh": 1,
        "sort": 1
      },
      {
        "name": "service",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": "label_values(http_requests_total{namespace=\"$namespace\"}, service)",
        "refresh": 2,
        "includeAll": true,
        "multi": true
      },
      {
        "name": "route",
        "type": "query",
        "datasource": { "type": "prometheus", "uid": "${datasource}" },
        "query": "label_values(http_requests_total{namespace=\"$namespace\", service=~\"$service\"}, route)",
        "refresh": 2,
        "includeAll": true,
        "multi": true
      }
    ]
  }
}

# 多选变量要以正则形式接收。用等号接收的话只有单个值时才生效
sum by (route) (
  rate(http_requests_total{namespace="$namespace", service=~"$service", route=~"$route"}[$__rate_interval])
)

# 仪表盘面板里要用 rate 窗口变量。
# 用固定的 [5m]，时间范围拉宽时会浪费分辨率，
# 时间范围收窄时又会因为样本不够在图上留下空洞
histogram_quantile(0.99,
  sum by (le, route) (
    rate(http_request_duration_seconds_bucket{namespace="$namespace", service=~"$service"}[$__rate_interval])
  )
)

# 只画排名前 10 的路由。想看其余的就用变量收窄范围
topk(10,
  sum by (route) (
    rate(http_requests_total{service=~"$service", status_class="5xx"}[$__rate_interval])
  )
)

# 把发布时间当作标注叠加上去，「从什么时候开始」这个问题立刻就解决了
changes(kube_deployment_status_observed_generation{namespace="$namespace"}[$__rate_interval]) > 0

# 基于原因 — 这类规则堆到 40 条左右，告警疲劳就开始了
- alert: HighCPU
  expr: rate(process_cpu_seconds_total[5m]) > 0.8
  for: 5m
- alert: HighMemory
  expr: process_resident_memory_bytes / container_spec_memory_limit_bytes > 0.9
  for: 5m
- alert: ManyGoroutines
  expr: go_goroutines > 10000
  for: 5m
- alert: DBConnectionsHigh
  expr: pg_stat_activity_count > 80
  for: 5m

# 基于症状 — 挂在用户实际体验到的东西上
groups:
  - name: symptom_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx"}[5m]))
            /
          sum by (service) (rate(http_requests_total[5m]))
            > 0.02
        for: 5m
        labels:
          severity: page
        annotations:
          summary: '5xx 比率已超过 2%'
          runbook_url: https://wiki.internal/runbook/high-error-rate

      - alert: HighLatency
        expr: |
          histogram_quantile(0.99,
            sum by (le, service) (rate(http_request_duration_seconds_bucket[5m]))
          ) > 1.5
        for: 10m
        labels:
          severity: page
        annotations:
          summary: 'p99 延迟已超过 1.5 秒'

      - alert: RequestsStopped
        expr: |
          sum by (service) (rate(http_requests_total[5m])) == 0
            and
          sum by (service) (rate(http_requests_total[5m] offset 1h)) > 1
        for: 5m
        labels:
          severity: page
        annotations:
          summary: '平时有流量的服务现在没有收到请求'

# 有问题的规则 —— 5 分钟平均值几乎不可能超过阈值
avg_over_time(http_request_duration_seconds_sum[5m])
  / avg_over_time(http_request_duration_seconds_count[5m]) > 5

# 为什么不会响: 平均值会掩盖长尾。p99 可能是 8 秒，平均值却只有 0.2 秒。
# 这条规则只有在整个服务都普遍耗时 8 秒时才会触发。

# 找出过去 30 天里一次都没触发过的规则
# ALERTS_FOR_STATE 只在告警处于激活状态时存在，所以要和规则列表交叉核对
count by (alertname) (max_over_time(ALERTS[30d]))

# 同时确认规则评估本身有没有失败
increase(prometheus_rule_evaluation_failures_total[1h]) > 0

# tests/alerts_test.yml
rule_files:
  - ../rules/symptom_alerts.yml

evaluation_interval: 30s

tests:
  - interval: 15s
    input_series:
      - series: 'http_requests_total{service="checkout", status_class="2xx"}'
        values: '0+90x60'
      - series: 'http_requests_total{service="checkout", status_class="5xx"}'
        values: '0+10x60'
    alert_rule_test:
      - eval_time: 10m
        alertname: HighErrorRate
        exp_alerts:
          - exp_labels:
              service: checkout
              severity: page
            exp_annotations:
              summary: '5xx 比率已超过 2%'
              runbook_url: https://wiki.internal/runbook/high-error-rate

# rules/slo.yml
groups:
  - name: slo_sli
    interval: 30s
    rules:
      # 好请求的比例 —— 排除健康检查
      - record: service:sli_availability:ratio_rate5m
        expr: |
          sum by (service) (
            rate(http_requests_total{status_class!="5xx", route!~"/healthz|/readyz"}[5m])
          )
          /
          sum by (service) (
            rate(http_requests_total{route!~"/healthz|/readyz"}[5m])
          )

      # 各个窗口的错误比率 —— 供燃烧率告警引用
      - record: service:http_error_ratio:rate5m
        expr: |
          1 - service:sli_availability:ratio_rate5m

      - record: service:http_error_ratio:rate1h
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[1h]))
          /
          sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[1h]))

      - record: service:http_error_ratio:rate6h
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[6h]))
          /
          sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[6h]))

      - record: service:http_error_ratio:rate30m
        expr: |
          sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[30m]))
          /
          sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[30m]))

      # 剩余错误预算比例 —— 仪表盘上的核心面板
      - record: service:error_budget_remaining:ratio30d
        expr: |
          1 - (
            (
              sum by (service) (rate(http_requests_total{status_class="5xx", route!~"/healthz|/readyz"}[30d]))
              /
              sum by (service) (rate(http_requests_total{route!~"/healthz|/readyz"}[30d]))
            )
            / 0.001
          )

groups:
  - name: slo_burn_rate
    rules:
      - alert: ErrorBudgetBurnFast
        expr: |
          service:http_error_ratio:rate1h > (14.4 * 0.001)
            and
          service:http_error_ratio:rate5m > (14.4 * 0.001)
        for: 2m
        labels:
          severity: page
          slo: availability
        annotations:
          summary: '正以 14.4 倍速度消耗错误预算，约 2 天后将全部耗尽'
          runbook_url: https://wiki.internal/runbook/slo-burn

      - alert: ErrorBudgetBurnSlow
        expr: |
          service:http_error_ratio:rate6h > (6 * 0.001)
            and
          service:http_error_ratio:rate30m > (6 * 0.001)
        for: 15m
        labels:
          severity: ticket
          slo: availability
        annotations:
          summary: '正以 6 倍速度消耗错误预算'

# alertmanager.yml
route:
  group_by: ['alertname', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: slack-default
  routes:
    - matchers: [severity = page]
      receiver: pagerduty
      group_wait: 10s
      repeat_interval: 1h
      continue: true
    - matchers: [severity = ticket]
      receiver: jira
      repeat_interval: 24h

inhibit_rules:
  # 整个集群故障期间，抑制单个服务的告警
  - source_matchers: [alertname = ClusterUnreachable]
    target_matchers: [severity =~ 'page|ticket']
    equal: [cluster]
  # 同一个服务如果已经有 page 在响，就抑制它的 ticket
  - source_matchers: [severity = page]
    target_matchers: [severity = ticket]
    equal: [service]

receivers:
  - name: pagerduty
    pagerduty_configs:
      - routing_key_file: /etc/alertmanager/pd_key
  - name: jira
    webhook_configs:
      - url: http://alert-to-jira.internal/hook
  - name: slack-default
    slack_configs:
      - channel: '#alerts'
        send_resolved: true

# provisioning/dashboards/dashboards.yaml
apiVersion: 1
providers:
  - name: gitops
    orgId: 1
    folder: Services
    type: file
    disableDeletion: true
    updateIntervalSeconds: 60
    allowUiUpdates: false        # 禁止界面修改，变更必须走仓库
    options:
      path: /etc/grafana/dashboards
      foldersFromFilesStructure: true

# provisioning/alerting/rules.yaml — 把 Grafana 管理的告警定义成文件
apiVersion: 1
groups:
  - orgId: 1
    name: checkout_slo
    folder: Alerts
    interval: 1m
    rules:
      - uid: checkout-burn-fast
        title: Checkout error budget burn (fast)
        condition: THRESHOLD
        data:
          - refId: BURN
            relativeTimeRange:
              from: 3600
              to: 0
            datasourceUid: prom-prod
            model:
              expr: service:http_error_ratio:rate1h{service="checkout"}
              instant: true
              refId: BURN
          - refId: THRESHOLD
            datasourceUid: __expr__
            model:
              type: threshold
              expression: BURN
              conditions:
                - evaluator:
                    type: gt
                    params: [0.0144]
              refId: THRESHOLD
        for: 2m
        noDataState: NoData
        execErrState: Alerting
        labels:
          severity: page
          slo: availability
        annotations:
          summary: Checkout 的错误预算正以 14.4 倍速度消耗
          runbook_url: https://wiki.internal/runbook/slo-burn

- alert: ScrapeTargetDown
  expr: up{job=~"checkout.*|payment.*"} == 0
  for: 5m
  labels:
    severity: page
  annotations:
    summary: '抓取目标已经 5 分钟没有响应'