设计能回答问题的 Prometheus 指标 —— 类型选择、基数预算，以及 rate 和分位数的陷阱

# 差 —— 只留下最后一次请求的延迟，既算不出分位数也算不出平均值
from prometheus_client import Gauge

last_latency = Gauge("http_request_duration_seconds", "请求处理时间")

def handle(req):
    t0 = time.monotonic()
    resp = process(req)
    last_latency.set(time.monotonic() - t0)   # 之前的值全部消失
    return resp

# 好 —— histogram 会把每一次观测都累加进桶里
from prometheus_client import Counter, Histogram

REQUESTS = Counter(
    "http_requests_total", "请求总数",
    ["method", "route", "status_class"],
)
LATENCY = Histogram(
    "http_request_duration_seconds", "请求处理时间",
    ["method", "route"],
    buckets=(0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0),
)

def handle(req):
    route = req.route_template          # "/v1/orders/:id" —— 不是真实的 ID
    with LATENCY.labels(req.method, route).time():
        resp = process(req)
    REQUESTS.labels(req.method, route, f"{resp.status // 100}xx").inc()
    return resp

JOB_RUNS = Counter("batch_job_runs_total", "批处理运行次数", ["job", "result"])
JOB_LAST_SUCCESS = Gauge("batch_job_last_success_timestamp_seconds", "最后一次成功的时间", ["job"])

def run_job(name, fn):
    try:
        fn()
        JOB_RUNS.labels(name, "success").inc()
        JOB_LAST_SUCCESS.labels(name).set(time.time())
    except Exception:
        JOB_RUNS.labels(name, "failure").inc()
        raise

http_request_duration_seconds_bucket
  route          120 个 (路由模板)
  method           5 个
  le              11 个 (桶边界 + Inf)
  instance        40 个 (Pod 数)
  ------------------------------------
  = 120 * 5 * 11 * 40 = 264,000 条时间序列

再加上 _sum 和 _count
  120 * 5 * 40 * 2 = 48,000
  ------------------------------------
合计约 312,000 条时间序列 —— 仅来自一个指标

# 时间序列数量排名前 10 的指标
topk(10, count by (__name__)({__name__=~".+"}))

# 某个指标的基数是由哪个标签造成的
count(count by (route) (http_requests_total))
count(count by (instance) (http_requests_total))
count(count by (status) (http_requests_total))

# 全部活跃时间序列数量的走势 —— 如果出现阶梯式跳变，说明有新标签被引入了
prometheus_tsdb_head_series

# 每个抓取目标发来多少样本 (超过上限抓取会被拒绝)
topk(20, scrape_samples_scraped)

# prometheus.yml
scrape_configs:
  - job_name: checkout-api
    sample_limit: 20000              # 该目标发来的样本超过这个数，本次抓取就按失败处理
    label_limit: 24
    label_value_length_limit: 256
    scrape_interval: 15s
    metric_relabel_configs:
      # 用于事故应对 —— 在采集这一步就把出问题的标签清掉。
      # 之所以不用 labeldrop、而是用 replace 写入空值:
      # labeldrop 只看标签的「名字」来匹配，会把这个 job 下所有指标里的
      # user_id 都清掉；像下面这样写，就只会从 http_requests_total 里清掉它。
      # 在 Prometheus 的数据模型里，空值的标签等同于不存在这个标签。
      - source_labels: [__name__]
        regex: 'http_requests_total'
        target_label: user_id
        replacement: ''
        action: replace
      # 完全不需要的指标直接丢弃
      - source_labels: [__name__]
        regex: 'go_gc_duration_seconds.*|python_gc_.*'
        action: drop

# 危险 —— 抓取间隔 15s 时，[20s] 会出现只有 1 个样本的瞬间
rate(http_requests_total[20s])

# 安全 —— 至少是抓取间隔的 4 倍。告警建议用 [5m] 以上
rate(http_requests_total[1m])
rate(http_requests_total[5m])

# 错误 —— 先把 Counter 加起来，实例重启(也就是重置)就检测不到了
rate(sum(http_requests_total) by (route)[5m:])

# 正确 —— 先 rate，再聚合
sum(rate(http_requests_total[5m])) by (route)

# 只会画出一条不断右上扬的直线，没有任何信息
http_requests_total

# 每秒增长率 —— 这才是真正想看的值
sum(rate(http_requests_total[5m])) by (route)

# 特定区间内的总增量 —— 很适合写进告警文案
sum(increase(http_requests_total{status_class="5xx"}[1h])) by (route)

# 错误 —— 丢掉 le，各个桶会被混在一起，算出一个毫无意义的数字
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (route))

# 正确 —— le 必须保留
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, route)
)

# 诊断 —— 看 Inf 桶和最后一个有限桶之间的差值
sum(rate(http_request_duration_seconds_bucket{le="+Inf"}[5m])) by (route)
-
sum(rate(http_request_duration_seconds_bucket{le="10.0"}[5m])) by (route)

# 错误 —— 分位数不支持算术平均
avg(histogram_quantile(0.99, ...))

# 正确 —— 先把桶计数器加起来，再算分位数
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))

# prometheus.yml
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: checkout-api
    # 原生直方图的抓取是显式 opt-in
    scrape_native_histograms: true
    static_configs:
      - targets: ['checkout-api:8000']

# 固定桶: _bucket 时间序列上带 le 标签
histogram_quantile(0.99, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, route))

# 原生直方图: 不带 le，直接把指标本身传进去
histogram_quantile(0.99, sum(rate(http_request_duration_seconds[5m])) by (route))

# rules/http.yml
groups:
  - name: http_sli
    interval: 30s
    rules:
      # 第一层 —— 只从原始数据聚合一次
      - record: route:http_requests:rate5m
        expr: sum(rate(http_requests_total[5m])) by (route)

      - record: route:http_requests_errors:rate5m
        expr: sum(rate(http_requests_total{status_class="5xx"}[5m])) by (route)

      # 第二层 —— 引用第一层，不重新扫描原始数据
      - record: route:http_error_ratio:rate5m
        expr: |
          route:http_requests_errors:rate5m
            /
          route:http_requests:rate5m

      - record: route:http_request_duration_seconds:p99_rate5m
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, route)
          )

      # 整个服务级别 —— 去掉了 route 维度的汇总
      - record: service:http_error_ratio:rate5m
        expr: |
          sum(rate(http_requests_total{status_class="5xx"}[5m]))
            /
          sum(rate(http_requests_total[5m]))

# 语法检查
promtool check rules rules/http.yml

# 单元测试 —— 输入一组时间序列，验证期望值
promtool test rules tests/http_test.yml

# 检查完整配置
promtool check config prometheus.yml

# tests/http_test.yml
rule_files:
  - ../rules/http.yml

evaluation_interval: 30s

tests:
  - interval: 15s
    input_series:
      - series: 'http_requests_total{route="/v1/orders", status_class="2xx"}'
        values: '0+150x40'
      - series: 'http_requests_total{route="/v1/orders", status_class="5xx"}'
        values: '0+3x40'
    promql_expr_test:
      - expr: route:http_error_ratio:rate5m
        eval_time: 8m
        exp_samples:
          - labels: 'route:http_error_ratio:rate5m{route="/v1/orders"}'
            value: 0.0196078431372549