分布式追踪真正回答的问题 — 跨度、采样，以及时间到底花在哪里

Trace 4bf92f3577b34da6a3ce929d0e0e4736                    共 1,847ms
├─ SERVER    api-gateway   POST /v1/orders                    1847ms
│  ├─ CLIENT    auth-svc     GET /verify                         38ms
│  ├─ CLIENT    checkout     POST /orders                      1782ms
│  │  ├─ SERVER   checkout    POST /orders                     1776ms
│  │  │  ├─ INTERNAL  cart.validate                               6ms
│  │  │  ├─ CLIENT    inventory  GET /stock                      41ms
│  │  │  ├─ CLIENT    pricing    POST /quote                     52ms
│  │  │  ├─ CLIENT    db.query   SELECT coupons WHERE ...      1612ms   <-- 就是这里
│  │  │  └─ CLIENT    payment    POST /charge                    64ms
│  │  └─ (网络 + 排队 6ms)
│  └─ INTERNAL  response.serialize                                9ms

# 网关调用 checkout 时实际发出去的请求头
curl -sD - -o /dev/null http://checkout.internal/v1/orders \
  -H 'traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'

# 00                                 版本
# 4bf92f3577b34da6a3ce929d0e0e4736   追踪 ID — 整个请求中保持一致
# 00f067aa0ba902b7                   父跨度 ID — 每一跳调用都会变
# 01                                 采样标志 — 如果是 00，这条追踪不会被保存

├─ SERVER  order-api  GET /v1/orders/A-99183                     1421ms
│  ├─ CLIENT  db.query  SELECT * FROM orders WHERE id = ?           5ms
│  ├─ CLIENT  db.query  SELECT * FROM order_items WHERE oid = ?     4ms
│  ├─ CLIENT  db.query  SELECT * FROM products WHERE id = ?         4ms
│  ├─ CLIENT  db.query  SELECT * FROM products WHERE id = ?         4ms
│  ├─ ... (同一条查询 340 次) ...

npm i @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node

OTEL_SERVICE_NAME=checkout-api \
OTEL_RESOURCE_ATTRIBUTES=service.version=1.42.3,deployment.environment=prod \
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.observability:4318 \
OTEL_TRACES_SAMPLER=parentbased_traceidratio \
OTEL_TRACES_SAMPLER_ARG=1.0 \
node --require @opentelemetry/auto-instrumentations-node/register server.js

├─ SERVER  report-api  GET /v1/reports/monthly              3204ms
│  ├─ CLIENT  db.query   SELECT ... FROM ledger              412ms
│  └─ CLIENT  s3.upload  PUT report-2026-07.pdf              260ms
│     self time = 3204 - 412 - 260 = 2532ms  <-- 未埋点的区间

import { trace, SpanStatusCode } from '@opentelemetry/api'

const tracer = trace.getTracer('checkout', '1.42.3')

export async function applyPromotions(cart, tenantId) {
  return tracer.startActiveSpan('checkout.applyPromotions', async (span) => {
    span.setAttribute('cart.item_count', cart.items.length)
    span.setAttribute('promotion.engine', 'rules-v3')
    span.setAttribute('tenant.id', tenantId)

    try {
      const cached = await rules.fromCache(tenantId)
      span.setAttribute('promotion.cache_hit', Boolean(cached))

      const result = await (cached ?? rules.compile(tenantId)).evaluate(cart)
      span.setAttribute('promotion.rules_evaluated', result.evaluated)
      span.setAttribute('promotion.matched_count', result.matched.length)
      return result
    } catch (err) {
      span.recordException(err)
      span.setStatus({ code: SpanStatusCode.ERROR, message: err.message })
      throw err
    } finally {
      span.end()
    }
  })
}

python3 - <<'PY'
rate = 0.01              # 头部采样 1%
errors_per_hour = 5      # 想排查的错误的真实发生频率
kept = errors_per_hour * rate
print(f"被保留的错误追踪: 每小时 {kept:.2f} 条")
print(f"想看到 1 条平均要等 {1/kept:.0f} 小时")
PY
# 被保留的错误追踪: 每小时 0.05 条
# 想看到 1 条平均要等 20 小时

# otel-collector-tailsampler.yaml
processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 2000
    policies:
      - name: keep-errors
        type: status_code
        status_code:
          status_codes: [ERROR]

      - name: keep-slow
        type: latency
        latency:
          threshold_ms: 800

      - name: keep-vip-tenants
        type: string_attribute
        string_attribute:
          key: tenant.tier
          values: [enterprise]

      - name: baseline
        type: probabilistic
        probabilistic:
          sampling_percentage: 2

python3 - <<'PY'
traces_per_sec, wait_sec, spans, span_kb = 2000, 10, 12, 1.2
mb = traces_per_sec * wait_sec * spans * span_kb / 1024
print(f"缓冲内存大约 {mb:.0f} MB (留两倍余量就是 {mb*2:.0f} MB)")
PY
# 缓冲内存大约 281 MB (留两倍余量就是 563 MB)

# otel-collector-gateway.yaml — 在前端按 trace ID 路由
exporters:
  loadbalancing:
    routing_key: traceID
    protocol:
      otlp:
        tls:
          insecure: true
    resolver:
      dns:
        hostname: otel-tailsampler.observability.svc.cluster.local
        port: 4317

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [loadbalancing]

connectors:
  spanmetrics:
    # 只有列在这里的才会变成指标标签 — 只放低基数的
    dimensions:
      - name: http.route
      - name: http.request.method
      - name: deployment.environment
    histogram:
      explicit:
        buckets: [10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2s, 5s]

OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT=64 \
OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT=2048 \
node --require @opentelemetry/auto-instrumentations-node/register server.js

import { propagation, context, trace, SpanKind } from '@opentelemetry/api'

// 生产者: 把上下文注入到消息头里
export async function publish(order) {
  return tracer.startActiveSpan('orders.publish', { kind: SpanKind.PRODUCER }, async (span) => {
    const headers = {}
    propagation.inject(context.active(), headers)
    await producer.send({ topic: 'orders', messages: [{ value: JSON.stringify(order), headers }] })
    span.end()
  })
}

// 批量消费者: 把每条消息的上下文作为链接而不是父级挂上去
export async function consumeBatch(messages) {
  const links = messages
    .map((m) => trace.getSpan(propagation.extract(context.active(), m.headers))?.spanContext())
    .filter(Boolean)
    .map((spanContext) => ({ context: spanContext }))

  return tracer.startActiveSpan(
    'orders.processBatch',
    { kind: SpanKind.CONSUMER, links },
    async (span) => {
      span.setAttribute('messaging.batch.message_count', messages.length)
      await Promise.all(messages.map(handle))
      span.end()
    }
  )
}