분산 트레이싱이 실제로 답하는 질문 — 스팬, 샘플링, 그리고 어디서 시간이 갔는가

Trace 4bf92f3577b34da6a3ce929d0e0e4736                    총 1,847ms
├─ SERVER    api-gateway   POST /v1/orders                    1847ms
│  ├─ CLIENT    auth-svc     GET /verify                         38ms
│  ├─ CLIENT    checkout     POST /orders                      1782ms
│  │  ├─ SERVER   checkout    POST /orders                     1776ms
│  │  │  ├─ INTERNAL  cart.validate                               6ms
│  │  │  ├─ CLIENT    inventory  GET /stock                      41ms
│  │  │  ├─ CLIENT    pricing    POST /quote                     52ms
│  │  │  ├─ CLIENT    db.query   SELECT coupons WHERE ...      1612ms   <-- 여기
│  │  │  └─ CLIENT    payment    POST /charge                    64ms
│  │  └─ (네트워크 + 큐잉 6ms)
│  └─ INTERNAL  response.serialize                                9ms

# 게이트웨이가 checkout 을 호출할 때 실제로 나가는 헤더
curl -sD - -o /dev/null http://checkout.internal/v1/orders \
  -H 'traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'

# 00                                 버전
# 4bf92f3577b34da6a3ce929d0e0e4736   트레이스 ID — 요청 전체에서 동일
# 00f067aa0ba902b7                   부모 스팬 ID — 호출 단계마다 바뀐다
# 01                                 샘플링 플래그 — 00 이면 이 트레이스는 저장되지 않는다

├─ SERVER  order-api  GET /v1/orders/A-99183                     1421ms
│  ├─ CLIENT  db.query  SELECT * FROM orders WHERE id = ?           5ms
│  ├─ CLIENT  db.query  SELECT * FROM order_items WHERE oid = ?     4ms
│  ├─ CLIENT  db.query  SELECT * FROM products WHERE id = ?         4ms
│  ├─ CLIENT  db.query  SELECT * FROM products WHERE id = ?         4ms
│  ├─ ... (같은 쿼리 340회) ...

npm i @opentelemetry/sdk-node @opentelemetry/auto-instrumentations-node

OTEL_SERVICE_NAME=checkout-api \
OTEL_RESOURCE_ATTRIBUTES=service.version=1.42.3,deployment.environment=prod \
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector.observability:4318 \
OTEL_TRACES_SAMPLER=parentbased_traceidratio \
OTEL_TRACES_SAMPLER_ARG=1.0 \
node --require @opentelemetry/auto-instrumentations-node/register server.js

├─ SERVER  report-api  GET /v1/reports/monthly              3204ms
│  ├─ CLIENT  db.query   SELECT ... FROM ledger              412ms
│  └─ CLIENT  s3.upload  PUT report-2026-07.pdf              260ms
│     self time = 3204 - 412 - 260 = 2532ms  <-- 계측되지 않은 구간

import { trace, SpanStatusCode } from '@opentelemetry/api'

const tracer = trace.getTracer('checkout', '1.42.3')

export async function applyPromotions(cart, tenantId) {
  return tracer.startActiveSpan('checkout.applyPromotions', async (span) => {
    span.setAttribute('cart.item_count', cart.items.length)
    span.setAttribute('promotion.engine', 'rules-v3')
    span.setAttribute('tenant.id', tenantId)

    try {
      const cached = await rules.fromCache(tenantId)
      span.setAttribute('promotion.cache_hit', Boolean(cached))

      const result = await (cached ?? rules.compile(tenantId)).evaluate(cart)
      span.setAttribute('promotion.rules_evaluated', result.evaluated)
      span.setAttribute('promotion.matched_count', result.matched.length)
      return result
    } catch (err) {
      span.recordException(err)
      span.setStatus({ code: SpanStatusCode.ERROR, message: err.message })
      throw err
    } finally {
      span.end()
    }
  })
}

python3 - <<'PY'
rate = 0.01              # 헤드 샘플링 1%
errors_per_hour = 5      # 조사하려는 오류의 실제 발생 빈도
kept = errors_per_hour * rate
print(f"보존되는 오류 트레이스: 시간당 {kept:.2f}건")
print(f"1건을 보려면 평균 {1/kept:.0f}시간 대기")
PY
# 보존되는 오류 트레이스: 시간당 0.05건
# 1건을 보려면 평균 20시간 대기

# otel-collector-tailsampler.yaml
processors:
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 2000
    policies:
      - name: keep-errors
        type: status_code
        status_code:
          status_codes: [ERROR]

      - name: keep-slow
        type: latency
        latency:
          threshold_ms: 800

      - name: keep-vip-tenants
        type: string_attribute
        string_attribute:
          key: tenant.tier
          values: [enterprise]

      - name: baseline
        type: probabilistic
        probabilistic:
          sampling_percentage: 2

python3 - <<'PY'
traces_per_sec, wait_sec, spans, span_kb = 2000, 10, 12, 1.2
mb = traces_per_sec * wait_sec * spans * span_kb / 1024
print(f"버퍼 메모리 대략 {mb:.0f} MB (여유 2배 잡으면 {mb*2:.0f} MB)")
PY
# 버퍼 메모리 대략 281 MB (여유 2배 잡으면 563 MB)

# otel-collector-gateway.yaml — 앞단에서 trace ID 로 라우팅한다
exporters:
  loadbalancing:
    routing_key: traceID
    protocol:
      otlp:
        tls:
          insecure: true
    resolver:
      dns:
        hostname: otel-tailsampler.observability.svc.cluster.local
        port: 4317

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [batch]
      exporters: [loadbalancing]

connectors:
  spanmetrics:
    # 여기에 나열한 것만 메트릭 레이블이 된다 — 저카디널리티만 넣는다
    dimensions:
      - name: http.route
      - name: http.request.method
      - name: deployment.environment
    histogram:
      explicit:
        buckets: [10ms, 25ms, 50ms, 100ms, 250ms, 500ms, 1s, 2s, 5s]

OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT=64 \
OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT=2048 \
node --require @opentelemetry/auto-instrumentations-node/register server.js

import { propagation, context, trace, SpanKind } from '@opentelemetry/api'

// 프로듀서: 메시지 헤더에 컨텍스트를 주입한다
export async function publish(order) {
  return tracer.startActiveSpan('orders.publish', { kind: SpanKind.PRODUCER }, async (span) => {
    const headers = {}
    propagation.inject(context.active(), headers)
    await producer.send({ topic: 'orders', messages: [{ value: JSON.stringify(order), headers }] })
    span.end()
  })
}

// 배치 컨슈머: 각 메시지의 컨텍스트를 부모가 아니라 링크로 붙인다
export async function consumeBatch(messages) {
  const links = messages
    .map((m) => trace.getSpan(propagation.extract(context.active(), m.headers))?.spanContext())
    .filter(Boolean)
    .map((spanContext) => ({ context: spanContext }))

  return tracer.startActiveSpan(
    'orders.processBatch',
    { kind: SpanKind.CONSUMER, links },
    async (span) => {
      span.setAttribute('messaging.batch.message_count', messages.length)
      await Promise.all(messages.map(handle))
      span.end()
    }
  )
}