분산 트레이싱 완전 가이드 2025: OpenTelemetry, Jaeger, Tempo, Span 분석, 샘플링 전략

요청 → [모놀리스 앱]
        ├─ Auth check
        ├─ DB query
        ├─ Cache lookup
        └─ Response

요청 → [API Gateway]
        ↓
       [Auth Service]
        ↓
       [User Service]
        ↓ ↘
       [DB]  [Cache]
              ↓
       [Email Service]
              ↓
       [SMS Service]

Total: 1245ms
├─ API Gateway (5ms)
├─ Auth Service (50ms)
│   └─ JWT verify (45ms)
├─ User Service (1180ms) ⚠️
│   ├─ DB query (1100ms) 🔴 ← 병목!
│   └─ Cache lookup (5ms)
└─ Response (5ms)

trace_id: abc123...
├─ Span A (root)
├─ Span B (child of A)
├─ Span C (child of B)
└─ Span D (child of A)

{
  "trace_id": "abc123def456...",
  "span_id": "789xyz...",
  "parent_span_id": "456abc...",
  "name": "GET /api/users/123",
  "start_time": "2025-04-15T10:00:00.000Z",
  "end_time": "2025-04-15T10:00:00.150Z",
  "duration_ms": 150,
  "status": { "code": "OK" },
  "attributes": {
    "http.method": "GET",
    "http.url": "/api/users/123",
    "http.status_code": 200,
    "user.id": "123",
    "db.query.count": 3
  },
  "events": [
    {
      "name": "cache_miss",
      "timestamp": "2025-04-15T10:00:00.020Z"
    }
  ]
}

GET /api/users/123 HTTP/1.1
traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01
tracestate: rojo=00f067aa0ba902b7,congo=t61rcWkgMzE

[Application]
   ↓ (OTel SDK)
[Spans/Metrics/Logs]
   ↓ (OTLP)
[OpenTelemetry Collector]
   ↓ (export)
[Jaeger / Tempo / Datadog / ...]

const { NodeSDK } = require('@opentelemetry/sdk-node')
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http')
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node')

const sdk = new NodeSDK({
  serviceName: 'my-service',
  traceExporter: new OTLPTraceExporter({
    url: 'http://localhost:4318/v1/traces'
  }),
  instrumentations: [getNodeAutoInstrumentations()]
})

sdk.start()

// 이제 HTTP, Express, MongoDB 등 모두 자동 트레이싱

const { trace } = require('@opentelemetry/api')

const tracer = trace.getTracer('my-service')

async function processOrder(orderId) {
  const span = tracer.startSpan('process_order')
  span.setAttribute('order.id', orderId)
  
  try {
    await chargePayment(orderId)
    span.addEvent('payment_charged')
    
    await updateInventory(orderId)
    span.addEvent('inventory_updated')
    
    span.setStatus({ code: SpanStatusCode.OK })
  } catch (error) {
    span.recordException(error)
    span.setStatus({ code: SpanStatusCode.ERROR, message: error.message })
    throw error
  } finally {
    span.end()
  }
}

{ resource.service.name = "checkout" && duration > 1s }

# 10% 샘플링
if random.random() < 0.1:
    span = tracer.start_span(...)

# OTel Collector
processors:
  probabilistic_sampler:
    sampling_percentage: 10

processors:
  tail_sampling:
    decision_wait: 10s
    policies:
      - name: error-traces
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow-traces
        type: latency
        latency: { threshold_ms: 1000 }
      - name: probabilistic
        type: probabilistic
        probabilistic: { sampling_percentage: 1 }

정상 트래픽: 1%
느린 트레이스 (>1s): 100%
에러: 100%
새 endpoint: 100% (1주일)
중요 endpoint (/checkout): 10%

java -javaagent:opentelemetry-javaagent.jar \
  -Dotel.service.name=my-app \
  -Dotel.exporter.otlp.endpoint=http://collector:4318 \
  -jar my-app.jar

pip install opentelemetry-distro opentelemetry-exporter-otlp
opentelemetry-bootstrap --action=install
opentelemetry-instrument python my_app.py

node --require @opentelemetry/auto-instrumentations-node/register my-app.js

require('@opentelemetry/auto-instrumentations-node/register')
// 이후 모든 import는 자동 트레이싱됨

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/trace"
)

tracer := otel.Tracer("my-service")

func handleRequest(ctx context.Context) {
    ctx, span := tracer.Start(ctx, "handle_request")
    defer span.End()
    // ...
}

parent_span (200ms)
├─ db_query (5ms)  ← user 정보
├─ db_query (5ms)  ← user 1의 posts
├─ db_query (5ms)  ← user 2의 posts
├─ db_query (5ms)  ← user 3의 posts
... (47 more)

parent_span (300ms)
├─ call_service_a (100ms)
├─ call_service_b (100ms)  ← 직렬!
└─ call_service_c (100ms)

parent_span (200ms)
├─ cache_check (5ms)        ← miss
└─ db_query (190ms)

processors:
  spanmetrics:
    metrics_exporter: prometheus

{ resource.service.name = "checkout" && duration > 5s }

checkout (28s)
├─ validate (10ms)
├─ inventory (50ms)
└─ payment (27s)
    ├─ db_save (10ms)
    └─ stripe_api_call (26.9s) ← !!
        └─ http_retry (3 attempts, 9s each)

"http.url": "https://api.stripe.com/v1/charges",
"http.status_code": 0,  ← timeout
"http.error": "EAI_AGAIN"

checkout (1.2s)  ✅
└─ payment (800ms)
    └─ stripe_api_call (750ms)  ← 정상

span.set_attribute("user.id", user_id)  # 개별 ID
span.set_attribute("user.tier", "premium")  # 분류
span.set_attribute("db.statement", query)  # 정보

try:
    do_something()
except Exception as e:
    span.record_exception(e)
    span.set_status(Status(StatusCode.ERROR, str(e)))
    raise

import logging
from opentelemetry import trace

current_span = trace.get_current_span()
ctx = current_span.get_span_context()

logger.info("Order processed", extra={
    "trace_id": format(ctx.trace_id, "032x"),
    "span_id": format(ctx.span_id, "016x"),
    "order_id": order_id
})