OpenTelemetry 분산 트레이싱 실전 가이드 — 계측부터 Jaeger 시각화까지

Trace (전체 요청의 여정):
├── Span A: API Gateway (50ms)
│   ├── Span B: Auth Service (10ms)
│   ├── Span C: Order Service (35ms)
│   │   ├── Span D: Database Query (15ms)
│   │   └── Span E: Payment Service (18ms)
│   │       └── Span F: Bank API Call (12ms)
│   └── Span G: Notification Service (5ms, async)

# Span의 구조
{
    "traceId": "abc123def456...",        # 전체 Trace 고유 ID (128-bit)
    "spanId": "span789...",              # 이 Span 고유 ID (64-bit)
    "parentSpanId": "parent456...",      # 부모 Span ID
    "name": "POST /api/orders",          # Span 이름
    "kind": "SERVER",                    # CLIENT, SERVER, PRODUCER, CONSUMER, INTERNAL
    "startTime": "2026-03-03T12:00:00Z",
    "endTime": "2026-03-03T12:00:00.050Z",
    "status": {"code": "OK"},
    "attributes": {                       # 메타데이터
        "http.method": "POST",
        "http.url": "/api/orders",
        "http.status_code": 201,
        "service.name": "order-service"
    },
    "events": [                           # Span 내 이벤트
        {
            "name": "order.validated",
            "timestamp": "2026-03-03T12:00:00.010Z",
            "attributes": {"order_id": "ORD-123"}
        }
    ]
}

서비스 간 컨텍스트 전파:

[Service A]                    [Service B]
    │                              │
    │ traceparent: 00-abc123-span1-01
    │ ─────────────────────────>   │
    │                              │ (같은 traceId로 자식 Span 생성)
    │                              │
    │                              │  traceparent: 00-abc123-span2-01
    │                              │  ──────────> [Service C]

HTTP Header:
traceparent: 00-{traceId}-{spanId}-{flags}
예: traceparent: 00-abc123def456789-span12345678-01

# 의존성 설치
pip install opentelemetry-distro opentelemetry-exporter-otlp
opentelemetry-bootstrap -a install  # 자동 계측 패키지 설치

# 자동 계측으로 앱 실행
opentelemetry-instrument \
  --service_name order-service \
  --traces_exporter otlp \
  --metrics_exporter otlp \
  --exporter_otlp_endpoint http://otel-collector:4317 \
  python app.py

from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource
from opentelemetry.semconv.resource import ResourceAttributes

# 1. TracerProvider 설정
resource = Resource.create({
    ResourceAttributes.SERVICE_NAME: "order-service",
    ResourceAttributes.SERVICE_VERSION: "1.2.0",
    ResourceAttributes.DEPLOYMENT_ENVIRONMENT: "production",
})

provider = TracerProvider(resource=resource)
processor = BatchSpanProcessor(
    OTLPSpanExporter(endpoint="http://otel-collector:4317")
)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

# 2. Tracer 생성
tracer = trace.get_tracer("order-service", "1.2.0")

# 3. Span 생성 및 사용
@tracer.start_as_current_span("create_order")
def create_order(order_data: dict) -> dict:
    span = trace.get_current_span()

    # 속성 추가
    span.set_attribute("order.customer_id", order_data["customer_id"])
    span.set_attribute("order.total_amount", order_data["total"])
    span.set_attribute("order.item_count", len(order_data["items"]))

    # 이벤트 추가
    span.add_event("order.validation_started")

    # 검증
    validate_order(order_data)
    span.add_event("order.validation_completed")

    # 결제 처리 (자식 Span 자동 생성)
    payment_result = process_payment(order_data)

    # 결과 기록
    span.set_attribute("order.id", payment_result["order_id"])
    span.set_status(trace.StatusCode.OK)

    return payment_result


@tracer.start_as_current_span("process_payment")
def process_payment(order_data: dict) -> dict:
    span = trace.get_current_span()
    span.set_attribute("payment.method", order_data.get("payment_method", "card"))

    try:
        result = payment_client.charge(order_data)
        span.set_attribute("payment.transaction_id", result["txn_id"])
        return result
    except Exception as e:
        span.set_status(trace.StatusCode.ERROR, str(e))
        span.record_exception(e)
        raise

from fastapi import FastAPI, Request
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor

app = FastAPI()

# 자동 계측 적용
FastAPIInstrumentor.instrument_app(app)
HTTPXClientInstrumentor().instrument()  # 외부 HTTP 호출
SQLAlchemyInstrumentor().instrument(engine=db_engine)  # DB 쿼리

@app.post("/api/orders")
async def create_order(request: Request, order: OrderRequest):
    # 현재 Span에 비즈니스 컨텍스트 추가
    span = trace.get_current_span()
    span.set_attribute("order.customer_id", order.customer_id)
    span.set_attribute("order.region", order.shipping_region)

    result = await order_service.create(order)
    return result

<dependency>
    <groupId>io.opentelemetry.instrumentation</groupId>
    <artifactId>opentelemetry-spring-boot-starter</artifactId>
    <version>2.11.0</version>
</dependency>

# application.yml
otel:
  service:
    name: order-service
  exporter:
    otlp:
      endpoint: http://otel-collector:4317
  traces:
    sampler: parentbased_traceidratio
    sampler.arg: '0.1' # 10% 샘플링 (프로덕션)

@RestController
@RequiredArgsConstructor
public class OrderController {

    private final Tracer tracer;
    private final OrderService orderService;

    @PostMapping("/api/orders")
    public ResponseEntity<OrderResponse> createOrder(@RequestBody OrderRequest request) {
        Span span = Span.current();
        span.setAttribute("order.customer_id", request.getCustomerId());
        span.setAttribute("order.total", request.getTotal().doubleValue());

        OrderResponse response = orderService.create(request);

        span.setAttribute("order.id", response.getOrderId());
        return ResponseEntity.status(HttpStatus.CREATED).body(response);
    }
}

// 수동 Span 생성
@Service
public class PaymentService {

    @Autowired
    private Tracer tracer;

    public PaymentResult processPayment(Order order) {
        Span span = tracer.spanBuilder("process_payment")
            .setAttribute("payment.amount", order.getTotal().doubleValue())
            .startSpan();

        try (Scope scope = span.makeCurrent()) {
            PaymentResult result = paymentGateway.charge(order);
            span.setAttribute("payment.txn_id", result.getTransactionId());
            return result;
        } catch (Exception e) {
            span.setStatus(StatusCode.ERROR, e.getMessage());
            span.recordException(e);
            throw e;
        } finally {
            span.end();
        }
    }
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
    send_batch_max_size: 2048

  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128

  attributes:
    actions:
      - key: environment
        value: production
        action: upsert

  tail_sampling:
    decision_wait: 10s
    policies:
      - name: errors
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow-traces
        type: latency
        latency: { threshold_ms: 1000 }
      - name: probabilistic
        type: probabilistic
        probabilistic: { sampling_percentage: 10 }

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true

  prometheus:
    endpoint: 0.0.0.0:8889

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch, attributes, tail_sampling]
      exporters: [otlp/jaeger]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [prometheus]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector
spec:
  replicas: 2
  selector:
    matchLabels:
      app: otel-collector
  template:
    spec:
      containers:
        - name: collector
          image: otel/opentelemetry-collector-contrib:0.96.0
          args: ['--config=/conf/config.yaml']
          ports:
            - containerPort: 4317 # gRPC
            - containerPort: 4318 # HTTP
            - containerPort: 8889 # Prometheus metrics
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 1000m
              memory: 512Mi
          volumeMounts:
            - name: config
              mountPath: /conf
      volumes:
        - name: config
          configMap:
            name: otel-collector-config
---
apiVersion: v1
kind: Service
metadata:
  name: otel-collector
spec:
  selector:
    app: otel-collector
  ports:
    - name: grpc
      port: 4317
    - name: http
      port: 4318

from opentelemetry.sdk.trace.sampling import (
    TraceIdRatioBased,
    ParentBasedTraceIdRatio,
    ALWAYS_ON,
    ALWAYS_OFF,
)

# 개발 환경: 전부 수집
dev_sampler = ALWAYS_ON

# 프로덕션: 10% 샘플링 (부모 Span 결정 따름)
prod_sampler = ParentBasedTraceIdRatio(0.1)

# 커스텀 샘플러: 에러는 100%, 정상은 5%
class SmartSampler:
    def should_sample(self, context, trace_id, name, kind, attributes, links):
        # 에러 Span은 무조건 수집
        if attributes and attributes.get("error") == True:
            return SamplingResult(Decision.RECORD_AND_SAMPLE)
        # 정상은 5%
        if trace_id % 100 < 5:
            return SamplingResult(Decision.RECORD_AND_SAMPLE)
        return SamplingResult(Decision.DROP)

# Docker로 Jaeger 실행
docker run -d --name jaeger \
  -p 16686:16686 \
  -p 4317:4317 \
  -p 4318:4318 \
  jaegertracing/all-in-one:1.62

# 브라우저에서 http://localhost:16686 접속

1. Trace 검색
   - Service 선택 → Operation 필터 → Duration 범위
   - Tags 검색: http.status_code=500

2. Trace 타임라인 (Waterfall)
   - 각 Span의 시작/종료 시간 시각화
   - 병렬 처리와 순차 처리 구분
   - 각 Span의 속성/이벤트/로그 상세 확인

3. Service Dependency Graph
   - 서비스 간 호출 관계 시각화
   - 호출 빈도와 에러율 표시

4. Compare
   - 두 Trace 비교하여 성능 차이 분석

from opentelemetry.semconv.trace import SpanAttributes

# 표준 속성 사용 (벤더 간 호환)
span.set_attribute(SpanAttributes.HTTP_METHOD, "POST")
span.set_attribute(SpanAttributes.HTTP_URL, "/api/orders")
span.set_attribute(SpanAttributes.HTTP_STATUS_CODE, 201)
span.set_attribute(SpanAttributes.DB_SYSTEM, "postgresql")
span.set_attribute(SpanAttributes.DB_STATEMENT, "SELECT * FROM orders")

# Collector에서 PII 제거
processors:
  attributes:
    actions:
      - key: http.request.header.authorization
        action: delete
      - key: db.statement
        action: hash  # SQL을 해시로 대체
      - key: user.email
        action: delete

try:
    result = external_api.call()
except Exception as e:
    span.set_status(trace.StatusCode.ERROR, str(e))
    span.record_exception(e)
    # record_exception은 자동으로 스택 트레이스 포함
    raise