분산 트레이싱 실전 가이드: OpenTelemetry, Jaeger, Grafana Tempo

traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
             |  |                                |                |
           버전  Trace ID (32자)                  Parent Span ID   Flags

pip install opentelemetry-api \
  opentelemetry-sdk \
  opentelemetry-exporter-otlp \
  opentelemetry-instrumentation-flask \
  opentelemetry-instrumentation-requests \
  opentelemetry-instrumentation-sqlalchemy

# tracing.py - OpenTelemetry 초기화 모듈
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.resources import Resource, SERVICE_NAME, SERVICE_VERSION
from opentelemetry.propagate import set_global_textmap
from opentelemetry.propagators.composite import CompositePropagator
from opentelemetry.trace.propagation import TraceContextTextMapPropagator
from opentelemetry.baggage.propagation import W3CBaggagePropagator

def init_tracer(service_name: str, otlp_endpoint: str = "localhost:4317"):
    """OpenTelemetry TracerProvider를 초기화하고 글로벌 설정을 적용한다."""
    resource = Resource.create({
        SERVICE_NAME: service_name,
        SERVICE_VERSION: "1.0.0",
        "deployment.environment": "production",
    })

    provider = TracerProvider(resource=resource)

    # OTLP gRPC Exporter 설정
    otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint, insecure=True)
    span_processor = BatchSpanProcessor(
        otlp_exporter,
        max_queue_size=2048,
        max_export_batch_size=512,
        schedule_delay_millis=5000,
    )
    provider.add_span_processor(span_processor)

    # 글로벌 TracerProvider 등록
    trace.set_tracer_provider(provider)

    # W3C Trace Context + Baggage 전파 설정
    set_global_textmap(CompositePropagator([
        TraceContextTextMapPropagator(),
        W3CBaggagePropagator(),
    ]))

    return provider

# app.py - Flask 애플리케이션에 트레이싱 적용
from flask import Flask, request, jsonify
from opentelemetry import trace
from opentelemetry.instrumentation.flask import FlaskInstrumentor
from opentelemetry.instrumentation.requests import RequestsInstrumentor
from tracing import init_tracer
import requests

# 트레이서 초기화
provider = init_tracer("order-service", "otel-collector:4317")

app = Flask(__name__)

# 자동 계측: Flask와 requests 라이브러리
FlaskInstrumentor().instrument_app(app)
RequestsInstrumentor().instrument()

# 수동 계측을 위한 트레이서 획득
tracer = trace.get_tracer("order-service", "1.0.0")

@app.route("/orders", methods=["POST"])
def create_order():
    """주문 생성 API - 수동 Span을 추가하여 세밀한 추적 수행"""
    with tracer.start_as_current_span(
        "validate_order",
        attributes={"order.type": "standard"}
    ) as span:
        order_data = request.get_json()
        span.set_attribute("order.item_count", len(order_data.get("items", [])))

        # 재고 확인 - 외부 서비스 호출 (자동 계측됨)
        inventory_resp = requests.post(
            "http://inventory-service:8080/check",
            json=order_data["items"]
        )

        if inventory_resp.status_code != 200:
            span.set_status(trace.StatusCode.ERROR, "Inventory check failed")
            span.record_exception(Exception("재고 부족"))
            return jsonify({"error": "insufficient_inventory"}), 400

    with tracer.start_as_current_span("process_payment") as span:
        payment_resp = requests.post(
            "http://payment-service:8080/charge",
            json={"amount": order_data["total"]}
        )
        span.set_attribute("payment.method", order_data.get("payment_method", "card"))

    return jsonify({"order_id": "ORD-12345", "status": "confirmed"}), 201

if __name__ == "__main__":
    app.run(host="0.0.0.0", port=8080)

// tracing.go - OpenTelemetry 초기화
package main

import (
    "context"
    "log"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/propagation"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
    "go.opentelemetry.io/otel/trace"
    "google.golang.org/grpc"
    "google.golang.org/grpc/credentials/insecure"
)

func initTracer(ctx context.Context, serviceName, endpoint string) (*sdktrace.TracerProvider, error) {
    // gRPC 연결로 OTLP Exporter 생성
    conn, err := grpc.NewClient(endpoint,
        grpc.WithTransportCredentials(insecure.NewCredentials()),
    )
    if err != nil {
        return nil, err
    }

    exporter, err := otlptracegrpc.New(ctx, otlptracegrpc.WithGRPCConn(conn))
    if err != nil {
        return nil, err
    }

    // Resource 정의 - 서비스 메타데이터
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName(serviceName),
            semconv.ServiceVersion("1.0.0"),
            semconv.DeploymentEnvironment("production"),
        ),
    )
    if err != nil {
        return nil, err
    }

    // TracerProvider 생성
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter,
            sdktrace.WithMaxQueueSize(2048),
            sdktrace.WithMaxExportBatchSize(512),
        ),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.ParentBased(
            sdktrace.TraceIDRatioBased(0.1), // 10% 샘플링
        )),
    )

    // 글로벌 설정
    otel.SetTracerProvider(tp)
    otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
        propagation.TraceContext{},
        propagation.Baggage{},
    ))

    return tp, nil
}

// handleOrder - HTTP 핸들러에서 수동 Span 생성
func handleOrder(w http.ResponseWriter, r *http.Request) {
    ctx := r.Context()
    tracer := otel.Tracer("order-handler")

    ctx, span := tracer.Start(ctx, "process-order",
        trace.WithSpanKind(trace.SpanKindServer),
    )
    defer span.End()

    // 비즈니스 로직에 컨텍스트 전달
    orderID, err := processOrder(ctx)
    if err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        http.Error(w, "order failed", http.StatusInternalServerError)
        return
    }

    span.SetAttributes(attribute.String("order.id", orderID))
    w.WriteHeader(http.StatusCreated)
}

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 1024
    send_batch_max_size: 2048

  memory_limiter:
    check_interval: 1s
    limit_mib: 1024
    spike_limit_mib: 256

  resource:
    attributes:
      - key: environment
        value: production
        action: upsert
      - key: cluster
        value: k8s-prod-01
        action: upsert

  # Tail-based 샘플링: 에러와 느린 요청은 100% 수집
  tail_sampling:
    decision_wait: 10s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      - name: error-policy
        type: status_code
        status_code:
          status_codes:
            - ERROR
      - name: latency-policy
        type: latency
        latency:
          threshold_ms: 1000
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

exporters:
  otlp/jaeger:
    endpoint: jaeger-collector:4317
    tls:
      insecure: true

  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true

  debug:
    verbosity: basic

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  zpages:
    endpoint: 0.0.0.0:55679

service:
  extensions: [health_check, zpages]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, resource, tail_sampling, batch]
      exporters: [otlp/jaeger, otlp/tempo, debug]
  telemetry:
    logs:
      level: info
    metrics:
      address: 0.0.0.0:8888

# docker-compose.yaml (Collector 부분)
services:
  otel-collector:
    image: otel/opentelemetry-collector-contrib:0.116.0
    command: ['--config=/etc/otel-collector-config.yaml']
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
    ports:
      - '4317:4317' # OTLP gRPC
      - '4318:4318' # OTLP HTTP
      - '8888:8888' # Collector 메트릭
      - '13133:13133' # Health check
      - '55679:55679' # zPages
    restart: unless-stopped
    deploy:
      resources:
        limits:
          memory: 2g
          cpus: '1.0'
    healthcheck:
      test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:13133']
      interval: 10s
      timeout: 5s
      retries: 3

# docker-compose-jaeger.yaml
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.17.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - 'ES_JAVA_OPTS=-Xms1g -Xmx1g'
    ports:
      - '9200:9200'
    volumes:
      - es_data:/usr/share/elasticsearch/data
    healthcheck:
      test: ['CMD-SHELL', 'curl -f http://localhost:9200/_cluster/health || exit 1']
      interval: 10s
      timeout: 5s
      retries: 10

  jaeger:
    image: jaegertracing/jaeger:2.4.0
    environment:
      - SPAN_STORAGE_TYPE=elasticsearch
      - ES_SERVER_URLS=http://elasticsearch:9200
      - ES_NUM_SHARDS=3
      - ES_NUM_REPLICAS=1
      - ES_INDEX_PREFIX=jaeger
      - ES_TAGS_AS_FIELDS_ALL=true
      - COLLECTOR_OTLP_ENABLED=true
    ports:
      - '16686:16686' # Jaeger UI
      - '4317:4317' # OTLP gRPC
      - '4318:4318' # OTLP HTTP
      - '14250:14250' # gRPC Collector
    depends_on:
      elasticsearch:
        condition: service_healthy
    restart: unless-stopped

volumes:
  es_data:
    driver: local

#!/bin/bash
# jaeger-index-cleaner.sh - 14일 이상 된 Jaeger 인덱스 삭제
ES_URL="http://elasticsearch:9200"
RETENTION_DAYS=14

# 삭제 대상 날짜 계산
CUTOFF_DATE=$(date -d "-${RETENTION_DAYS} days" +%Y-%m-%d)

echo "Deleting Jaeger indices older than ${CUTOFF_DATE}..."

# jaeger-span-* 및 jaeger-service-* 인덱스 조회 및 삭제
for INDEX_TYPE in "jaeger-span" "jaeger-service"; do
  INDICES=$(curl -s "${ES_URL}/_cat/indices/${INDEX_TYPE}-*" \
    | awk '{print $3}' \
    | sort)

  for INDEX in ${INDICES}; do
    # 인덱스명에서 날짜 추출 (jaeger-span-2026-03-01 형식)
    INDEX_DATE=$(echo "${INDEX}" | grep -oP '\d{4}-\d{2}-\d{2}')
    if [[ "${INDEX_DATE}" < "${CUTOFF_DATE}" ]]; then
      echo "Deleting index: ${INDEX}"
      curl -s -X DELETE "${ES_URL}/${INDEX}"
    fi
  done
done

echo "Cleanup completed."

# tempo-config.yaml
server:
  http_listen_port: 3200

distributor:
  receivers:
    otlp:
      protocols:
        grpc:
          endpoint: 0.0.0.0:4317
        http:
          endpoint: 0.0.0.0:4318

ingester:
  max_block_duration: 5m
  max_block_bytes: 1073741824 # 1GB
  flush_check_period: 10s

compactor:
  compaction:
    block_retention: 336h # 14일
  ring:
    kvstore:
      store: memberlist

storage:
  trace:
    backend: s3
    s3:
      bucket: tempo-traces-prod
      endpoint: s3.ap-northeast-2.amazonaws.com
      region: ap-northeast-2
      # IAM Role 기반 인증 권장 (access_key 직접 입력 지양)
    wal:
      path: /var/tempo/wal
    local:
      path: /var/tempo/blocks
    pool:
      max_workers: 100
      queue_depth: 10000

querier:
  search:
    query_timeout: 30s
  frontend_worker:
    frontend_address: tempo-query-frontend:9095

metrics_generator:
  registry:
    external_labels:
      source: tempo
      cluster: k8s-prod
  storage:
    path: /var/tempo/generator/wal
    remote_write:
      - url: http://prometheus:9090/api/v1/write
        send_exemplars: true
  traces_storage:
    path: /var/tempo/generator/traces
  processor:
    service_graphs:
      dimensions:
        - service.namespace
        - deployment.environment
    span_metrics:
      dimensions:
        - http.method
        - http.status_code
        - http.route

overrides:
  defaults:
    metrics_generator:
      processors: [service-graphs, span-metrics]

// 에러 상태인 Span을 가진 트레이스 검색
{ status = error }

// 특정 서비스에서 1초 이상 걸린 Span 검색
{ resource.service.name = "order-service" && duration > 1s }

// HTTP 500 에러를 반환한 Span 검색
{ span.http.status_code = 500 }

// 두 서비스 간의 호출 관계 검색
{ resource.service.name = "api-gateway" } >> { resource.service.name = "payment-service" }

# Python에서 Head-based 확률적 샘플링 설정
from opentelemetry.sdk.trace.sampling import TraceIdRatioBased, ParentBased

# ParentBased: 부모 Span이 샘플링되었으면 자식도 샘플링
# 부모가 없는 루트 Span은 10% 확률로 샘플링
sampler = ParentBased(root=TraceIdRatioBased(0.1))

provider = TracerProvider(
    resource=resource,
    sampler=sampler,
)