OpenTelemetry Collector 운영 완벽 가이드 — 파이프라인 구성부터 백엔드 연동까지

[애플리케이션 + OTel SDK]
        |
        | OTLP (gRPC/HTTP)
        v
+========================+
|   OpenTelemetry        |
|   Collector            |
|                        |
|  Receiver -> Processor |
|          -> Exporter   |
+========================+
    |         |         |
    v         v         v
 [Jaeger]  [Tempo]  [Datadog]

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 512
    spike_limit_mib: 128
  batch:
    send_batch_size: 8192
    timeout: 200ms
    send_batch_max_size: 0

exporters:
  otlp:
    endpoint: tempo.monitoring.svc.cluster.local:4317
    tls:
      insecure: true
  debug:
    verbosity: detailed

extensions:
  health_check:
    endpoint: 0.0.0.0:13133
  zpages:
    endpoint: 0.0.0.0:55679

service:
  extensions: [health_check, zpages]
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [otlp, debug]

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        max_recv_msg_size_mib: 4
        keepalive:
          server_parameters:
            max_connection_idle: 11s
            max_connection_age: 30s
      http:
        endpoint: 0.0.0.0:4318
        cors:
          allowed_origins:
            - 'https://*.example.com'

receivers:
  jaeger:
    protocols:
      grpc:
        endpoint: 0.0.0.0:14250
      thrift_http:
        endpoint: 0.0.0.0:14268
      thrift_compact:
        endpoint: 0.0.0.0:6831

receivers:
  prometheus:
    config:
      scrape_configs:
        - job_name: 'otel-collector-internal'
          scrape_interval: 15s
          static_configs:
            - targets: ['0.0.0.0:8888']
        - job_name: 'kubernetes-pods'
          kubernetes_sd_configs:
            - role: pod
          relabel_configs:
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
              action: keep
              regex: true
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
              action: replace
              target_label: __address__
              regex: (.+)
              replacement: $$1

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 1024 # 하드 리밋 (1GB)
    spike_limit_mib: 256 # 스파이크 허용치 (하드 리밋의 약 20%)

processors:
  batch:
    send_batch_size: 8192 # 이 개수에 도달하면 즉시 전송
    timeout: 200ms # 크기에 상관없이 이 시간이 지나면 전송
    send_batch_max_size: 16384 # 배치 최대 크기 상한 (0이면 무제한)

processors:
  filter:
    error_mode: ignore
    traces:
      span:
        - 'attributes["http.route"] == "/healthz"'
        - 'attributes["http.route"] == "/readyz"'
        - 'name == "health_check"'
    metrics:
      metric:
        - 'name == "rpc.server.duration" and resource.attributes["service.name"] == "debug-svc"'

processors:
  attributes:
    actions:
      - key: environment
        value: production
        action: upsert
      - key: db.statement
        action: delete
      - key: http.request.header.authorization
        action: delete

exporters:
  otlp/jaeger:
    endpoint: jaeger-collector.monitoring.svc.cluster.local:4317
    tls:
      insecure: true
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
      max_elapsed_time: 300s
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 5000

exporters:
  otlp/tempo:
    endpoint: tempo-distributor.monitoring.svc.cluster.local:4317
    tls:
      insecure: true
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 10000
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s

exporters:
  datadog:
    api:
      key: ${env:DD_API_KEY}
      site: datadoghq.com
    traces:
      span_name_as_resource_name: true
    metrics:
      histograms:
        mode: distributions
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 5000

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, filter, attributes, batch]
      exporters: [otlp/tempo, datadog]
    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, batch]
      exporters: [prometheusremotewrite]
    logs:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [loki]

# otel-agent-daemonset.yaml
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: otel-collector-agent
  namespace: monitoring
spec:
  selector:
    matchLabels:
      app: otel-collector-agent
  template:
    metadata:
      labels:
        app: otel-collector-agent
    spec:
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.120.0
          args: ['--config=/etc/otelcol/config.yaml']
          env:
            - name: GOMEMLIMIT
              value: '400MiB'
            - name: K8S_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          ports:
            - containerPort: 4317 # OTLP gRPC
              hostPort: 4317
              protocol: TCP
            - containerPort: 4318 # OTLP HTTP
              hostPort: 4318
              protocol: TCP
            - containerPort: 13133 # Health Check
              protocol: TCP
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: 500m
              memory: 512Mi
          livenessProbe:
            httpGet:
              path: /
              port: 13133
            initialDelaySeconds: 10
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /
              port: 13133
            initialDelaySeconds: 5
            periodSeconds: 5
          volumeMounts:
            - name: config
              mountPath: /etc/otelcol
      volumes:
        - name: config
          configMap:
            name: otel-agent-config

# otel-gateway-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-collector-gateway
  namespace: monitoring
spec:
  replicas: 3
  selector:
    matchLabels:
      app: otel-collector-gateway
  template:
    metadata:
      labels:
        app: otel-collector-gateway
    spec:
      containers:
        - name: otel-collector
          image: otel/opentelemetry-collector-contrib:0.120.0
          args: ['--config=/etc/otelcol/config.yaml']
          env:
            - name: GOMEMLIMIT
              value: '3200MiB'
          ports:
            - containerPort: 4317
              protocol: TCP
            - containerPort: 13133
              protocol: TCP
          resources:
            requests:
              cpu: 1000m
              memory: 2Gi
            limits:
              cpu: 2000m
              memory: 4Gi
          livenessProbe:
            httpGet:
              path: /
              port: 13133
          readinessProbe:
            httpGet:
              path: /
              port: 13133
          volumeMounts:
            - name: config
              mountPath: /etc/otelcol
      volumes:
        - name: config
          configMap:
            name: otel-gateway-config
---
apiVersion: v1
kind: Service
metadata:
  name: otel-collector-gateway
  namespace: monitoring
spec:
  selector:
    app: otel-collector-gateway
  ports:
    - name: otlp-grpc
      port: 4317
      targetPort: 4317
    - name: otlp-http
      port: 4318
      targetPort: 4318
  type: ClusterIP

# Agent의 exporter 설정
exporters:
  otlp/gateway:
    endpoint: otel-collector-gateway.monitoring.svc.cluster.local:4317
    tls:
      insecure: true
    sending_queue:
      enabled: true
      queue_size: 2000
    retry_on_failure:
      enabled: true

# SDK 설정에서 TraceIdRatioBased Sampler 사용
# 10%의 트레이스만 샘플링
processors:
  probabilistic_sampler:
    sampling_percentage: 10

processors:
  tail_sampling:
    decision_wait: 30s
    num_traces: 100000
    expected_new_traces_per_sec: 1000
    policies:
      # 정책 1: 에러가 포함된 트레이스는 100% 샘플링
      - name: errors-policy
        type: status_code
        status_code:
          status_codes:
            - ERROR
      # 정책 2: 지연이 500ms를 초과하는 트레이스는 100% 샘플링
      - name: latency-policy
        type: latency
        latency:
          threshold_ms: 500
      # 정책 3: 나머지 트레이스는 5%만 샘플링
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 5

# 성능 최적화 설정 예시
processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 1638 # 컨테이너 메모리 2GB의 ~80%
    spike_limit_mib: 328 # limit_mib의 ~20%
  batch:
    send_batch_size: 10000
    timeout: 500ms
    send_batch_max_size: 20000

exporters:
  otlp/backend:
    endpoint: backend:4317
    sending_queue:
      enabled: true
      num_consumers: 20 # 병렬 전송 워커 수
      queue_size: 10000 # 큐 크기 (배치 단위)
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
      max_elapsed_time: 300s
    timeout: 30s

# 컨테이너 메모리 제한이 2GB일 때
# GOMEMLIMIT = limit_mib * 0.8 = 1638 * 0.8 ~ 1310MiB
export GOMEMLIMIT=1310MiB

# PodDisruptionBudget 설정
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: otel-gateway-pdb
  namespace: monitoring
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: otel-collector-gateway

# 1단계: Collector 로그 확인
kubectl logs -n monitoring deploy/otel-collector-gateway --tail=100

# 2단계: zPages에서 파이프라인 상태 확인
kubectl port-forward -n monitoring svc/otel-collector-gateway 55679:55679
# 브라우저에서 http://localhost:55679/debug/tracez 접속

# 3단계: debug exporter를 임시 추가하여 데이터 흐름 확인
# exporters에 debug: { verbosity: detailed } 추가 후
# pipeline에 debug exporter 추가

# Collector 내부 메트릭으로 큐 상태 확인
curl http://localhost:8888/metrics | grep otelcol_exporter_queue
# otelcol_exporter_queue_size: 현재 큐 크기
# otelcol_exporter_queue_capacity: 큐 최대 용량
# queue_size가 capacity에 근접하면 큐 포화 상태

# ConfigMap 롤백
kubectl rollout undo configmap/otel-gateway-config -n monitoring
# 또는 이전 버전의 ConfigMap을 직접 적용
kubectl apply -f otel-gateway-config-backup.yaml

# Gateway Deployment 재시작
kubectl rollout restart deployment/otel-collector-gateway -n monitoring

# 복구 상태 확인
kubectl rollout status deployment/otel-collector-gateway -n monitoring