OpenTelemetry Collector 파이프라인 설계와 운영 가이드: 수집부터 백엔드 연동까지

[Application / Infrastructure]
        |
        v
+-------------------+
|    Receivers       |  <-- OTLP, Prometheus, Filelog, Kafka, etc.
+-------------------+
        |
        v
+-------------------+
|    Processors      |  <-- Memory Limiter, Batch, Attributes, Tail Sampling
+-------------------+
        |
        v
+-------------------+
|    Exporters       |  <-- OTLP, Prometheus Remote Write, Loki, Kafka, etc.
+-------------------+

+-------------------+
|    Extensions      |  <-- Health Check, zPages, pprof, Bearer Token Auth
+-------------------+

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
        max_recv_msg_size_mib: 8 # 대용량 배치 허용
        max_concurrent_streams: 256 # 동시 스트림 수
        keepalive:
          server_parameters:
            max_connection_idle: 30s
            max_connection_age: 60s
            max_connection_age_grace: 10s
          enforcement_policy:
            min_time: 10s
            permit_without_stream: true
      http:
        endpoint: 0.0.0.0:4318
        cors:
          allowed_origins:
            - 'https://*.company.com'
          allowed_headers:
            - 'Content-Type'
            - 'X-Custom-Header'
          max_age: 600

receivers:
  prometheus:
    config:
      scrape_configs:
        - job_name: 'kubernetes-pods'
          scrape_interval: 30s
          scrape_timeout: 10s
          kubernetes_sd_configs:
            - role: pod
          relabel_configs:
            # prometheus.io/scrape 어노테이션이 있는 Pod만 스크래핑
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
              action: keep
              regex: true
            # 커스텀 포트 지정
            - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_port]
              action: replace
              target_label: __address__
              regex: (.+)
              replacement: '$$1'
            # 네임스페이스 레이블 추가
            - source_labels: [__meta_kubernetes_namespace]
              action: replace
              target_label: namespace
            # Pod 이름 레이블 추가
            - source_labels: [__meta_kubernetes_pod_name]
              action: replace
              target_label: pod
        - job_name: 'node-exporter'
          scrape_interval: 15s
          static_configs:
            - targets: ['node-exporter.monitoring.svc:9100']

receivers:
  filelog:
    include:
      - /var/log/pods/*/*/*.log
    exclude:
      - /var/log/pods/*/otel-collector*/*.log
      - /var/log/pods/kube-system_*/*/*.log
    start_at: end # 신규 로그만 수집 (beginning이면 기존 로그부터)
    include_file_path: true
    include_file_name: false
    retry_on_failure:
      enabled: true
      initial_interval: 1s
      max_interval: 30s
    operators:
      # CRI 로그 포맷 파싱 (containerd)
      - type: regex_parser
        id: parser-cri
        regex: '^(?P<time>[^ Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
        timestamp:
          parse_from: attributes.time
          layout: '%Y-%m-%dT%H:%M:%S.%fZ'
      # JSON 로그 본문 파싱
      - type: json_parser
        id: parser-json
        parse_from: attributes.log
        parse_to: body
        on_error: send_quiet # 파싱 실패 시 원본 유지
      # 심각도 매핑
      - type: severity_parser
        parse_from: attributes.level
        mapping:
          fatal: [FATAL, fatal, F]
          error: [ERROR, error, E]
          warn: [WARN, warn, W]
          info: [INFO, info, I]
          debug: [DEBUG, debug, D]

memory_limiter -> k8sattributes -> resourcedetection -> attributes -> filter -> tail_sampling -> batch

processors:
  memory_limiter:
    check_interval: 1s
    limit_mib: 1800 # 하드 리밋 (컨테이너 limit의 80%)
    spike_limit_mib: 500 # 갑작스런 스파이크 허용량
    # limit_percentage: 80     # 또는 비율 기반 설정 (cgroup 인식)
    # spike_limit_percentage: 25

processors:
  batch:
    timeout: 5s # 최대 대기 시간
    send_batch_size: 8192 # 배치 크기 (레코드 수)
    send_batch_max_size: 16384 # 최대 배치 크기 (이 크기를 넘으면 분할)
  # Gateway에서는 더 큰 배치
  batch/gateway:
    timeout: 10s
    send_batch_size: 16384
    send_batch_max_size: 32768

processors:
  attributes/security:
    actions:
      # 민감 HTTP 헤더 삭제
      - key: http.request.header.authorization
        action: delete
      - key: http.request.header.cookie
        action: delete
      # DB 쿼리 해싱 (민감 데이터 보호)
      - key: db.statement
        action: hash
      # 환경 태그 추가
      - key: deployment.environment
        action: upsert
        value: production
      # IP 주소 마스킹
      - key: net.peer.ip
        action: extract
        pattern: '^(?P<subnet>\d+\.\d+\.\d+)\.\d+$'
      - key: net.peer.ip
        action: delete
      - key: net.peer.subnet
        from_attribute: subnet
        action: upsert

processors:
  tail_sampling:
    decision_wait: 30s # 트레이스 완료 대기 시간
    num_traces: 200000 # 메모리에 유지할 최대 트레이스 수
    expected_new_traces_per_sec: 5000
    policies:
      # 정책 1: 에러가 포함된 트레이스는 100% 보존
      - name: error-traces
        type: status_code
        status_code:
          status_codes: [ERROR]
      # 정책 2: 2초 이상 걸린 트레이스는 100% 보존
      - name: high-latency
        type: latency
        latency:
          threshold_ms: 2000
          upper_threshold_ms: 0 # 0이면 상한 없음
      # 정책 3: 핵심 서비스는 50% 보존
      - name: critical-services
        type: and
        and:
          and_sub_policy:
            - name: service-match
              type: string_attribute
              string_attribute:
                key: service.name
                values:
                  - payment-service
                  - auth-service
                  - order-service
            - name: sample-half
              type: probabilistic
              probabilistic:
                sampling_percentage: 50
      # 정책 4: 특정 HTTP 경로 제외 (health check 등)
      - name: drop-health-checks
        type: string_attribute
        string_attribute:
          key: http.route
          values:
            - /healthz
            - /readyz
            - /livez
          invert_match: true
      # 정책 5: 나머지 트래픽은 5%만 샘플링
      - name: default-sampling
        type: probabilistic
        probabilistic:
          sampling_percentage: 5

exporters:
  # Traces -> Grafana Tempo
  otlp/tempo:
    endpoint: tempo-distributor.observability.svc:4317
    tls:
      insecure: true # 클러스터 내부 통신
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
      max_elapsed_time: 300s
    sending_queue:
      enabled: true
      num_consumers: 10
      queue_size: 5000
    timeout: 30s

  # Traces -> Jaeger (OTLP 네이티브 지원)
  otlp/jaeger:
    endpoint: jaeger-collector.observability.svc:4317
    tls:
      cert_file: /certs/client.crt
      key_file: /certs/client.key
      ca_file: /certs/ca.crt

exporters:
  prometheusremotewrite/mimir:
    endpoint: https://mimir.observability.svc:9009/api/v1/push
    tls:
      insecure: false
      cert_file: /certs/client.crt
      key_file: /certs/client.key
    headers:
      X-Scope-OrgID: 'tenant-production'
    resource_to_telemetry_conversion:
      enabled: true # Resource 속성을 메트릭 레이블로 변환
    external_labels:
      cluster: 'prod-ap-northeast-2'
      region: 'ap-northeast-2'
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 60s
    sending_queue:
      enabled: true
      num_consumers: 5
      queue_size: 10000

exporters:
  loki:
    endpoint: https://loki-gateway.observability.svc:3100/loki/api/v1/push
    headers:
      X-Scope-OrgID: 'tenant-production'
    default_labels_enabled:
      exporter: false
      job: true
      instance: true
      level: true
    retry_on_failure:
      enabled: true
      initial_interval: 5s
      max_interval: 30s
    sending_queue:
      enabled: true
      num_consumers: 5
      queue_size: 5000

[Node 1]                    [Node 2]                    [Node N]
+----------+               +----------+               +----------+
| App Pods |               | App Pods |               | App Pods |
+----+-----+               +----+-----+               +----+-----+
     |                          |                          |
+----+-----+               +----+-----+               +----+-----+
| OTel     |               | OTel     |               | OTel     |
| Agent    |               | Agent    |               | Agent    |
| (DaemonSet)              | (DaemonSet)              | (DaemonSet)
+----+-----+               +----+-----+               +----+-----+
     |                          |                          |
     +------------+-------------+-----------+--------------+
                  |                         |
           +------+------+          +------+------+
           | OTel Gateway |          | OTel Gateway |
           | (Deployment) |          | (Deployment) |
           +------+------+          +------+------+
                  |                         |
     +------------+-------------------------+
     |             |              |
+----+----+  +----+----+  +-----+-----+
|  Tempo  |  |  Mimir  |  |   Loki    |
+---------+  +---------+  +-----------+

apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: otel-agent
  namespace: observability
spec:
  mode: daemonset
  image: otel/opentelemetry-collector-contrib:0.120.0
  serviceAccount: otel-collector-agent
  env:
    - name: K8S_NODE_NAME
      valueFrom:
        fieldRef:
          fieldPath: spec.nodeName
    - name: K8S_POD_IP
      valueFrom:
        fieldRef:
          fieldPath: status.podIP
  resources:
    requests:
      cpu: 200m
      memory: 256Mi
    limits:
      cpu: 500m
      memory: 512Mi
  volumeMounts:
    - name: varlogpods
      mountPath: /var/log/pods
      readOnly: true
    - name: varlibdockercontainers
      mountPath: /var/lib/docker/containers
      readOnly: true
  volumes:
    - name: varlogpods
      hostPath:
        path: /var/log/pods
    - name: varlibdockercontainers
      hostPath:
        path: /var/lib/docker/containers
  tolerations:
    - operator: Exists
  config:
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
          http:
            endpoint: 0.0.0.0:4318
      filelog:
        include:
          - /var/log/pods/*/*/*.log
        exclude:
          - /var/log/pods/observability_otel-*/*/*.log
        start_at: end
        include_file_path: true
        operators:
          - type: regex_parser
            id: parser-cri
            regex: '^(?P<time>[^ Z]+Z) (?P<stream>stdout|stderr) (?P<logtag>[^ ]*) ?(?P<log>.*)$'
            timestamp:
              parse_from: attributes.time
              layout: '%Y-%m-%dT%H:%M:%S.%fZ'
      hostmetrics:
        collection_interval: 30s
        scrapers:
          cpu: {}
          memory: {}
          disk: {}
          network: {}
          load: {}
          filesystem:
            exclude_mount_points:
              mount_points: ['/dev/*', '/proc/*', '/sys/*']
              match_type: regexp

    processors:
      memory_limiter:
        check_interval: 1s
        limit_mib: 400
        spike_limit_mib: 100
      k8sattributes:
        auth_type: serviceAccount
        passthrough: false
        extract:
          metadata:
            - k8s.namespace.name
            - k8s.deployment.name
            - k8s.statefulset.name
            - k8s.daemonset.name
            - k8s.pod.name
            - k8s.pod.uid
            - k8s.node.name
            - k8s.container.name
          labels:
            - tag_name: app.label.team
              key: team
              from: pod
        pod_association:
          - sources:
              - from: resource_attribute
                name: k8s.pod.ip
          - sources:
              - from: connection
      batch:
        timeout: 5s
        send_batch_size: 4096

    exporters:
      otlp/gateway:
        endpoint: otel-gateway.observability.svc.cluster.local:4317
        tls:
          insecure: true
        retry_on_failure:
          enabled: true
          initial_interval: 5s
          max_interval: 30s
        sending_queue:
          enabled: true
          queue_size: 2000

    extensions:
      health_check:
        endpoint: 0.0.0.0:13133

    service:
      extensions: [health_check]
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, k8sattributes, batch]
          exporters: [otlp/gateway]
        metrics:
          receivers: [otlp, hostmetrics]
          processors: [memory_limiter, k8sattributes, batch]
          exporters: [otlp/gateway]
        logs:
          receivers: [otlp, filelog]
          processors: [memory_limiter, k8sattributes, batch]
          exporters: [otlp/gateway]
      telemetry:
        logs:
          level: warn
        metrics:
          address: 0.0.0.0:8888

apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: otel-gateway
  namespace: observability
spec:
  mode: deployment
  replicas: 3
  image: otel/opentelemetry-collector-contrib:0.120.0
  serviceAccount: otel-collector-gateway
  resources:
    requests:
      cpu: '1'
      memory: 2Gi
    limits:
      cpu: '2'
      memory: 4Gi
  autoscaler:
    minReplicas: 3
    maxReplicas: 10
    targetCPUUtilization: 70
    targetMemoryUtilization: 80
  podDisruptionBudget:
    minAvailable: 2
  config:
    receivers:
      otlp:
        protocols:
          grpc:
            endpoint: 0.0.0.0:4317
            max_recv_msg_size_mib: 16

    processors:
      memory_limiter:
        check_interval: 1s
        limit_mib: 3200
        spike_limit_mib: 800
      resourcedetection:
        detectors: [env, system, gcp, aws, azure]
        timeout: 5s
        override: false
      attributes/security:
        actions:
          - key: http.request.header.authorization
            action: delete
          - key: db.statement
            action: hash
      filter/metrics:
        metrics:
          exclude:
            match_type: regexp
            metric_names:
              - 'go_.*'
              - 'process_.*'
              - 'promhttp_.*'
      tail_sampling:
        decision_wait: 30s
        num_traces: 200000
        expected_new_traces_per_sec: 5000
        policies:
          - name: error-traces
            type: status_code
            status_code:
              status_codes: [ERROR]
          - name: high-latency
            type: latency
            latency:
              threshold_ms: 2000
          - name: default-sampling
            type: probabilistic
            probabilistic:
              sampling_percentage: 10
      batch:
        timeout: 10s
        send_batch_size: 16384
        send_batch_max_size: 32768

    exporters:
      otlp/tempo:
        endpoint: tempo-distributor.observability.svc:4317
        tls:
          insecure: true
        sending_queue:
          enabled: true
          num_consumers: 10
          queue_size: 10000
      prometheusremotewrite/mimir:
        endpoint: http://mimir-distributor.observability.svc:8080/api/v1/push
        headers:
          X-Scope-OrgID: 'production'
        resource_to_telemetry_conversion:
          enabled: true
        sending_queue:
          enabled: true
          num_consumers: 5
          queue_size: 10000
      loki:
        endpoint: http://loki-gateway.observability.svc:3100/loki/api/v1/push
        headers:
          X-Scope-OrgID: 'production'
        sending_queue:
          enabled: true
          queue_size: 5000

    extensions:
      health_check:
        endpoint: 0.0.0.0:13133
      zpages:
        endpoint: 0.0.0.0:55679
      pprof:
        endpoint: 0.0.0.0:1777

    service:
      extensions: [health_check, zpages, pprof]
      pipelines:
        traces:
          receivers: [otlp]
          processors: [memory_limiter, resourcedetection, attributes/security, tail_sampling, batch]
          exporters: [otlp/tempo]
        metrics:
          receivers: [otlp]
          processors: [memory_limiter, resourcedetection, filter/metrics, batch]
          exporters: [prometheusremotewrite/mimir]
        logs:
          receivers: [otlp]
          processors: [memory_limiter, resourcedetection, attributes/security, batch]
          exporters: [loki]
      telemetry:
        logs:
          level: info
        metrics:
          address: 0.0.0.0:8888

# Agent 측 Exporter 설정 (Gateway로 전송 시)
exporters:
  loadbalancing:
    protocol:
      otlp:
        tls:
          insecure: true
        timeout: 10s
    resolver:
      dns:
        hostname: otel-gateway-headless.observability.svc.cluster.local
        port: 4317
      # 또는 Kubernetes resolver 사용
      # k8s:
      #   service: otel-gateway
      #   ports:
      #     - 4317
    routing_key: traceID # TraceID 기반 일관된 해싱

총 메모리 = Go 런타임 기본 (약 50MB)
          + Receiver 버퍼 (연결 수 x 메시지 크기)
          + Processor 버퍼
            - Batch: send_batch_max_size x 레코드 평균 크기
            - Tail Sampling: num_traces x 트레이스 평균 크기
          + Exporter 큐: queue_size x 배치 크기 x 레코드 평균 크기
          + 내부 오버헤드 (약 10-20%)

env:
  - name: GOGC
    value: '80' # 기본값 100에서 80으로 낮춤
  - name: GOMEMLIMIT
    value: '3600MiB' # 소프트 메모리 상한 (limit의 90%)

# 수신 메트릭
otelcol_receiver_accepted_spans          # Receiver가 수락한 Span 수
otelcol_receiver_refused_spans           # Receiver가 거부한 Span 수 (백프레셔)
otelcol_receiver_accepted_metric_points  # 수락된 메트릭 포인트 수
otelcol_receiver_accepted_log_records    # 수락된 로그 레코드 수

# Processor 메트릭
otelcol_processor_dropped_spans          # Processor에서 드롭된 Span 수
otelcol_processor_batch_batch_send_size  # 실제 전송된 배치 크기

# Exporter 메트릭
otelcol_exporter_sent_spans              # Exporter가 전송 성공한 Span 수
otelcol_exporter_send_failed_spans       # 전송 실패한 Span 수
otelcol_exporter_queue_size              # 현재 큐에 대기 중인 항목 수
otelcol_exporter_queue_capacity          # 큐 최대 용량

apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-collector
rules:
  - apiGroups: ['']
    resources: ['pods', 'namespaces', 'nodes']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['apps']
    resources: ['replicasets', 'deployments', 'statefulsets', 'daemonsets']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['batch']
    resources: ['jobs', 'cronjobs']
    verbs: ['get', 'list', 'watch']
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-collector
subjects:
  - kind: ServiceAccount
    name: otel-collector-agent
    namespace: observability
roleRef:
  kind: ClusterRole
  name: otel-collector
  apiGroup: rbac.authorization.k8s.io

# 설정 파일 문법 검증
otelcol validate --config=config.yaml

# 드라이런 모드로 기동 테스트
otelcol --config=config.yaml --dry-run

# Docker를 활용한 로컬 테스트
docker run --rm \
  -v $(pwd)/config.yaml:/etc/otelcol/config.yaml \
  otel/opentelemetry-collector-contrib:0.120.0 \
  validate --config=/etc/otelcol/config.yaml

connectors:
  routing:
    table:
      - statement: route() where attributes["team"] == "platform"
        pipelines: [traces/platform]
      - statement: route() where attributes["team"] == "payments"
        pipelines: [traces/payments]
    default_pipelines: [traces/default]

service:
  pipelines:
    traces/ingress:
      receivers: [otlp]
      processors: [memory_limiter, batch]
      exporters: [routing]
    traces/platform:
      receivers: [routing]
      processors: [batch]
      exporters: [otlp/tempo-platform]
    traces/payments:
      receivers: [routing]
      processors: [batch]
      exporters: [otlp/tempo-payments]
    traces/default:
      receivers: [routing]
      processors: [batch]
      exporters: [otlp/tempo-default]