eBPF 기반 제로 계측 Kubernetes 옵저버빌리티: Cilium Hubble과 Grafana Beyla 실전 가이드

# 현재 노드의 커널 버전 및 BTF 지원 여부 확인
uname -r
# 출력 예: 5.15.0-91-generic

# BTF 지원 확인
ls /sys/kernel/btf/vmlinux
# 파일이 존재하면 BTF 지원 활성화 상태

# eBPF 기능 프로브 (bpftool 사용)
bpftool feature probe kernel | grep -E "map_type|program_type|helper"

# Kubernetes 노드의 커널 버전 일괄 확인
kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kernelVersion}{"\n"}{end}'

# cilium-hubble-values.yaml
# Cilium + Hubble 프로덕션 배포를 위한 Helm values
hubble:
  enabled: true
  relay:
    enabled: true
    replicas: 3 # 고가용성을 위한 Relay 복제본
    resources:
      requests:
        cpu: 100m
        memory: 128Mi
      limits:
        cpu: 500m
        memory: 512Mi
    retryTimeout: 30s
    sortBufferLenMax: 5000
    sortBufferDrainTimeout: 1s
  ui:
    enabled: true
    replicas: 2
    ingress:
      enabled: true
      annotations:
        kubernetes.io/ingress.class: nginx
        cert-manager.io/cluster-issuer: letsencrypt-prod
      hosts:
        - hubble.internal.example.com
      tls:
        - secretName: hubble-ui-tls
          hosts:
            - hubble.internal.example.com
  metrics:
    enableOpenMetrics: true
    enabled:
      - dns
      - drop
      - tcp
      - flow
      - port-distribution
      - icmp
      - httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction
    serviceMonitor:
      enabled: true # Prometheus Operator ServiceMonitor 자동 생성
      interval: 15s
  tls:
    enabled: true # Hubble Server-Relay 간 mTLS 활성화
    auto:
      enabled: true
      method: certmanager
      certManagerIssuerRef:
        group: cert-manager.io
        kind: ClusterIssuer
        name: hubble-issuer

# Cilium Agent 최적화 설정
operator:
  replicas: 2
  resources:
    requests:
      cpu: 100m
      memory: 128Mi

# 모니터 버퍼 크기 (노드당 이벤트 캐시)
bpf:
  monitorAggregation: medium # none/low/medium/high - 집계 수준
  monitorInterval: 5s
  monitorFlags: all
  mapDynamicSizeRatio: 0.0025

# Cilium + Hubble 설치 (Helm)
helm repo add cilium https://helm.cilium.io
helm repo update

helm upgrade --install cilium cilium/cilium \
  --version 1.16.5 \
  --namespace kube-system \
  --values cilium-hubble-values.yaml \
  --wait

# 설치 검증
cilium status --wait
cilium hubble port-forward &

# Hubble CLI로 클러스터 전체 흐름 확인
hubble observe --since 1m --output compact

# 특정 네임스페이스의 HTTP 트래픽만 필터링
hubble observe \
  --namespace production \
  --protocol http \
  --http-status 500-599 \
  --output json | jq '{
    source: .flow.source.labels,
    destination: .flow.destination.labels,
    http: .flow.l7.http
  }'

# 서비스 간 드롭 패킷 분석
hubble observe \
  --verdict DROPPED \
  --namespace production \
  --output table

# 서비스별 HTTP 요청 성공률 (SLI)
(
  sum(rate(hubble_http_requests_total{http_status_code=~"2.."}[5m])) by (destination_workload, destination_namespace)
  /
  sum(rate(hubble_http_requests_total[5m])) by (destination_workload, destination_namespace)
) * 100

# 네트워크 정책에 의해 드롭된 패킷 비율
sum(rate(hubble_drop_total{reason="POLICY_DENIED"}[5m])) by (source_workload, destination_workload)
/ on(source_workload) group_left
sum(rate(hubble_flows_processed_total[5m])) by (source_workload)

# TCP 연결 설정 P99 지연 시간 (서비스별)
histogram_quantile(0.99,
  sum(rate(hubble_tcp_connect_duration_seconds_bucket[5m])) by (le, destination_workload, destination_namespace)
)

# DNS 쿼리 실패율 상위 10개 워크로드
topk(10,
  sum(rate(hubble_dns_responses_total{rcode!="No Error"}[5m])) by (source_workload, source_namespace, qtypes, rcode)
)

# 네임스페이스 간 트래픽 볼륨 매트릭스
sum(rate(hubble_flows_processed_total[5m])) by (source_namespace, destination_namespace)

# beyla-daemonset.yaml
# Grafana Beyla DaemonSet 배포 매니페스트
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: beyla
  namespace: monitoring
  labels:
    app.kubernetes.io/name: beyla
    app.kubernetes.io/component: auto-instrumentation
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: beyla
  template:
    metadata:
      labels:
        app.kubernetes.io/name: beyla
      annotations:
        prometheus.io/scrape: 'true'
        prometheus.io/port: '9090'
        prometheus.io/path: '/metrics'
    spec:
      serviceAccountName: beyla
      hostPID: true # 호스트 PID 네임스페이스 접근 필수
      tolerations:
        - operator: Exists # 모든 노드(마스터 포함)에 배포
      containers:
        - name: beyla
          image: grafana/beyla:1.8.2
          securityContext:
            privileged: true # eBPF 프로그램 로드를 위해 필요
            runAsUser: 0
          ports:
            - containerPort: 9090
              name: metrics
              protocol: TCP
          env:
            - name: BEYLA_OPEN_PORT
              value: '80,443,8080,8443,3000,5000,9090'
            - name: BEYLA_SERVICE_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
            - name: BEYLA_KUBE_METADATA_ENABLE
              value: 'autodetect'
          volumeMounts:
            - name: beyla-config
              mountPath: /config
            - name: sys-kernel-security
              mountPath: /sys/kernel/security
              readOnly: true
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
            limits:
              cpu: 500m
              memory: 512Mi
      volumes:
        - name: beyla-config
          configMap:
            name: beyla-config
        - name: sys-kernel-security
          hostPath:
            path: /sys/kernel/security
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: beyla-config
  namespace: monitoring
data:
  beyla-config.yml: |
    # Beyla 메인 설정 파일
    log_level: info

    # 서비스 자동 탐지 설정
    discovery:
      services:
        - k8s_namespace: "production|staging"
          k8s_pod_labels:
            app.kubernetes.io/part-of: ".*"
        - k8s_namespace: ".*"
          k8s_deployment_name: ".*"

    # 메트릭 내보내기 설정
    otel_metrics_export:
      endpoint: http://mimir-distributor.monitoring:4317
      protocol: grpc
      interval: 15s
      features:
        - application
        - application_process
        - application_service_graph
      histograms:
        - explicit:
            boundaries:
              - 0.005
              - 0.01
              - 0.025
              - 0.05
              - 0.1
              - 0.25
              - 0.5
              - 1
              - 2.5
              - 5
              - 10

    # 트레이스 내보내기 설정
    otel_traces_export:
      endpoint: http://tempo-distributor.monitoring:4317
      protocol: grpc
      sampler:
        name: parentbased_traceidratio
        arg: "0.1"                       # 10% 샘플링

    # Prometheus 엔드포인트 설정
    prometheus_export:
      port: 9090
      path: /metrics
      features:
        - application
        - application_process
        - application_service_graph

    # 네트워크 메트릭 (실험적 기능)
    network:
      enable: true
      cidrs:
        - 10.0.0.0/8
        - 172.16.0.0/12

    # 속성 설정
    attributes:
      kubernetes:
        enable: true
      host_id:
        fetch_timeout: 5s
      select:
        beyla_network_flow_bytes:
          include:
            - k8s.src.namespace
            - k8s.dst.namespace
            - direction
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: beyla
  namespace: monitoring
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: beyla
rules:
  - apiGroups: ['']
    resources: ['pods', 'nodes', 'services', 'replicationcontrollers']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['apps']
    resources: ['deployments', 'replicasets', 'statefulsets', 'daemonsets']
    verbs: ['get', 'list', 'watch']
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: beyla
subjects:
  - kind: ServiceAccount
    name: beyla
    namespace: monitoring
roleRef:
  kind: ClusterRole
  name: beyla
  apiGroup: rbac.authorization.k8s.io

# Beyla DaemonSet 배포
kubectl apply -f beyla-daemonset.yaml

# 배포 상태 확인
kubectl rollout status daemonset/beyla -n monitoring

# Beyla 로그에서 자동 탐지된 서비스 확인
kubectl logs -n monitoring -l app.kubernetes.io/name=beyla --tail=50 | grep "instrumenting"
# 출력 예:
# msg="instrumenting process" pid=12345 service=frontend-deployment
# msg="instrumenting process" pid=12346 service=api-server-deployment
# msg="instrumenting process" pid=12347 service=payment-service-deployment

# Beyla가 생성하는 메트릭 확인
kubectl exec -n monitoring $(kubectl get pod -n monitoring -l app.kubernetes.io/name=beyla -o jsonpath='{.items[0].metadata.name}') \
  -- wget -qO- http://localhost:9090/metrics | head -40

# eBPF 프로그램 로드 상태 확인
kubectl exec -n monitoring $(kubectl get pod -n monitoring -l app.kubernetes.io/name=beyla -o jsonpath='{.items[0].metadata.name}') \
  -- bpftool prog list | grep beyla

# grafana-datasources.yaml
# Grafana 데이터소스 ConfigMap (Hubble + Beyla 통합)
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasources
  namespace: monitoring
  labels:
    grafana_datasource: '1'
data:
  datasources.yaml: |
    apiVersion: 1
    datasources:
      # Mimir - Hubble 네트워크 메트릭 + Beyla APM 메트릭 통합 저장
      - name: Mimir
        type: prometheus
        url: http://mimir-query-frontend.monitoring:8080/prometheus
        access: proxy
        isDefault: true
        jsonData:
          timeInterval: 15s
          exemplarTraceIdDestinations:
            - name: traceID
              datasourceUid: tempo
              urlDisplayLabel: View Trace
          httpMethod: POST
          prometheusType: Mimir

      # Tempo - Beyla 자동 생성 트레이스 저장
      - name: Tempo
        type: tempo
        uid: tempo
        url: http://tempo-query-frontend.monitoring:3200
        access: proxy
        jsonData:
          tracesToMetrics:
            datasourceUid: mimir
            spanStartTimeShift: "-1h"
            spanEndTimeShift: "1h"
            tags:
              - key: service.name
                value: service
              - key: http.method
                value: method
            queries:
              - name: Request Rate
                query: "sum(rate(http_server_request_duration_seconds_count{service=\"${__span.tags.service.name}\"}[5m]))"
              - name: Error Rate
                query: "sum(rate(http_server_request_duration_seconds_count{service=\"${__span.tags.service.name}\",http_status_code=~\"5..\"}[5m]))"
          tracesToLogs:
            datasourceUid: loki
            spanStartTimeShift: "-5m"
            spanEndTimeShift: "5m"
            filterByTraceID: true
            filterBySpanID: false
            tags:
              - key: k8s.namespace.name
                value: namespace
              - key: k8s.pod.name
                value: pod
          serviceMap:
            datasourceUid: mimir
          nodeGraph:
            enabled: true
          search:
            hide: false
          lokiSearch:
            datasourceUid: loki

      # Loki - Hubble 플로우 로그 + 애플리케이션 로그
      - name: Loki
        type: loki
        uid: loki
        url: http://loki-read.monitoring:3100
        access: proxy
        jsonData:
          derivedFields:
            - datasourceUid: tempo
              matcherRegex: "traceID=(\\w+)"
              name: TraceID
              url: "$${__value.raw}"

# 최소 권한 보안 컨텍스트 (privileged 대신)
securityContext:
  privileged: false
  runAsUser: 0
  capabilities:
    add:
      - BPF # eBPF 프로그램 로드
      - PERFMON # perf 이벤트 접근
      - NET_RAW # raw 소켓 접근 (네트워크 관측)
      - SYS_PTRACE # 프로세스 추적 (uprobes)
      - DAC_READ_SEARCH # 파일 시스템 탐색
    drop:
      - ALL