Cilium Service Mesh: eBPF 기반 사이드카 없는 서비스 메시 구축과 운영 가이드

# 커널 버전 확인 (5.10+ 필수, 6.1+ 권장)
uname -r

# eBPF 지원 확인
cat /boot/config-$(uname -r) | grep CONFIG_BPF
# CONFIG_BPF=y
# CONFIG_BPF_SYSCALL=y
# CONFIG_BPF_JIT=y

# Cilium CLI 설치
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
curl -L --fail --remote-name-all \
  https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz
sudo tar xzvf cilium-linux-amd64.tar.gz -C /usr/local/bin

# Helm repo 추가
helm repo add cilium https://helm.cilium.io/
helm repo update

# Cilium Service Mesh 설치 (kube-proxy 대체 + 서비스 메시 활성화)
helm install cilium cilium/cilium --version 1.19.0 \
  --namespace kube-system \
  --set kubeProxyReplacement=true \
  --set k8sServiceHost="API_SERVER_IP" \
  --set k8sServicePort=6443 \
  --set envoyConfig.enabled=true \
  --set ingressController.enabled=true \
  --set ingressController.loadbalancerMode=shared \
  --set hubble.enabled=true \
  --set hubble.relay.enabled=true \
  --set hubble.ui.enabled=true \
  --set encryption.enabled=true \
  --set encryption.type=wireguard \
  --set authentication.mutual.spire.enabled=true \
  --set authentication.mutual.spire.install.enabled=true

# Cilium 전체 상태 확인
cilium status --wait

# 예상 출력:
#     /¯¯\
#  /¯¯\__/¯¯\    Cilium:             OK
#  \__/¯¯\__/    Operator:           OK
#  /¯¯\__/¯¯\    Envoy DaemonSet:    OK
#  \__/¯¯\__/    Hubble Relay:       OK
#     \__/        ClusterMesh:        disabled
#                 SPIRE Server:       OK
#                 SPIRE Agent:        OK

# 서비스 메시 기능 확인
cilium config view | grep -E "envoy|mesh|mutual"

# 연결 테스트 (서비스 메시 포함)
cilium connectivity test

# mtls-authentication-policy.yaml
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: require-mutual-auth
  namespace: production
spec:
  endpointSelector:
    matchLabels:
      app: payment-service
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: order-service
      authentication:
        mode: required # mTLS 인증 필수
      toPorts:
        - ports:
            - port: '8080'
              protocol: TCP

# cluster-wide-mtls.yaml
apiVersion: cilium.io/v2
kind: CiliumClusterwideNetworkPolicy
metadata:
  name: enforce-mtls-cluster-wide
spec:
  endpointSelector:
    matchExpressions:
      - key: io.kubernetes.pod.namespace
        operator: NotIn
        values:
          - kube-system
  ingress:
    - fromEndpoints:
        - {}
      authentication:
        mode: required

# SPIRE에 등록된 워크로드 확인
kubectl exec -n kube-system spire-server-0 -- \
  /opt/spire/bin/spire-server entry show

# mTLS 인증 상태 확인
cilium-dbg identity list | grep -i auth
hubble observe --namespace production --verdict DROPPED -o json | \
  jq 'select(.drop_reason_desc == "Authentication required")'

# l7-traffic-split.yaml
apiVersion: cilium.io/v2
kind: CiliumEnvoyConfig
metadata:
  name: api-traffic-split
  namespace: production
spec:
  services:
    - name: api-service
      namespace: production
  backendServices:
    - name: api-service-v1
      namespace: production
    - name: api-service-v2
      namespace: production
  resources:
    - '@type': type.googleapis.com/envoy.config.listener.v3.Listener
      name: api-listener
      filter_chains:
        - filters:
            - name: envoy.filters.network.http_connection_manager
              typed_config:
                '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
                stat_prefix: api-traffic
                route_config:
                  name: api-routes
                  virtual_hosts:
                    - name: api-host
                      domains: ['*']
                      routes:
                        - match:
                            prefix: '/'
                            headers:
                              - name: 'x-canary'
                                exact_match: 'true'
                          route:
                            cluster: 'production/api-service-v2'
                        - match:
                            prefix: '/'
                          route:
                            weighted_clusters:
                              clusters:
                                - name: 'production/api-service-v1'
                                  weight: 90
                                - name: 'production/api-service-v2'
                                  weight: 10
                http_filters:
                  - name: envoy.filters.http.router
                    typed_config:
                      '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router

# l4-lb-policy.yaml
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: backend-l4-policy
  namespace: production
spec:
  endpointSelector:
    matchLabels:
      app: backend-service
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: api-gateway
      toPorts:
        - ports:
            - port: '8080'
              protocol: TCP
            - port: '9090'
              protocol: TCP
  egress:
    - toEndpoints:
        - matchLabels:
            app: database
      toPorts:
        - ports:
            - port: '5432'
              protocol: TCP
    - toEndpoints:
        - matchLabels:
            app: cache
      toPorts:
        - ports:
            - port: '6379'
              protocol: TCP

# l7-rate-limiting.yaml
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: api-l7-with-ratelimit
  namespace: production
spec:
  endpointSelector:
    matchLabels:
      app: public-api
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: frontend
      toPorts:
        - ports:
            - port: '8080'
              protocol: TCP
          rules:
            http:
              - method: GET
                path: '/api/v1/products.*'
              - method: GET
                path: '/api/v1/categories.*'
              - method: POST
                path: '/api/v1/orders'
                headers:
                  - 'Content-Type: application/json'

# fortio를 이용한 성능 측정
kubectl run fortio-client --rm -it --image=fortio/fortio -- \
  load -c 50 -qps 1000 -t 60s -json - \
  http://api-service.production:8080/api/v1/health

# 결과 분석 포인트:
# - P50, P90, P99 레이턴시
# - 최대 QPS (Queries Per Second)
# - 에러율
# - CPU/메모리 사용량 (kubectl top pods로 별도 측정)

# Hubble 메트릭으로 L7 지연 시간 확인
hubble observe --namespace production --protocol http -o json | \
  jq '.l7.latency_ns / 1000000'

# 인증 실패 트래픽 확인
hubble observe --namespace production --verdict DROPPED -o compact

# SPIRE 에이전트 상태 확인
kubectl get pods -n kube-system -l app=spire-agent
kubectl logs -n kube-system -l app=spire-agent --tail=50

# 특정 워크로드의 SVID(인증서) 상태 확인
kubectl exec -n kube-system spire-server-0 -- \
  /opt/spire/bin/spire-server entry show -selector k8s:ns:production

# Cilium 엔드포인트의 인증 상태 확인
kubectl exec -n kube-system ds/cilium -- \
  cilium-dbg endpoint list -o json | jq '.[].status.policy.realized.auth'

# Envoy DaemonSet 상태 확인
kubectl get ds -n kube-system cilium-envoy
kubectl describe ds -n kube-system cilium-envoy

# 특정 노드의 Envoy 로그 확인
kubectl logs -n kube-system -l k8s-app=cilium-envoy --tail=100

# Cilium Agent와 Envoy 간 연결 확인
kubectl exec -n kube-system ds/cilium -- \
  cilium-dbg status --verbose | grep -A5 "Envoy"

# CiliumEnvoyConfig 상태 확인
kubectl get cec -n production -o yaml

# Cilium Agent 롤링 재시작 상태 확인
kubectl rollout status ds/cilium -n kube-system

# eBPF 맵 동기화 상태 확인
kubectl exec -n kube-system ds/cilium -- \
  cilium-dbg bpf endpoint list

# 엔드포인트 복구 대기
kubectl exec -n kube-system ds/cilium -- \
  cilium-dbg endpoint list | grep -v ready

# Hubble Relay 상태 확인
kubectl get pods -n kube-system -l k8s-app=hubble-relay
kubectl logs -n kube-system -l k8s-app=hubble-relay --tail=50

# Hubble 연결 테스트
cilium hubble port-forward &
hubble status

# Cilium Agent의 Hubble 모니터링 상태
kubectl exec -n kube-system ds/cilium -- \
  cilium-dbg monitor --type drop --type trace

# 네임스페이스별 Istio 사이드카 주입 비활성화
kubectl label namespace staging istio-injection-
kubectl rollout restart deployment -n staging

# Cilium mTLS 정책 적용
kubectl apply -f cilium-mtls-policy-staging.yaml

# 전환 후 확인
hubble observe --namespace staging --protocol http

# Prometheus에서 수집해야 할 핵심 메트릭

# 1. Cilium Agent 상태
# cilium_agent_api_process_time_seconds - API 처리 시간
# cilium_agent_bootstrap_seconds - Agent 시작 시간
# cilium_bpf_map_ops_total - BPF 맵 작업 수

# 2. 서비스 메시 관련
# cilium_proxy_upstream_reply_seconds - L7 프록시 업스트림 응답 시간
# cilium_proxy_redirects - L7 프록시로 리다이렉트된 연결 수
# cilium_auth_map_entries - mTLS 인증 맵 엔트리 수

# 3. Hubble 관측성
# hubble_flows_processed_total - 처리된 흐름 수
# hubble_tcp_flags_total - TCP 플래그별 카운트

# Grafana 대시보드 임포트
# Cilium 공식 대시보드: https://grafana.com/grafana/dashboards/16611