Kyverno 프로덕션 운영: HA, 모니터링, 트러블슈팅

helm install kyverno kyverno/kyverno -n kyverno --create-namespace \
  --set admissionController.replicas=3 \
  --set backgroundController.replicas=2 \
  --set cleanupController.replicas=2 \
  --set reportsController.replicas=2

# values.yaml
admissionController:
  replicas: 3
  container:
    resources:
      limits:
        cpu: '2'
        memory: 2Gi
      requests:
        cpu: 500m
        memory: 512Mi
  topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: kubernetes.io/hostname
      whenUnsatisfiable: DoNotSchedule
      labelSelector:
        matchLabels:
          app.kubernetes.io/component: admission-controller

backgroundController:
  replicas: 2
  resources:
    limits:
      cpu: '1'
      memory: 1Gi
    requests:
      cpu: 200m
      memory: 256Mi

# 프로덕션 권장: 핵심 정책은 Fail, 비핵심은 Ignore
# Helm values
admissionController:
  failurePolicy: Fail # Webhook 장애 시 요청 거부 (보안 우선)

# 또는 개별 정책에서 설정
apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: critical-policy
spec:
  failurePolicy: Fail
  validationFailureAction: Enforce

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: kyverno-admission-controller
  namespace: kyverno
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app.kubernetes.io/component: admission-controller

# 네임스페이스별 정책 보고서 조회
kubectl get policyreport -n production

# 클러스터 레벨 보고서
kubectl get clusterpolicyreport

# 상세 보고서 조회
kubectl get policyreport -n production -o yaml

apiVersion: wgpolicyk8s.io/v1alpha2
kind: PolicyReport
metadata:
  name: polr-ns-production
  namespace: production
results:
  - message: "Validation rule 'require-labels' passed"
    policy: require-labels
    rule: check-app-label
    result: pass
    source: kyverno
    category: Best Practices
    timestamp:
      nanos: 0
      seconds: 1710950400
    resources:
      - apiVersion: apps/v1
        kind: Deployment
        name: web-app
        namespace: production
summary:
  pass: 45
  fail: 3
  warn: 2
  error: 0
  skip: 0

# Policy Reporter 설치 (선택적 UI)
helm repo add policy-reporter https://kyverno.github.io/policy-reporter
helm install policy-reporter policy-reporter/policy-reporter -n kyverno \
  --set ui.enabled=true \
  --set kyvernoPlugin.enabled=true

# 메트릭 엔드포인트
# http://kyverno-svc.kyverno:8000/metrics

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: kyverno
  namespace: kyverno
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: kyverno
  endpoints:
    - port: metrics
      interval: 30s
      path: /metrics

# 알림 규칙 예시 (PrometheusRule)
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: kyverno-alerts
  namespace: kyverno
spec:
  groups:
    - name: kyverno
      rules:
        - alert: KyvernoWebhookHighLatency
          expr: histogram_quantile(0.99, rate(kyverno_admission_review_duration_seconds_bucket[5m])) > 10
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: 'Kyverno webhook latency is high'
        - alert: KyvernoHighFailureRate
          expr: rate(kyverno_admission_requests_total{success="false"}[5m]) > 0.1
          for: 5m
          labels:
            severity: critical

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: check-existing-resources
spec:
  validationFailureAction: Audit # 기존 리소스는 Audit 모드 권장
  background: true # 백그라운드 스캐닝 활성화 (기본값: true)
  rules:
    - name: check-labels
      match:
        any:
          - resources:
              kinds:
                - Deployment
      validate:
        message: 'Required label missing'
        pattern:
          metadata:
            labels:
              app.kubernetes.io/name: '?*'

# Helm values로 스캐닝 주기 조정
helm upgrade kyverno kyverno/kyverno -n kyverno \
  --set backgroundController.backgroundScanInterval=1h

apiVersion: kyverno.io/v2beta1
kind: PolicyException
metadata:
  name: allow-privileged-monitoring
  namespace: monitoring
spec:
  exceptions:
    - policyName: disallow-privileged
      ruleNames:
        - check-privileged
  match:
    any:
      - resources:
          kinds:
            - Pod
          namespaces:
            - monitoring
          names:
            - 'prometheus-*'
            - 'node-exporter-*'

# PolicyException 기능 활성화 (Helm)
helm upgrade kyverno kyverno/kyverno -n kyverno \
  --set features.policyExceptions.enabled=true \
  --set features.policyExceptions.namespace="*"

# Webhook 타임아웃 확인
kubectl get mutatingwebhookconfigurations kyverno-resource-mutating-webhook-cfg -o yaml | grep timeout

# 타임아웃 증가
helm upgrade kyverno kyverno/kyverno -n kyverno \
  --set admissionController.webhookTimeout=30

# Kyverno Pod 로그 확인
kubectl logs -n kyverno -l app.kubernetes.io/component=admission-controller --tail=100

# Webhook 상태 확인
kubectl get mutatingwebhookconfigurations
kubectl get validatingwebhookconfigurations

# Kyverno 서비스 엔드포인트 확인
kubectl get endpoints -n kyverno kyverno-svc

# 정책 상태 확인
kubectl get clusterpolicy -o wide
kubectl describe clusterpolicy require-labels

# 정책 보고서에서 실패 원인 확인
kubectl get policyreport -A -o jsonpath='{range .items[*].results[?(@.result=="fail")]}{.policy}: {.message}{"\n"}{end}'

# dry-run으로 정책 테스트
kubectl apply -f test-resource.yaml --dry-run=server

# Admission 리뷰 레이턴시 확인
kubectl exec -n kyverno deploy/kyverno-admission-controller -- wget -qO- localhost:8000/metrics | grep admission_review_duration

# 리소스 사용량 확인
kubectl top pods -n kyverno

# Background Controller 스캔 상태
kubectl logs -n kyverno -l app.kubernetes.io/component=background-controller --tail=50

# 현재 버전 확인
helm list -n kyverno

# 업그레이드 (Helm)
helm repo update
helm upgrade kyverno kyverno/kyverno -n kyverno \
  --reuse-values \
  --version X.Y.Z

# CRD 수동 업그레이드 (필요 시)
kubectl apply -f https://raw.githubusercontent.com/kyverno/kyverno/release-X.Y/config/crds/kyverno.io_clusterpolicies.yaml