Kyverno Production Operations: HA, Monitoring, Troubleshooting

helm install kyverno kyverno/kyverno -n kyverno --create-namespace \
  --set admissionController.replicas=3 \
  --set backgroundController.replicas=2 \
  --set reportsController.replicas=2

admissionController:
  replicas: 3
  container:
    resources:
      limits:
        cpu: '2'
        memory: 2Gi
      requests:
        cpu: 500m
        memory: 512Mi
  topologySpreadConstraints:
    - maxSkew: 1
      topologyKey: kubernetes.io/hostname
      whenUnsatisfiable: DoNotSchedule

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: kyverno-admission-controller
  namespace: kyverno
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app.kubernetes.io/component: admission-controller

kubectl get policyreport -n production
kubectl get clusterpolicyreport

helm install policy-reporter policy-reporter/policy-reporter -n kyverno \
  --set ui.enabled=true --set kyvernoPlugin.enabled=true

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: kyverno-alerts
spec:
  groups:
    - name: kyverno
      rules:
        - alert: KyvernoWebhookHighLatency
          expr: histogram_quantile(0.99, rate(kyverno_admission_review_duration_seconds_bucket[5m])) > 10
          for: 5m
        - alert: KyvernoHighFailureRate
          expr: rate(kyverno_admission_requests_total{success="false"}[5m]) > 0.1
          for: 5m

apiVersion: kyverno.io/v1
kind: ClusterPolicy
spec:
  validationFailureAction: Audit
  background: true # Enable background scanning (default: true)
  rules:
    - name: check-labels
      validate:
        pattern:
          metadata:
            labels:
              app.kubernetes.io/name: '?*'

apiVersion: kyverno.io/v2beta1
kind: PolicyException
metadata:
  name: allow-privileged-monitoring
  namespace: monitoring
spec:
  exceptions:
    - policyName: disallow-privileged
      ruleNames:
        - check-privileged
  match:
    any:
      - resources:
          kinds: ['Pod']
          namespaces: ['monitoring']
          names: ['prometheus-*']

# Check Kyverno logs
kubectl logs -n kyverno -l app.kubernetes.io/component=admission-controller --tail=100

# Check webhook status
kubectl get mutatingwebhookconfigurations
kubectl get validatingwebhookconfigurations

# Check policy status
kubectl get clusterpolicy -o wide
kubectl describe clusterpolicy require-labels

# Check admission latency
kubectl exec -n kyverno deploy/kyverno-admission-controller -- wget -qO- localhost:8000/metrics | grep admission_review_duration