Kubernetes v1.33 운영 실전 플레이북: 업그레이드와 보안 자동화

#!/bin/bash
# pre-upgrade-check.sh - 업그레이드 전 클러스터 상태 점검

echo "=== 1. 현재 버전 확인 ==="
kubectl version --short 2>/dev/null || kubectl version

echo ""
echo "=== 2. 노드 상태 확인 ==="
kubectl get nodes -o wide
NOT_READY=$(kubectl get nodes --no-headers | grep -v " Ready " | wc -l)
if [ "$NOT_READY" -gt 0 ]; then
  echo "WARNING: $NOT_READY 개 노드가 NotReady 상태입니다. 업그레이드 전 해결 필요."
fi

echo ""
echo "=== 3. Deprecated API 사용 확인 ==="
# v1.33에서 제거된 API를 사용하는 리소스 탐색
kubectl get --raw /metrics | grep -E 'apiserver_requested_deprecated_apis'

echo ""
echo "=== 4. PodDisruptionBudget 확인 ==="
kubectl get pdb --all-namespaces -o wide
echo "PDB가 없는 critical 워크로드가 있으면 업그레이드 중 downtime 발생 가능"

echo ""
echo "=== 5. etcd 상태 확인 ==="
kubectl -n kube-system exec -it etcd-$(hostname) -- \
  etcdctl endpoint health --cluster \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

echo ""
echo "=== 6. 버전 skew 확인 ==="
echo "Control Plane:"
kubectl get nodes -l node-role.kubernetes.io/control-plane -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}'
echo ""
echo "Workers:"
kubectl get nodes -l '!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[*].status.nodeInfo.kubeletVersion}'

# Endpoints API를 사용하는 리소스 찾기
kubectl get endpoints --all-namespaces -o name | wc -l

# EndpointSlice로 읽는 방법
kubectl get endpointslices --all-namespaces -o wide

# Before: Endpoints API
from kubernetes import client

v1 = client.CoreV1Api()
endpoints = v1.read_namespaced_endpoints("my-service", "default")
for subset in endpoints.subsets:
    for address in subset.addresses:
        print(address.ip)

# After: EndpointSlice API
discovery_v1 = client.DiscoveryV1Api()
slices = discovery_v1.list_namespaced_endpoint_slice(
    "default",
    label_selector="kubernetes.io/service-name=my-service"
)
for ep_slice in slices.items:
    for endpoint in ep_slice.endpoints:
        for address in endpoint.addresses:
            print(address)

# etcd 스냅샷 생성
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-pre-v133-$(date +%Y%m%d%H%M).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# 스냅샷 검증
ETCDCTL_API=3 etcdctl snapshot status /backup/etcd-pre-v133-*.db --write-out=table

# 1. kubeadm 업그레이드
sudo apt-get update
sudo apt-get install -y kubeadm=1.33.0-1.1

# 2. 업그레이드 계획 확인
sudo kubeadm upgrade plan v1.33.0

# 3. 업그레이드 실행 (첫 번째 control plane 노드)
sudo kubeadm upgrade apply v1.33.0

# 4. kubelet과 kubectl 업그레이드
sudo apt-get install -y kubelet=1.33.0-1.1 kubectl=1.33.0-1.1
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# 5. 추가 control plane 노드 (두 번째, 세 번째)
sudo kubeadm upgrade node

# API Server 버전 확인
kubectl version

# 컴포넌트 상태 확인
kubectl get componentstatuses 2>/dev/null || \
  kubectl get --raw /readyz?verbose

# etcd 클러스터 상태
kubectl -n kube-system exec -it etcd-cp-01 -- \
  etcdctl member list --write-out=table \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

#!/bin/bash
# worker-upgrade.sh <node-name>
NODE=$1

echo "=== $NODE 업그레이드 시작 ==="

# 1. 노드 cordon (새 Pod 스케줄링 차단)
kubectl cordon "$NODE"

# 2. 노드 drain (기존 Pod 퇴거)
kubectl drain "$NODE" \
  --ignore-daemonsets \
  --delete-emptydir-data \
  --timeout=300s \
  --grace-period=120

# 3. 노드에서 패키지 업그레이드 (SSH)
ssh "$NODE" "sudo apt-get update && \
  sudo apt-get install -y kubeadm=1.33.0-1.1 && \
  sudo kubeadm upgrade node && \
  sudo apt-get install -y kubelet=1.33.0-1.1 && \
  sudo systemctl daemon-reload && \
  sudo systemctl restart kubelet"

# 4. 노드 uncordon
kubectl uncordon "$NODE"

# 5. 노드 상태 확인
kubectl get node "$NODE" -o wide
echo "=== $NODE 업그레이드 완료 ==="

# 현재 리소스 확인
kubectl get pod my-app-xyz -o jsonpath='{.spec.containers[0].resources}'

# 리사이즈 요청 (resize subresource 사용)
kubectl patch pod my-app-xyz --subresource=resize --type=merge -p '{
  "spec": {
    "containers": [{
      "name": "app",
      "resources": {
        "requests": {"cpu": "500m", "memory": "512Mi"},
        "limits": {"cpu": "1000m", "memory": "1Gi"}
      }
    }]
  }
}'

# 리사이즈 상태 확인
kubectl get pod my-app-xyz -o jsonpath='{.status.resize}'
# "Proposed" → "InProgress" → "Completed" 순서로 전환

apiVersion: v1
kind: Pod
metadata:
  name: resizable-app
spec:
  containers:
    - name: app
      image: nginx:1.27
      resources:
        requests:
          cpu: '250m'
          memory: '256Mi'
        limits:
          cpu: '500m'
          memory: '512Mi'
      resizePolicy:
        - resourceName: cpu
          restartPolicy: NotRequired # CPU는 재시작 없이 조정
        - resourceName: memory
          restartPolicy: RestartContainer # Memory는 컨테이너 재시작 필요

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: my-app-vpa
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-app
  updatePolicy:
    updateMode: 'InPlaceOrRecreate' # v1.33에서 추가된 모드
  resourcePolicy:
    containerPolicies:
      - containerName: app
        minAllowed:
          cpu: '100m'
          memory: '128Mi'
        maxAllowed:
          cpu: '2000m'
          memory: '4Gi'

apiVersion: v1
kind: Pod
metadata:
  name: secure-app
spec:
  hostUsers: false # User Namespaces 활성화
  containers:
    - name: app
      image: myapp:v2.0
      securityContext:
        runAsUser: 0 # 컨테이너 내부에서는 root
        # 하지만 hostUsers: false이므로 호스트에서는 고유 UID로 매핑

# Namespace에 Pod Security 레벨 적용
apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-user-namespaces
spec:
  validationFailureAction: Enforce
  rules:
    - name: require-hostusers-false
      match:
        any:
          - resources:
              kinds: ['Pod']
              namespaces: ['production', 'staging']
      validate:
        message: '프로덕션/스테이징 Pod는 hostUsers: false를 설정해야 합니다.'
        pattern:
          spec:
            hostUsers: false

# 현재 kube-proxy 모드 확인
kubectl -n kube-system get configmap kube-proxy -o yaml | grep mode

# nftables 지원 확인 (노드에서)
nft --version
# nftables v1.0.6 이상 필요

apiVersion: v1
kind: ConfigMap
metadata:
  name: kube-proxy
  namespace: kube-system
data:
  config.conf: |
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    kind: KubeProxyConfiguration
    mode: nftables     # iptables → nftables
    nftables:
      masqueradeAll: false
      syncPeriod: 30s
      minSyncPeriod: 1s

kubectl -n kube-system rollout restart daemonset kube-proxy
kubectl -n kube-system rollout status daemonset kube-proxy

#!/bin/bash
# post-upgrade-verify.sh

ERRORS=0

echo "=== Post-Upgrade Verification ==="

# 1. 모든 노드 Ready 확인
echo "[1/7] Node Status..."
NOT_READY=$(kubectl get nodes --no-headers | grep -v " Ready " | wc -l)
if [ "$NOT_READY" -gt 0 ]; then
  echo "FAIL: $NOT_READY nodes not ready"
  ERRORS=$((ERRORS + 1))
else
  echo "PASS: All nodes ready"
fi

# 2. 모든 시스템 Pod Running 확인
echo "[2/7] System Pods..."
FAILING=$(kubectl -n kube-system get pods --no-headers | grep -v "Running\|Completed" | wc -l)
if [ "$FAILING" -gt 0 ]; then
  echo "FAIL: $FAILING system pods not running"
  kubectl -n kube-system get pods | grep -v "Running\|Completed"
  ERRORS=$((ERRORS + 1))
else
  echo "PASS: All system pods healthy"
fi

# 3. CoreDNS 동작 확인
echo "[3/7] DNS Resolution..."
kubectl run dns-test --image=busybox:1.36 --restart=Never --rm -it -- \
  nslookup kubernetes.default.svc.cluster.local 2>/dev/null
if [ $? -eq 0 ]; then
  echo "PASS: DNS working"
else
  echo "FAIL: DNS resolution failed"
  ERRORS=$((ERRORS + 1))
fi

# 4. API Server 버전 확인
echo "[4/7] API Server Version..."
SERVER_VERSION=$(kubectl version -o json | jq -r '.serverVersion.gitVersion')
echo "Server version: $SERVER_VERSION"
if [[ "$SERVER_VERSION" == *"1.33"* ]]; then
  echo "PASS: Correct version"
else
  echo "FAIL: Unexpected version"
  ERRORS=$((ERRORS + 1))
fi

# 5. 핵심 워크로드 상태 확인
echo "[5/7] Critical Workloads..."
for ns in production staging; do
  UNAVAIL=$(kubectl -n "$ns" get deployments --no-headers 2>/dev/null | \
    awk '{if ($2 != $4) print $1}')
  if [ -n "$UNAVAIL" ]; then
    echo "FAIL: Unavailable deployments in $ns: $UNAVAIL"
    ERRORS=$((ERRORS + 1))
  fi
done
echo "PASS: Critical workloads healthy"

# 6. Ingress/Service 동작 확인
echo "[6/7] Service Connectivity..."
kubectl get svc -A --no-headers | awk '{print $1"/"$2}' | head -5 | while read svc; do
  echo "  Checking $svc..."
done
echo "PASS: Services enumerated"

# 7. 메트릭 파이프라인 확인
echo "[7/7] Metrics Pipeline..."
kubectl top nodes 2>/dev/null
if [ $? -eq 0 ]; then
  echo "PASS: Metrics working"
else
  echo "WARN: Metrics pipeline may need time to recover"
fi

echo ""
echo "=== Verification Complete: $ERRORS errors ==="
exit $ERRORS

# Sonobuoy로 Conformance Test 실행
sonobuoy run --mode=certified-conformance --wait

# 결과 확인
sonobuoy status
sonobuoy results $(sonobuoy retrieve)

# Worker 노드를 이전 버전으로 다운그레이드
ssh worker-node-01 "sudo apt-get install -y \
  kubeadm=1.32.x-1.1 kubelet=1.32.x-1.1 && \
  sudo kubeadm upgrade node && \
  sudo systemctl daemon-reload && \
  sudo systemctl restart kubelet"

# etcd 스냅샷에서 복원
ETCDCTL_API=3 etcdctl snapshot restore /backup/etcd-pre-v133-*.db \
  --data-dir=/var/lib/etcd-restored

# etcd 데이터 디렉토리 교체 (모든 control plane 노드에서)
sudo systemctl stop etcd
sudo mv /var/lib/etcd /var/lib/etcd-broken
sudo mv /var/lib/etcd-restored /var/lib/etcd
sudo systemctl start etcd