Split View: etcd 백업 & 복원 완전 가이드 — CKA 시험 필수 핸즈온

┌─────────────────────────────────────────────┐
│            Kubernetes Control Plane          │
│                                             │
│  kube-apiserver ──── etcd (all state stored) │
│       │                                     │
│  kube-scheduler    kube-controller-manager   │
└─────────────────────────────────────────────┘

# Check etcd Pod
kubectl get pods -n kube-system | grep etcd

# Example output
# etcd-controlplane   1/1   Running   0   45m

# Find certificate paths from etcd Pod arguments
kubectl describe pod etcd-controlplane -n kube-system | grep -E "cert|key|cacert|data-dir"

# Check etcdctl version
ETCDCTL_API=3 etcdctl version

# Install if missing (Ubuntu)
apt-get install etcd-client

# Or run inside the etcd Pod
kubectl exec -it etcd-controlplane -n kube-system -- etcdctl version

# Set environment variables (to avoid typing them every time)
export ETCDCTL_API=3
export ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/ca.crt
export ETCDCTL_CERT=/etc/kubernetes/pki/etcd/server.crt
export ETCDCTL_KEY=/etc/kubernetes/pki/etcd/server.key
export ETCDCTL_ENDPOINTS=https://127.0.0.1:2379

# Cluster health check
etcdctl endpoint health
# Output: https://127.0.0.1:2379 is healthy: took = 2.345ms

# List members
etcdctl member list --write-out=table

# Check the number of stored keys
etcdctl get / --prefix --keys-only | wc -l

ETCDCTL_API=3 etcdctl snapshot save /opt/etcd-backup.db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

ETCDCTL_API=3 etcdctl snapshot status /opt/etcd-backup.db --write-out=table

+----------+----------+------------+------------+
|   HASH   | REVISION | TOTAL KEYS | TOTAL SIZE |
+----------+----------+------------+------------+
| 3e9a843c |    12450 |       1287 |     5.8 MB |
+----------+----------+------------+------------+

apiVersion: batch/v1
kind: CronJob
metadata:
  name: etcd-backup
  namespace: kube-system
spec:
  schedule: '0 */6 * * *'
  jobTemplate:
    spec:
      template:
        spec:
          hostNetwork: true
          containers:
            - name: backup
              image: bitnami/etcd:3.5
              command:
                - /bin/sh
                - -c
                - |
                  etcdctl snapshot save /backup/etcd-$(date +%Y%m%d-%H%M%S).db \
                    --endpoints=https://127.0.0.1:2379 \
                    --cacert=/etc/kubernetes/pki/etcd/ca.crt \
                    --cert=/etc/kubernetes/pki/etcd/server.crt \
                    --key=/etc/kubernetes/pki/etcd/server.key
                  find /backup -name "etcd-*.db" -mtime +7 -delete
              env:
                - name: ETCDCTL_API
                  value: '3'
              volumeMounts:
                - name: etcd-certs
                  mountPath: /etc/kubernetes/pki/etcd
                  readOnly: true
                - name: backup
                  mountPath: /backup
          restartPolicy: OnFailure
          nodeSelector:
            node-role.kubernetes.io/control-plane: ''
          tolerations:
            - effect: NoSchedule
              operator: Exists
          volumes:
            - name: etcd-certs
              hostPath:
                path: /etc/kubernetes/pki/etcd
            - name: backup
              hostPath:
                path: /opt/etcd-backups

ETCDCTL_API=3 etcdctl snapshot restore /opt/etcd-backup.db \
  --data-dir=/var/lib/etcd-from-backup \
  --initial-cluster=controlplane=https://127.0.0.1:2380 \
  --initial-cluster-token=etcd-cluster-1 \
  --initial-advertise-peer-urls=https://127.0.0.1:2380

vi /etc/kubernetes/manifests/etcd.yaml

# Before
volumes:
- hostPath:
    path: /var/lib/etcd
    type: DirectoryOrCreate
  name: etcd-data

# After — change to the new path!
volumes:
- hostPath:
    path: /var/lib/etcd-from-backup
    type: DirectoryOrCreate
  name: etcd-data

# Since it's a Static Pod, kubelet will restart it automatically
watch "kubectl get pods -n kube-system | grep etcd"

# Or check directly with crictl
crictl ps | grep etcd

kubectl get nodes
kubectl get pods --all-namespaces

ETCDCTL_API=3 etcdctl endpoint health \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# Common in CKA exams!
# Solution 1: Use the full path
/usr/local/bin/etcdctl version

# Solution 2: Run inside the etcd Pod
kubectl exec -it etcd-controlplane -n kube-system -- sh

# Solution 3: SSH and sudo
sudo -i
export PATH=$PATH:/usr/local/bin

# Check logs
crictl logs $(crictl ps -a | grep etcd | awk '{print $1}')

# Common cause 1: Permission issues
chown -R etcd:etcd /var/lib/etcd-from-backup

# Common cause 2: data-dir path mismatch
# Verify that etcd.yaml's volumes.hostPath.path matches the actual restore path

# Common cause 3: initial-cluster-token mismatch

# Cause: etcd hasn't fully started yet
# Solution: Wait 30-60 seconds and retry

# 1. Backup
ETCDCTL_API=3 etcdctl snapshot save /opt/backup.db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# 2. Restore
ETCDCTL_API=3 etcdctl snapshot restore /opt/backup.db \
  --data-dir=/var/lib/etcd-from-backup

# 3. Change data-dir in etcd.yaml (one-liner)
sed -i 's|/var/lib/etcd|/var/lib/etcd-from-backup|g' \
  /etc/kubernetes/manifests/etcd.yaml

# 4. Verify status
ETCDCTL_API=3 etcdctl snapshot status /opt/backup.db -w table

# Find the etcd endpoint from kube-apiserver
cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep etcd-servers

# SSH to the etcd node and perform backup/restore
# Certificate paths may differ (/etc/etcd/pki/)
# After restore: systemctl restart etcd