Kubernetes Velero 백업·복구 완전 가이드: 클러스터 DR부터 멀티 리전 마이그레이션까지

# Velero CLI 설치 (최신 안정 버전)
curl -fsSL -o velero-v1.15.0-linux-amd64.tar.gz \
  https://github.com/vmware-tanzu/velero/releases/download/v1.15.0/velero-v1.15.0-linux-amd64.tar.gz
tar -xvf velero-v1.15.0-linux-amd64.tar.gz
sudo mv velero-v1.15.0-linux-amd64/velero /usr/local/bin/

# 버전 확인
velero version --client-only

# CSI 스냅샷을 사용할 경우 CRD 설치 확인
kubectl get crd | grep volumesnapshot
# volumesnapshotclasses.snapshot.storage.k8s.io
# volumesnapshotcontents.snapshot.storage.k8s.io
# volumesnapshots.snapshot.storage.k8s.io

# S3 버킷 생성
aws s3api create-bucket \
  --bucket velero-backup-prod \
  --region ap-northeast-2 \
  --create-bucket-configuration LocationConstraint=ap-northeast-2

# IAM 자격 증명 파일 생성
cat > credentials-velero <<EOF
[default]
aws_access_key_id=<YOUR_ACCESS_KEY>
aws_secret_access_key=<YOUR_SECRET_KEY>
EOF

# Velero 설치 (CSI 스냅샷 + Node Agent 활성화)
velero install \
  --provider aws \
  --plugins velero/velero-plugin-for-aws:v1.11.0 \
  --bucket velero-backup-prod \
  --backup-location-config region=ap-northeast-2 \
  --snapshot-location-config region=ap-northeast-2 \
  --secret-file ./credentials-velero \
  --use-node-agent \
  --features EnableCSI \
  --wait

# 설치 확인
kubectl get pods -n velero
# NAME                      READY   STATUS    RESTARTS   AGE
# node-agent-xxxxx          1/1     Running   0          30s
# node-agent-yyyyy          1/1     Running   0          30s
# velero-xxxxxxxxx-zzzzz    1/1     Running   0          30s

# values-velero.yaml
configuration:
  backupStorageLocation:
    - name: default
      provider: aws
      bucket: velero-backup-prod
      config:
        region: ap-northeast-2
  volumeSnapshotLocation:
    - name: default
      provider: aws
      config:
        region: ap-northeast-2
  features: EnableCSI
  defaultVolumesToFsBackup: false

credentials:
  useSecret: true
  secretContents:
    cloud: |
      [default]
      aws_access_key_id=<YOUR_ACCESS_KEY>
      aws_secret_access_key=<YOUR_SECRET_KEY>

initContainers:
  - name: velero-plugin-for-aws
    image: velero/velero-plugin-for-aws:v1.11.0
    volumeMounts:
      - mountPath: /target
        name: plugins
  - name: velero-plugin-for-csi
    image: velero/velero-plugin-for-csi:v0.8.0
    volumeMounts:
      - mountPath: /target
        name: plugins

deployNodeAgent: true

nodeAgent:
  resources:
    requests:
      cpu: 500m
      memory: 512Mi
    limits:
      cpu: '2'
      memory: 2Gi

resources:
  requests:
    cpu: 500m
    memory: 256Mi
  limits:
    cpu: '1'
    memory: 1Gi

schedules:
  daily-production:
    disabled: false
    schedule: '0 2 * * *'
    useOwnerReferencesInBackup: false
    template:
      ttl: '168h'
      includedNamespaces:
        - production
        - staging
      snapshotVolumes: true
      storageLocation: default
      volumeSnapshotLocations:
        - default

# Helm으로 설치
helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
helm repo update
helm install velero vmware-tanzu/velero \
  --namespace velero \
  --create-namespace \
  -f values-velero.yaml

# Tier 1: 미션 크리티컬 워크로드 - 매 시간 백업
apiVersion: velero.io/v1
kind: Schedule
metadata:
  name: tier1-hourly-backup
  namespace: velero
spec:
  schedule: '0 * * * *'
  useOwnerReferencesInBackup: false
  template:
    ttl: 72h0m0s
    includedNamespaces:
      - payment
      - order-service
    snapshotVolumes: true
    storageLocation: default
    volumeSnapshotLocations:
      - default
    defaultVolumesToFsBackup: false
    metadata:
      labels:
        backup-tier: 'tier1'
        environment: 'production'
---
# Tier 3: 일반 워크로드 - 일일 백업
apiVersion: velero.io/v1
kind: Schedule
metadata:
  name: tier3-daily-backup
  namespace: velero
spec:
  schedule: '0 2 * * *'
  useOwnerReferencesInBackup: false
  template:
    ttl: 720h0m0s
    includedNamespaces:
      - monitoring
      - logging
      - internal-tools
    snapshotVolumes: false
    defaultVolumesToFsBackup: true
    storageLocation: default
    metadata:
      labels:
        backup-tier: 'tier3'
        environment: 'production'

# 시간별 백업 (24시간 보존)
velero schedule create hourly-backup \
  --schedule="0 * * * *" \
  --ttl 24h0m0s \
  --include-namespaces production \
  --snapshot-volumes

# 일별 백업 (7일 보존)
velero schedule create daily-backup \
  --schedule="0 3 * * *" \
  --ttl 168h0m0s \
  --include-namespaces production,staging \
  --snapshot-volumes

# 주별 백업 (30일 보존)
velero schedule create weekly-backup \
  --schedule="0 4 * * 0" \
  --ttl 720h0m0s \
  --snapshot-volumes

# 월별 백업 (365일 보존)
velero schedule create monthly-backup \
  --schedule="0 5 1 * *" \
  --ttl 8760h0m0s \
  --snapshot-volumes

# CSI 스냅샷 CRD 설치
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.2.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.2.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.2.0/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml

# 스냅샷 컨트롤러 설치
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.2.0/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/v8.2.0/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml

# AWS EBS CSI 드라이버용 VolumeSnapshotClass
apiVersion: snapshot.storage.k8s.io/v1
kind: VolumeSnapshotClass
metadata:
  name: ebs-csi-snapclass
  labels:
    velero.io/csi-volumesnapshot-class: 'true'
driver: ebs.csi.aws.com
deletionPolicy: Retain
parameters:
  tagSpecification_1: 'velero-backup=true'

apiVersion: velero.io/v1
kind: Backup
metadata:
  name: production-with-datamover
  namespace: velero
spec:
  includedNamespaces:
    - production
  snapshotMoveData: true
  storageLocation: default
  datamover: velero
  snapshotVolumes: true

# [소스 클러스터] 마이그레이션 백업 생성
velero backup create migration-app-v2 \
  --include-namespaces app-v2 \
  --snapshot-volumes \
  --snapshot-move-data \
  --wait

# 백업 상태 확인
velero backup describe migration-app-v2 --details

# [타겟 클러스터] BSL 동기화 (Velero가 동일 S3를 바라보고 있어야 함)
# 기본적으로 1분 간격 동기화, 즉시 동기화는 아래 명령으로 실행
kubectl -n velero patch backupstoragelocation default \
  --type merge \
  --patch '{"spec":{"accessMode":"ReadOnly"}}'

# 잠시 후 ReadWrite로 복귀
kubectl -n velero patch backupstoragelocation default \
  --type merge \
  --patch '{"spec":{"accessMode":"ReadWrite"}}'

# 백업 목록 확인
velero backup get

# [타겟 클러스터] 복원 수행
velero restore create migration-restore \
  --from-backup migration-app-v2 \
  --namespace-mappings app-v2:app-production \
  --wait

# 복원 결과 확인
velero restore describe migration-restore --details
kubectl get pods -n app-production

# 네임스페이스 이름 변경 복원
velero restore create --from-backup my-backup \
  --namespace-mappings old-namespace:new-namespace

# 특정 리소스 유형만 복원
velero restore create --from-backup my-backup \
  --include-resources deployments,services,configmaps,secrets

# 특정 리소스 유형 제외 복원
velero restore create --from-backup my-backup \
  --exclude-resources storageclasses,persistentvolumes

# 레이블 셀렉터로 특정 리소스만 복원
velero restore create --from-backup my-backup \
  --selector app=frontend

# Primary BSL - 서울 리전
apiVersion: velero.io/v1
kind: BackupStorageLocation
metadata:
  name: primary-seoul
  namespace: velero
spec:
  provider: aws
  objectStorage:
    bucket: velero-backup-ap-northeast-2
    prefix: cluster-prod
  config:
    region: ap-northeast-2
  accessMode: ReadWrite
  default: true
---
# Secondary BSL - 도쿄 리전 (DR용)
apiVersion: velero.io/v1
kind: BackupStorageLocation
metadata:
  name: secondary-tokyo
  namespace: velero
spec:
  provider: aws
  objectStorage:
    bucket: velero-backup-ap-northeast-1
    prefix: cluster-prod-dr
  config:
    region: ap-northeast-1
  accessMode: ReadWrite
  credential:
    name: velero-dr-credentials
    key: cloud

# 두 BSL 모두에 백업 생성
velero backup create dr-backup-$(date +%Y%m%d) \
  --include-namespaces production \
  --snapshot-volumes \
  --snapshot-move-data \
  --storage-location primary-seoul

# DR 사이트에도 복제 백업
velero backup create dr-backup-$(date +%Y%m%d)-replica \
  --include-namespaces production \
  --snapshot-move-data \
  --storage-location secondary-tokyo

# 실수로 삭제된 네임스페이스 확인
kubectl get ns production
# Error from server (NotFound): namespaces "production" not found

# 최신 백업 확인
velero backup get --selector backup-tier=tier1 | head -5

# 네임스페이스 복원
velero restore create ns-recovery-$(date +%s) \
  --from-backup tier1-hourly-backup-20260308020000 \
  --include-namespaces production \
  --wait

# 복원 상태 확인
velero restore describe ns-recovery-$(date +%s) --details

# 잘못된 Deployment만 이전 상태로 복원
velero restore create deployment-rollback \
  --from-backup daily-backup-20260307 \
  --include-namespaces production \
  --include-resources deployments \
  --selector app=api-server \
  --existing-resource-policy update \
  --wait

apiVersion: velero.io/v1
kind: Backup
metadata:
  name: db-consistent-backup
  namespace: velero
spec:
  includedNamespaces:
    - database
  snapshotVolumes: true
  hooks:
    resources:
      - name: postgresql-hook
        includedNamespaces:
          - database
        labelSelector:
          matchLabels:
            app: postgresql
        pre:
          - exec:
              container: postgresql
              command:
                - /bin/bash
                - -c
                - 'pg_dump -U postgres -d myapp > /var/lib/postgresql/backup/pre_backup.sql && sync'
              onError: Fail
              timeout: 120s
        post:
          - exec:
              container: postgresql
              command:
                - /bin/bash
                - -c
                - 'rm -f /var/lib/postgresql/backup/pre_backup.sql'
              onError: Continue
              timeout: 30s

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: velero
  namespace: velero
  labels:
    app.kubernetes.io/name: velero
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: velero
  namespaceSelector:
    matchNames:
      - velero
  endpoints:
    - port: monitoring
      interval: 30s

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: velero-alerts
  namespace: velero
spec:
  groups:
    - name: velero.rules
      rules:
        - alert: VeleroBackupFailure
          expr: increase(velero_backup_failure_total[1h]) > 0
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: 'Velero 백업 실패 감지'
            description: '최근 1시간 내 Velero 백업이 실패했습니다. 즉시 확인이 필요합니다.'
        - alert: VeleroBackupNotRunning
          expr: time() - velero_backup_last_successful_timestamp{schedule!=""} > 86400
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: 'Velero 스케줄 백업이 24시간 이상 실행되지 않음'
            description: '스케줄 {{ $labels.schedule }}의 마지막 성공 백업이 24시간을 초과했습니다.'
        - alert: VeleroBackupPartialFailure
          expr: increase(velero_backup_partial_failure_total[6h]) > 0
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: 'Velero 백업 부분 실패 감지'
            description: '일부 리소스가 백업에 포함되지 않았습니다. 볼륨 스냅샷 또는 훅 실행 상태를 점검하십시오.'

# 상세 원인 확인
velero backup describe my-backup --details

# 로그에서 구체적 오류 확인
velero backup logs my-backup | grep -i "error\|warning"

# 복원된 PVC 상태 확인
kubectl get pvc -n restored-namespace

# StorageClass 확인
kubectl get sc

# ConfigMap으로 StorageClass 매핑
kubectl -n velero create configmap change-storage-class-config \
  --from-literal=old-storage-class=new-storage-class

# 매핑을 적용한 복원
velero restore create --from-backup my-backup \
  --include-namespaces production

# BSL 상태 확인
velero backup-location get
# NAME      PROVIDER   BUCKET/PREFIX              PHASE         LAST VALIDATED
# default   aws        velero-backup-prod/        Unavailable   2026-03-08 00:00:00

# 자격 증명 확인
kubectl -n velero get secret cloud-credentials -o jsonpath='{.data.cloud}' | base64 -d

# BSL 접근 수동 테스트
kubectl -n velero exec deploy/velero -- \
  aws s3 ls s3://velero-backup-prod/ --region ap-northeast-2

# 버전 확인
velero version
# Client:
#   Version: v1.15.0
# Server:
#   Version: v1.14.1

# 서버와 동일한 CLI 버전으로 재설치하거나 서버를 업그레이드합니다.