Split View: ArgoCD GitOps 멀티 클러스터 배포 전략과 운영 자동화 가이드

┌─────────────────────────────────────────────────┐
│                  ArgoCD Server                   │
│  ┌──────────┐  ┌───────────┐  ┌──────────────┐  │
│  │ API Server│  │ Repo Server│  │ App Controller│  │
│  └──────────┘  └───────────┘  └──────────────┘  │
│  ┌──────────────────┐  ┌─────────────────────┐  │
│  │ ApplicationSet    │  │ Notification         │  │
│  │ Controller        │  │ Controller           │  │
│  └──────────────────┘  └─────────────────────┘  │
└─────────────────────────────────────────────────┘
         │                    │
    ┌────┴────┐          ┌───┴────┐
    │ Git Repo │          │ K8s    │
    │ (Source) │          │Clusters│
    └─────────┘          └────────┘

# Check kubeconfig contexts
kubectl config get-contexts

# Add cluster (ArgoCD auto-creates ServiceAccount)
argocd cluster add staging-cluster \
  --name staging \
  --label env=staging \
  --label region=ap-northeast-2

argocd cluster add production-apne2 \
  --name production-apne2 \
  --label env=production \
  --label region=ap-northeast-2

argocd cluster add production-use1 \
  --name production-use1 \
  --label env=production \
  --label region=us-east-1

apiVersion: v1
kind: Secret
metadata:
  name: production-apne2-cluster
  namespace: argocd
  labels:
    argocd.argoproj.io/secret-type: cluster
    env: production
    region: ap-northeast-2
type: Opaque
stringData:
  name: production-apne2
  server: 'https://kubernetes.production-apne2.internal:6443'
  config: |
    {
      "bearerToken": "<service-account-token>",
      "tlsClientConfig": {
        "insecure": false,
        "caData": "<base64-encoded-ca-cert>"
      }
    }

apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: database-credentials
  namespace: production
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: aws-secrets-manager
    kind: ClusterSecretStore
  target:
    name: database-credentials
    creationPolicy: Owner
  data:
    - secretKey: DB_HOST
      remoteRef:
        key: production/database
        property: host
    - secretKey: DB_PASSWORD
      remoteRef:
        key: production/database
        property: password

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: prometheus-stack
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - clusters:
        selector:
          matchLabels:
            env: production
        values:
          helmReleaseName: kube-prometheus
  template:
    metadata:
      name: 'prometheus-{{.name}}'
    spec:
      project: infrastructure
      source:
        repoURL: 'https://github.com/org/k8s-infrastructure.git'
        targetRevision: main
        path: 'clusters/{{.metadata.labels.region}}/prometheus'
        helm:
          releaseName: '{{.values.helmReleaseName}}'
          valueFiles:
            - 'values.yaml'
            - 'values-{{.metadata.labels.env}}.yaml'
      destination:
        server: '{{.server}}'
        namespace: monitoring
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true
          - ServerSideApply=true

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: microservices
  namespace: argocd
spec:
  goTemplate: true
  generators:
    - matrix:
        generators:
          - clusters:
              selector:
                matchLabels:
                  env: production
          - list:
              elements:
                - service: order-service
                  replicas: '3'
                  memory: '512Mi'
                - service: payment-service
                  replicas: '2'
                  memory: '1Gi'
                - service: notification-service
                  replicas: '2'
                  memory: '256Mi'
  template:
    metadata:
      name: '{{.service}}-{{.name}}'
      annotations:
        notifications.argoproj.io/subscribe.on-sync-failed.slack: deploy-alerts
    spec:
      project: applications
      source:
        repoURL: 'https://github.com/org/k8s-apps.git'
        targetRevision: main
        path: 'apps/{{.service}}'
        helm:
          parameters:
            - name: replicaCount
              value: '{{.replicas}}'
            - name: resources.requests.memory
              value: '{{.memory}}'
      destination:
        server: '{{.server}}'
        namespace: '{{.service}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

k8s-gitops/
├── root-apps/
│   ├── infrastructure.yaml      # Infrastructure App of Apps
│   ├── platform.yaml            # Platform App of Apps
│   └── applications.yaml        # Business app App of Apps
├── infrastructure/
│   ├── cert-manager.yaml
│   ├── external-secrets.yaml
│   ├── ingress-nginx.yaml
│   └── prometheus-stack.yaml
├── platform/
│   ├── istio.yaml
│   ├── kafka.yaml
│   └── redis.yaml
└── applications/
    ├── order-service.yaml
    ├── payment-service.yaml
    └── notification-service.yaml

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: root-infrastructure
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: '-2'
spec:
  project: default
  source:
    repoURL: 'https://github.com/org/k8s-gitops.git'
    targetRevision: main
    path: infrastructure
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: root-platform
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: '-1'
spec:
  project: default
  source:
    repoURL: 'https://github.com/org/k8s-gitops.git'
    targetRevision: main
    path: platform
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
---
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: root-applications
  namespace: argocd
  annotations:
    argocd.argoproj.io/sync-wave: '0'
spec:
  project: default
  source:
    repoURL: 'https://github.com/org/k8s-gitops.git'
    targetRevision: main
    path: applications
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true

# Wave -1: Create Namespace and RBAC first
apiVersion: v1
kind: Namespace
metadata:
  name: payment-service
  annotations:
    argocd.argoproj.io/sync-wave: '-1'
---
# Wave 0: ConfigMap and Secret
apiVersion: v1
kind: ConfigMap
metadata:
  name: payment-config
  namespace: payment-service
  annotations:
    argocd.argoproj.io/sync-wave: '0'
data:
  DATABASE_URL: 'postgresql://db.internal:5432/payments'
  KAFKA_BROKERS: 'kafka-0.kafka:9092'
---
# Wave 1: Main Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-service
  namespace: payment-service
  annotations:
    argocd.argoproj.io/sync-wave: '1'
spec:
  replicas: 3
  selector:
    matchLabels:
      app: payment-service
  template:
    metadata:
      labels:
        app: payment-service
    spec:
      containers:
        - name: payment
          image: registry.internal/payment-service:v2.3.1
          envFrom:
            - configMapRef:
                name: payment-config
---
# Wave 2: Service exposure
apiVersion: v1
kind: Service
metadata:
  name: payment-service
  namespace: payment-service
  annotations:
    argocd.argoproj.io/sync-wave: '2'
spec:
  selector:
    app: payment-service
  ports:
    - port: 8080
      targetPort: 8080

# PreSync Hook: Database migration before deployment
apiVersion: batch/v1
kind: Job
metadata:
  name: db-migration
  namespace: payment-service
  annotations:
    argocd.argoproj.io/hook: PreSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
    argocd.argoproj.io/sync-wave: '-1'
spec:
  template:
    spec:
      containers:
        - name: migrate
          image: registry.internal/payment-service:v2.3.1
          command: ['python', 'manage.py', 'migrate']
          envFrom:
            - secretRef:
                name: database-credentials
      restartPolicy: Never
  backoffLimit: 3
---
# PostSync Hook: Smoke test after deployment
apiVersion: batch/v1
kind: Job
metadata:
  name: smoke-test
  namespace: payment-service
  annotations:
    argocd.argoproj.io/hook: PostSync
    argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
  template:
    spec:
      containers:
        - name: smoke-test
          image: registry.internal/smoke-tester:latest
          command: ['./run-tests.sh']
          env:
            - name: TARGET_URL
              value: 'http://payment-service.payment-service:8080'
            - name: TEST_SUITE
              value: 'smoke'
      restartPolicy: Never
  backoffLimit: 1

# Check Application status
argocd app get payment-service --show-operation

# Force terminate Sync operation
argocd app terminate-op payment-service

# Retry Sync
argocd app sync payment-service --retry-limit 3 --retry-backoff-duration 10s

# Check Repo Server logs
kubectl logs -n argocd deployment/argocd-repo-server --tail=100

# Test Git connection
argocd repo list

# Re-register repository
argocd repo add https://github.com/org/k8s-gitops.git \
  --username deploy-bot \
  --password "$GITHUB_TOKEN"

# Check cluster status
argocd cluster list

# All apps for a specific cluster
argocd app list --dest-server https://kubernetes.production-apne2.internal:6443

# Renew cluster credentials
argocd cluster set production-apne2 \
  --server https://kubernetes.production-apne2.internal:6443 \
  --bearer-token "new-token-here"

# Check Application history
argocd app history payment-service

# Rollback to a specific revision
argocd app rollback payment-service 42

# Or Git revert followed by auto Sync
git revert HEAD
git push origin main

apiVersion: v1
kind: ConfigMap
metadata:
  name: argocd-notifications-cm
  namespace: argocd
data:
  trigger.on-sync-failed: |
    - description: Application sync has failed
      send:
        - slack-deploy-alert
      when: app.status.operationState.phase in ['Error', 'Failed']
  trigger.on-health-degraded: |
    - description: Application health has degraded
      send:
        - slack-deploy-alert
      when: app.status.health.status == 'Degraded'
  template.slack-deploy-alert: |
    message: |
      Application *{{.app.metadata.name}}* sync {{.app.status.operationState.phase}}.
      Revision: {{.app.status.sync.revision}}
      Cluster: {{.app.spec.destination.server}}
    slack:
      attachments: |
        [{
          "color": "#E96D76",
          "title": "{{.app.metadata.name}}",
          "title_link": "https://argocd.internal/applications/{{.app.metadata.name}}",
          "fields": [
            {"title": "Sync Status", "value": "{{.app.status.sync.status}}", "short": true},
            {"title": "Health", "value": "{{.app.status.health.status}}", "short": true}
          ]
        }]
  service.slack: |
    token: $slack-token
    username: ArgoCD
    icon: ":argocd:"

# policy.csv in argocd-rbac-cm ConfigMap
p, role:platform-team, applications, *, infrastructure/*, allow
p, role:platform-team, clusters, get, *, allow

p, role:dev-team-order, applications, get, applications/order-*, allow
p, role:dev-team-order, applications, sync, applications/order-*, allow
p, role:dev-team-order, applications, action/*, applications/order-*, allow

p, role:dev-team-payment, applications, get, applications/payment-*, allow
p, role:dev-team-payment, applications, sync, applications/payment-*, allow

g, platform-admins, role:platform-team
g, order-team, role:dev-team-order
g, payment-team, role:dev-team-payment