GitOps와 ArgoCD 기반 지속적 배포 파이프라인 구축 가이드

Push 모델 (전통적 CI/CD)
========================
[개발자] --> [Git Push] --> [CI 서버] --kubectl apply--> [K8s 클러스터]
                                  |
                          (클러스터 자격증명 보유)

Pull 모델 (GitOps)
========================
[개발자] --> [Git Push] --> [Git 저장소]
                                  |
                          [ArgoCD Agent] <--polling/webhook-- [Git 저장소]
                                  |
                          --reconcile--> [K8s 클러스터]

gitops-manifests/
├── apps/
│   ├── frontend/
│   │   ├── base/
│   │   │   ├── deployment.yaml
│   │   │   ├── service.yaml
│   │   │   └── kustomization.yaml
│   │   └── overlays/
│   │       ├── dev/
│   │       │   └── kustomization.yaml
│   │       ├── staging/
│   │       │   └── kustomization.yaml
│   │       └── prod/
│   │           └── kustomization.yaml
│   └── backend/
│       ├── base/
│       └── overlays/
├── infrastructure/
│   ├── cert-manager/
│   ├── ingress-nginx/
│   └── monitoring/
└── clusters/
    ├── dev-cluster/
    ├── staging-cluster/
    └── prod-cluster/

# ArgoCD Helm 저장소 추가
helm repo add argo https://argoproj.github.io/argo-helm
helm repo update

# 네임스페이스 생성
kubectl create namespace argocd

# HA 모드로 ArgoCD 설치 (프로덕션 권장)
helm install argocd argo/argo-cd \
  --namespace argocd \
  --set redis-ha.enabled=true \
  --set controller.replicas=2 \
  --set server.replicas=2 \
  --set repoServer.replicas=2 \
  --set applicationSet.replicas=2 \
  --set server.service.type=LoadBalancer \
  --wait

# 초기 admin 패스워드 확인
kubectl -n argocd get secret argocd-initial-admin-secret \
  -o jsonpath='{.data.password}' | base64 -d && echo

# ArgoCD CLI 로그인
argocd login <ARGOCD_SERVER> --username admin --password <PASSWORD>

# admin 패스워드 변경 (필수)
argocd account update-password

# argocd-values.yaml
global:
  image:
    tag: 'v3.3.3'

server:
  replicas: 2
  ingress:
    enabled: true
    ingressClassName: nginx
    hosts:
      - argocd.example.com
    tls:
      - secretName: argocd-tls
        hosts:
          - argocd.example.com

  config:
    # OIDC 연동 (SSO)
    url: https://argocd.example.com
    oidc.config: |
      name: Keycloak
      issuer: https://keycloak.example.com/realms/devops
      clientID: argocd
      clientSecret: $oidc.keycloak.clientSecret
      requestedScopes:
        - openid
        - profile
        - email
        - groups

  rbacConfig:
    policy.csv: |
      p, role:developer, applications, get, */*, allow
      p, role:developer, applications, sync, */dev-*, allow
      p, role:admin, applications, *, */*, allow
      g, dev-team, role:developer
      g, platform-team, role:admin
    policy.default: role:readonly

controller:
  replicas: 2
  resources:
    requests:
      cpu: 500m
      memory: 1Gi
    limits:
      cpu: 2000m
      memory: 4Gi

repoServer:
  replicas: 2
  resources:
    requests:
      cpu: 250m
      memory: 512Mi
    limits:
      cpu: 1000m
      memory: 2Gi

redis-ha:
  enabled: true

applicationSet:
  replicas: 2

# 커스텀 values로 설치/업그레이드
helm upgrade --install argocd argo/argo-cd \
  --namespace argocd \
  -f argocd-values.yaml \
  --wait

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: frontend-app
  namespace: argocd
  finalizers:
    - resources-finalizer.argocd.argoproj.io
spec:
  project: default
  source:
    repoURL: https://github.com/my-org/gitops-manifests.git
    targetRevision: main
    path: apps/frontend/overlays/prod
  destination:
    server: https://kubernetes.default.svc
    namespace: frontend
  syncPolicy:
    automated:
      prune: true # Git에서 삭제된 리소스를 클러스터에서도 삭제
      selfHeal: true # 수동 변경을 자동으로 원복
      allowEmpty: false # 빈 리소스 목록 배포 방지
    syncOptions:
      - CreateNamespace=true
      - PrunePropagationPolicy=foreground
      - PruneLast=true
    retry:
      limit: 3
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m0s
  ignoreDifferences:
    - group: apps
      kind: Deployment
      jsonPointers:
        - /spec/replicas # HPA가 관리하는 필드는 무시

# 데이터베이스 마이그레이션을 먼저 실행
apiVersion: batch/v1
kind: Job
metadata:
  name: db-migration
  annotations:
    argocd.argoproj.io/hook: PreSync
    argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
  template:
    spec:
      containers:
        - name: migrate
          image: my-org/db-migrator:v1.2.0
          command: ['./migrate', 'up']
      restartPolicy: Never
---
# Wave 0: ConfigMap, Secret
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
  annotations:
    argocd.argoproj.io/sync-wave: '0'
data:
  DATABASE_HOST: 'postgres.db.svc.cluster.local'
---
# Wave 1: Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend
  annotations:
    argocd.argoproj.io/sync-wave: '1'
spec:
  replicas: 3
  selector:
    matchLabels:
      app: backend
  template:
    metadata:
      labels:
        app: backend
    spec:
      containers:
        - name: backend
          image: my-org/backend:v2.1.0
---
# Wave 2: Service, Ingress
apiVersion: v1
kind: Service
metadata:
  name: backend-svc
  annotations:
    argocd.argoproj.io/sync-wave: '2'
spec:
  selector:
    app: backend
  ports:
    - port: 80
      targetPort: 8080

# apps/backend/base/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: backend
spec:
  replicas: 1
  selector:
    matchLabels:
      app: backend
  template:
    metadata:
      labels:
        app: backend
    spec:
      containers:
        - name: backend
          image: my-org/backend:latest
          ports:
            - containerPort: 8080
          resources:
            requests:
              cpu: 100m
              memory: 128Mi
            limits:
              cpu: 500m
              memory: 512Mi
          livenessProbe:
            httpGet:
              path: /healthz
              port: 8080
            initialDelaySeconds: 10
          readinessProbe:
            httpGet:
              path: /ready
              port: 8080
            initialDelaySeconds: 5

# apps/backend/base/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
  - deployment.yaml
  - service.yaml
  - hpa.yaml
commonLabels:
  app.kubernetes.io/managed-by: argocd

# apps/backend/overlays/prod/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: backend-prod
resources:
  - ../../base
patches:
  - target:
      kind: Deployment
      name: backend
    patch: |
      - op: replace
        path: /spec/replicas
        value: 5
      - op: replace
        path: /spec/template/spec/containers/0/resources/requests/cpu
        value: "500m"
      - op: replace
        path: /spec/template/spec/containers/0/resources/requests/memory
        value: "512Mi"
      - op: replace
        path: /spec/template/spec/containers/0/resources/limits/cpu
        value: "2000m"
      - op: replace
        path: /spec/template/spec/containers/0/resources/limits/memory
        value: "2Gi"
images:
  - name: my-org/backend
    newTag: v2.1.0

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: prometheus-stack
  namespace: argocd
spec:
  project: infrastructure
  source:
    repoURL: https://prometheus-community.github.io/helm-charts
    chart: kube-prometheus-stack
    targetRevision: 65.1.0
    helm:
      releaseName: prometheus
      valuesObject:
        grafana:
          enabled: true
          ingress:
            enabled: true
            hosts:
              - grafana.example.com
        prometheus:
          prometheusSpec:
            retention: 30d
            storageSpec:
              volumeClaimTemplate:
                spec:
                  accessModes:
                    - ReadWriteOnce
                  resources:
                    requests:
                      storage: 100Gi
        alertmanager:
          alertmanagerSpec:
            replicas: 2
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true
      - ServerSideApply=true

# 외부 클러스터 등록
argocd cluster add dev-cluster-context --name dev-cluster \
  --label env=dev --label region=ap-northeast-2

argocd cluster add staging-cluster-context --name staging-cluster \
  --label env=staging --label region=ap-northeast-2

argocd cluster add prod-cluster-context --name prod-cluster \
  --label env=prod --label region=ap-northeast-2

# 등록된 클러스터 확인
argocd cluster list

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: backend-appset
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - list:
        elements:
          - cluster: dev-cluster
            url: https://dev-k8s.example.com
            namespace: backend-dev
            replicas: '1'
          - cluster: staging-cluster
            url: https://staging-k8s.example.com
            namespace: backend-staging
            replicas: '2'
          - cluster: prod-cluster
            url: https://prod-k8s.example.com
            namespace: backend-prod
            replicas: '5'
  template:
    metadata:
      name: 'backend-{{.cluster}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/my-org/gitops-manifests.git
        targetRevision: main
        path: 'apps/backend/overlays/{{.cluster}}'
      destination:
        server: '{{.url}}'
        namespace: '{{.namespace}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: infra-monitoring
  namespace: argocd
spec:
  goTemplate: true
  generators:
    - clusters:
        selector:
          matchLabels:
            env: prod
        values:
          revision: main
  template:
    metadata:
      name: 'monitoring-{{.name}}'
    spec:
      project: infrastructure
      source:
        repoURL: https://github.com/my-org/gitops-manifests.git
        targetRevision: '{{.values.revision}}'
        path: infrastructure/monitoring
      destination:
        server: '{{.server}}'
        namespace: monitoring
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

# .github/workflows/ci-cd.yaml
name: CI - Build and Update Manifests

on:
  push:
    branches: [main]
    paths:
      - 'src/**'
      - 'Dockerfile'

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: my-org/backend
  GITOPS_REPO: my-org/gitops-manifests

jobs:
  build-and-push:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write
    outputs:
      image_tag: ${{ steps.meta.outputs.version }}

    steps:
      - name: Checkout source code
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/my-org/backend
          tags: |
            type=sha,prefix=
            type=ref,event=branch

      - name: Build and push Docker image
        uses: docker/build-push-action@v6
        with:
          context: .
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

      - name: Run Trivy vulnerability scan
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: 'ghcr.io/my-org/backend:${{ steps.meta.outputs.version }}'
          format: 'sarif'
          output: 'trivy-results.sarif'

  update-manifests:
    needs: build-and-push
    runs-on: ubuntu-latest
    permissions:
      contents: write

    steps:
      - name: Checkout GitOps manifests repo
        uses: actions/checkout@v4
        with:
          repository: my-org/gitops-manifests
          token: ${{ secrets.GITOPS_PAT }}
          path: gitops-manifests

      - name: Update image tag in Kustomize
        working-directory: gitops-manifests
        run: |
          cd apps/backend/overlays/staging
          kustomize edit set image \
            my-org/backend=ghcr.io/my-org/backend:${{ needs.build-and-push.outputs.image_tag }}

      - name: Commit and push changes
        working-directory: gitops-manifests
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add .
          git diff --staged --quiet || \
            git commit -m "chore: update backend image to ${{ needs.build-and-push.outputs.image_tag }}"
          git push

apiVersion: v1
kind: ConfigMap
metadata:
  name: argocd-notifications-cm
  namespace: argocd
data:
  service.slack: |
    token: $slack-token
  trigger.on-sync-succeeded: |
    - when: app.status.operationState.phase in ['Succeeded']
      send: [app-sync-succeeded]
  trigger.on-sync-failed: |
    - when: app.status.operationState.phase in ['Error', 'Failed']
      send: [app-sync-failed]
  template.app-sync-succeeded: |
    slack:
      attachments: |
        [{
          "color": "#18be52",
          "title": "{{.app.metadata.name}} 배포 성공",
          "text": "Revision: {{.app.status.sync.revision}}\nEnvironment: {{.app.spec.destination.namespace}}"
        }]
  template.app-sync-failed: |
    slack:
      attachments: |
        [{
          "color": "#E96D76",
          "title": "{{.app.metadata.name}} 배포 실패",
          "text": "Revision: {{.app.status.sync.revision}}\nMessage: {{.app.status.operationState.message}}"
        }]

# External Secrets Operator 예시
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: backend-secrets
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: aws-secrets-manager
    kind: ClusterSecretStore
  target:
    name: backend-secrets
  data:
    - secretKey: DATABASE_PASSWORD
      remoteRef:
        key: prod/backend/database
        property: password
    - secretKey: API_KEY
      remoteRef:
        key: prod/backend/api-keys
        property: main-key

# Repo Server 로그 확인
kubectl logs -n argocd deployment/argocd-repo-server --tail=100

# 매니페스트를 로컬에서 미리 렌더링하여 검증
kustomize build apps/backend/overlays/prod | kubectl apply --dry-run=server -f -

# Helm 차트의 경우
helm template my-release ./chart -f values-prod.yaml | kubectl apply --dry-run=server -f -

# Application Controller 로그 확인
kubectl logs -n argocd deployment/argocd-application-controller --tail=100

# 캐시 초기화
argocd app get frontend-app --hard-refresh

# Application 상태 강제 새로고침
argocd app get frontend-app --refresh

# 저장소 연결 상태 확인
argocd repo list

# SSH 키 문제 시 재등록
argocd repo add git@github.com:my-org/gitops-manifests.git \
  --ssh-private-key-path ~/.ssh/argocd_deploy_key

# HTTPS 인증 문제 시
argocd repo add https://github.com/my-org/gitops-manifests.git \
  --username git --password <PERSONAL_ACCESS_TOKEN>

# 네임스페이스 리소스 쿼터 확인
kubectl describe resourcequota -n backend-prod

# Application 이벤트 확인
kubectl describe application frontend-app -n argocd

metadata:
  annotations:
    argocd.argoproj.io/sync-options: Prune=false

# 문제가 된 커밋 확인
git log --oneline -10

# 특정 커밋 되돌리기
git revert <COMMIT_SHA> --no-edit
git push origin main

# ArgoCD가 자동 Sync하거나 수동 Sync 트리거
argocd app sync frontend-app

# 배포 히스토리 확인
argocd app history frontend-app

# 특정 리비전으로 롤백
argocd app rollback frontend-app <REVISION_NUMBER>

# 주의: 이 방법은 Git과 클러스터 상태가 불일치하게 된다
# 반드시 이후에 Git에서도 해당 변경을 반영해야 한다

# GitOps 매니페스트 저장소에서 이미지 태그를 이전 버전으로 변경
cd gitops-manifests/apps/backend/overlays/prod
kustomize edit set image my-org/backend=ghcr.io/my-org/backend:<PREVIOUS_TAG>

git add .
git commit -m "hotfix: rollback backend to <PREVIOUS_TAG> due to OOM issue"
git push origin main

# ArgoCD Sync 대기 또는 수동 트리거
argocd app sync backend-prod-cluster --prune=false