ArgoCD GitOps Complete Guide: Declarative Kubernetes Deployment with ApplicationSet, Sync Waves, and Hooks

# Create namespace and install ArgoCD
kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

# Or install via Helm (recommended for production)
helm repo add argo https://argoproj.github.io/argo-helm
helm repo update

helm install argocd argo/argo-cd \
  --namespace argocd \
  --create-namespace \
  --set server.service.type=LoadBalancer \
  --set configs.params."server\.insecure"=true \
  --set controller.replicas=2 \
  --set repoServer.replicas=2 \
  --set redis-ha.enabled=true

# Retrieve initial admin password
kubectl -n argocd get secret argocd-initial-admin-secret \
  -o jsonpath='{.data.password}' | base64 -d

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: my-application
  namespace: argocd
  # Finalizer: delete cluster resources when Application is deleted
  finalizers:
    - resources-finalizer.argocd.argoproj.io
spec:
  project: production
  source:
    repoURL: https://github.com/myorg/k8s-manifests.git
    targetRevision: main
    path: apps/my-app/overlays/production
    # Kustomize options
    kustomize:
      namePrefix: prod-
      commonLabels:
        env: production
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune: true # Delete cluster resources removed from Git
      selfHeal: true # Auto-revert manual changes
      allowEmpty: false # Prevent deploying empty manifests
    syncOptions:
      - CreateNamespace=true
      - PrunePropagationPolicy=foreground
      - PruneLast=true
      - ServerSideApply=true
    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m
  ignoreDifferences:
    - group: apps
      kind: Deployment
      jsonPointers:
        - /spec/replicas # Ignore since HPA manages this

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: prometheus-stack
  namespace: argocd
spec:
  project: monitoring
  source:
    repoURL: https://prometheus-community.github.io/helm-charts
    chart: kube-prometheus-stack
    targetRevision: 65.1.0
    helm:
      releaseName: prometheus
      valuesObject:
        grafana:
          enabled: true
          adminPassword: vault:secret/grafana#password
        prometheus:
          prometheusSpec:
            retention: 30d
            storageSpec:
              volumeClaimTemplate:
                spec:
                  storageClassName: gp3
                  resources:
                    requests:
                      storage: 100Gi
  destination:
    server: https://kubernetes.default.svc
    namespace: monitoring

apiVersion: argoproj.io/v1alpha1
kind: AppProject
metadata:
  name: production
  namespace: argocd
spec:
  description: 'Production environment project'
  # Allowed source repositories
  sourceRepos:
    - 'https://github.com/myorg/k8s-manifests.git'
    - 'https://github.com/myorg/helm-charts.git'
  # Allowed deployment targets
  destinations:
    - namespace: 'production'
      server: 'https://kubernetes.default.svc'
    - namespace: 'production-*'
      server: 'https://kubernetes.default.svc'
  # Allowed cluster resources
  clusterResourceWhitelist:
    - group: ''
      kind: Namespace
    - group: 'rbac.authorization.k8s.io'
      kind: ClusterRole
    - group: 'rbac.authorization.k8s.io'
      kind: ClusterRoleBinding
  # Blocked namespace resources
  namespaceResourceBlacklist:
    - group: ''
      kind: ResourceQuota
    - group: ''
      kind: LimitRange
  # RBAC role definitions
  roles:
    - name: deployer
      description: 'Can sync and manage applications'
      policies:
        - p, proj:production:deployer, applications, get, production/*, allow
        - p, proj:production:deployer, applications, sync, production/*, allow
        - p, proj:production:deployer, applications, action/*, production/*, allow
      groups:
        - platform-team
    - name: viewer
      description: 'Read-only access'
      policies:
        - p, proj:production:viewer, applications, get, production/*, allow
      groups:
        - dev-team

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: app-of-apps
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - git:
        repoURL: https://github.com/myorg/k8s-manifests.git
        revision: main
        directories:
          - path: 'apps/*/overlays/production'
          - path: 'apps/deprecated-*'
            exclude: true
  template:
    metadata:
      name: '{{.path.basename}}'
      namespace: argocd
      labels:
        app.kubernetes.io/managed-by: applicationset
    spec:
      project: production
      source:
        repoURL: https://github.com/myorg/k8s-manifests.git
        targetRevision: main
        path: '{{.path.path}}'
      destination:
        server: https://kubernetes.default.svc
        namespace: '{{.path.basename}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: multi-cluster-apps
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - clusters:
        selector:
          matchLabels:
            env: production
          matchExpressions:
            - key: region
              operator: In
              values:
                - ap-northeast-2
                - us-west-2
                - eu-west-1
  template:
    metadata:
      name: 'app-{{.name}}'
      namespace: argocd
    spec:
      project: production
      source:
        repoURL: https://github.com/myorg/k8s-manifests.git
        targetRevision: main
        path: 'apps/my-app/overlays/{{.metadata.labels.env}}'
        kustomize:
          commonLabels:
            cluster: '{{.name}}'
            region: '{{.metadata.labels.region}}'
      destination:
        server: '{{.server}}'
        namespace: production

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: matrix-deployment
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - matrix:
        generators:
          # Generator 1: Cluster list
          - clusters:
              selector:
                matchLabels:
                  env: production
          # Generator 2: App list from Git directories
          - git:
              repoURL: https://github.com/myorg/k8s-manifests.git
              revision: main
              directories:
                - path: 'apps/*'
  template:
    metadata:
      name: '{{.path.basename}}-{{.name}}'
      namespace: argocd
    spec:
      project: production
      source:
        repoURL: https://github.com/myorg/k8s-manifests.git
        targetRevision: main
        path: '{{.path.path}}/overlays/production'
      destination:
        server: '{{.server}}'
        namespace: '{{.path.basename}}'

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: environment-apps
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - list:
        elements:
          - env: dev
            cluster: https://dev-k8s.example.com
            revision: develop
            replicas: '1'
          - env: staging
            cluster: https://staging-k8s.example.com
            revision: release/v2.5
            replicas: '2'
          - env: production
            cluster: https://kubernetes.default.svc
            revision: main
            replicas: '3'
  template:
    metadata:
      name: 'my-app-{{.env}}'
      namespace: argocd
    spec:
      project: '{{.env}}'
      source:
        repoURL: https://github.com/myorg/k8s-manifests.git
        targetRevision: '{{.revision}}'
        path: 'apps/my-app/overlays/{{.env}}'
      destination:
        server: '{{.cluster}}'
        namespace: 'my-app-{{.env}}'

# Wave -2: Namespace and RBAC (prerequisites)
apiVersion: v1
kind: Namespace
metadata:
  name: production
  annotations:
    argocd.argoproj.io/sync-wave: '-2'
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: app-service-account
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '-2'
---
# Wave -1: ConfigMap and Secret (configuration)
apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '-1'
data:
  APP_ENV: production
  LOG_LEVEL: info
---
# Wave 0: Core application (default)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-app
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '0'
spec:
  replicas: 3
  selector:
    matchLabels:
      app: my-app
  template:
    metadata:
      labels:
        app: my-app
    spec:
      serviceAccountName: app-service-account
      containers:
        - name: app
          image: myorg/my-app:v2.5.0
          ports:
            - containerPort: 8080
          envFrom:
            - configMapRef:
                name: app-config
---
apiVersion: v1
kind: Service
metadata:
  name: my-app
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '0'
spec:
  selector:
    app: my-app
  ports:
    - port: 8080
      targetPort: 8080
---
# Wave 1: Dependent resources (Ingress, HPA)
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: my-app-hpa
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '1'
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-app
  minReplicas: 3
  maxReplicas: 10
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
---
# Wave 2: Monitoring
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: my-app-monitor
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '2'
spec:
  selector:
    matchLabels:
      app: my-app
  endpoints:
    - port: http
      interval: 15s

apiVersion: batch/v1
kind: Job
metadata:
  name: db-migration
  namespace: production
  annotations:
    argocd.argoproj.io/hook: PreSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
    argocd.argoproj.io/sync-wave: '-1'
spec:
  backoffLimit: 3
  template:
    spec:
      containers:
        - name: migrate
          image: myorg/db-migrator:v2.5.0
          command: ['./migrate', 'up']
          env:
            - name: DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: db-credentials
                  key: url
      restartPolicy: Never

apiVersion: batch/v1
kind: Job
metadata:
  name: smoke-test
  namespace: production
  annotations:
    argocd.argoproj.io/hook: PostSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
  backoffLimit: 1
  template:
    spec:
      containers:
        - name: smoke-test
          image: myorg/smoke-tester:latest
          command:
            - /bin/sh
            - -c
            - |
              echo "Running smoke tests..."
              curl -sf http://my-app.production.svc:8080/healthz || exit 1
              curl -sf http://my-app.production.svc:8080/api/v1/status || exit 1
              echo "All smoke tests passed!"
      restartPolicy: Never

apiVersion: batch/v1
kind: Job
metadata:
  name: sync-fail-notification
  namespace: production
  annotations:
    argocd.argoproj.io/hook: SyncFail
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
  backoffLimit: 1
  template:
    spec:
      containers:
        - name: notify
          image: curlimages/curl:latest
          command:
            - /bin/sh
            - -c
            - |
              curl -X POST "$SLACK_WEBHOOK_URL" \
                -H 'Content-Type: application/json' \
                -d '{"text":"ArgoCD Sync FAILED for my-app in production!"}'
          env:
            - name: SLACK_WEBHOOK_URL
              valueFrom:
                secretKeyRef:
                  name: slack-webhook
                  key: url
      restartPolicy: Never

# Install Sealed Secrets controller
helm repo add sealed-secrets https://bitnami-labs.github.io/sealed-secrets
helm install sealed-secrets sealed-secrets/sealed-secrets \
  --namespace kube-system

# Encrypt secret with kubeseal CLI
kubectl create secret generic db-credentials \
  --namespace production \
  --from-literal=url='postgresql://user:pass@db:5432/mydb' \
  --dry-run=client -o yaml | \
  kubeseal --format yaml > sealed-db-credentials.yaml

# SealedSecret safe to commit to Git
apiVersion: bitnami.com/v1alpha1
kind: SealedSecret
metadata:
  name: db-credentials
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '-2'
spec:
  encryptedData:
    url: AgA2X5N0Q...encrypted...base64==
  template:
    metadata:
      name: db-credentials
      namespace: production
    type: Opaque

# ExternalSecret: Sync secrets from AWS Secrets Manager
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
  namespace: production
  annotations:
    argocd.argoproj.io/sync-wave: '-2'
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: aws-secretsmanager
    kind: ClusterSecretStore
  target:
    name: db-credentials
    creationPolicy: Owner
  data:
    - secretKey: url
      remoteRef:
        key: production/database
        property: connection_url
    - secretKey: password
      remoteRef:
        key: production/database
        property: password
---
# ClusterSecretStore definition
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: aws-secretsmanager
spec:
  provider:
    aws:
      service: SecretsManager
      region: ap-northeast-2
      auth:
        jwt:
          serviceAccountRef:
            name: external-secrets-sa
            namespace: external-secrets

# Register target clusters (kubeconfig context-based)
argocd cluster add staging-cluster \
  --name staging \
  --label env=staging \
  --label region=ap-northeast-2

argocd cluster add production-cluster \
  --name production \
  --label env=production \
  --label region=ap-northeast-2

# Verify registered clusters
argocd cluster list

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: platform-services
  namespace: argocd
spec:
  goTemplate: true
  goTemplateOptions: ['missingkey=error']
  generators:
    - matrix:
        generators:
          - clusters:
              selector:
                matchLabels:
                  env: production
          - list:
              elements:
                - app: cert-manager
                  namespace: cert-manager
                  chart: cert-manager
                  repoURL: https://charts.jetstack.io
                  targetRevision: v1.16.0
                - app: external-secrets
                  namespace: external-secrets
                  chart: external-secrets
                  repoURL: https://charts.external-secrets.io
                  targetRevision: 0.12.0
                - app: metrics-server
                  namespace: kube-system
                  chart: metrics-server
                  repoURL: https://kubernetes-sigs.github.io/metrics-server
                  targetRevision: 3.12.0
  template:
    metadata:
      name: '{{.app}}-{{.name}}'
      namespace: argocd
    spec:
      project: platform
      source:
        repoURL: '{{.repoURL}}'
        chart: '{{.chart}}'
        targetRevision: '{{.targetRevision}}'
        helm:
          releaseName: '{{.app}}'
      destination:
        server: '{{.server}}'
        namespace: '{{.namespace}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: my-app
  namespace: argocd
spec:
  project: production
  source:
    repoURL: https://github.com/myorg/k8s-manifests.git
    targetRevision: main
    path: apps/my-app/overlays/production
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    retry:
      limit: 3
      backoff:
        duration: 10s
        factor: 2
        maxDuration: 5m

# Check deployment history
argocd app history my-app

# Rollback to a specific revision
argocd app rollback my-app 5

# Or sync to a specific Git commit
argocd app sync my-app --revision abc123def

# Verify sync status
argocd app get my-app
argocd app wait my-app --health

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: argocd-metrics
  namespace: argocd
spec:
  selector:
    matchLabels:
      app.kubernetes.io/part-of: argocd
  endpoints:
    - port: metrics
      interval: 30s
---
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: argocd-alerts
  namespace: argocd
spec:
  groups:
    - name: argocd
      rules:
        - alert: ArgoCDAppOutOfSync
          expr: |
            argocd_app_info{sync_status="OutOfSync"} == 1
          for: 10m
          labels:
            severity: warning
          annotations:
            summary: 'Application {{ "{{" }} $labels.name {{ "}}" }} is OutOfSync for more than 10 minutes'
        - alert: ArgoCDAppDegraded
          expr: |
            argocd_app_info{health_status="Degraded"} == 1
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: 'Application {{ "{{" }} $labels.name {{ "}}" }} is Degraded'
        - alert: ArgoCDSyncFailed
          expr: |
            increase(argocd_app_sync_total{phase="Failed"}[10m]) > 0
          labels:
            severity: critical
          annotations:
            summary: 'ArgoCD sync failed for {{ "{{" }} $labels.name {{ "}}" }}'

# argocd-notifications-cm ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: argocd-notifications-cm
  namespace: argocd
data:
  service.slack: |
    token: $slack-token
  trigger.on-sync-failed: |
    - when: app.status.operationState.phase in ['Error', 'Failed']
      send: [app-sync-failed]
  trigger.on-health-degraded: |
    - when: app.status.health.status == 'Degraded'
      send: [app-health-degraded]
  trigger.on-sync-succeeded: |
    - when: app.status.operationState.phase in ['Succeeded']
      oncePer: app.status.sync.revision
      send: [app-sync-succeeded]
  template.app-sync-failed: |
    slack:
      attachments: |
        [{
          "color": "#E96D76",
          "title": "Sync Failed: {{.app.metadata.name}}",
          "text": "Application {{.app.metadata.name}} sync failed.\nRevision: {{.app.status.sync.revision}}\nMessage: {{.app.status.operationState.message}}"
        }]
  template.app-health-degraded: |
    slack:
      attachments: |
        [{
          "color": "#f4c030",
          "title": "Health Degraded: {{.app.metadata.name}}",
          "text": "Application {{.app.metadata.name}} health is degraded."
        }]
  template.app-sync-succeeded: |
    slack:
      attachments: |
        [{
          "color": "#18BE52",
          "title": "Sync Succeeded: {{.app.metadata.name}}",
          "text": "Application {{.app.metadata.name}} synced successfully.\nRevision: {{.app.status.sync.revision}}"
        }]

# Symptom: Application keeps cycling through Syncing state
# Cause: Drift detection due to missing ignoreDifferences

# Diagnosis
argocd app diff my-app
argocd app get my-app --show-operation

# Recovery: Add ignoreDifferences
kubectl patch application my-app -n argocd --type merge -p '{
  "spec": {
    "ignoreDifferences": [
      {
        "group": "apps",
        "kind": "Deployment",
        "jsonPointers": ["/spec/replicas"]
      }
    ]
  }
}'

# Symptom: Application in Unknown state, manifest generation fails
# Diagnosis
kubectl logs -n argocd deploy/argocd-repo-server --previous
kubectl top pods -n argocd

# Recovery: Increase resource limits
kubectl patch deployment argocd-repo-server -n argocd --type json -p '[
  {
    "op": "replace",
    "path": "/spec/template/spec/containers/0/resources/limits/memory",
    "value": "4Gi"
  }
]'

# Restart Repo Server
kubectl rollout restart deployment/argocd-repo-server -n argocd

# Symptom: Application is Healthy but Pod is in CrashLoopBackOff
# Cause: ExternalSecret has not yet synchronized

# Diagnosis
kubectl get externalsecret -n production
kubectl describe externalsecret db-credentials -n production

# Recovery: Force ExternalSecret sync
kubectl annotate externalsecret db-credentials \
  -n production \
  force-sync=$(date +%s) --overwrite

# Or ensure ordering with Sync Waves
# ExternalSecret: sync-wave=-2, Deployment: sync-wave=0

# Symptom: Existing Applications deleted after ApplicationSet modification
# Cause: Generator config error reducing matched items

# Prevention: Set preserveResourcesOnDeletion
kubectl patch applicationset my-appset -n argocd --type merge -p '{
  "spec": {
    "syncPolicy": {
      "preserveResourcesOnDeletion": true
    }
  }
}'

# Recovery: Restore previous ApplicationSet config from Git history
git log --oneline -- applicationsets/my-appset.yaml
git checkout HEAD~1 -- applicationsets/my-appset.yaml
git commit -m "Revert ApplicationSet to restore applications"
git push