CI/CD Best Practices 2025: Pipeline Design, Automation, and Security for Teams

Traditional approach:
Code → Build → Test → Security → Deploy → Monitor
                              ↑ Problems found here

Shift Left:
Code + Test + Security → Build → Deploy → Monitor
↑ Problems found here (10x cost reduction)

# .github/workflows/ci.yml
name: CI Pipeline

on:
  pull_request:
    branches: [main]
  push:
    branches: [main]

concurrency:
  group: ci-${{ '{{' }} github.ref {{ '}}' }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run lint

  test:
    runs-on: ubuntu-latest
    needs: lint
    strategy:
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm test -- --shard=${{ '{{' }} matrix.shard {{ '}}' }}/4

  build:
    runs-on: ubuntu-latest
    needs: test
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: myapp:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max

// Jenkinsfile (Declarative Pipeline)
pipeline {
    agent any

    environment {
        DOCKER_REGISTRY = 'registry.example.com'
        IMAGE_NAME = 'myapp'
    }

    stages {
        stage('Checkout') {
            steps {
                checkout scm
            }
        }

        stage('Lint & Test') {
            parallel {
                stage('Lint') {
                    steps {
                        sh 'npm run lint'
                    }
                }
                stage('Unit Test') {
                    steps {
                        sh 'npm test -- --coverage'
                    }
                    post {
                        always {
                            junit 'reports/junit.xml'
                        }
                    }
                }
            }
        }

        stage('Build & Push') {
            steps {
                script {
                    def image = docker.build("${DOCKER_REGISTRY}/${IMAGE_NAME}:${BUILD_NUMBER}")
                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'registry-credentials') {
                        image.push()
                        image.push('latest')
                    }
                }
            }
        }
    }

    post {
        failure {
            slackSend(
                channel: '#ci-alerts',
                color: 'danger',
                message: "Build FAILED: ${env.JOB_NAME} #${env.BUILD_NUMBER}"
            )
        }
    }
}

# .gitlab-ci.yml
stages:
  - lint
  - test
  - build
  - deploy

variables:
  DOCKER_HOST: tcp://docker:2376

lint:
  stage: lint
  image: node:20-alpine
  cache:
    key: npm-cache
    paths:
      - node_modules/
  script:
    - npm ci
    - npm run lint

test:
  stage: test
  image: node:20-alpine
  parallel: 4
  script:
    - npm ci
    - npm test -- --shard=$CI_NODE_INDEX/$CI_NODE_TOTAL
  coverage: '/Statements\s*:\s*(\d+\.?\d*)%/'
  artifacts:
    reports:
      junit: reports/junit.xml

build:
  stage: build
  image: docker:24
  services:
    - docker:24-dind
  script:
    - docker build -t myapp:latest .
    - docker push myapp:latest
  only:
    - main

Target times:
├── Lint + format check: under 30s
├── Unit tests: under 2min
├── Integration tests: under 5min
├── Build: under 3min
└── Full pipeline: under 10min

Reality (before optimization): 30min+
Reality (after optimization): 8-10min

# Parallel pipeline example
jobs:
  # Phase 1: Lint/security run independently in parallel
  lint:
    runs-on: ubuntu-latest
    # ...
  security-scan:
    runs-on: ubuntu-latest
    # ...

  # Phase 2: Tests split into shards
  test:
    needs: [lint]
    strategy:
      matrix:
        shard: [1, 2, 3, 4]
    # ...

  # Phase 3: Build after tests pass
  build:
    needs: [test, security-scan]
    # ...

# npm caching (GitHub Actions)
- uses: actions/cache@v4
  with:
    path: ~/.npm
    key: npm-${{ '{{' }} hashFiles('**/package-lock.json') {{ '}}' }}
    restore-keys: |
      npm-

# Docker layer caching
- uses: docker/build-push-action@v5
  with:
    cache-from: type=gha
    cache-to: type=gha,mode=max

# Gradle caching
- uses: actions/cache@v4
  with:
    path: |
      ~/.gradle/caches
      ~/.gradle/wrapper
    key: gradle-${{ '{{' }} hashFiles('**/*.gradle*') {{ '}}' }}

# Bad: Timestamp-based tag (different result on re-run)
# IMAGE_TAG: my-app:build-20250323-142000

# Good: Commit SHA-based tag (always the same)
# IMAGE_TAG: my-app:abc1234

# Good: Semantic version (deterministic)
# IMAGE_TAG: my-app:v1.2.3

          /    E2E    \          Slow but high confidence
         /  (5-10%)    \
        / Integration   \       Medium speed, medium confidence
       /   (15-25%)      \
      /    Unit Tests      \    Fast and numerous
     /     (65-80%)         \
    /_________________________\

# Jest test sharding
test:
  strategy:
    matrix:
      shard: [1, 2, 3, 4]
  steps:
    - run: npx jest --shard=${{ '{{' }} matrix.shard {{ '}}' }}/4

# Cypress parallel execution
e2e:
  strategy:
    matrix:
      container: [1, 2, 3]
  steps:
    - uses: cypress-io/github-action@v6
      with:
        record: true
        parallel: true
        group: 'e2e-tests'

// Flaky test detection and isolation strategy
// jest.config.js
module.exports = {
  // Auto-retry on failure
  retryTimes: 2,

  // Flaky test reporter
  reporters: [
    'default',
    ['jest-flaky-reporter', {
      outputFile: 'flaky-tests.json',
      threshold: 3  // Report if flaky 3+ times
    }]
  ]
};

# Isolate flaky tests in CI
test-stable:
  runs-on: ubuntu-latest
  steps:
    - run: npx jest --testPathIgnorePatterns="flaky"

test-flaky:
  runs-on: ubuntu-latest
  continue-on-error: true  # Pipeline continues on failure
  steps:
    - run: npx jest --testPathPattern="flaky" --retries=3

# Set coverage thresholds
test:
  steps:
    - run: npx jest --coverage
    - name: Check coverage threshold
      run: |
        COVERAGE=$(cat coverage/coverage-summary.json | jq '.total.statements.pct')
        if (( $(echo "$COVERAGE < 80" | bc -l) )); then
          echo "Coverage $COVERAGE% is below 80% threshold"
          exit 1
        fi

# Stage 1: Install dependencies
FROM node:20-alpine AS deps
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci --production

# Stage 2: Build
FROM node:20-alpine AS builder
WORKDIR /app
COPY --from=deps /app/node_modules ./node_modules
COPY . .
RUN npm run build

# Stage 3: Production image
FROM node:20-alpine AS runner
WORKDIR /app
ENV NODE_ENV=production

# Security: non-root user
RUN addgroup --system --gid 1001 nodejs && \
    adduser --system --uid 1001 nextjs

COPY --from=builder --chown=nextjs:nodejs /app/.next ./.next
COPY --from=deps --chown=nextjs:nodejs /app/node_modules ./node_modules
COPY --from=builder /app/package.json ./

USER nextjs
EXPOSE 3000
CMD ["npm", "start"]

# Bad: Source changes trigger npm ci re-run
COPY . .
RUN npm ci
RUN npm run build

# Good: Copy dependency files first
COPY package.json package-lock.json ./
RUN npm ci
COPY . .
RUN npm run build

# Using BuildKit in GitHub Actions
- uses: docker/setup-buildx-action@v3

- uses: docker/build-push-action@v5
  with:
    context: .
    push: true
    tags: myapp:latest
    cache-from: type=gha
    cache-to: type=gha,mode=max
    platforms: linux/amd64,linux/arm64

# Build images with Kaniko in Kubernetes
apiVersion: v1
kind: Pod
metadata:
  name: kaniko-build
spec:
  containers:
    - name: kaniko
      image: gcr.io/kaniko-project/executor:latest
      args:
        - "--dockerfile=Dockerfile"
        - "--context=git://github.com/myorg/myapp"
        - "--destination=registry.example.com/myapp:latest"
        - "--cache=true"
        - "--cache-repo=registry.example.com/myapp/cache"

Image size comparison:
├── node:20          → 1.1GB
├── node:20-slim     → 220MB
├── node:20-alpine   → 140MB
├── distroless/nodejs → 120MB
└── Multi-stage optimized → 80-100MB

GitOps workflow:
1. Developer pushes changes to Git
2. CI builds and tests the image
3. CI updates image tag in deployment manifests
4. ArgoCD compares Git vs cluster state
5. Auto-sync on drift (or manual approval)
6. Cluster matches Git state

┌────────┐    Push     ┌────────┐   Detect   ┌────────┐
│  Dev   │ ──────────> │  Git   │ <────────> │ ArgoCD │
└────────┘             └────────┘            └───┬────┘
                                                 │ Sync
                                            ┌────▼────┐
                                            │  K8s    │
                                            │ Cluster │
                                            └─────────┘

# apps/root-app.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: root-app
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/myorg/gitops-config
    targetRevision: main
    path: apps
  destination:
    server: https://kubernetes.default.svc
    namespace: argocd
  syncPolicy:
    automated:
      prune: true
      selfHeal: true

# apps/api-service.yaml
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: api-service
spec:
  project: default
  source:
    repoURL: https://github.com/myorg/gitops-config
    path: services/api
    targetRevision: main
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
    syncOptions:
      - CreateNamespace=true

apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: api-service
spec:
  replicas: 10
  strategy:
    canary:
      canaryService: api-canary
      stableService: api-stable
      trafficRouting:
        istio:
          virtualService:
            name: api-vsvc
      steps:
        - setWeight: 10
        - pause:
            duration: 5m
        - analysis:
            templates:
              - templateName: success-rate
        - setWeight: 30
        - pause:
            duration: 5m
        - analysis:
            templates:
              - templateName: success-rate
        - setWeight: 60
        - pause:
            duration: 5m
        - setWeight: 100

# AnalysisTemplate
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: success-rate
spec:
  metrics:
    - name: success-rate
      interval: 60s
      successCondition: result[0] >= 0.95
      provider:
        prometheus:
          address: http://prometheus:9090
          query: |
            sum(rate(http_requests_total{status=~"2.*",app="api-service",version="canary"}[5m]))
            /
            sum(rate(http_requests_total{app="api-service",version="canary"}[5m]))

CI/CD Security Layers:
┌─────────────────────────────────────────────┐
│ Layer 1: Pre-commit                         │
│  - Secret scanning (gitleaks, detect-secrets)│
│  - Lint (security rules)                    │
├─────────────────────────────────────────────┤
│ Layer 2: PR / Build                         │
│  - SAST (Semgrep, CodeQL, SonarQube)        │
│  - SCA (Dependabot, Snyk, Trivy)            │
│  - License compliance                       │
├─────────────────────────────────────────────┤
│ Layer 3: Container Build                    │
│  - Image scanning (Trivy, Grype)            │
│  - Base image policy (distroless, alpine)   │
│  - SBOM generation (Syft)                   │
├─────────────────────────────────────────────┤
│ Layer 4: Deploy                             │
│  - Policy enforcement (OPA/Kyverno)         │
│  - Signing (cosign, Sigstore)               │
│  - Runtime security (Falco)                 │
└─────────────────────────────────────────────┘

# GitHub Actions secret usage
deploy:
  steps:
    - name: Deploy
      env:
        AWS_ACCESS_KEY_ID: ${{ '{{' }} secrets.AWS_ACCESS_KEY_ID {{ '}}' }}
        AWS_SECRET_ACCESS_KEY: ${{ '{{' }} secrets.AWS_SECRET_ACCESS_KEY {{ '}}' }}
      run: |
        aws ecs update-service --cluster prod --service api

# OIDC-based authentication (secretless - recommended)
permissions:
  id-token: write
  contents: read

steps:
  - uses: aws-actions/configure-aws-credentials@v4
    with:
      role-to-assume: arn:aws:iam::123456789012:role/github-actions
      aws-region: ap-northeast-2

# Generate SBOM with Syft
- name: Generate SBOM
  uses: anchore/sbom-action@v0
  with:
    image: myapp:latest
    format: spdx-json
    output-file: sbom.spdx.json

# Sign image with cosign
- name: Sign image
  run: |
    cosign sign --key env://COSIGN_PRIVATE_KEY myapp:latest

# Verify signature with cosign
- name: Verify signature
  run: |
    cosign verify --key cosign.pub myapp:latest

# .pre-commit-config.yaml
repos:
  - repo: https://github.com/gitleaks/gitleaks
    rev: v8.18.0
    hooks:
      - id: gitleaks

# Run gitleaks in CI
security:
  runs-on: ubuntu-latest
  steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - uses: gitleaks/gitleaks-action@v2
      env:
        GITHUB_TOKEN: ${{ '{{' }} secrets.GITHUB_TOKEN {{ '}}' }}

# Kubernetes Blue-Green deployment
apiVersion: v1
kind: Service
metadata:
  name: api-service
spec:
  selector:
    app: api
    version: green  # Switch from blue to green
  ports:
    - port: 80
      targetPort: 8080

Blue-Green switchover process:
1. Blue(v1) running → Deploy Green(v2)
2. Green health check and smoke test
3. Switch service selector to Green
4. Roll back to Blue immediately on issues
5. Clean up Blue resources after stabilization

[Users] → [LB] → [Blue v1] Active
                  [Green v2] ← Preparing

[Users] → [LB] → [Blue v1] ← Standby
                  [Green v2] Active

# Canary deployment with Istio VirtualService
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: api-service
spec:
  hosts:
    - api-service
  http:
    - route:
        - destination:
            host: api-service
            subset: stable
          weight: 90
        - destination:
            host: api-service
            subset: canary
          weight: 10

// LaunchDarkly or custom feature flag system
import { featureFlags } from './feature-flags';

async function handleRequest(req: Request) {
  const userId = req.user.id;

  if (await featureFlags.isEnabled('new-checkout-flow', userId)) {
    return newCheckoutFlow(req);
  }

  return legacyCheckoutFlow(req);
}

Feature flag-based deployment:
1. Deploy code with new feature wrapped in a flag
2. Enable for internal users only
3. Gradually increase rollout (1% → 5% → 25% → 100%)
4. Disable flag immediately if issues arise
5. Decouple deployment from release

# Argo Rollouts automatic rollback
spec:
  strategy:
    canary:
      steps:
        - setWeight: 10
        - analysis:
            templates:
              - templateName: error-rate-check
      # Auto-rollback on analysis failure
      abortScaleDownDelaySeconds: 30

# Kubernetes Deployment automatic rollback
apiVersion: apps/v1
kind: Deployment
spec:
  progressDeadlineSeconds: 300  # Fails if not done in 5min
  minReadySeconds: 30
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 0

// Deployment circuit breaker
class DeploymentCircuitBreaker {
  private errorThreshold = 0.05; // 5% error rate
  private windowSize = 300;       // 5 minute window

  async shouldRollback(metrics: DeploymentMetrics): Promise<boolean> {
    const errorRate = metrics.errors / metrics.totalRequests;
    const p99Latency = metrics.p99LatencyMs;

    return (
      errorRate > this.errorThreshold ||
      p99Latency > 3000 // Over 3 seconds
    );
  }

  async executeRollback(deployment: string) {
    console.log(`Rolling back ${deployment}`);
    await exec(`kubectl rollout undo deployment/${deployment}`);

    await notify({
      channel: '#deployments',
      message: `Auto-rollback triggered for ${deployment}`,
      severity: 'critical'
    });
  }
}

Safe DB migration strategy:
1. Expand-Contract Pattern
   Phase 1 (Expand): Add new columns, write to both
   Phase 2 (Migrate): Migrate existing data
   Phase 3 (Contract): Remove old columns

2. Only apply rollback-safe migrations
   - Add column (rollback-safe)
   - Add index (rollback-safe)
   - Drop column (NOT safe → use Expand-Contract)
   - Change type (NOT safe → add new column, then switch)

-- Safe migration example
-- Step 1: Add new column (rollback-safe)
ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT FALSE;

-- Step 2: Data migration (background)
UPDATE users SET email_verified = TRUE
WHERE verified_at IS NOT NULL;

-- Step 3: Switch app code to use new column
-- Step 4: Drop old column (separate migration)
-- ALTER TABLE users DROP COLUMN verified_at;

Pipeline Health Dashboard:
┌─────────────────────────────────────────┐
│  Build Time Trend                       │
│  ██████████████ 8m (avg)                │
│  Target: < 10m                          │
├─────────────────────────────────────────┤
│  Success Rate                           │
│  ████████████████████ 94%               │
│  Target: > 95%                          │
├─────────────────────────────────────────┤
│  Flaky Test Rate                        │
│  ██ 3%                                  │
│  Target: < 2%                           │
├─────────────────────────────────────────┤
│  Mean Time to Recovery (MTTR)           │
│  ████ 25min                             │
│  Target: < 30min                        │
└─────────────────────────────────────────┘

# Report build metrics to Datadog
- name: Report build metrics
  if: always()
  run: |
    END_TIME=$(date +%s)
    DURATION=$((END_TIME - START_TIME))
    curl -X POST "https://api.datadoghq.com/api/v1/series" \
      -H "DD-API-KEY: $DD_API_KEY" \
      -d "{
        \"series\": [{
          \"metric\": \"ci.build.duration\",
          \"points\": [[$END_TIME, $DURATION]],
          \"tags\": [
            \"repo:myapp\",
            \"branch:$GITHUB_REF_NAME\",
            \"status:$JOB_STATUS\"
          ]
        }]
      }"

# Automated build failure classification script
import re
from enum import Enum

class FailureCategory(Enum):
    FLAKY_TEST = "flaky_test"
    DEPENDENCY = "dependency"
    COMPILATION = "compilation"
    INFRASTRUCTURE = "infrastructure"
    TIMEOUT = "timeout"
    UNKNOWN = "unknown"

def categorize_failure(log: str) -> FailureCategory:
    patterns = {
        FailureCategory.FLAKY_TEST: [
            r"retry.*failed",
            r"intermittent",
            r"flaky"
        ],
        FailureCategory.DEPENDENCY: [
            r"npm ERR!.*404",
            r"Could not resolve dependencies",
            r"ECONNRESET"
        ],
        FailureCategory.COMPILATION: [
            r"error TS\d+",
            r"SyntaxError",
            r"TypeError"
        ],
        FailureCategory.INFRASTRUCTURE: [
            r"runner.*offline",
            r"disk space",
            r"out of memory"
        ],
        FailureCategory.TIMEOUT: [
            r"timed out",
            r"deadline exceeded"
        ]
    }

    for category, regexes in patterns.items():
        for pattern in regexes:
            if re.search(pattern, log, re.IGNORECASE):
                return category

    return FailureCategory.UNKNOWN

# .github/workflows/production.yml
name: Production Pipeline

on:
  push:
    branches: [main]

permissions:
  id-token: write
  contents: read
  packages: write

jobs:
  # Phase 1: Code quality
  quality:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run lint
      - run: npm run type-check

  # Phase 2: Security scanning
  security:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run Semgrep
        uses: returntocorp/semgrep-action@v1
      - name: Run Trivy (filesystem)
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: 'fs'
          severity: 'HIGH,CRITICAL'

  # Phase 3: Tests
  test:
    needs: quality
    runs-on: ubuntu-latest
    strategy:
      matrix:
        shard: [1, 2, 3, 4]
    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_DB: testdb
          POSTGRES_PASSWORD: testpass
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm test -- --shard=${{ '{{' }} matrix.shard {{ '}}' }}/4
        env:
          DATABASE_URL: postgresql://postgres:testpass@localhost:5432/testdb

  # Phase 4: Build and push
  build:
    needs: [test, security]
    runs-on: ubuntu-latest
    outputs:
      image-tag: ${{ '{{' }} steps.meta.outputs.tags {{ '}}' }}
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ '{{' }} github.actor {{ '}}' }}
          password: ${{ '{{' }} secrets.GITHUB_TOKEN {{ '}}' }}
      - id: meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/myorg/myapp
      - uses: docker/build-push-action@v5
        with:
          push: true
          tags: ${{ '{{' }} steps.meta.outputs.tags {{ '}}' }}
          cache-from: type=gha
          cache-to: type=gha,mode=max

  # Phase 5: Update deployment manifests (GitOps)
  update-manifest:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          repository: myorg/gitops-config
          token: ${{ '{{' }} secrets.GITOPS_TOKEN {{ '}}' }}
      - name: Update image tag
        run: |
          cd services/api
          kustomize edit set image myapp=${{ '{{' }} needs.build.outputs.image-tag {{ '}}' }}
      - name: Commit and push
        run: |
          git config user.name "CI Bot"
          git config user.email "ci@example.com"
          git add .
          git commit -m "chore: update api-service image"
          git push