Split View: SSL 인증서 운영 플레이북: 무중단 갱신과 만료 사고 예방

┌──────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐
│ Issuance │ → │  Deploy  │ → │ Monitor  │ → │ Renewal  │ → │  Revoke  │
└──────────┘    └──────────┘    └──────────┘    └──────────┘    └──────────┘
      ↑                                              │
      └──────────────────────────────────────────────┘
                    (auto-renewal cycle)

# Issue Let's Encrypt certificate with ECDSA key
sudo certbot certonly \
  --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \
  --key-type ecdsa \
  --elliptic-curve secp256r1 \
  -d "*.example.com" \
  -d "example.com"

#!/bin/bash
# /usr/local/bin/deploy-cert.sh
# Certificate deployment script (multi-server)

CERT_DIR="/etc/letsencrypt/live/example.com"
SERVERS=("web01" "web02" "web03")
REMOTE_CERT_DIR="/etc/nginx/ssl"
DEPLOY_LOG="/var/log/cert-deploy.log"

deploy_cert() {
    local server=$1
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] Deploying to $server" >> "$DEPLOY_LOG"

    # Transfer certificate files
    scp -q "$CERT_DIR/fullchain.pem" "$server:$REMOTE_CERT_DIR/fullchain.pem.new"
    scp -q "$CERT_DIR/privkey.pem" "$server:$REMOTE_CERT_DIR/privkey.pem.new"

    # Atomic replacement (mv is atomic on the same filesystem)
    ssh "$server" "
        mv $REMOTE_CERT_DIR/fullchain.pem.new $REMOTE_CERT_DIR/fullchain.pem
        mv $REMOTE_CERT_DIR/privkey.pem.new $REMOTE_CERT_DIR/privkey.pem
        nginx -t && systemctl reload nginx
    "

    if [ $? -eq 0 ]; then
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] $server: OK" >> "$DEPLOY_LOG"
    else
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] $server: FAILED" >> "$DEPLOY_LOG"
        return 1
    fi
}

for server in "${SERVERS[@]}"; do
    deploy_cert "$server"
done

# Revoke a Let's Encrypt certificate
sudo certbot revoke --cert-path /etc/letsencrypt/live/example.com/cert.pem \
  --reason keycompromise

# Delete certificate files after revocation
sudo certbot delete --cert-name example.com

# Immediately issue a new certificate
sudo certbot certonly --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \
  -d "*.example.com" -d "example.com"

# restart vs reload difference
# restart: stops and restarts the process → potential request loss
# reload:  spawns new workers → graceful shutdown of old workers → zero downtime

# Automate reload with certbot deploy hook
sudo certbot renew --deploy-hook "systemctl reload nginx"

#!/bin/bash
# /usr/local/bin/rolling-cert-renewal.sh

SERVERS=("web01" "web02" "web03")
LB_API="http://lb-admin.internal:8080/api"
HEALTH_CHECK_URL="https://example.com/healthz"
WAIT_SECONDS=30

for server in "${SERVERS[@]}"; do
    echo "=== Processing $server ==="

    # 1. Remove server from load balancer
    curl -s -X POST "$LB_API/drain" -d "server=$server"
    echo "Draining $server from load balancer..."
    sleep $WAIT_SECONDS  # Wait for existing connections to complete

    # 2. Deploy certificate and apply
    scp /etc/letsencrypt/live/example.com/fullchain.pem "$server:/etc/nginx/ssl/"
    scp /etc/letsencrypt/live/example.com/privkey.pem "$server:/etc/nginx/ssl/"
    ssh "$server" "nginx -t && systemctl reload nginx"

    # 3. Health check
    for i in $(seq 1 10); do
        status=$(curl -s -o /dev/null -w "%{http_code}" "https://$server/healthz" --resolve "example.com:443:$(dig +short $server)")
        if [ "$status" = "200" ]; then
            echo "$server health check passed"
            break
        fi
        sleep 2
    done

    # 4. Re-add server to load balancer
    curl -s -X POST "$LB_API/enable" -d "server=$server"
    echo "$server re-enabled in load balancer"
    sleep 5
done

echo "=== Rolling renewal complete ==="

# /etc/nginx/conf.d/ssl-blue-green.conf
# Blue-Green certificate switching via symlinks

# Active certificate (symlink)
# /etc/nginx/ssl/active/ -> /etc/nginx/ssl/blue/ or /etc/nginx/ssl/green/

server {
    listen 443 ssl http2;
    server_name example.com;

    ssl_certificate     /etc/nginx/ssl/active/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/active/privkey.pem;

    # ... other settings
}

#!/bin/bash
# /usr/local/bin/blue-green-cert-switch.sh

ACTIVE_LINK="/etc/nginx/ssl/active"
BLUE_DIR="/etc/nginx/ssl/blue"
GREEN_DIR="/etc/nginx/ssl/green"

# Determine current active slot
current=$(readlink "$ACTIVE_LINK")
if [ "$current" = "$BLUE_DIR" ]; then
    target="$GREEN_DIR"
    target_name="green"
else
    target="$BLUE_DIR"
    target_name="blue"
fi

echo "Current: $current"
echo "Deploying new cert to: $target ($target_name)"

# Deploy new certificate to inactive slot
cp /etc/letsencrypt/live/example.com/fullchain.pem "$target/fullchain.pem"
cp /etc/letsencrypt/live/example.com/privkey.pem "$target/privkey.pem"

# Validate certificate
openssl x509 -in "$target/fullchain.pem" -noout -checkend 86400
if [ $? -ne 0 ]; then
    echo "ERROR: New certificate expires within 24 hours. Aborting."
    exit 1
fi

# Verify key match
CERT_MD5=$(openssl x509 -noout -modulus -in "$target/fullchain.pem" | openssl md5)
KEY_MD5=$(openssl rsa -noout -modulus -in "$target/privkey.pem" 2>/dev/null | openssl md5)
if [ "$CERT_MD5" != "$KEY_MD5" ]; then
    echo "ERROR: Certificate and key do not match. Aborting."
    exit 1
fi

# Atomic symlink switch
ln -sfn "$target" "${ACTIVE_LINK}.new"
mv -T "${ACTIVE_LINK}.new" "$ACTIVE_LINK"

# Nginx reload
nginx -t && systemctl reload nginx
echo "Switched to $target_name slot. Reload complete."

server {
    listen 443 ssl http2;
    server_name example.com;

    # RSA certificate
    ssl_certificate     /etc/nginx/ssl/rsa/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/rsa/privkey.pem;

    # ECDSA certificate
    ssl_certificate     /etc/nginx/ssl/ecdsa/fullchain.pem;
    ssl_certificate_key /etc/nginx/ssl/ecdsa/privkey.pem;

    # Nginx automatically selects based on client support
    # ECDSA preferred, RSA fallback for unsupported clients
}

# /etc/systemd/system/certbot-renewal.service
[Unit]
Description=Certbot SSL Certificate Renewal
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
ExecStart=/usr/bin/certbot renew --quiet \
  --pre-hook "/usr/local/bin/cert-pre-hook.sh" \
  --deploy-hook "/usr/local/bin/cert-deploy-hook.sh"
ExecStartPost=/usr/local/bin/cert-renewal-notify.sh
TimeoutStartSec=300

# /etc/systemd/system/certbot-renewal.timer
[Unit]
Description=Run certbot renewal twice daily

[Timer]
OnCalendar=*-*-* 02,14:00:00
RandomizedDelaySec=3600
Persistent=true
AccuracySec=1s

[Install]
WantedBy=timers.target

# Enable timer and check status
sudo systemctl daemon-reload
sudo systemctl enable --now certbot-renewal.timer
sudo systemctl list-timers certbot-renewal.timer

# Manual test (dry-run)
sudo certbot renew --dry-run

# Manual trigger
sudo systemctl start certbot-renewal.service

#!/bin/bash
# /usr/local/bin/cert-pre-hook.sh
# Executed before renewal

LOG="/var/log/cert-hooks.log"

echo "[$(date '+%Y-%m-%d %H:%M:%S')] PRE-HOOK: Starting renewal process" >> "$LOG"

# Back up current certificate information
for domain_dir in /etc/letsencrypt/live/*/; do
    domain=$(basename "$domain_dir")
    expiry=$(openssl x509 -in "${domain_dir}fullchain.pem" -noout -enddate 2>/dev/null | cut -d= -f2)
    echo "[PRE] $domain expires: $expiry" >> "$LOG"
done

#!/bin/bash
# /usr/local/bin/cert-deploy-hook.sh
# Executed after successful renewal
# Environment variables $RENEWED_DOMAINS and $RENEWED_LINEAGE are available

LOG="/var/log/cert-hooks.log"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] DEPLOY-HOOK: Certificate renewed" >> "$LOG"
echo "  Domains: $RENEWED_DOMAINS" >> "$LOG"
echo "  Lineage: $RENEWED_LINEAGE" >> "$LOG"

# 1. Validate Nginx config then reload
if nginx -t 2>/dev/null; then
    systemctl reload nginx
    echo "  Nginx reloaded successfully" >> "$LOG"
else
    echo "  ERROR: Nginx config test failed!" >> "$LOG"
    curl -s -X POST "$SLACK_WEBHOOK" \
      -H 'Content-type: application/json' \
      -d '{"text":"CRITICAL: Nginx config test failed after cert renewal!"}'
    exit 1
fi

# 2. If HAProxy is running, combine cert and reload
if systemctl is-active haproxy > /dev/null 2>&1; then
    cat "$RENEWED_LINEAGE/fullchain.pem" "$RENEWED_LINEAGE/privkey.pem" \
      > /etc/haproxy/certs/$(basename "$RENEWED_LINEAGE").pem
    systemctl reload haproxy
    echo "  HAProxy reloaded" >> "$LOG"
fi

# 3. Sync certificates to other servers (if needed)
# /usr/local/bin/sync-certs-to-peers.sh "$RENEWED_LINEAGE"

#!/bin/bash
# /usr/local/bin/cert-renewal-notify.sh
# Renewal result notification

SLACK_WEBHOOK="https://hooks.slack.com/services/YOUR/WEBHOOK/URL"
LOG="/var/log/cert-hooks.log"

# Check last renewal result
last_renewal=$(journalctl -u certbot-renewal.service --since "5 minutes ago" --no-pager 2>/dev/null)

if echo "$last_renewal" | grep -q "Congratulations"; then
    # Renewal succeeded
    renewed_domains=$(echo "$last_renewal" | grep "renewed" | head -5)
    curl -s -X POST "$SLACK_WEBHOOK" \
      -H 'Content-type: application/json' \
      -d "{\"text\":\"SSL certificate renewal succeeded\\n${renewed_domains}\"}"
elif echo "$last_renewal" | grep -q "No renewals were attempted"; then
    # No certificates due for renewal (normal)
    echo "[$(date)] No certificates due for renewal" >> "$LOG"
else
    # Renewal failed
    curl -s -X POST "$SLACK_WEBHOOK" \
      -H 'Content-type: application/json' \
      -d '{"text":"WARNING: Please check certbot renewal execution results!"}'
fi

# Add to /etc/systemd/system/certbot-renewal.service
[Service]
# Retry after 5 minutes on failure, up to 3 times
Restart=on-failure
RestartSec=300
StartLimitBurst=3
StartLimitIntervalSec=3600

# Install ssl_exporter
wget https://github.com/ribbybibby/ssl_exporter/releases/download/v2.4.3/ssl_exporter-2.4.3.linux-amd64.tar.gz
tar xzf ssl_exporter-2.4.3.linux-amd64.tar.gz
sudo mv ssl_exporter-2.4.3.linux-amd64/ssl_exporter /usr/local/bin/

# systemd service
sudo tee /etc/systemd/system/ssl-exporter.service << 'EOF'
[Unit]
Description=SSL Certificate Exporter
After=network-online.target

[Service]
ExecStart=/usr/local/bin/ssl_exporter
Restart=on-failure
User=ssl-exporter

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl enable --now ssl-exporter

# Prometheus scrape config
# /etc/prometheus/prometheus.yml

scrape_configs:
  - job_name: 'ssl'
    metrics_path: /probe
    static_configs:
      - targets:
          - example.com:443
          - api.example.com:443
          - admin.example.com:443
          - staging.example.com:443
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: ssl-exporter:9219 # ssl_exporter address

# Calculate days until certificate expiry
(ssl_cert_not_after - time()) / 86400

# Find certificates expiring within 30 days
(ssl_cert_not_after - time()) / 86400 < 30

# Find certificates expiring within 7 days
(ssl_cert_not_after - time()) / 86400 < 7

# /etc/prometheus/rules/ssl-alerts.yml
groups:
  - name: ssl_certificate_alerts
    rules:
      # Expiring within 30 days - Warning
      - alert: SSLCertExpiringSoon
        expr: (ssl_cert_not_after - time()) / 86400 < 30
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: 'SSL certificate expiring soon ({{ $labels.instance }})'
          description: '{{ $labels.instance }} certificate expires in {{ $value | printf "%.0f" }} days.'
          runbook_url: 'https://wiki.internal/runbooks/ssl-renewal'

      # Expiring within 7 days - Critical
      - alert: SSLCertExpiryCritical
        expr: (ssl_cert_not_after - time()) / 86400 < 7
        for: 10m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: 'SSL certificate expiry urgent ({{ $labels.instance }})'
          description: '{{ $labels.instance }} certificate expires in {{ $value | printf "%.0f" }} days. Immediate action required.'
          runbook_url: 'https://wiki.internal/runbooks/ssl-emergency-renewal'

      # Already expired certificate
      - alert: SSLCertExpired
        expr: (ssl_cert_not_after - time()) < 0
        for: 0m
        labels:
          severity: critical
          escalation: pagerduty
        annotations:
          summary: 'SSL certificate expired ({{ $labels.instance }})'
          description: '{{ $labels.instance }} certificate has expired! Service outage may occur.'

      # Probe failure (connection failed)
      - alert: SSLProbeFailure
        expr: ssl_probe_success == 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'SSL probe failed ({{ $labels.instance }})'
          description: 'Cannot establish TLS connection to {{ $labels.instance }}.'

# Alertmanager routing config
# /etc/alertmanager/alertmanager.yml

route:
  group_by: ['alertname', 'instance']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'slack-warning'
  routes:
    - match:
        severity: critical
        escalation: pagerduty
      receiver: 'pagerduty-critical'
      repeat_interval: 30m
    - match:
        severity: critical
      receiver: 'slack-critical'
      repeat_interval: 1h

receivers:
  - name: 'slack-warning'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/WEBHOOK/URL'
        channel: '#ssl-alerts'
        title: '{{ .CommonAnnotations.summary }}'
        text: '{{ .CommonAnnotations.description }}'

  - name: 'slack-critical'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/WEBHOOK/URL'
        channel: '#incident'
        title: '{{ .CommonAnnotations.summary }}'
        text: '{{ .CommonAnnotations.description }}'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_SERVICE_KEY'
        severity: critical

{
  "title": "SSL Certificate Dashboard",
  "panels": [
    {
      "title": "Days Until Certificate Expiry",
      "type": "table",
      "targets": [
        {
          "expr": "sort_desc((ssl_cert_not_after - time()) / 86400)",
          "legendFormat": "{{ instance }}"
        }
      ]
    },
    {
      "title": "Certificates Expiring Soon (within 30 days)",
      "type": "stat",
      "targets": [
        {
          "expr": "count((ssl_cert_not_after - time()) / 86400 < 30)"
        }
      ],
      "thresholds": [
        { "value": 0, "color": "green" },
        { "value": 1, "color": "orange" },
        { "value": 3, "color": "red" }
      ]
    }
  ]
}

#!/bin/bash
# /usr/local/bin/ssl-expiry-check.sh
# Certificate expiry monitoring + Slack/Email alerts

set -euo pipefail

DOMAINS=(
    "example.com"
    "api.example.com"
    "admin.example.com"
    "staging.example.com"
)

WARNING_DAYS=30
CRITICAL_DAYS=7
SLACK_WEBHOOK="${SLACK_WEBHOOK:-}"
ALERT_EMAIL="ops-team@example.com"

check_cert() {
    local domain=$1
    local port=${2:-443}

    # Get certificate expiry date
    local expiry_date
    expiry_date=$(echo | timeout 10 openssl s_client \
        -servername "$domain" \
        -connect "${domain}:${port}" 2>/dev/null \
        | openssl x509 -noout -enddate 2>/dev/null \
        | cut -d= -f2)

    if [ -z "$expiry_date" ]; then
        echo "UNKNOWN|${domain}|Connection failed"
        return
    fi

    # Calculate remaining days (Linux/macOS compatible)
    local expiry_epoch days_left
    if date --version >/dev/null 2>&1; then
        # GNU date (Linux)
        expiry_epoch=$(date -d "$expiry_date" +%s)
    else
        # BSD date (macOS)
        expiry_epoch=$(date -j -f "%b %d %T %Y %Z" "$expiry_date" +%s)
    fi
    days_left=$(( (expiry_epoch - $(date +%s)) / 86400 ))

    if [ "$days_left" -lt 0 ]; then
        echo "EXPIRED|${domain}|${days_left}|${expiry_date}"
    elif [ "$days_left" -lt "$CRITICAL_DAYS" ]; then
        echo "CRITICAL|${domain}|${days_left}|${expiry_date}"
    elif [ "$days_left" -lt "$WARNING_DAYS" ]; then
        echo "WARNING|${domain}|${days_left}|${expiry_date}"
    else
        echo "OK|${domain}|${days_left}|${expiry_date}"
    fi
}

# Check all domains
alerts=""
for domain in "${DOMAINS[@]}"; do
    result=$(check_cert "$domain")
    status=$(echo "$result" | cut -d'|' -f1)
    days=$(echo "$result" | cut -d'|' -f3)

    case $status in
        OK)
            printf "%-30s %-10s %s days\n" "$domain" "[OK]" "$days"
            ;;
        WARNING)
            printf "%-30s %-10s %s days\n" "$domain" "[WARNING]" "$days"
            alerts="${alerts}WARNING: ${domain} - ${days} days remaining\n"
            ;;
        CRITICAL|EXPIRED)
            printf "%-30s %-10s %s days\n" "$domain" "[$status]" "$days"
            alerts="${alerts}${status}: ${domain} - ${days} days remaining\n"
            ;;
        UNKNOWN)
            printf "%-30s %-10s\n" "$domain" "[UNKNOWN]"
            alerts="${alerts}UNKNOWN: ${domain} - Connection failed\n"
            ;;
    esac
done

# Send alerts
if [ -n "$alerts" ]; then
    if [ -n "$SLACK_WEBHOOK" ]; then
        curl -s -X POST "$SLACK_WEBHOOK" \
          -H 'Content-type: application/json' \
          -d "{\"text\":\"SSL Certificate Check Results:\\n${alerts}\"}"
    fi

    # Email alert (requires mailutils)
    echo -e "$alerts" | mail -s "[SSL Alert] Certificate Expiry Warning" "$ALERT_EMAIL" 2>/dev/null || true
fi

# Register in cron (daily at 9 AM)
echo "0 9 * * * /usr/local/bin/ssl-expiry-check.sh >> /var/log/ssl-check.log 2>&1" | sudo crontab -

┌─────────────────────────────────────────────────────────────┐
│  Certificate Expiry Incident Response Flow                  │
├─────────────────────────────────────────────────────────────┤
│                                                             │
│  1. Detection & Confirmation (0-5 min)                      │
│     └→ Assess blast radius, list affected domains           │
│                                                             │
│  2. Immediate Mitigation (5-15 min)                         │
│     └→ Apply temporary certificate or reroute traffic       │
│                                                             │
│  3. Formal Certificate Renewal (15-30 min)                  │
│     └→ certbot renewal or emergency issuance                │
│                                                             │
│  4. Service Verification (30-45 min)                        │
│     └→ Confirm TLS connectivity on all endpoints            │
│                                                             │
│  5. Postmortem (within 48 hours)                            │
│     └→ Root cause analysis, preventive measures             │
│                                                             │
└─────────────────────────────────────────────────────────────┘

# 1-1. Immediately check expiry status
echo | openssl s_client -servername example.com -connect example.com:443 2>/dev/null \
  | openssl x509 -noout -dates

# 1-2. Check multiple domains at once
for domain in example.com api.example.com admin.example.com; do
    echo -n "$domain: "
    echo | openssl s_client -servername "$domain" -connect "${domain}:443" 2>/dev/null \
      | openssl x509 -noout -enddate 2>/dev/null || echo "CONNECTION FAILED"
done

# 1-3. Check local certificate files
for cert in /etc/letsencrypt/live/*/fullchain.pem; do
    domain=$(basename $(dirname "$cert"))
    expiry=$(openssl x509 -in "$cert" -noout -enddate | cut -d= -f2)
    echo "$domain: $expiry"
done

# 1-4. Share situation in incident channel
# "SSL certificate expiry confirmed. Blast radius: example.com, api.example.com. Response initiated."

# 2-1. Force renew Let's Encrypt certificate
sudo certbot renew --force-renewal --cert-name example.com
sudo systemctl reload nginx

# 2-2. If renewal fails - emergency issuance via standalone
sudo systemctl stop nginx
sudo certbot certonly --standalone -d example.com -d "*.example.com"
sudo systemctl start nginx

# 2-3. If hitting Let's Encrypt rate limit - temporary self-signed cert
openssl req -x509 -nodes -days 1 -newkey rsa:2048 \
  -keyout /tmp/emergency.key \
  -out /tmp/emergency.crt \
  -subj "/CN=example.com"

# Note: Self-signed certs show browser warnings but can serve
# as a temporary measure for API servers or internal communication

# 2-4. For AWS environments using ACM certificates
# ACM auto-renews, so the issue is usually on the ALB/CloudFront side
aws elbv2 describe-listeners --load-balancer-arn $ALB_ARN \
  --query 'Listeners[].Certificates[].CertificateArn'

# Check ACM certificate status
aws acm describe-certificate --certificate-arn $CERT_ARN \
  --query 'Certificate.{Status:Status,NotAfter:NotAfter}'

# 3-1. Validate renewed certificate chain
openssl verify -CAfile /etc/ssl/certs/ca-certificates.crt \
  /etc/letsencrypt/live/example.com/fullchain.pem

# 3-2. Verify certificate and key match
diff <(openssl x509 -noout -modulus -in /etc/letsencrypt/live/example.com/fullchain.pem | openssl md5) \
     <(openssl rsa -noout -modulus -in /etc/letsencrypt/live/example.com/privkey.pem | openssl md5)

# 3-3. Deploy to all servers
/usr/local/bin/deploy-cert.sh

# 4-1. TLS connection test
curl -vI https://example.com 2>&1 | grep -E "SSL|expire|subject"

# 4-2. Full endpoint check
for url in https://example.com https://api.example.com/healthz https://admin.example.com; do
    status=$(curl -s -o /dev/null -w "%{http_code}" "$url")
    echo "$url: HTTP $status"
done

# 4-3. External verification (SSL Labs)
echo "Check: https://www.ssllabs.com/ssltest/analyze.html?d=example.com"

## Incident Postmortem: SSL Certificate Expiry

### Timeline

- HH:MM - Initial alert received (source: Prometheus/user report)
- HH:MM - Incident confirmed, response started
- HH:MM - Certificate renewal completed
- HH:MM - Service confirmed normal

### Blast Radius

- Affected domains: example.com, api.example.com
- Duration: XX minutes
- Affected users: approximately N

### Root Cause

- (e.g.) certbot timer was disabled, so auto-renewal was not running
- (e.g.) DNS validation kept failing but no failure alerts were configured

### Preventive Measures

- [ ] Add monitoring for auto-renewal timer status
- [ ] Set up immediate alerts on renewal failure
- [ ] Confirm 30-day warning alerts are configured

# Install mkcert (macOS)
brew install mkcert
mkcert -install  # Add local CA to system certificate store

# Create development certificates
mkcert "*.dev.example.com" localhost 127.0.0.1 ::1

# Output files
# _wildcard.dev.example.com+3.pem (certificate)
# _wildcard.dev.example.com+3-key.pem (key)

# Issue certificate from staging server (no rate limits, not browser-trusted)
sudo certbot certonly --staging \
  --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \
  -d "*.staging.example.com"

# Use -k (insecure) flag for API testing
curl -k https://staging.example.com/api/healthz

example.com                  → Single domain + wildcard
├── www.example.com          → Covered by *.example.com
├── api.example.com          → Covered by *.example.com
├── admin.example.com        → Covered by *.example.com
├── staging.example.com      → Separate certificate (staging env)
│   ├── api.staging.example.com → Covered by *.staging.example.com
│   └── admin.staging.example.com → Covered by *.staging.example.com
└── internal.example.com     → Internal only (consider mTLS)

# Production wildcard certificate (apex + wildcard)
sudo certbot certonly \
  --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \
  -d "example.com" \
  -d "*.example.com" \
  --cert-name prod-wildcard

# Staging wildcard certificate (separate)
sudo certbot certonly \
  --dns-cloudflare \
  --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \
  -d "staging.example.com" \
  -d "*.staging.example.com" \
  --cert-name staging-wildcard

# cert-manager ClusterIssuer (Let's Encrypt)
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:
  name: letsencrypt-prod
spec:
  acme:
    server: https://acme-v02.api.letsencrypt.org/directory
    email: admin@example.com
    privateKeySecretRef:
      name: letsencrypt-prod-account-key
    solvers:
      - dns01:
          cloudflare:
            email: admin@example.com
            apiTokenSecretRef:
              name: cloudflare-api-token
              key: api-token

# Certificate resource
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: example-com-tls
  namespace: istio-system
spec:
  secretName: example-com-tls
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer
  dnsNames:
    - example.com
    - '*.example.com'
  # Auto-renewal: 30 days before expiry
  renewBefore: 720h # 30 days

# Ingress with automatic certificate issuance
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: example-ingress
  annotations:
    cert-manager.io/cluster-issuer: 'letsencrypt-prod'
spec:
  tls:
    - hosts:
        - example.com
        - api.example.com
      secretName: example-com-tls
  rules:
    - host: example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: web
                port:
                  number: 80

# Check certificate status
kubectl get certificates -A
kubectl describe certificate example-com-tls -n istio-system

# Check certificate events
kubectl get events --field-selector reason=IssueError -A

# cert-manager logs
kubectl logs -n cert-manager deploy/cert-manager -f