The Complete HTTP/HTTPS Troubleshooting Guide - Production Debugging Techniques

100 Continue    → Server acknowledges the client may send a large request body
101 Switching   → Must verify during WebSocket upgrade negotiations
103 Early Hints → Used for browser preload optimization

200 OK          → The baseline. But an empty body deserves suspicion
201 Created     → Always check the Location header after POST
204 No Content  → Appropriate for DELETE responses. A body here is a bug
206 Partial     → Range requests. Essential for large file download debugging

301 Moved Permanently  → Browsers cache this aggressively. Misconfigurations are hard to undo
302 Found              → Temporary redirect. Watch for SEO confusion with 301
303 See Other          → Redirect POST to GET (Post/Redirect/Get pattern)
307 Temporary Redirect → Preserves the HTTP method
308 Permanent Redirect → Permanent redirect that preserves the HTTP method

# Debugging 400 Bad Request
curl -v -X POST https://api.example.com/users \
  -H "Content-Type: application/json" \
  -d '{"name": "test"' # Malformed JSON - missing closing brace

# Understanding 401 vs 403
# 401: Authentication failed (no token / token expired)
# 403: Authenticated but not authorized (insufficient permissions)

# 405 Method Not Allowed
curl -v -X PATCH https://api.example.com/users/1
# Check the Allow header for permitted methods

# 429 Too Many Requests
# Inspect Retry-After and X-RateLimit-* headers
curl -v https://api.example.com/data 2>&1 | grep -i "rate\|retry"

500 Internal Server Error → Check server-side logs first
502 Bad Gateway           → Upstream server returned an invalid response
503 Service Unavailable   → Server overloaded or in maintenance mode
504 Gateway Timeout       → Upstream server response timed out

Client                                Server
  |                                     |
  |--- ClientHello (ciphers, SNI) ----->|
  |                                     |
  |<-- ServerHello (chosen cipher) -----|
  |<-- Certificate (cert chain) --------|
  |<-- ServerKeyExchange ---------------|
  |<-- ServerHelloDone -----------------|
  |                                     |
  |--- ClientKeyExchange -------------->|
  |--- ChangeCipherSpec --------------->|
  |--- Finished ----------------------->|
  |                                     |
  |<-- ChangeCipherSpec ----------------|
  |<-- Finished ------------------------|
  |                                     |
  |====== Encrypted communication ======|

# Basic connection test
openssl s_client -connect example.com:443 -servername example.com

# Test specific TLS versions
openssl s_client -connect example.com:443 -tls1_2
openssl s_client -connect example.com:443 -tls1_3

# Dump the full certificate chain
openssl s_client -connect example.com:443 -showcerts 2>/dev/null | \
  openssl x509 -noout -subject -issuer -dates

# Debug SNI (Server Name Indication) issues
openssl s_client -connect 1.2.3.4:443 -servername mysite.com

# Test a specific cipher suite
openssl s_client -connect example.com:443 -cipher ECDHE-RSA-AES256-GCM-SHA384

# Error: SSL_ERROR_HANDSHAKE_FAILURE
# Cause: No matching cipher suites between client and server
# Resolution: Check the server's supported cipher suites
nmap --script ssl-enum-ciphers -p 443 example.com

# Error: CERTIFICATE_VERIFY_FAILED
# Cause: Incomplete certificate chain or untrusted root CA
# Diagnosis:
openssl s_client -connect example.com:443 2>&1 | grep "verify"

# Error: hostname mismatch
# Cause: Certificate CN/SAN does not match the requested domain
openssl s_client -connect example.com:443 2>/dev/null | \
  openssl x509 -noout -text | grep -A1 "Subject Alternative Name"

Root CA (self-signed, embedded in OS/browser trust stores)
  └── Intermediate CA (signed by Root CA)
        └── Server Certificate (signed by Intermediate CA)

#!/bin/bash
# Certificate chain verification script
DOMAIN=$1

echo "=== Certificate Chain Verification: $DOMAIN ==="

# Retrieve the certificate chain from the server
echo | openssl s_client -connect ${DOMAIN}:443 -servername ${DOMAIN} \
  -showcerts 2>/dev/null | \
  awk '/BEGIN CERTIFICATE/,/END CERTIFICATE/{ print }' > /tmp/chain.pem

# Split and display each certificate
csplit -f /tmp/cert- -b '%02d.pem' /tmp/chain.pem \
  '/BEGIN CERTIFICATE/' '{*}' 2>/dev/null

for cert in /tmp/cert-*.pem; do
  [ -s "$cert" ] || continue
  echo "--- Certificate: $cert ---"
  openssl x509 -in "$cert" -noout \
    -subject -issuer -dates -fingerprint 2>/dev/null
  echo ""
done

# Verify the chain
openssl verify -verbose -CAfile /etc/ssl/certs/ca-certificates.crt \
  /tmp/chain.pem 2>&1

# 1. Check certificate expiration
echo | openssl s_client -connect example.com:443 2>/dev/null | \
  openssl x509 -noout -dates

# 2. Detect missing intermediate certificates
# The most reliable method is SSL Labs:
# https://www.ssllabs.com/ssltest/

# 3. Using curl with self-signed certificates
curl --cacert /path/to/ca.crt https://internal.example.com

# 4. Verify certificate renewal was applied
echo | openssl s_client -connect example.com:443 2>/dev/null | \
  openssl x509 -noout -serial -fingerprint

# -v (verbose): The most fundamental debugging flag
curl -v https://api.example.com/health

# -vv: Even more detail
curl -vv https://api.example.com/health

# --trace: Full byte-level communication dump
curl --trace /tmp/curl-trace.log https://api.example.com/health

# --trace-ascii: Readable ASCII dump
curl --trace-ascii /tmp/curl-trace.txt https://api.example.com/health

# --trace-time: Include timestamps
curl --trace-time --trace-ascii - https://api.example.com/health

# DNS override (--resolve): Send requests to a specific IP
# Useful for testing a new server before DNS cutover
curl --resolve api.example.com:443:10.0.1.50 \
  https://api.example.com/health

# Measure connection timing breakdown
curl -o /dev/null -s -w "\
  DNS Lookup:    %{time_namelookup}s\n\
  TCP Connect:   %{time_connect}s\n\
  TLS Handshake: %{time_appconnect}s\n\
  Start Transfer: %{time_starttransfer}s\n\
  Total Time:    %{time_total}s\n\
  HTTP Code:     %{http_code}\n\
  Download Size: %{size_download} bytes\n" \
  https://api.example.com/health

# Trace redirect chains
curl -v -L --max-redirs 10 https://example.com 2>&1 | grep "< Location"

# Force specific HTTP version
curl --http1.1 https://api.example.com/health
curl --http2 https://api.example.com/health
curl --http3 https://api.example.com/health

# Debug through a proxy
curl -v --proxy http://proxy.internal:8080 https://api.example.com/health

# Client certificate authentication (mTLS)
curl --cert /path/to/client.crt --key /path/to/client.key \
  https://mtls.example.com/api

# Monitor response times with repeated requests
for i in $(seq 1 10); do
  echo -n "Request $i: "
  curl -o /dev/null -s -w "%{http_code} %{time_total}s" \
    https://api.example.com/health
  echo ""
  sleep 1
done

# Quick header inspection
curl -I https://api.example.com/health

# Debug POST requests with filtered output
curl -v -X POST https://api.example.com/users \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $TOKEN" \
  -d '{"name": "test", "email": "test@example.com"}' \
  2>&1 | grep -E "^[<>*]"

# Cause: Upstream server returned an invalid response

# 1. Test the upstream server directly
curl -v http://upstream-server:8080/health

# 2. Check Nginx error logs
tail -f /var/log/nginx/error.log | grep "502\|upstream"

# 3. Verify upstream server is listening
ss -tlnp | grep 8080
netstat -tlnp | grep 8080

# 4. Inspect Nginx upstream configuration
nginx -T | grep -A5 "upstream"

upstream backend {
    server 10.0.1.10:8080 max_fails=3 fail_timeout=30s;
    server 10.0.1.11:8080 max_fails=3 fail_timeout=30s;
    keepalive 32;
}

server {
    location /api/ {
        proxy_pass http://backend;
        proxy_next_upstream error timeout http_502 http_503;
        proxy_connect_timeout 5s;
        proxy_read_timeout 60s;
        proxy_send_timeout 60s;

        # Adjust buffer sizes for large response headers
        proxy_buffer_size 16k;
        proxy_buffers 4 32k;
        proxy_busy_buffers_size 64k;
    }
}

# Root cause analysis
# 1. Check server load
top -bn1 | head -5
free -m
df -h

# 2. Check active connections
ss -s
ss -tn state established | wc -l

# 3. Check process status
systemctl status nginx
systemctl status your-app

# 4. Check resource limits
ulimit -n  # File descriptor limit
cat /proc/sys/net/core/somaxconn  # Socket backlog

# Cause: Upstream server timed out

# 1. Diagnose timeout at each stage
curl -o /dev/null -s -w "connect: %{time_connect}s\nttfb: %{time_starttransfer}s\ntotal: %{time_total}s\n" \
  https://api.example.com/slow-endpoint

# 2. Check for slow queries (if the root cause is the database)
# MySQL
SHOW PROCESSLIST;
# PostgreSQL
SELECT pid, now() - pg_stat_activity.query_start AS duration, query
FROM pg_stat_activity
WHERE state = 'active' AND now() - query_start > interval '5 seconds';

# 3. Adjust timeout settings
# Nginx: proxy_read_timeout 120s;
# HAProxy: timeout server 120s

Browser                            Server
  |                                  |
  |--- Preflight (OPTIONS) --------->|
  |    Origin: https://app.com       |
  |    Access-Control-Request-Method: POST
  |                                  |
  |<-- 200 OK ----------------------|
  |    Access-Control-Allow-Origin: https://app.com
  |    Access-Control-Allow-Methods: POST, GET
  |    Access-Control-Allow-Headers: Content-Type
  |    Access-Control-Max-Age: 86400
  |                                  |
  |--- Actual Request (POST) ------->|
  |    Origin: https://app.com       |
  |                                  |
  |<-- 200 OK ----------------------|
  |    Access-Control-Allow-Origin: https://app.com

# Simulate a preflight request
curl -v -X OPTIONS https://api.example.com/data \
  -H "Origin: https://app.example.com" \
  -H "Access-Control-Request-Method: POST" \
  -H "Access-Control-Request-Headers: Content-Type, Authorization"

# Inspect CORS headers in the response
curl -v https://api.example.com/data \
  -H "Origin: https://app.example.com" \
  2>&1 | grep -i "access-control"

location /api/ {
    # Handle preflight requests
    if ($request_method = 'OPTIONS') {
        add_header 'Access-Control-Allow-Origin' '$http_origin' always;
        add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always;
        add_header 'Access-Control-Allow-Headers' 'Authorization, Content-Type, X-Requested-With' always;
        add_header 'Access-Control-Allow-Credentials' 'true' always;
        add_header 'Access-Control-Max-Age' 86400;
        add_header 'Content-Length' 0;
        return 204;
    }

    # Add CORS headers to actual responses
    add_header 'Access-Control-Allow-Origin' '$http_origin' always;
    add_header 'Access-Control-Allow-Credentials' 'true' always;

    proxy_pass http://backend;
}

# Trace the redirect chain
curl -v -L --max-redirs 20 https://example.com 2>&1 | \
  grep -E "^< (HTTP|Location)" | head -30

# Example output (infinite loop):
# < HTTP/1.1 301 Moved Permanently
# < Location: https://www.example.com/
# < HTTP/1.1 301 Moved Permanently
# < Location: https://example.com/
# < HTTP/1.1 301 Moved Permanently
# < Location: https://www.example.com/   ← Loop detected!

# Common causes:
# 1. HTTP→HTTPS redirect conflicting with HTTPS→HTTP redirect
# 2. www ↔ non-www redirect conflict
# 3. Load balancer SSL termination + application HTTPS enforcement

# Solution: Use the X-Forwarded-Proto header
# Pass protocol information from Nginx to the backend
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

# Check HTTP/2 support
curl -v --http2 https://example.com 2>&1 | grep "ALPN"

# Frame-level HTTP/2 debugging (using nghttp2)
nghttp -v https://example.com

# Verify HTTP/2 multiplexing
nghttp -v -m 10 https://example.com/style.css \
  https://example.com/script.js \
  https://example.com/image.png

# Check HTTP/2 server push
nghttp -v --stat https://example.com 2>&1 | grep "push"

# Test h2c (HTTP/2 Cleartext)
curl -v --http2-prior-knowledge http://localhost:8080/health

# Check HTTP/3 support
curl --http3 -v https://example.com 2>&1 | head -20

# Inspect Alt-Svc header (HTTP/3 advertisement)
curl -sI https://example.com | grep -i "alt-svc"
# Example: alt-svc: h3=":443"; ma=86400

# QUIC connection debugging (with quiche tools)
# Test 0-RTT connection
curl --http3 -v --connect-to example.com:443:server-ip:443 \
  https://example.com

# 1. HTTP/2 Head-of-Line (HOL) blocking
# HTTP/2 can suffer HOL blocking at the TCP layer
# Resolution: Evaluate migration to HTTP/3 (QUIC)

# 2. HTTP/2 GOAWAY frame debugging
nghttp -v https://example.com 2>&1 | grep "GOAWAY"

# 3. HPACK header compression issues
# Check dynamic table size for requests with large headers
nghttp -v --header-table-size=65536 https://example.com

# 4. Stream concurrency limits
# Check SETTINGS_MAX_CONCURRENT_STREAMS
nghttp -v https://example.com 2>&1 | grep "MAX_CONCURRENT"

# 1. Test the health endpoint directly
curl -v http://backend-server:8080/health

# 2. Test under the same conditions the load balancer uses
curl -v -H "Host: api.example.com" \
  --resolve api.example.com:80:10.0.1.10 \
  http://api.example.com/health

# 3. Check HAProxy status
echo "show stat" | socat stdio /var/run/haproxy/admin.sock | \
  awk -F',' '{print $1, $2, $18, $19}'

# 4. Nginx upstream status (with nginx-module-vts)
curl http://localhost/status/format/json | jq '.upstreamZones'

# Inspect cookie-based session affinity
curl -v -c cookies.txt https://app.example.com/login
curl -v -b cookies.txt https://app.example.com/dashboard

# Verify routing consistency to the same backend
for i in $(seq 1 5); do
  curl -s -b cookies.txt https://app.example.com/api/server-id
  echo ""
done

# Inspect proxy chain headers
curl -v https://api.example.com/debug-headers 2>&1

# Expected headers:
# X-Forwarded-For: client-ip, proxy1-ip, proxy2-ip
# X-Forwarded-Proto: https
# X-Forwarded-Host: api.example.com
# X-Real-IP: client-ip

# Test whether the proxy correctly forwards the original IP
curl -H "X-Forwarded-For: 1.2.3.4" https://api.example.com/my-ip

#!/bin/bash
# Intermittent 502 monitoring script
URL="https://api.example.com/health"
LOG_FILE="/tmp/502-monitor.log"

echo "=== 502 Monitoring Started: $(date) ===" >> $LOG_FILE

while true; do
  RESPONSE=$(curl -s -o /dev/null -w "%{http_code}|%{time_total}|%{time_connect}|%{time_starttransfer}" $URL)
  HTTP_CODE=$(echo $RESPONSE | cut -d'|' -f1)
  TOTAL_TIME=$(echo $RESPONSE | cut -d'|' -f2)
  CONNECT_TIME=$(echo $RESPONSE | cut -d'|' -f3)
  TTFB=$(echo $RESPONSE | cut -d'|' -f4)

  if [ "$HTTP_CODE" != "200" ]; then
    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
    echo "[$TIMESTAMP] HTTP $HTTP_CODE | Total: ${TOTAL_TIME}s | Connect: ${CONNECT_TIME}s | TTFB: ${TTFB}s" >> $LOG_FILE

    # Collect detailed information on 502 occurrence
    if [ "$HTTP_CODE" = "502" ]; then
      echo "  Detailed trace:" >> $LOG_FILE
      curl -v $URL 2>> $LOG_FILE
      echo "---" >> $LOG_FILE
    fi
  fi

  sleep 5
done

#!/bin/bash
# Certificate expiry monitoring script
DOMAINS=(
  "api.example.com"
  "app.example.com"
  "admin.example.com"
)
WARNING_DAYS=30

for domain in "${DOMAINS[@]}"; do
  EXPIRY=$(echo | openssl s_client -connect ${domain}:443 -servername ${domain} 2>/dev/null | \
    openssl x509 -noout -enddate 2>/dev/null | cut -d= -f2)

  if [ -z "$EXPIRY" ]; then
    echo "[ERROR] $domain: Connection failed"
    continue
  fi

  EXPIRY_EPOCH=$(date -d "$EXPIRY" +%s 2>/dev/null || date -j -f "%b %d %T %Y %Z" "$EXPIRY" +%s 2>/dev/null)
  NOW_EPOCH=$(date +%s)
  DAYS_LEFT=$(( (EXPIRY_EPOCH - NOW_EPOCH) / 86400 ))

  if [ $DAYS_LEFT -lt 0 ]; then
    echo "[CRITICAL] $domain: Certificate EXPIRED! ($EXPIRY)"
  elif [ $DAYS_LEFT -lt $WARNING_DAYS ]; then
    echo "[WARNING] $domain: Expires in ${DAYS_LEFT} days ($EXPIRY)"
  else
    echo "[OK] $domain: ${DAYS_LEFT} days remaining ($EXPIRY)"
  fi
done

#!/bin/bash
# Comprehensive HTTP diagnostic script
URL=${1:-"https://example.com"}
echo "============================================"
echo "HTTP Comprehensive Diagnostics: $URL"
echo "Timestamp: $(date)"
echo "============================================"

# 1. DNS resolution
echo -e "\n[1] DNS Resolution"
DOMAIN=$(echo $URL | awk -F[/:] '{print $4}')
dig +short $DOMAIN A
dig +short $DOMAIN AAAA

# 2. TCP connectivity
echo -e "\n[2] TCP Connection Check"
nc -zv $DOMAIN 443 2>&1

# 3. TLS information
echo -e "\n[3] TLS Information"
echo | openssl s_client -connect ${DOMAIN}:443 -servername ${DOMAIN} 2>/dev/null | \
  grep -E "Protocol|Cipher|Verify"

# 4. Certificate details
echo -e "\n[4] Certificate Details"
echo | openssl s_client -connect ${DOMAIN}:443 -servername ${DOMAIN} 2>/dev/null | \
  openssl x509 -noout -subject -issuer -dates 2>/dev/null

# 5. HTTP response timing
echo -e "\n[5] HTTP Response"
curl -s -o /dev/null -w "\
  HTTP Code:      %{http_code}\n\
  DNS Lookup:     %{time_namelookup}s\n\
  TCP Connect:    %{time_connect}s\n\
  TLS Handshake:  %{time_appconnect}s\n\
  TTFB:           %{time_starttransfer}s\n\
  Total Time:     %{time_total}s\n\
  Download Size:  %{size_download} bytes\n\
  HTTP Version:   %{http_version}\n\
  Remote IP:      %{remote_ip}\n" $URL

# 6. Response headers
echo -e "\n[6] Key Response Headers"
curl -sI $URL | grep -iE "^(server|x-|content-type|cache-control|strict|location|set-cookie)"

# 7. Security headers audit
echo -e "\n[7] Security Headers Audit"
HEADERS=$(curl -sI $URL)
for header in "Strict-Transport-Security" "X-Content-Type-Options" "X-Frame-Options" "Content-Security-Policy" "X-XSS-Protection"; do
  if echo "$HEADERS" | grep -qi "$header"; then
    echo "  [PASS] $header"
  else
    echo "  [FAIL] $header (missing)"
  fi
done

echo -e "\n============================================"
echo "Diagnostics Complete"
echo "============================================"