The Complete Guide to Container & Kubernetes Network Debugging

# Inspect the bridge network details
docker network inspect bridge

# Check a container's IP address
docker inspect --format '{{.NetworkSettings.IPAddress}}' <container_id>

# List veth pairs
ip link show type veth

# Show interfaces connected to docker0
brctl show docker0

# Check iptables NAT rules
sudo iptables -t nat -L -n -v

# Verify IP forwarding is enabled
cat /proc/sys/net/ipv4/ip_forward

# Check port bindings
docker port <container_id>

# Run a container in host network mode
docker run --network host nginx

# Verify the container sees host interfaces
docker exec <container_id> ip addr show

# Confirm shared network stack
docker exec <container_id> ss -tlnp

# Create an overlay network
docker network create --driver overlay my-overlay

# Check VXLAN tunnel status
ip -d link show type vxlan

# Inspect overlay network peer information
docker network inspect my-overlay --format '{{json .Peers}}'

# Check the cluster network CIDR
kubectl cluster-info dump | grep -m 1 cluster-cidr

# Get Pod CIDRs assigned to each node
kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}'

# Identify the CNI plugin in use
ls /etc/cni/net.d/
cat /etc/cni/net.d/*.conflist

# Locate CNI binaries
ls /opt/cni/bin/

# Inspect the CNI configuration
cat /etc/cni/net.d/10-calico.conflist

# Check kubelet logs for CNI-related events
journalctl -u kubelet | grep -i cni

# Get Pod IPs
kubectl get pods -o wide

# Ping from one Pod to another
kubectl exec -it <pod-a> -- ping <pod-b-ip>

# Trace the route between Pods on the same node
kubectl exec -it <pod-a> -- traceroute <pod-b-ip>

# Inspect veth interfaces on the node
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- ip link show

# Check the routing table on the node
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- route -n

# Verify VXLAN or IPIP tunnel status
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- ip tunnel show

# Capture packets with tcpdump
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- \
  tcpdump -i any -nn host <target-pod-ip>

# Test for MTU issues (Path MTU Discovery)
kubectl exec -it <pod> -- ping -M do -s 1400 <target-pod-ip>

# Check the Service's ClusterIP and Endpoints
kubectl get svc <service-name> -o wide
kubectl get endpoints <service-name>

# Inspect the Endpoints in detail
kubectl get endpoints <service-name> -o yaml

# Test connectivity to the Service
kubectl exec -it <test-pod> -- curl -v http://<service-name>:<port>

# Verify Pods matching the Service selector
kubectl get pods -l <label-selector>

# Determine the kube-proxy mode
kubectl get configmap kube-proxy -n kube-system -o yaml | grep mode

# iptables mode: inspect Service-related rules
sudo iptables -t nat -L KUBE-SERVICES -n
sudo iptables -t nat -L KUBE-SVC-<hash> -n
sudo iptables -t nat -L KUBE-SEP-<hash> -n

# IPVS mode: list virtual servers
sudo ipvsadm -Ln
sudo ipvsadm -Ln -t <cluster-ip>:<port>

# Check kube-proxy logs
kubectl logs -n kube-system -l k8s-app=kube-proxy --tail=100

PREROUTING -> KUBE-SERVICES -> KUBE-SVC-xxx -> KUBE-SEP-xxx (DNAT)

# Trace the full iptables chain for a specific Service
SERVICE_IP=$(kubectl get svc <service-name> -o jsonpath='{.spec.clusterIP}')
sudo iptables -t nat -L KUBE-SERVICES -n | grep $SERVICE_IP

# Inspect conntrack entries for the Service
sudo conntrack -L -d $SERVICE_IP

# Check CoreDNS Pod status
kubectl get pods -n kube-system -l k8s-app=kube-dns

# Inspect CoreDNS configuration
kubectl get configmap coredns -n kube-system -o yaml

# Test DNS resolution
kubectl exec -it <test-pod> -- nslookup <service-name>
kubectl exec -it <test-pod> -- nslookup <service-name>.<namespace>.svc.cluster.local

# Measure DNS response times
kubectl exec -it <test-pod> -- dig <service-name>.default.svc.cluster.local +stats

# Check the Pod's DNS configuration
kubectl exec -it <pod> -- cat /etc/resolv.conf

# Review CoreDNS logs for errors
kubectl logs -n kube-system -l k8s-app=kube-dns --tail=200

# Deploy a DNS debugging Pod
kubectl run dnsutils --image=gcr.io/kubernetes-e2e-test-images/dnsutils:1.3 \
  --restart=Never -- sleep 3600

# Test external domain resolution
kubectl exec -it dnsutils -- nslookup google.com
kubectl exec -it dnsutils -- nslookup kubernetes.default

# Demonstrate the ndots effect: compare query counts
kubectl exec -it <pod> -- dig myservice.default.svc.cluster.local +search +showsearch
kubectl exec -it <pod> -- dig myservice.default.svc.cluster.local. +search +showsearch

# List all Ingress resources
kubectl get ingress -A
kubectl describe ingress <ingress-name>

# Check Ingress Controller logs
kubectl logs -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx --tail=100

# Verify LoadBalancer Service external IPs
kubectl get svc -l type=LoadBalancer

# Test NodePort access externally
curl -v http://<node-ip>:<node-port>

# Verify outbound connectivity from a Pod
kubectl exec -it <pod> -- curl -v https://httpbin.org/ip

# Check NAT gateway and routing
kubectl exec -it <pod> -- traceroute 8.8.8.8

# Inspect SNAT rules
sudo iptables -t nat -L POSTROUTING -n -v

# List all Network Policies
kubectl get networkpolicy -A
kubectl describe networkpolicy <policy-name> -n <namespace>

# Find Network Policies applying to a specific Pod
kubectl get networkpolicy -n <namespace> -o json | \
  jq '.items[] | select(.spec.podSelector.matchLabels | to_entries[] |
  .key == "app" and .value == "myapp")'

# Test connectivity before and after policy application
kubectl exec -it <source-pod> -- nc -zv <target-pod-ip> <port>

# Default deny-all policy
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress

# Test connectivity after applying the policy
kubectl exec -it <pod> -- wget -qO- --timeout=2 http://<target-service>

# Check CNI-specific Network Policy logs
# Calico
kubectl logs -n calico-system -l k8s-app=calico-node --tail=100

# Cilium
kubectl exec -n kube-system <cilium-pod> -- cilium policy get
kubectl exec -n kube-system <cilium-pod> -- cilium endpoint list

# Verify Calico node health
kubectl get pods -n calico-system
calicoctl node status

# Check BGP peer status
calicoctl get bgpPeer
calicoctl node status | grep -A 5 "BGP"

# Inspect IP pools
calicoctl get ippool -o wide

# List Calico-specific network policies
calicoctl get networkpolicy -A
calicoctl get globalnetworkpolicy

# Increase Felix log level for debugging
calicoctl patch felixconfiguration default \
  --patch '{"spec":{"logSeverityScreen":"Debug"}}'

# Check BIRD routing daemon status
kubectl exec -n calico-system <calico-node-pod> -- birdcl show route

# Inspect IP-in-IP tunnel status
kubectl exec -n calico-system <calico-node-pod> -- ip tunnel show

# Review Calico-programmed iptables rules
sudo iptables -L -n | grep cali

# Get detailed Cilium agent status
kubectl exec -n kube-system <cilium-pod> -- cilium status --verbose

# List all Cilium endpoints
kubectl exec -n kube-system <cilium-pod> -- cilium endpoint list

# Inspect BPF maps
kubectl exec -n kube-system <cilium-pod> -- cilium bpf lb list
kubectl exec -n kube-system <cilium-pod> -- cilium bpf ct list global

# Observe network flows with Hubble
hubble observe --namespace <namespace>
hubble observe --pod <pod-name> --protocol TCP

# Run Cilium connectivity test
kubectl exec -n kube-system <cilium-pod> -- cilium connectivity test

# Check eBPF program status
kubectl exec -n kube-system <cilium-pod> -- cilium bpf prog list

# Trace policy decisions between Pods
kubectl exec -n kube-system <cilium-pod> -- cilium policy trace \
  --src-k8s-pod <namespace>:<src-pod> \
  --dst-k8s-pod <namespace>:<dst-pod> \
  --dport <port>

# Monitor events in real time
kubectl exec -n kube-system <cilium-pod> -- cilium monitor --type drop
kubectl exec -n kube-system <cilium-pod> -- cilium monitor --type policy-verdict

# Attach a debug container to a running Pod
kubectl debug -it <pod-name> --image=nicolaka/netshoot --target=<container-name>

# Debug at the node level
kubectl debug node/<node-name> -it --image=ubuntu

# Create a copy of a Pod for debugging (no impact on original)
kubectl debug <pod-name> -it --copy-to=debug-pod --image=nicolaka/netshoot

# Debug container with shared process namespace
kubectl debug -it <pod-name> --image=busybox --share-processes

# Deploy a netshoot container
kubectl run netshoot --image=nicolaka/netshoot --restart=Never -- sleep 3600

# Enter the container for comprehensive diagnostics
kubectl exec -it netshoot -- bash

# Commands to run inside the container:
# TCP connectivity test
nc -zv <service-ip> <port>

# HTTP request test
curl -v http://<service-name>.<namespace>.svc.cluster.local:<port>/health

# DNS lookup
dig +short <service-name>.<namespace>.svc.cluster.local

# Packet capture
tcpdump -i eth0 -nn port 80

# Network path tracing
mtr <target-ip>

# SSL/TLS connection verification
openssl s_client -connect <service-ip>:443

# Step 1: Verify the Service and its Endpoints
kubectl get svc <service-name> -o yaml
kubectl get endpoints <service-name>

# Step 2: If Endpoints are empty, check the selector
kubectl get pods -l <label-selector> --show-labels

# Step 3: Verify target Pods are Ready
kubectl get pods -l <label-selector> -o wide
kubectl describe pod <target-pod> | grep -A 5 "Conditions"

# Step 4: Inspect kube-proxy rules
sudo iptables -t nat -L KUBE-SERVICES -n | grep <cluster-ip>

# Step 5: Test direct Pod IP connectivity
kubectl exec -it <source-pod> -- curl -v http://<pod-ip>:<target-port>

# Step 1: Check conntrack table saturation
sudo sysctl net.netfilter.nf_conntrack_count
sudo sysctl net.netfilter.nf_conntrack_max

# Step 2: Look for packet drops
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- \
  netstat -s | grep -i drop

# Step 3: Check network interface errors
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- \
  ip -s link show

# Step 4: Monitor TCP retransmissions
kubectl debug node/<node-name> -it --image=nicolaka/netshoot -- \
  netstat -s | grep -i retrans

# Step 1: Measure DNS response times
kubectl exec -it <pod> -- dig <service-name>.default.svc.cluster.local +stats | grep "Query time"

# Step 2: Check CoreDNS resource usage
kubectl top pods -n kube-system -l k8s-app=kube-dns

# Step 3: Apply ndots optimization
# Add to Pod spec:
# dnsConfig:
#   options:
#   - name: ndots
#     value: "2"

# Step 4: Check CoreDNS cache hit rates
kubectl exec -n kube-system <coredns-pod> -- \
  wget -qO- http://localhost:9153/metrics | grep coredns_cache

[ ] Pod status verification (Running, Ready)
[ ] Service Endpoints verification (not empty)
[ ] DNS resolution test (nslookup/dig)
[ ] Direct Pod IP connectivity test
[ ] kube-proxy rules inspection (iptables/IPVS)
[ ] Network Policy review (Ingress/Egress rules)
[ ] CNI plugin health check
[ ] Cross-node network connectivity
[ ] MTU configuration verification
[ ] Firewall / security group rules
[ ] conntrack table status
[ ] CoreDNS health and logs