필사 모드: 隔离网络 Kubernetes 的 Day 2 运维 — 阻止集群因证书到期而死掉的方法

# 在控制平面节点上
sudo kubeadm certs check-expiration

# kubelet 证书要单独确认
sudo ls -la /var/lib/kubelet/pki/
sudo openssl x509 -in /var/lib/kubelet/pki/kubelet-client-current.pem -noout -enddate

sudo k3s certificate check --output table

# 查询到期警告事件
sudo k3s kubectl get events -A --field-selector reason=CertificateExpirationWarning

#!/usr/bin/env bash
# cert-watch.sh — 每天执行。距到期 60 天以内就用非正常退出来告警。
set -uo pipefail
THRESHOLD_DAYS=60
NOW=$(date +%s)
WARN=0

check_pem() {
  local f="$1"
  [ -f "${f}" ] || return 0
  local end epoch left
  end=$(openssl x509 -in "${f}" -noout -enddate 2>/dev/null | cut -d= -f2) || return 0
  epoch=$(date -d "${end}" +%s 2>/dev/null) || return 0
  left=$(( (epoch - NOW) / 86400 ))
  if [ "${left}" -lt "${THRESHOLD_DAYS}" ]; then
    echo "WARN  剩余 ${left} 天  ${f}  (${end})"
    WARN=$((WARN+1))
  else
    echo "ok    剩余 ${left} 天  ${f}"
  fi
}

# kubeadm 系
for f in /etc/kubernetes/pki/*.crt /etc/kubernetes/pki/etcd/*.crt; do
  check_pem "${f}"
done
# kubelet
check_pem /var/lib/kubelet/pki/kubelet-client-current.pem
# k3s 系
for f in /var/lib/rancher/k3s/server/tls/*.crt; do
  check_pem "${f}"
done

echo "警告 ${WARN} 件"
exit "${WARN}"

# 全部续期
sudo kubeadm certs renew all

# 也可以单个续期
sudo kubeadm certs renew apiserver
sudo kubeadm certs renew admin.conf

# 让静态 Pod 重启的最简单方法 — 把清单挪走一下再放回来
sudo mkdir -p /tmp/kube-manifests
sudo mv /etc/kubernetes/manifests/*.yaml /tmp/kube-manifests/
sleep 20
sudo mv /tmp/kube-manifests/*.yaml /etc/kubernetes/manifests/

# 让新的 admin.conf 生效
sudo cp /etc/kubernetes/admin.conf "${HOME}/.kube/config"
sudo chown "$(id -u):$(id -g)" "${HOME}/.kube/config"

# 确认
sudo kubeadm certs check-expiration
kubectl get nodes

apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
certificateValidityPeriod: 8760h # 默认 365 天
caCertificateValidityPeriod: 87600h # 默认 3650 天

# 诱发自动续期的最简单方法 — 重启
sudo systemctl restart k3s

# 显式轮换
sudo systemctl stop k3s
sudo k3s certificate rotate
sudo systemctl start k3s

# 只轮换特定服务
sudo k3s certificate rotate --service api-server,etcd,kubelet

# 自签名 CA 轮换 — 用交叉签名维持信任链
# （脚本必须在带入时一起带进来。隔离网络里没法用 curl 取。）
sudo bash /opt/staging/rotate-default-ca-certs.sh
sudo k3s certificate rotate-ca --path=/var/lib/rancher/k3s/server/rotate-ca

# 换成内部 CA 的情况 — 在单独的目录里准备好之后再应用
sudo k3s certificate rotate-ca --path=/opt/k3s/server
# 应用后要在所有节点上重启 k3s。
# 要换掉根 CA 本身则需要 --force，并且必须先重设已加入节点的 token。

# 确认当前的同步状态 — 这里是隔离网络点检的第 1 项
timedatectl status
chronyc sources -v 2>/dev/null || ntpq -p 2>/dev/null
chronyc tracking 2>/dev/null

# 固定到内部 NTP 服务器
sudo tee /etc/chrony.d/internal.conf > /dev/null <<'EOF'
server 10.10.10.11 iburst prefer
server 10.10.10.12 iburst
makestep 1.0 3
EOF

# 必须删掉指向公共池的默认配置（路径因发行版而异）
sudo grep -rn "pool\|ntp.org" /etc/chrony.conf /etc/chrony.d/ 2>/dev/null

sudo systemctl restart chronyd
sleep 5
chronyc sources -v

# 一次性查看集群全部节点的时刻偏差
for n in $(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'); do
  printf "%-20s " "${n}"
  ssh "${n}" "chronyc tracking 2>/dev/null | awk '/System time/{print \$4, \$5, \$6}'" || echo "unreachable"
done

# 确认 CoreDNS 在看着什么
kubectl -n kube-system get configmap coredns -o yaml

# 节点的解析器
cat /etc/resolv.conf

# 确认内部 DNS 的应答
dig @10.10.10.53 registry.internal.example +short

# /etc/rancher/k3s/config.yaml — 隔离网络推荐配置
etcd-snapshot-schedule-cron: '0 */6 * * *'
etcd-snapshot-retention: 12
etcd-snapshot-compress: true
etcd-snapshot-dir: /var/backups/k3s-snapshots

# 按需快照 — 升级或危险变更之前务必执行
sudo k3s etcd-snapshot save

# 列表与清理
sudo k3s etcd-snapshot ls
sudo k3s etcd-snapshot prune --snapshot-retention 12
sudo k3s etcd-snapshot delete on-demand-node1-1753900000

# 把快照和 token 当作一套来保管
sudo install -m 0600 /var/lib/rancher/k3s/server/token \
  /var/backups/k3s-snapshots/token-$(date +%Y%m%d)

# 把备份集推到集群之外的存储（比如隔离网络内部另外的 NAS）
sudo rsync -a --delete /var/backups/k3s-snapshots/ backup-nas:/k3s/prod/

# 单台 Server
sudo systemctl stop k3s
sudo k3s server \
  --cluster-reset \
  --cluster-reset-restore-path=/var/backups/k3s-snapshots/etcd-snapshot-node1-1753900000
# 上面的命令打印出完成消息之后用 Ctrl-C 退出，然后
sudo systemctl start k3s

# 多台 Server — 必须守住顺序
# 1) 在所有 Server 上停止 k3s
sudo systemctl stop k3s

# 2) 只在第一台 Server 上执行 cluster-reset 恢复（上面的命令）

# 3) 启动第一台 Server
sudo systemctl start k3s

# 4) 在其余 Server 上删除数据目录 — 漏掉这一步它们就合流不进来
sudo rm -rf /var/lib/rancher/k3s/server/db/

# 5) 启动其余 Server
sudo systemctl start k3s

# 快照
sudo ETCDCTL_API=3 etcdctl snapshot save /var/backups/etcd-$(date +%Y%m%d%H%M).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# 确认快照的完整性 — 只保存却不确认的团队真的很多
sudo ETCDCTL_API=3 etcdctl snapshot status \
  /var/backups/etcd-*.db --write-out=table

# 带入计划的核算 — 确认当前版本与目标版本之间的阶段
kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.nodeInfo.kubeletVersion}{"\n"}{end}'
kubectl version -o json | jq -r '.serverVersion.gitVersion'

/opt/staging/upgrade-2026Q3/
├── VERSION                       # 目标版本一行
├── MANIFEST.sha256
├── step1-v1.35.7/
│   ├── k3s
│   └── k3s-airgap-images-amd64.tar.zst
├── step2-v1.36.2/
│   ├── k3s
│   └── k3s-airgap-images-amd64.tar.zst
├── addons/                       # 这一轮一起升级的附加组件镜像
└── ROLLBACK.md                   # 各阶段的回滚判断基准与命令

#!/usr/bin/env bash
# pre-upgrade.sh
set -uo pipefail
ERR=0

echo "[1] 快照"
sudo k3s etcd-snapshot save || ERR=$((ERR+1))

echo "[2] token 备份"
sudo cp /var/lib/rancher/k3s/server/token "/var/backups/token-$(date +%Y%m%d%H%M)" || ERR=$((ERR+1))

echo "[3] 打包完整性"
(cd /opt/staging/upgrade-2026Q3 && sha256sum -c MANIFEST.sha256) || ERR=$((ERR+1))

echo "[4] 节点状态"
kubectl get nodes --no-headers | grep -v " Ready " && ERR=$((ERR+1))

echo "[5] PodDisruptionBudget"
kubectl get pdb -A

echo "[6] 证书剩余期限"
sudo k3s certificate check --output table

echo "错误 ${ERR} 件"
exit "${ERR}"

sudo systemctl stop k3s
sudo rm -f /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64.tar.zst
sudo rm -f /var/lib/rancher/k3s/agent/images/.cache.json
sudo cp /opt/staging/upgrade-2026Q3/step2-v1.36.2/k3s-airgap-images-amd64.tar.zst \
  /var/lib/rancher/k3s/agent/images/
sudo install -m 0755 /opt/staging/upgrade-2026Q3/step2-v1.36.2/k3s /usr/local/bin/k3s
sudo systemctl start k3s
sudo k3s kubectl get nodes -o wide

# DMZ 采集阶段 — 把仪表盘 JSON 落成文件
mkdir -p dashboards
curl -fL -o dashboards/node-exporter.json \
  "https://grafana.com/api/dashboards/1860/revisions/latest/download"
sha256sum dashboards/*.json >> MANIFEST.sha256

# 隔离网络内部 — 用 provisioning 部署（ConfigMap 挂载方式）
apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-dashboard-provider
  namespace: monitoring
data:
  provider.yaml: |
    apiVersion: 1
    providers:
      - name: airgap
        orgId: 1
        folder: 'Airgap'
        type: file
        disableDeletion: true
        options:
          path: /var/lib/grafana/dashboards

# Alertmanager — 只走内部 SMTP 出去的配置
route:
  receiver: internal-mail
  group_wait: 30s
  repeat_interval: 4h
receivers:
  - name: internal-mail
    email_configs:
      - to: 'k8s-ops@internal.example'
        from: 'alertmanager@internal.example'
        smarthost: 'smtp.internal.example:25'
        require_tls: false

#!/usr/bin/env bash
# support-bundle.sh — 考虑了带出审批的诊断包
set -uo pipefail
TS="$(date +%Y%m%d-%H%M)"
DIR="/var/tmp/support-${TS}"
mkdir -p "${DIR}"/{cluster,nodes,logs}
K="sudo k3s kubectl"

echo "== 集群资源（不含 Secret）"
${K} get nodes -o wide            > "${DIR}/cluster/nodes.txt" 2>&1
${K} get pods -A -o wide          > "${DIR}/cluster/pods.txt" 2>&1
${K} get events -A --sort-by=.lastTimestamp > "${DIR}/cluster/events.txt" 2>&1
${K} describe nodes               > "${DIR}/cluster/nodes-describe.txt" 2>&1
${K} api-resources                > "${DIR}/cluster/api-resources.txt" 2>&1
${K} version -o json              > "${DIR}/cluster/version.json" 2>&1
# Secret 只取名字 — 值绝对不装进去
${K} get secrets -A -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,TYPE:.type \
                                  > "${DIR}/cluster/secrets-list.txt" 2>&1

echo "== 只取失败中的 Pod 的日志"
${K} get pods -A --field-selector=status.phase!=Running --no-headers 2>/dev/null \
| while read -r ns name _; do
    ${K} -n "${ns}" logs "${name}" --all-containers --tail=500 \
      > "${DIR}/logs/${ns}_${name}.log" 2>&1 || true
    ${K} -n "${ns}" describe pod "${name}" \
      > "${DIR}/logs/${ns}_${name}.describe" 2>&1 || true
  done

echo "== 节点信息"
uname -a                          > "${DIR}/nodes/uname.txt" 2>&1
free -h                           > "${DIR}/nodes/mem.txt" 2>&1
df -h                             > "${DIR}/nodes/disk.txt" 2>&1
timedatectl status                > "${DIR}/nodes/time.txt" 2>&1
chronyc tracking                  > "${DIR}/nodes/chrony.txt" 2>&1
sudo journalctl -u k3s -n 5000 --no-pager   > "${DIR}/logs/k3s.log" 2>&1
sudo tail -n 5000 /var/lib/rancher/k3s/agent/containerd/containerd.log \
                                  > "${DIR}/logs/containerd.log" 2>&1
sudo k3s certificate check --output table   > "${DIR}/cluster/certs.txt" 2>&1

echo "== 带出前的点检 — 检测 token 与密钥模式"
grep -rIlE 'BEGIN (RSA |EC )?PRIVATE KEY|K10[0-9a-f]{16,}|password:\s*\S+' "${DIR}" \
  > "${DIR}/REDACTION-REVIEW.txt" 2>/dev/null || true
echo "带出之前请务必审阅下面这些文件:"
cat "${DIR}/REDACTION-REVIEW.txt"

tar -czf "/var/tmp/support-${TS}.tar.gz" -C /var/tmp "support-${TS}"
sha256sum "/var/tmp/support-${TS}.tar.gz"
echo "生成完成: /var/tmp/support-${TS}.tar.gz"