k3s 隔离网络安装完全指南 — 从镜像 tarball 带入到私有仓库、Agent 加入

#!/usr/bin/env bash
# collect-k3s.sh — 在能上网的暂存机上执行
set -euo pipefail

K3S_VERSION="v1.36.2+k3s1"
ARCH="amd64"
# URL 路径中的 + 必须编码为 %2B
URLVER="${K3S_VERSION/+/%2B}"
BASE="https://github.com/k3s-io/k3s/releases/download/${URLVER}"
OUT="./k3s-airgap-${K3S_VERSION}"

mkdir -p "${OUT}"
cd "${OUT}"

# 1) 系统镜像归档包（推荐 zstd，节省介质容量）
curl -fL -o "k3s-airgap-images-${ARCH}.tar.zst" \
  "${BASE}/k3s-airgap-images-${ARCH}.tar.zst"

# 2) k3s 二进制文件
curl -fL -o k3s "${BASE}/k3s"

# 3) 安装脚本（与 SKIP_DOWNLOAD 搭配使用，让执行时不动用网络）
curl -fL -o install.sh https://get.k3s.io

# 4) 校验和文件（发布页面有提供的情况下）
curl -fL -o "sha256sum-${ARCH}.txt" "${BASE}/sha256sum-${ARCH}.txt" || \
  echo "WARN: 发布页面没有 sha256sum-${ARCH}.txt。请用自行计算的值代替。"

ls -la

# 生成自制校验和清单（连通网络区段）
cd "./k3s-airgap-v1.36.2+k3s1"
sha256sum k3s k3s-airgap-images-amd64.tar.zst install.sh > MANIFEST.sha256
cat MANIFEST.sha256

# 在隔离网络节点上
cd /opt/staging/k3s-airgap-v1.36.2+k3s1

# 1) 校验和验证 — 这一步失败就不再往下走
sha256sum -c MANIFEST.sha256

# 2) 确认归档包本身能打开（需要 zstd）
zstd -t k3s-airgap-images-amd64.tar.zst && echo "archive OK"

# 3) 确认归档包里装着的镜像列表
zstd -dc k3s-airgap-images-amd64.tar.zst | tar -tf - | grep -E 'manifest|repositories' | head

#!/usr/bin/env bash
# install-k3s-server.sh — 隔离网络 Server 节点
set -euo pipefail

STAGE=/opt/staging/k3s-airgap-v1.36.2+k3s1

# 1) 放置系统镜像归档包
sudo mkdir -p /var/lib/rancher/k3s/agent/images/
sudo cp "${STAGE}/k3s-airgap-images-amd64.tar.zst" /var/lib/rancher/k3s/agent/images/

# 2) 启用条件式导入缓存（v1.33.1+k3s1 以上）
#    有这个文件的话，只要归档包没变，重启时就会跳过重新导入
sudo touch /var/lib/rancher/k3s/agent/images/.cache.json

# 3) 放置二进制文件
sudo cp "${STAGE}/k3s" /usr/local/bin/k3s
sudo chmod +x /usr/local/bin/k3s

# 4) 执行安装脚本 — 指示它跳过下载
sudo chmod +x "${STAGE}/install.sh"
sudo INSTALL_K3S_SKIP_DOWNLOAD=true "${STAGE}/install.sh"

sudo INSTALL_K3S_SKIP_DOWNLOAD=true \
  INSTALL_K3S_EXEC="server --cluster-init --tls-san 10.10.20.10 --tls-san k8s-api.internal.example --write-kubeconfig-mode 0644 --disable traefik" \
  /opt/staging/k3s-airgap-v1.36.2+k3s1/install.sh

# /etc/rancher/k3s/config.yaml
cluster-init: true
tls-san:
  - 10.10.20.10
  - k8s-api.internal.example
write-kubeconfig-mode: '0644'
disable:
  - traefik
node-label:
  - topology.kubernetes.io/zone=dc-a

sudo systemctl status k3s --no-pager
sudo k3s kubectl get nodes -o wide
sudo k3s kubectl -n kube-system get pods

# 直接在 containerd 存储里确认镜像导入是否真的发生了
sudo k3s ctr images ls | awk '{print $1}' | sort -u | head -20

# 在 Server 节点上确认 token
sudo cat /var/lib/rancher/k3s/server/node-token

#!/usr/bin/env bash
# install-k3s-agent.sh — 隔离网络 Agent 节点
set -euo pipefail

STAGE=/opt/staging/k3s-airgap-v1.36.2+k3s1
SERVER_URL="https://10.10.20.10:6443"
JOIN_TOKEN="K10xxxxxxxx::server:xxxxxxxx"

sudo mkdir -p /var/lib/rancher/k3s/agent/images/
sudo cp "${STAGE}/k3s-airgap-images-amd64.tar.zst" /var/lib/rancher/k3s/agent/images/
sudo touch /var/lib/rancher/k3s/agent/images/.cache.json
sudo cp "${STAGE}/k3s" /usr/local/bin/k3s
sudo chmod +x /usr/local/bin/k3s

sudo INSTALL_K3S_SKIP_DOWNLOAD=true \
  K3S_URL="${SERVER_URL}" \
  K3S_TOKEN="${JOIN_TOKEN}" \
  "${STAGE}/install.sh"

# Server 1 — 集群初始化（在 config.yaml 里写 cluster-init: true 也一样）
sudo INSTALL_K3S_SKIP_DOWNLOAD=true \
  K3S_TOKEN="共享密钥" \
  INSTALL_K3S_EXEC="server --cluster-init --tls-san 10.10.20.9" \
  ./install.sh

# Server 2、3 — 加入既有的 Server
sudo INSTALL_K3S_SKIP_DOWNLOAD=true \
  K3S_TOKEN="共享密钥" \
  INSTALL_K3S_EXEC="server --server https://10.10.20.10:6443 --tls-san 10.10.20.9" \
  ./install.sh

# /etc/rancher/k3s/registries.yaml
mirrors:
  docker.io:
    endpoint:
      - 'https://registry.internal.example:5000'
  registry.k8s.io:
    endpoint:
      - 'https://registry.internal.example:5000'
  ghcr.io:
    endpoint:
      - 'https://registry.internal.example:5000'
    rewrite:
      '^(.*)': 'mirror/ghcr/$1'

configs:
  'registry.internal.example:5000':
    auth:
      username: k3s-puller
      password: '带入时替换'
    tls:
      ca_file: /etc/rancher/k3s/certs/internal-ca.crt

# 把所有仓库都指向内部镜像仓库（通配符）
mirrors:
  '*':
    endpoint:
      - 'https://registry.internal.example:5000'

configs:
  'registry.internal.example:5000':
    tls:
      ca_file: /etc/rancher/k3s/certs/internal-ca.crt

# 1) 给 containerd 用 — registries.yaml 里 ca_file 指向的位置
sudo mkdir -p /etc/rancher/k3s/certs
sudo cp /opt/staging/internal-ca.crt /etc/rancher/k3s/certs/internal-ca.crt
sudo chmod 644 /etc/rancher/k3s/certs/internal-ca.crt

# 2) OS 信任存储 — 给 helm、skopeo、crictl、curl 等其他客户端用
#    RHEL 系
sudo cp /opt/staging/internal-ca.crt /etc/pki/ca-trust/source/anchors/
sudo update-ca-trust extract
#    Debian 系
sudo cp /opt/staging/internal-ca.crt /usr/local/share/ca-certificates/internal-ca.crt
sudo update-ca-certificates

# 3) registries.yaml 的变更必须重启才会生效 — 在所有节点上
sudo systemctl restart k3s        # Server 节点
sudo systemctl restart k3s-agent  # Agent 节点

# 确认仓库配置是否已反映到 containerd
sudo k3s ctr --namespace k8s.io images pull \
  registry.internal.example:5000/library/busybox:1.36

# 失败的话就看 containerd 日志（而不是 kubelet 的消息）
sudo tail -n 100 /var/lib/rancher/k3s/agent/containerd/containerd.log

# /etc/rancher/k3s/config.yaml（所有 Server 节点）
embedded-registry: true

# /etc/rancher/k3s/registries.yaml（所有节点）
mirrors:
  '*':

# 在连通网络区段：把需要的镜像打成一个归档包
docker pull myapp/api:1.4.2
docker pull myapp/worker:1.4.2
docker save -o myapp-images.tar myapp/api:1.4.2 myapp/worker:1.4.2
sha256sum myapp-images.tar >> MANIFEST.sha256

# 在隔离网络节点上：方法 A — 放到镜像目录里然后重启
sudo cp myapp-images.tar /var/lib/rancher/k3s/agent/images/
sudo systemctl restart k3s

# 方法 B — 不重启，立刻导入
sudo k3s ctr --namespace k8s.io images import myapp-images.tar
sudo k3s ctr --namespace k8s.io images ls | grep myapp

# 应对 A — 真正的原因在 containerd 日志里
sudo grep -iE 'failed|error' /var/lib/rancher/k3s/agent/containerd/containerd.log | tail -40

# 应对 B — /etc/rancher/k3s/config.yaml
# 对已配置镜像源的仓库，关掉默认端点回退
disable-default-registry-endpoint: true

# 确认 CoreDNS 把什么当作 forward 对象
sudo k3s kubectl -n kube-system get configmap coredns -o yaml

# 确认节点的解析器
cat /etc/resolv.conf
sudo resolvectl status 2>/dev/null | head -30

# 从节点上直接确认内部 DNS 是否真的会应答
dig @10.10.10.53 registry.internal.example +short

# 单独放一份隔离网络专用的解析器文件
sudo tee /etc/rancher/k3s/resolv.conf > /dev/null <<'EOF'
nameserver 10.10.10.53
nameserver 10.10.10.54
search internal.example
options timeout:1 attempts:2
EOF

# /etc/rancher/k3s/config.yaml
resolv-conf: /etc/rancher/k3s/resolv.conf

# 1) 二进制文件版本
k3s -v

# 2) 镜像目录里堆了些什么
ls -la /var/lib/rancher/k3s/agent/images/

# 3) containerd 存储里的系统镜像标签
sudo k3s ctr --namespace k8s.io images ls | grep -E 'coredns|pause|local-path|metrics-server'

# 清理 — 删掉旧归档包，只留下新归档包，然后重启
sudo systemctl stop k3s
sudo rm -f /var/lib/rancher/k3s/agent/images/k3s-airgap-images-amd64-v1.35.6.tar.zst
sudo rm -f /var/lib/rancher/k3s/agent/images/.cache.json
sudo cp /opt/staging/k3s-airgap-images-amd64.tar.zst /var/lib/rancher/k3s/agent/images/
sudo systemctl start k3s

# 在带入目录名与文件名两边都刻上版本
/opt/staging/k3s-v1.36.2+k3s1/
├── MANIFEST.sha256
├── VERSION            # v1.36.2+k3s1 一行
├── install.sh
├── k3s
└── k3s-airgap-images-amd64.tar.zst

# 在安装脚本里放一道版本闸门
EXPECTED="$(cat "${STAGE}/VERSION")"
ACTUAL="$(/usr/local/bin/k3s -v | awk '/^k3s version/{print $3}')"
if [ "${ACTUAL}" != "${EXPECTED}" ]; then
  echo "FATAL: 二进制文件 ${ACTUAL} 与带入包 ${EXPECTED} 不一致" >&2
  exit 1
fi

#!/usr/bin/env bash
# verify-airgap-k3s.sh
set -uo pipefail
ERR=0
K="sudo k3s kubectl"

echo "== 1. 节点状态"
${K} get nodes -o wide
NR=$(${K} get nodes --no-headers | grep -cv " Ready ") || true
[ "${NR}" -gt 0 ] && { echo "FAIL: NotReady 节点 ${NR} 台"; ERR=$((ERR+1)); }

echo "== 2. 系统 Pod"
BAD=$(${K} -n kube-system get pods --no-headers | grep -cvE "Running|Completed") || true
[ "${BAD}" -gt 0 ] && { ${K} -n kube-system get pods | grep -vE "Running|Completed"; ERR=$((ERR+1)); }

echo "== 3. 是否有往外部仓库跑的尝试"
if sudo grep -qiE 'docker\.io|registry\.k8s\.io|ghcr\.io' \
     /var/lib/rancher/k3s/agent/containerd/containerd.log 2>/dev/null; then
  echo "WARN: containerd 日志里有连接外部仓库的痕迹"
  echo "      请确认 registries.yaml 的镜像源配置与 disable-default-registry-endpoint"
fi

echo "== 4. DNS"
${K} run dnscheck --rm -i --restart=Never --image=registry.internal.example:5000/library/busybox:1.36 -- \
  nslookup kubernetes.default.svc.cluster.local || { echo "FAIL: 集群 DNS"; ERR=$((ERR+1)); }

echo "== 5. 证书到期"
sudo k3s certificate check --output table

echo "== 6. 数据存储"
if [ -d /var/lib/rancher/k3s/server/db/etcd ]; then
  echo "datastore: embedded etcd"
  ls -la /var/lib/rancher/k3s/server/db/snapshots/ 2>/dev/null | tail -5
else
  echo "datastore: sqlite（无法扩容 Server 节点）"
fi

echo "== 结果: 错误 ${ERR} 件"
exit "${ERR}"