필사 모드: 用 vCluster 实现 Kubernetes 多租户：虚拟集群隔离与运维指南

# 安装 vCluster CLI
curl -L -o vcluster "https://github.com/loft-sh/vcluster/releases/latest/download/vcluster-linux-amd64"
chmod +x vcluster
sudo mv vcluster /usr/local/bin/

# 创建虚拟集群（默认基于 k3s）
vcluster create my-vcluster --namespace team-alpha

# 连接虚拟集群（自动配置 kubeconfig）
vcluster connect my-vcluster --namespace team-alpha

# 确认连接
kubectl get namespaces
kubectl get nodes

# 断开虚拟集群连接
vcluster disconnect

# 删除虚拟集群
vcluster delete my-vcluster --namespace team-alpha

# vcluster.yaml - 生产配置示例
controlPlane:
  # 使用 k8s 而非 k3s（生产推荐）
  distro:
    k8s:
      enabled: true
      apiServer:
        extraArgs:
          - '--audit-log-path=/var/log/kubernetes/audit.log'
          - '--audit-log-maxage=30'
          - '--audit-log-maxbackup=10'
          - '--audit-log-maxsize=100'
      controllerManager:
        extraArgs:
          - '--terminated-pod-gc-threshold=50'

  # 控制平面资源限制
  statefulSet:
    resources:
      limits:
        cpu: '2'
        memory: 4Gi
      requests:
        cpu: '500m'
        memory: 1Gi
    persistence:
      size: 20Gi

  # 使用外部 etcd（高可用）
  backingStore:
    etcd:
      deploy:
        enabled: true
        statefulSet:
          resources:
            limits:
              cpu: '1'
              memory: 2Gi
            requests:
              cpu: '200m'
              memory: 512Mi

# 资源同步配置
sync:
  toHost:
    pods:
      enabled: true
    services:
      enabled: true
    configmaps:
      enabled: true
    secrets:
      enabled: true
    endpoints:
      enabled: true
    persistentvolumeclaims:
      enabled: true
    ingresses:
      enabled: true
    storageClasses:
      enabled: false
  fromHost:
    nodes:
      enabled: true
      selector:
        labels:
          team: alpha
    storageClasses:
      enabled: true
    ingressClasses:
      enabled: true

# 网络配置
networking:
  replicateServices:
    fromHost:
      - from: monitoring/prometheus-server
        to: monitoring/prometheus-server
  resolveDNS:
    - hostname: '*.team-alpha.svc.cluster.local'
      target:
        hostNamespace: team-alpha

# 安全策略
policies:
  resourceQuota:
    enabled: true
    quota:
      requests.cpu: '8'
      requests.memory: '16Gi'
      limits.cpu: '16'
      limits.memory: '32Gi'
      pods: '50'
      services: '20'
      persistentvolumeclaims: '10'
  limitRange:
    enabled: true
    default:
      cpu: '500m'
      memory: '512Mi'
    defaultRequest:
      cpu: '100m'
      memory: '128Mi'

# 添加 Helm 仓库
helm repo add loft https://charts.loft.sh
helm repo update

# 创建命名空间
kubectl create namespace team-alpha

# 部署 vCluster
helm upgrade --install my-vcluster loft/vcluster \
  --namespace team-alpha \
  --values vcluster.yaml \
  --version 0.24.1

# 确认部署状态
kubectl get pods -n team-alpha
kubectl get statefulset -n team-alpha

# 连接 vCluster
vcluster connect my-vcluster -n team-alpha

# vcluster-tenant-rbac.yaml
# 在虚拟集群内部给租户授予 admin 权限
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: tenant-admin-binding
subjects:
  - kind: Group
    name: team-alpha-developers
    apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
---
# 在宿主集群中限制对 vCluster 命名空间的访问
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: vcluster-namespace-viewer
  namespace: team-alpha
rules:
  - apiGroups: ['']
    resources: ['pods', 'services', 'configmaps']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['']
    resources: ['pods/log']
    verbs: ['get']
  - apiGroups: ['apps']
    resources: ['statefulsets']
    verbs: ['get', 'list', 'watch']
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: vcluster-namespace-viewer-binding
  namespace: team-alpha
subjects:
  - kind: Group
    name: team-alpha-developers
    apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: vcluster-namespace-viewer
  apiGroup: rbac.authorization.k8s.io

# host-resource-quota.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
  name: vcluster-team-alpha-quota
  namespace: team-alpha
spec:
  hard:
    requests.cpu: '8'
    requests.memory: '16Gi'
    limits.cpu: '16'
    limits.memory: '32Gi'
    pods: '50'
    services: '20'
    services.loadbalancers: '2'
    services.nodeports: '5'
    persistentvolumeclaims: '10'
    requests.storage: '100Gi'
    count/deployments.apps: '30'
    count/statefulsets.apps: '10'
    count/jobs.batch: '20'
    count/cronjobs.batch: '10'
---
apiVersion: v1
kind: LimitRange
metadata:
  name: vcluster-team-alpha-limits
  namespace: team-alpha
spec:
  limits:
    - type: Container
      default:
        cpu: '500m'
        memory: '512Mi'
      defaultRequest:
        cpu: '100m'
        memory: '128Mi'
      max:
        cpu: '4'
        memory: '8Gi'
      min:
        cpu: '50m'
        memory: '64Mi'
    - type: Pod
      max:
        cpu: '8'
        memory: '16Gi'
    - type: PersistentVolumeClaim
      max:
        storage: '50Gi'
      min:
        storage: '1Gi'

# host-networkpolicy.yaml
# 阻断虚拟集群命名空间之间的通信
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: isolate-vcluster-namespace
  namespace: team-alpha
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress
  ingress:
    # 允许同一命名空间内的通信
    - from:
        - podSelector: {}
    # 允许来自 Ingress 控制器的流量
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: ingress-nginx
          podSelector:
            matchLabels:
              app.kubernetes.io/name: ingress-nginx
    # 允许监控系统采集指标
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: monitoring
          podSelector:
            matchLabels:
              app: prometheus
  egress:
    # 允许 DNS 查询
    - to:
        - namespaceSelector: {}
          podSelector:
            matchLabels:
              k8s-app: kube-dns
      ports:
        - protocol: UDP
          port: 53
        - protocol: TCP
          port: 53
    # 允许同一命名空间内的通信
    - to:
        - podSelector: {}
    # 允许访问外部互联网（按需）
    - to:
        - ipBlock:
            cidr: 0.0.0.0/0
            except:
              - 10.0.0.0/8
              - 172.16.0.0/12
              - 192.168.0.0/16

# 1. 在宿主集群中识别孤儿 Pod
kubectl get pods -n team-alpha -l vcluster.loft.sh/managed-by=my-vcluster \
  --field-selector=status.phase=Running

# 2. 确认虚拟集群中该 Pod 是否存在
vcluster connect my-vcluster -n team-alpha
kubectl get pods --all-namespaces

# 3. 手动删除虚拟集群中不存在的孤儿 Pod
vcluster disconnect
kubectl delete pod <orphan-pod-name> -n team-alpha --grace-period=30

# 4. 重启 Syncer Pod 以恢复同步状态
kubectl rollout restart statefulset my-vcluster -n team-alpha

# 5. 确认同步状态（查看日志）
kubectl logs statefulset/my-vcluster -n team-alpha -c syncer --tail=100