Kubernetes DRA(Dynamic Resource Allocation)로 GPU 워크로드 스케줄링 최적화 가이드

apiVersion: v1
kind: Pod
metadata:
  name: gpu-training-legacy
spec:
  containers:
    - name: trainer
      image: nvcr.io/nvidia/pytorch:24.01-py3
      resources:
        limits:
          nvidia.com/gpu: 2

# 클러스터 버전 확인
kubectl version --short

# DRA 피처 게이트 활성화 확인 (kube-apiserver, kube-scheduler, kubelet 모두 필요)
# kubeadm 기반 클러스터의 경우 ClusterConfiguration에서 설정
kubectl get configmap -n kube-system kubeadm-config -o yaml | grep -A 5 "featureGates"

# DRA 관련 API 리소스 확인
kubectl api-resources | grep resource.k8s.io
# 출력 예시:
# deviceclasses              resource.k8s.io/v1beta1   false   DeviceClass
# resourceclaims             resource.k8s.io/v1beta1   true    ResourceClaim
# resourceclaimtemplates     resource.k8s.io/v1beta1   true    ResourceClaimTemplate
# resourceslices             resource.k8s.io/v1beta1   false   ResourceSlice

# NVIDIA DRA 드라이버 Helm 차트 저장소 추가
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

# nvidia-dra-driver 설치
# 기존 NVIDIA GPU Operator가 설치되어 있다면 device-plugin은 비활성화해야 합니다
helm install nvidia-dra-driver nvidia/nvidia-dra-driver \
  --namespace nvidia-dra-driver \
  --create-namespace \
  --set controller.enabled=true \
  --set kubeletPlugin.enabled=true \
  --version v0.8.0

# 설치 확인
kubectl get pods -n nvidia-dra-driver
# NAME                                              READY   STATUS    RESTARTS   AGE
# nvidia-dra-driver-controller-7d8f9b6c4d-k2xvn    1/1     Running   0          2m
# nvidia-dra-driver-kubelet-plugin-node1-abcde      1/1     Running   0          2m
# nvidia-dra-driver-kubelet-plugin-node2-fghij      1/1     Running   0          2m

# ResourceSlice 생성 확인 - 드라이버가 노드의 GPU 정보를 보고
kubectl get resourceslices -o wide
# NAME                              DRIVER               NODE    POOL        DEVICES
# gpu-node1-nvidia-gpu-slice-0      gpu.nvidia.com       node1   nvidia-gpu  4
# gpu-node2-nvidia-gpu-slice-0      gpu.nvidia.com       node2   nvidia-gpu  8

# 특정 노드의 ResourceSlice 상세 보기
kubectl get resourceslice gpu-node1-nvidia-gpu-slice-0 -o yaml

# gpu-deviceclasses.yaml
apiVersion: resource.k8s.io/v1beta1
kind: DeviceClass
metadata:
  name: gpu.nvidia.com-a100
spec:
  selectors:
    - cel:
        expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].productName == 'NVIDIA A100 80GB PCIe'"
  config:
    - opaque:
        driver: gpu.nvidia.com
        parameters:
          apiVersion: gpu.nvidia.com/v1alpha1
          kind: GpuConfig
          sharing:
            strategy: TimeSlicing
            timeSlicingConfig:
              interval: Long
---
apiVersion: resource.k8s.io/v1beta1
kind: DeviceClass
metadata:
  name: gpu.nvidia.com-h100
spec:
  selectors:
    - cel:
        expression: "device.driver == 'gpu.nvidia.com' && device.attributes['gpu.nvidia.com'].productName == 'NVIDIA H100 80GB HBM3'"
  config:
    - opaque:
        driver: gpu.nvidia.com
        parameters:
          apiVersion: gpu.nvidia.com/v1alpha1
          kind: GpuConfig
          sharing:
            strategy: TimeSlicing
            timeSlicingConfig:
              interval: Short
---
apiVersion: resource.k8s.io/v1beta1
kind: DeviceClass
metadata:
  name: gpu.nvidia.com-a100-mig-3g20gb
spec:
  selectors:
    - cel:
        expression: >-
          device.driver == 'gpu.nvidia.com' &&
          device.attributes['gpu.nvidia.com'].productName == 'NVIDIA A100 80GB PCIe' &&
          device.attributes['gpu.nvidia.com'].migProfile == '3g.20gb'

# 독립적인 ResourceClaim 생성
apiVersion: resource.k8s.io/v1beta1
kind: ResourceClaim
metadata:
  name: training-gpu-claim
  namespace: ml-workloads
spec:
  devices:
    requests:
      - name: gpu
        deviceClassName: gpu.nvidia.com-a100
        count: 4
    constraints:
      - requests: ['gpu']
        matchAttribute: 'gpu.nvidia.com/nvlinkInterconnect'
---
# ResourceClaim을 참조하는 Pod
apiVersion: v1
kind: Pod
metadata:
  name: distributed-training
  namespace: ml-workloads
spec:
  containers:
    - name: trainer
      image: nvcr.io/nvidia/pytorch:24.01-py3
      command: ['torchrun', '--nproc_per_node=4', 'train.py']
      resources:
        claims:
          - name: training-gpus
  resourceClaims:
    - name: training-gpus
      resourceClaimName: training-gpu-claim
  restartPolicy: Never

# MIG 기반 추론 서비스 배포
apiVersion: apps/v1
kind: Deployment
metadata:
  name: llm-inference-service
  namespace: ml-workloads
spec:
  replicas: 6
  selector:
    matchLabels:
      app: llm-inference
  template:
    metadata:
      labels:
        app: llm-inference
    spec:
      containers:
        - name: inference-server
          image: nvcr.io/nvidia/tritonserver:24.01-py3
          ports:
            - containerPort: 8000
              name: http
            - containerPort: 8001
              name: grpc
          resources:
            claims:
              - name: inference-gpu
      resourceClaims:
        - name: inference-gpu
          resourceClaimTemplateName: mig-inference-template
---
apiVersion: resource.k8s.io/v1beta1
kind: ResourceClaimTemplate
metadata:
  name: mig-inference-template
  namespace: ml-workloads
spec:
  spec:
    devices:
      requests:
        - name: mig-gpu
          deviceClassName: gpu.nvidia.com-a100-mig-3g20gb
          count: 1

apiVersion: v1
kind: Pod
metadata:
  name: quick-gpu-job
  namespace: ml-workloads
spec:
  containers:
    - name: compute
      image: nvcr.io/nvidia/cuda:12.3.1-runtime-ubuntu22.04
      command: ['python3', 'benchmark.py']
      resources:
        claims:
          - name: gpu-req
  resourceClaims:
    - name: gpu-req
      resourceClaimTemplateName: ''
      source:
        devices:
          requests:
            - name: gpu
              deviceClassName: gpu.nvidia.com-h100
              count: 1
  restartPolicy: Never

# GKE에서 DRA 지원 GPU 노드 풀 생성
gcloud container node-pools create gpu-pool-h100 \
  --cluster=ml-cluster \
  --zone=us-central1-c \
  --machine-type=a3-highgpu-8g \
  --accelerator=type=nvidia-h100-80gb,count=8 \
  --num-nodes=2 \
  --enable-autoscaling \
  --min-nodes=0 \
  --max-nodes=4 \
  --node-labels="gpu-type=h100" \
  --metadata="install-nvidia-driver=True"

# DRA 피처 게이트 활성화 (GKE Rapid 채널)
gcloud container clusters update ml-cluster \
  --zone=us-central1-c \
  --release-channel=rapid \
  --enable-kubernetes-unstable-apis=resource.k8s.io/v1beta1

# EKS 클러스터에 NVIDIA DRA 드라이버 애드온 설치
aws eks create-addon \
  --cluster-name ml-cluster \
  --addon-name nvidia-dra-driver \
  --addon-version v0.8.0-eksbuild.1 \
  --region us-east-1

# GPU 노드 그룹 생성 (Karpenter NodePool)
cat <<EOF | kubectl apply -f -
apiVersion: karpenter.sh/v1
kind: NodePool
metadata:
  name: gpu-h100-pool
spec:
  template:
    spec:
      requirements:
        - key: "node.kubernetes.io/instance-type"
          operator: In
          values: ["p5.48xlarge"]
        - key: "karpenter.sh/capacity-type"
          operator: In
          values: ["on-demand"]
      nodeClassRef:
        group: karpenter.k8s.aws
        kind: EC2NodeClass
        name: gpu-nodes
  limits:
    cpu: "1000"
    memory: 4000Gi
    nvidia.com/gpu: "64"
  disruption:
    consolidationPolicy: WhenEmpty
    consolidateAfter: 30m
EOF

# 현재 Extended Resource 방식으로 GPU를 사용하는 모든 Pod 조회
kubectl get pods --all-namespaces -o json | \
  jq -r '.items[] |
    select(.spec.containers[].resources.limits["nvidia.com/gpu"] != null) |
    [.metadata.namespace, .metadata.name,
     (.spec.containers[].resources.limits["nvidia.com/gpu"] // "0")] |
    @tsv' | \
  column -t -s $'\t'

# 노드별 GPU 할당 현황
kubectl get nodes -l nvidia.com/gpu.present=true -o json | \
  jq -r '.items[] |
    [.metadata.name,
     .status.capacity["nvidia.com/gpu"],
     .status.allocatable["nvidia.com/gpu"]] |
    @tsv' | \
  column -t -s $'\t' -N "NODE,CAPACITY,ALLOCATABLE"

# ResourceClaim 상태 모니터링
kubectl get resourceclaims --all-namespaces -o custom-columns=\
  'NAMESPACE:.metadata.namespace,'\
  'NAME:.metadata.name,'\
  'DEVICECLASS:.spec.devices.requests[0].deviceClassName,'\
  'ALLOCATED:.status.allocation.devices.results[0].device'

# Pending 상태의 ResourceClaim 확인 (할당 실패 감지)
kubectl get resourceclaims --all-namespaces \
  --field-selector="status.allocation=" \
  -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,AGE:.metadata.creationTimestamp'

# DeviceClass별 사용률 확인
kubectl get resourceslices -o json | \
  jq -r '[.items[].spec.devices[] |
    {driver: .basic.attributes["gpu.nvidia.com"].productName.stringValue}] |
    group_by(.driver) |
    map({gpu_model: .[0].driver, total: length}) |
    .[] | [.gpu_model, .total] | @tsv'

# 1. ResourceClaim 상태 확인
kubectl describe resourceclaim training-gpu-claim -n ml-workloads

# 2. 요청한 DeviceClass와 매칭되는 디바이스 존재 여부 확인
kubectl get resourceslices -o json | \
  jq '.items[].spec.devices[] |
    select(.basic.attributes["gpu.nvidia.com"].productName.stringValue == "NVIDIA A100 80GB PCIe")'

# 3. DRA 드라이버 상태 확인
kubectl get pods -n nvidia-dra-driver
kubectl logs -n nvidia-dra-driver -l app=nvidia-dra-driver-kubelet-plugin --tail=50

# 4. 스케줄러 로그에서 DRA 관련 오류 확인
kubectl logs -n kube-system -l component=kube-scheduler --tail=100 | grep -i "dra\|resourceclaim\|deviceclass"

# 5. 필요 시 ResourceClaim 삭제 후 재생성
kubectl delete resourceclaim training-gpu-claim -n ml-workloads
kubectl apply -f training-gpu-claim.yaml

# 해당 노드의 DRA 드라이버 플러그인 Pod 재시작
kubectl delete pod -n nvidia-dra-driver -l app=nvidia-dra-driver-kubelet-plugin \
  --field-selector spec.nodeName=problematic-node

# ResourceSlice가 올바르게 업데이트되었는지 확인 (재시작 후 30초 대기)
sleep 30
kubectl get resourceslices --field-selector spec.nodeName=problematic-node -o yaml

# 실제 디바이스가 보고하는 속성 이름과 값 확인
kubectl get resourceslices -o json | \
  jq '.items[0].spec.devices[0].basic.attributes'

# 출력 예시:
# {
#   "gpu.nvidia.com": {
#     "driverVersion": {"stringValue": "550.54.15"},
#     "productName": {"stringValue": "NVIDIA A100 80GB PCIe"},
#     "architecture": {"stringValue": "Ampere"},
#     "cudaComputeCapability": {"versionValue": "8.0"},
#     "memory": {"quantityValue": "80Gi"}
#   }
# }
# DeviceClass의 CEL 표현식에서 사용하는 속성 이름이 위 출력과 정확히 일치하는지 확인합니다.