Kubernetes GPU Workload Management: The Complete NVIDIA GPU Operator Guide

# 1. Create Namespace and configure PSA
kubectl create ns gpu-operator
kubectl label --overwrite ns gpu-operator \
  pod-security.kubernetes.io/enforce=privileged

# 2. Add NVIDIA Helm repository
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

# 3. Install GPU Operator (default configuration)
helm install --wait --generate-name \
  -n gpu-operator --create-namespace \
  nvidia/gpu-operator \
  --version=v25.10.1

helm install --wait --generate-name \
  -n gpu-operator --create-namespace \
  nvidia/gpu-operator \
  --version=v25.10.1 \
  --set driver.enabled=false

helm install --wait --generate-name \
  -n gpu-operator --create-namespace \
  nvidia/gpu-operator \
  --version=v25.10.1 \
  --set driver.enabled=false \
  --set toolkit.enabled=false

apiVersion: v1
kind: Pod
metadata:
  name: cuda-vectoradd
spec:
  restartPolicy: OnFailure
  containers:
    - name: cuda-vectoradd
      image: 'nvcr.io/nvidia/k8s/cuda-sample:vectoradd-cuda11.7.1-ubuntu20.04'
      resources:
        limits:
          nvidia.com/gpu: 1

kubectl apply -f cuda-vectoradd.yaml
kubectl logs pod/cuda-vectoradd
# [Vector addition of 50000 elements]
# Test PASSED

apiVersion: v1
kind: Pod
metadata:
  name: gpu-pod
spec:
  containers:
    - name: gpu-container
      image: nvcr.io/nvidia/pytorch:24.01-py3
      resources:
        limits:
          nvidia.com/gpu: 2 # Request 2 GPUs

resources:
  limits:
    nvidia.com/gpu: 1 # Exclusively allocate 1 GPU
    memory: '16Gi' # System memory is set separately from the GPU
    cpu: '4'

apiVersion: v1
kind: ResourceQuota
metadata:
  name: gpu-quota
  namespace: team-ml
spec:
  hard:
    requests.nvidia.com/gpu: '4' # Maximum 4 GPUs
    limits.nvidia.com/gpu: '4'

apiVersion: v1
kind: LimitRange
metadata:
  name: gpu-limit-range
  namespace: team-ml
spec:
  limits:
    - type: Container
      default:
        nvidia.com/gpu: '1'
      defaultRequest:
        nvidia.com/gpu: '1'
      max:
        nvidia.com/gpu: '2'

helm install --wait --generate-name \
  -n gpu-operator --create-namespace \
  nvidia/gpu-operator \
  --version=v25.10.1 \
  --set mig.strategy=single

# Apply MIG configuration Label to the node
kubectl label nodes gpu-node-01 \
  nvidia.com/mig.config=all-1g.10gb --overwrite

apiVersion: v1
kind: ConfigMap
metadata:
  name: custom-mig-config
  namespace: gpu-operator
data:
  config.yaml: |
    version: v1
    mig-configs:
      custom-profile:
        - devices: all
          mig-enabled: true
          mig-devices:
            "1g.10gb": 5
            "2g.20gb": 1

kubectl patch clusterpolicies.nvidia.com/cluster-policy \
  --type='json' \
  -p='[{"op":"replace","path":"/spec/migManager/config/name","value":"custom-mig-config"}]'

kubectl get node gpu-node-01 -o=jsonpath='{.metadata.labels}' | jq .
# Verify "nvidia.com/mig.config.state": "success"

apiVersion: v1
kind: Pod
metadata:
  name: mig-workload
spec:
  containers:
    - name: cuda-app
      image: nvcr.io/nvidia/pytorch:24.01-py3
      resources:
        limits:
          nvidia.com/mig-1g.10gb: 1 # Request 1 MIG 1g.10gb instance

apiVersion: v1
kind: ConfigMap
metadata:
  name: time-slicing-config
  namespace: gpu-operator
data:
  any: |-
    version: v1
    flags:
      migStrategy: none
    sharing:
      timeSlicing:
        renameByDefault: false
        failRequestsGreaterThanOne: false
        resources:
          - name: nvidia.com/gpu
            replicas: 4

kubectl apply -f time-slicing-config.yaml

kubectl patch clusterpolicies.nvidia.com/cluster-policy \
  -n gpu-operator --type merge \
  -p '{"spec":{"devicePlugin":{"config":{"name":"time-slicing-config","default":"any"}}}}'

# When per-GPU-model configurations are defined in the ConfigMap
kubectl patch clusterpolicies.nvidia.com/cluster-policy \
  -n gpu-operator --type merge \
  -p '{"spec":{"devicePlugin":{"config":{"name":"time-slicing-config"}}}}'

# Apply Label to the target node
kubectl label node gpu-node-01 \
  nvidia.com/device-plugin.config=tesla-t4

kubectl get node gpu-node-01 -o jsonpath='{.status.allocatable}' | jq .
# "nvidia.com/gpu": "4"  (1 physical GPU * 4 replicas)

kubectl get node gpu-node-01 --show-labels | grep nvidia
# nvidia.com/gpu.replicas=4
# nvidia.com/gpu.product=Tesla-T4-SHARED

kubectl rollout restart -n gpu-operator \
  daemonset/nvidia-device-plugin-daemonset

# MPS usage example on GKE
apiVersion: v1
kind: Pod
metadata:
  name: mps-workload
spec:
  containers:
    - name: cuda-app
      image: nvcr.io/nvidia/pytorch:24.01-py3
      resources:
        limits:
          nvidia.com/gpu: 1

# Add Taint to GPU nodes
kubectl taint nodes gpu-node-01 \
  nvidia.com/gpu=present:NoSchedule

kubectl taint nodes gpu-node-02 \
  nvidia.com/gpu=present:NoSchedule

apiVersion: v1
kind: Pod
metadata:
  name: gpu-training-job
spec:
  tolerations:
    - key: 'nvidia.com/gpu'
      operator: 'Equal'
      value: 'present'
      effect: 'NoSchedule'
  containers:
    - name: training
      image: nvcr.io/nvidia/pytorch:24.01-py3
      resources:
        limits:
          nvidia.com/gpu: 1

apiVersion: v1
kind: Pod
metadata:
  name: a100-training
spec:
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: nvidia.com/gpu.product
                operator: In
                values:
                  - 'A100-SXM4-80GB'
                  - 'A100-SXM4-40GB'
              - key: nvidia.com/gpu.memory
                operator: Gt
                values:
                  - '40000'
  tolerations:
    - key: 'nvidia.com/gpu'
      operator: 'Equal'
      value: 'present'
      effect: 'NoSchedule'
  containers:
    - name: llm-training
      image: nvcr.io/nvidia/pytorch:24.01-py3
      command: ['python', 'train.py']
      resources:
        limits:
          nvidia.com/gpu: 4

# Add Prometheus community Helm repository
helm repo add prometheus-community \
  https://prometheus-community.github.io/helm-charts
helm repo update

# kube-prometheus-stack-values.yaml
prometheus:
  service:
    type: NodePort
    nodePort: 30090
  prometheusSpec:
    serviceMonitorSelectorNilUsesHelmValues: false
    additionalScrapeConfigs:
      - job_name: gpu-metrics
        scrape_interval: 1s
        metrics_path: /metrics
        scheme: http
        kubernetes_sd_configs:
          - role: endpoints
            namespaces:
              names:
                - gpu-operator
        relabel_configs:
          - source_labels: [__meta_kubernetes_pod_node_name]
            action: replace
            target_label: kubernetes_node

grafana:
  service:
    type: NodePort
    nodePort: 32322

helm install prometheus-community/kube-prometheus-stack \
  --create-namespace --namespace prometheus \
  --generate-name \
  --values kube-prometheus-stack-values.yaml

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: gpu-alerts
  namespace: prometheus
spec:
  groups:
    - name: gpu.rules
      rules:
        - alert: GPUHighTemperature
          expr: DCGM_FI_DEV_GPU_TEMP > 85
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: 'GPU temperature is high on {{ $labels.kubernetes_node }}'
            description: 'GPU {{ $labels.gpu }} temperature is {{ $value }}C'
        - alert: GPUMemoryAlmostFull
          expr: (DCGM_FI_DEV_FB_USED / (DCGM_FI_DEV_FB_USED + DCGM_FI_DEV_FB_FREE)) > 0.95
          for: 5m
          labels:
            severity: critical
          annotations:
            summary: 'GPU memory usage above 95% on {{ $labels.kubernetes_node }}'
        - alert: GPUHighUtilization
          expr: DCGM_FI_DEV_GPU_UTIL > 95
          for: 30m
          labels:
            severity: info
          annotations:
            summary: 'Sustained high GPU utilization on {{ $labels.kubernetes_node }}'

# Run only on Hopper architecture GPUs
apiVersion: batch/v1
kind: Job
metadata:
  name: h100-inference
spec:
  template:
    spec:
      nodeSelector:
        nvidia.com/gpu.family: hopper
      containers:
        - name: inference
          image: nvcr.io/nvidia/tritonserver:24.01-py3
          resources:
            limits:
              nvidia.com/gpu: 1
      restartPolicy: Never

# Large model training requiring 80GB+ GPU memory
apiVersion: batch/v1
kind: Job
metadata:
  name: large-model-training
spec:
  template:
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: nvidia.com/gpu.memory
                    operator: Gt
                    values:
                      - '79000'
                  - key: nvidia.com/gpu.count
                    operator: Gt
                    values:
                      - '7'
      containers:
        - name: training
          image: nvcr.io/nvidia/pytorch:24.01-py3
          resources:
            limits:
              nvidia.com/gpu: 8
      restartPolicy: Never

apiVersion: batch/v1
kind: Job
metadata:
  name: distributed-training
spec:
  parallelism: 2
  completions: 2
  template:
    metadata:
      labels:
        app: distributed-training
    spec:
      tolerations:
        - key: 'nvidia.com/gpu'
          operator: 'Equal'
          value: 'present'
          effect: 'NoSchedule'
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: nvidia.com/gpu.product
                    operator: In
                    values:
                      - 'A100-SXM4-80GB'
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - distributed-training
                topologyKey: kubernetes.io/hostname
      containers:
        - name: pytorch-trainer
          image: nvcr.io/nvidia/pytorch:24.01-py3
          command:
            - torchrun
            - --nproc_per_node=4
            - --nnodes=2
            - train.py
          resources:
            limits:
              nvidia.com/gpu: 4
              memory: '64Gi'
              cpu: '16'
            requests:
              memory: '64Gi'
              cpu: '16'
          env:
            - name: NCCL_DEBUG
              value: 'INFO'
          volumeMounts:
            - name: dshm
              mountPath: /dev/shm
            - name: training-data
              mountPath: /data
      volumes:
        - name: dshm
          emptyDir:
            medium: Memory
            sizeLimit: '16Gi'
        - name: training-data
          persistentVolumeClaim:
            claimName: training-data-pvc
      restartPolicy: Never
  backoffLimit: 3

apiVersion: apps/v1
kind: Deployment
metadata:
  name: triton-inference
spec:
  replicas: 2
  selector:
    matchLabels:
      app: triton-inference
  template:
    metadata:
      labels:
        app: triton-inference
    spec:
      tolerations:
        - key: 'nvidia.com/gpu'
          operator: 'Equal'
          value: 'present'
          effect: 'NoSchedule'
      nodeSelector:
        nvidia.com/gpu.family: ampere
      containers:
        - name: triton
          image: nvcr.io/nvidia/tritonserver:24.01-py3
          args:
            - tritonserver
            - --model-repository=/models
            - --strict-model-config=false
          ports:
            - containerPort: 8000
              name: http
            - containerPort: 8001
              name: grpc
            - containerPort: 8002
              name: metrics
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: '16Gi'
              cpu: '4'
          readinessProbe:
            httpGet:
              path: /v2/health/ready
              port: 8000
            initialDelaySeconds: 30
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /v2/health/live
              port: 8000
            initialDelaySeconds: 30
            periodSeconds: 15
          volumeMounts:
            - name: model-store
              mountPath: /models
      volumes:
        - name: model-store
          persistentVolumeClaim:
            claimName: model-store-pvc
---
apiVersion: v1
kind: Service
metadata:
  name: triton-inference
spec:
  selector:
    app: triton-inference
  ports:
    - name: http
      port: 8000
      targetPort: 8000
    - name: grpc
      port: 8001
      targetPort: 8001
    - name: metrics
      port: 8002
      targetPort: 8002

# 1. Check GPU Operator Pod status
kubectl get pods -n gpu-operator

# 2. Check Driver Pod logs
kubectl logs -n gpu-operator -l app=nvidia-driver-daemonset

# 3. Check Device Plugin Pod logs
kubectl logs -n gpu-operator -l app=nvidia-device-plugin-daemonset

# 4. Check node's allocatable resources
kubectl describe node gpu-node-01 | grep -A 5 "Allocatable"

# 5. Check NFD Labels
kubectl get node gpu-node-01 -o json | \
  jq '.metadata.labels | to_entries[] | select(.key | startswith("nvidia"))'

# 1. Check Pod events
kubectl describe pod <pod-name>
# Look for "Insufficient nvidia.com/gpu" message

# 2. Check node's GPU allocation status
kubectl describe node gpu-node-01 | grep -A 3 "Allocated resources"

# 3. List Pods currently using GPUs
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.containers[].resources.limits["nvidia.com/gpu"] != null) | .metadata.name'

# 1. Check MIG Manager logs
kubectl logs -n gpu-operator -l app=nvidia-mig-manager

# 2. Check MIG configuration state Label
kubectl get node gpu-node-01 -o jsonpath='{.metadata.labels.nvidia\.com/mig\.config\.state}'
# If not "success", configuration failed

# 3. Check GPU mode (on the node)
nvidia-smi -L
nvidia-smi --query-gpu=mig.mode.current --format=csv

# 1. Check ConfigMap content
kubectl get configmap -n gpu-operator time-slicing-config -o yaml

# 2. Restart Device Plugin
kubectl rollout restart -n gpu-operator \
  daemonset/nvidia-device-plugin-daemonset

# 3. Check node resources after restart
kubectl get node gpu-node-01 -o jsonpath='{.status.allocatable.nvidia\.com/gpu}'