NVIDIA Triton Inference Server 生产指南：GPU 模型服务优化策略

Client (HTTP/gRPC/C API)
        │
        ▼
┌─────────────────────────────┐
│   Request Handler           │
│   (HTTP: 8000, gRPC: 8001)  │
│   (Metrics: 8002)           │
├─────────────────────────────┤
│   Scheduler Layer           │
│   ┌───────────────────────┐ │
│   │ Dynamic Batcher       │ │
│   │ Sequence Batcher      │ │
│   │ Ensemble Scheduler    │ │
│   └───────────────────────┘ │
├─────────────────────────────┤
│   Backend Layer             │
│   ┌──────┬──────┬────────┐  │
│   │TRT   │ONNX  │PyTorch │  │
│   │TF    │Python│OpenVINO│  │
│   └──────┴──────┴────────┘  │
├─────────────────────────────┤
│   GPU/CPU Execution Engine  │
│   (CUDA Streams, MIG, MPS)  │
└─────────────────────────────┘

model_repository/
├── text_classifier/
│   ├── config.pbtxt
│   ├── 1/
│   │   └── model.onnx
│   └── 2/
│       └── model.onnx
├── image_detector/
│   ├── config.pbtxt
│   └── 1/
│       └── model.plan          # TensorRT engine
├── embedding_model/
│   ├── config.pbtxt
│   └── 1/
│       └── model.pt            # PyTorch TorchScript
└── preprocessing/
    ├── config.pbtxt
    └── 1/
        └── model.py            # Python backend

# 运行 NVIDIA Triton Inference Server 容器
# 从 NGC 拉取最新镜像（以 25.02 版本为准）
docker pull nvcr.io/nvidia/tritonserver:25.02-py3

# 连同模型仓库一起启动服务器
docker run --gpus all \
  --rm \
  -p 8000:8000 \
  -p 8001:8001 \
  -p 8002:8002 \
  -v $(pwd)/model_repository:/models \
  nvcr.io/nvidia/tritonserver:25.02-py3 \
  tritonserver \
    --model-repository=/models \
    --strict-model-config=false \
    --log-verbose=1

# 检查服务器状态
curl -v localhost:8000/v2/health/ready

# text_classifier/config.pbtxt
name: "text_classifier"
platform: "onnxruntime_onnx"
max_batch_size: 64

input [
  {
    name: "input_ids"
    data_type: TYPE_INT64
    dims: [ 512 ]
  },
  {
    name: "attention_mask"
    data_type: TYPE_INT64
    dims: [ 512 ]
  }
]

output [
  {
    name: "logits"
    data_type: TYPE_FP32
    dims: [ 3 ]
  }
]

# Dynamic Batching 配置
dynamic_batching {
  preferred_batch_size: [ 8, 16, 32 ]
  max_queue_delay_microseconds: 100
}

# GPU 实例配置
instance_group [
  {
    count: 2
    kind: KIND_GPU
    gpus: [ 0 ]
  }
]

# 模型版本策略
version_policy: { latest { num_versions: 2 } }

# 优化配置（ONNX Runtime）
optimization {
  execution_accelerators {
    gpu_execution_accelerator: [
      {
        name: "tensorrt"
        parameters {
          key: "precision_mode"
          value: "FP16"
        }
        parameters {
          key: "max_workspace_size_bytes"
          value: "1073741824"
        }
      }
    ]
  }
}

# 高级 Dynamic Batching 配置
dynamic_batching {
  # 首选批大小：达到该大小时立即执行
  preferred_batch_size: [ 4, 8, 16, 32 ]

  # 最大队列等待时间（微秒）
  # 超过该时间后，用当前已聚集的请求执行批处理
  max_queue_delay_microseconds: 200

  # 请求优先级设置
  priority_levels: 3
  default_priority_level: 2

  # 队列策略：超过最大队列大小时拒绝
  default_queue_policy {
    timeout_action: REJECT
    default_timeout_microseconds: 5000000
    allow_timeout_override: true
    max_queue_size: 100
  }
}

# 使用 perf_analyzer 测量 Dynamic Batching 性能
# 将并发请求数从 1 依次调高到 32 进行测量
perf_analyzer \
  -m text_classifier \
  -u localhost:8001 \
  -i grpc \
  --concurrency-range 1:32:4 \
  --measurement-interval 10000 \
  -b 1 \
  --percentile=99 \
  -f results_dynamic_batch.csv

# 检查结果：吞吐量（infer/sec）与 p99 延迟的变化趋势
# concurrency=1:  throughput=120 infer/sec, p99=8.3ms
# concurrency=4:  throughput=450 infer/sec, p99=9.1ms
# concurrency=8:  throughput=820 infer/sec, p99=10.2ms
# concurrency=16: throughput=1400 infer/sec, p99=12.5ms
# concurrency=32: throughput=1650 infer/sec, p99=22.7ms  <- 开始饱和

# ensemble_pipeline/config.pbtxt
name: "ensemble_pipeline"
platform: "ensemble"
max_batch_size: 32

input [
  {
    name: "RAW_TEXT"
    data_type: TYPE_STRING
    dims: [ 1 ]
  }
]

output [
  {
    name: "PREDICTION"
    data_type: TYPE_FP32
    dims: [ 3 ]
  }
]

ensemble_scheduling {
  step [
    {
      model_name: "tokenizer"
      model_version: -1
      input_map {
        key: "TEXT_INPUT"
        value: "RAW_TEXT"
      }
      output_map {
        key: "INPUT_IDS"
        value: "tokenized_ids"
      }
      output_map {
        key: "ATTENTION_MASK"
        value: "tokenized_mask"
      }
    },
    {
      model_name: "text_classifier"
      model_version: -1
      input_map {
        key: "input_ids"
        value: "tokenized_ids"
      }
      input_map {
        key: "attention_mask"
        value: "tokenized_mask"
      }
      output_map {
        key: "logits"
        value: "raw_logits"
      }
    },
    {
      model_name: "postprocessor"
      model_version: -1
      input_map {
        key: "LOGITS"
        value: "raw_logits"
      }
      output_map {
        key: "RESULT"
        value: "PREDICTION"
      }
    }
  ]
}

# preprocessing/tokenizer/1/model.py
import triton_python_backend_utils as pb_utils
import numpy as np
from transformers import AutoTokenizer
import json


class TritonPythonModel:
    def initialize(self, args):
        """模型加载时调用一次。初始化分词器。"""
        self.model_config = json.loads(args["model_config"])
        self.tokenizer = AutoTokenizer.from_pretrained(
            "bert-base-uncased",
            cache_dir="/models/cache"
        )
        self.max_length = 512

    def execute(self, requests):
        """按批次处理推理请求。"""
        responses = []
        for request in requests:
            # 提取输入文本
            text_input = pb_utils.get_input_tensor_by_name(
                request, "TEXT_INPUT"
            )
            texts = [
                t.decode("utf-8")
                for t in text_input.as_numpy().flatten()
            ]

            # 执行分词
            encoded = self.tokenizer(
                texts,
                padding="max_length",
                truncation=True,
                max_length=self.max_length,
                return_tensors="np"
            )

            # 生成输出张量
            input_ids_tensor = pb_utils.Tensor(
                "INPUT_IDS",
                encoded["input_ids"].astype(np.int64)
            )
            attention_mask_tensor = pb_utils.Tensor(
                "ATTENTION_MASK",
                encoded["attention_mask"].astype(np.int64)
            )

            response = pb_utils.InferenceResponse(
                output_tensors=[input_ids_tensor, attention_mask_tensor]
            )
            responses.append(response)

        return responses

    def finalize(self):
        """模型卸载时执行清理工作。"""
        print("Tokenizer model finalized.")

# 将 ONNX 模型转换为 TensorRT 引擎
# 注意：必须在与 Triton 服务器相同 TensorRT 版本的容器中进行转换
docker run --gpus all --rm \
  -v $(pwd):/workspace \
  nvcr.io/nvidia/tensorrt:25.02-py3 \
  trtexec \
    --onnx=/workspace/model.onnx \
    --saveEngine=/workspace/model.plan \
    --fp16 \
    --workspace=4096 \
    --minShapes=input_ids:1x512,attention_mask:1x512 \
    --optShapes=input_ids:16x512,attention_mask:16x512 \
    --maxShapes=input_ids:64x512,attention_mask:64x512 \
    --verbose

# 检查 Dynamic Shape 配置
# minShapes：最小批大小（单个请求）
# optShapes：最优批大小（最常见的场景）
# maxShapes：最大批大小（应与 Dynamic Batching 的最大值一致）

# image_detector/config.pbtxt
name: "image_detector"
platform: "tensorrt_plan"
max_batch_size: 32

input [
  {
    name: "images"
    data_type: TYPE_FP32
    dims: [ 3, 640, 640 ]
  }
]

output [
  {
    name: "detections"
    data_type: TYPE_FP32
    dims: [ 100, 6 ]
  }
]

dynamic_batching {
  preferred_batch_size: [ 4, 8, 16 ]
  max_queue_delay_microseconds: 100
}

instance_group [
  {
    count: 1
    kind: KIND_GPU
    gpus: [ 0 ]
  }
]

# TensorRT 专用优化参数
parameters {
  key: "TRT_ENGINE_CACHE_ENABLE"
  value: { string_value: "1" }
}
parameters {
  key: "TRT_ENGINE_CACHE_PATH"
  value: { string_value: "/models/cache/trt" }
}

# 多 GPU 分布式部署配置
instance_group [
  {
    # GPU 0 上部署 2 个实例
    count: 2
    kind: KIND_GPU
    gpus: [ 0 ]
  },
  {
    # GPU 1 上部署 1 个实例
    count: 1
    kind: KIND_GPU
    gpus: [ 1 ]
  },
  {
    # CPU 回退实例（GPU 故障时使用）
    count: 1
    kind: KIND_CPU
  }
]

# 使用 Model Analyzer 进行 GPU 内存画像分析
# 测量各个模型占用多少 GPU 内存
model-analyzer profile \
  --model-repository=/models \
  --profile-models text_classifier,image_detector,embedding_model \
  --triton-launch-mode=docker \
  --triton-docker-image=nvcr.io/nvidia/tritonserver:25.02-py3 \
  --output-model-repository-path=/output/models \
  --export-path=/output/results \
  --run-config-search-max-concurrency 16 \
  --run-config-search-max-instance-count 4

# 结果示例（以 A100 80GB 为准）：
# text_classifier:  每实例约 2.1GB，最优实例数：4
# image_detector:   每实例约 4.8GB，最优实例数：2
# embedding_model:  每实例约 1.5GB，最优实例数：6
# 总使用量：2.1*4 + 4.8*2 + 1.5*6 = 8.4 + 9.6 + 9.0 = 27.0GB
# A100 80GB 剩余可用内存：53GB（已考虑 CUDA context、框架开销）

# 在 A100 上启用 MIG 并创建实例
sudo nvidia-smi -i 0 -mig 1

# 创建 2 个 3g.40gb 配置文件（以 A100 80GB 为准）
sudo nvidia-smi mig -i 0 -cgi 9,9 -C

# 检查 MIG 实例
nvidia-smi -L
# GPU 0: NVIDIA A100-SXM4-80GB
#   MIG 3g.40gb Device 0: (UUID: MIG-xxx-xxx)
#   MIG 3g.40gb Device 1: (UUID: MIG-xxx-xxx)

# 在 Triton 中按 MIG 实例分配模型
# 运行 Docker 时仅暴露特定 MIG 实例
docker run --gpus '"device=0:0"' \
  -v $(pwd)/models_group_a:/models \
  nvcr.io/nvidia/tritonserver:25.02-py3 \
  tritonserver --model-repository=/models

# 以 EXPLICIT 模式启动服务器（手动管理模型）
tritonserver \
  --model-repository=/models \
  --model-control-mode=explicit

# 通过 Model Control API 加载模型
curl -X POST localhost:8000/v2/repository/models/text_classifier/load

# 卸载模型（释放 GPU 内存）
curl -X POST localhost:8000/v2/repository/models/text_classifier/unload

# 检查当前已加载的模型列表
curl localhost:8000/v2/repository/index | python -m json.tool

# Triton gRPC 客户端示例
import tritonclient.grpc as grpcclient
import numpy as np
from functools import partial
import queue


def run_inference():
    """同步 gRPC 推理请求"""
    # 创建 gRPC 客户端
    triton_client = grpcclient.InferenceServerClient(
        url="localhost:8001",
        verbose=False
    )

    # 检查服务器状态
    if not triton_client.is_server_ready():
        raise RuntimeError("Triton server is not ready")

    # 查询模型元数据
    metadata = triton_client.get_model_metadata("text_classifier")
    print(f"Model: {metadata.name}, Versions: {metadata.versions}")

    # 准备输入数据
    input_ids = np.random.randint(0, 30000, size=(1, 512)).astype(np.int64)
    attention_mask = np.ones((1, 512), dtype=np.int64)

    # 创建输入张量
    inputs = [
        grpcclient.InferInput("input_ids", input_ids.shape, "INT64"),
        grpcclient.InferInput("attention_mask", attention_mask.shape, "INT64"),
    ]
    inputs[0].set_data_from_numpy(input_ids)
    inputs[1].set_data_from_numpy(attention_mask)

    # 指定输出张量
    outputs = [
        grpcclient.InferRequestedOutput("logits"),
    ]

    # 执行推理（同步）
    result = triton_client.infer(
        model_name="text_classifier",
        inputs=inputs,
        outputs=outputs,
        client_timeout=5.0,  # 5秒超时
        headers={"request-id": "req-001"}
    )

    # 提取结果
    logits = result.as_numpy("logits")
    print(f"Logits shape: {logits.shape}")
    print(f"Predictions: {np.argmax(logits, axis=-1)}")

    return logits


def run_async_inference():
    """异步 gRPC 推理请求（高吞吐量场景）"""
    triton_client = grpcclient.InferenceServerClient(
        url="localhost:8001"
    )

    result_queue = queue.Queue()

    def callback(result, error):
        if error:
            result_queue.put(error)
        else:
            result_queue.put(result.as_numpy("logits"))

    # 异步发送 100 个请求
    num_requests = 100
    for i in range(num_requests):
        input_ids = np.random.randint(0, 30000, size=(1, 512)).astype(np.int64)
        attention_mask = np.ones((1, 512), dtype=np.int64)

        inputs = [
            grpcclient.InferInput("input_ids", input_ids.shape, "INT64"),
            grpcclient.InferInput("attention_mask", attention_mask.shape, "INT64"),
        ]
        inputs[0].set_data_from_numpy(input_ids)
        inputs[1].set_data_from_numpy(attention_mask)

        outputs = [grpcclient.InferRequestedOutput("logits")]

        triton_client.async_infer(
            model_name="text_classifier",
            inputs=inputs,
            outputs=outputs,
            callback=partial(callback),
            request_id=f"async-req-{i}"
        )

    # 收集全部结果
    results = []
    for _ in range(num_requests):
        res = result_queue.get()
        if isinstance(res, Exception):
            print(f"Error: {res}")
        else:
            results.append(res)

    print(f"Completed {len(results)}/{num_requests} requests")
    return results


if __name__ == "__main__":
    run_inference()
    run_async_inference()

import tritonclient.grpc as grpcclient
from tritonclient import utils
import tritonclient.utils.cuda_shared_memory as cudashm
import numpy as np


def inference_with_cuda_shared_memory():
    """使用 CUDA Shared Memory 实现零拷贝推理"""
    triton_client = grpcclient.InferenceServerClient(url="localhost:8001")

    # 输入数据
    image_data = np.random.rand(1, 3, 640, 640).astype(np.float32)
    input_byte_size = image_data.nbytes

    # 创建 CUDA Shared Memory 区域
    shm_handle = cudashm.create_shared_memory_region(
        "input_images_shm", input_byte_size, 0  # GPU device 0
    )

    # 将数据复制到 CUDA Shared Memory
    cudashm.set_shared_memory_region(shm_handle, [image_data])

    # 向 Triton 服务器注册 Shared Memory 区域
    triton_client.register_cuda_shared_memory(
        "input_images_shm",
        cudashm.get_raw_handle(shm_handle),
        0,  # GPU device 0
        input_byte_size
    )

    # 将输入张量引用到 Shared Memory
    inputs = [grpcclient.InferInput("images", [1, 3, 640, 640], "FP32")]
    inputs[0].set_shared_memory("input_images_shm", input_byte_size)

    # 输出也通过 CUDA Shared Memory 接收
    output_byte_size = 100 * 6 * 4  # 100 detections * 6 values * float32
    out_shm_handle = cudashm.create_shared_memory_region(
        "output_detections_shm", output_byte_size, 0
    )
    triton_client.register_cuda_shared_memory(
        "output_detections_shm",
        cudashm.get_raw_handle(out_shm_handle),
        0,
        output_byte_size
    )

    outputs = [grpcclient.InferRequestedOutput("detections")]
    outputs[0].set_shared_memory("output_detections_shm", output_byte_size)

    # 执行推理（数据直接在 GPU 内存中传递）
    result = triton_client.infer(
        model_name="image_detector",
        inputs=inputs,
        outputs=outputs
    )

    # 从 Shared Memory 读取结果
    detections = cudashm.get_contents_as_numpy(
        out_shm_handle,
        utils.triton_to_np_dtype("FP32"),
        [100, 6]
    )

    # 清理
    triton_client.unregister_cuda_shared_memory("input_images_shm")
    triton_client.unregister_cuda_shared_memory("output_detections_shm")
    cudashm.destroy_shared_memory_region(shm_handle)
    cudashm.destroy_shared_memory_region(out_shm_handle)

    return detections

# 基本性能测量（基于并发数）
perf_analyzer \
  -m text_classifier \
  -u localhost:8001 \
  -i grpc \
  --concurrency-range 1:64:8 \
  --measurement-interval 10000 \
  --percentile=95 \
  --stability-percentage 10 \
  -v

# 基于请求速率的性能测量（模拟真实流量模式）
perf_analyzer \
  -m text_classifier \
  -u localhost:8001 \
  -i grpc \
  --request-rate-range 100:1000:100 \
  --measurement-interval 15000 \
  --percentile=99

# 使用真实数据进行性能测量
perf_analyzer \
  -m text_classifier \
  -u localhost:8001 \
  -i grpc \
  --input-data real_data.json \
  --concurrency-range 1:32 \
  --measurement-interval 10000

# 检查 Triton 指标端点
curl localhost:8002/metrics

# 核心指标：
# nv_inference_request_success     - 成功的推理请求数
# nv_inference_request_failure     - 失败的推理请求数
# nv_inference_count               - 总推理执行次数
# nv_inference_exec_count          - 批处理执行次数
# nv_inference_request_duration_us - 请求处理时间（微秒）
# nv_inference_queue_duration_us   - 队列等待时间（微秒）
# nv_inference_compute_infer_duration_us - 实际推理时间
# nv_gpu_utilization               - GPU 利用率
# nv_gpu_memory_used_bytes         - GPU 内存使用量
# nv_gpu_power_usage               - GPU 功耗

# prometheus/triton-alerts.yaml
# Triton 运营必备告警规则
groups:
  - name: triton-inference-alerts
    rules:
      # 推理失败率超过 1% 时告警
      - alert: TritonHighFailureRate
        expr: |
          rate(nv_inference_request_failure[5m])
          / (rate(nv_inference_request_success[5m]) + rate(nv_inference_request_failure[5m]))
          > 0.01
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: 'Triton 推理失败率超过 1%'
          description: '模型 {{ $labels.model }} 失败率：{{ $value | humanizePercentage }}'

      # p99 延迟超过 SLA
      - alert: TritonHighLatency
        expr: |
          histogram_quantile(0.99,
            rate(nv_inference_request_duration_us_bucket[5m])
          ) > 50000
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: 'Triton p99 延迟超过 50ms'

      # GPU 内存使用率超过 90%
      - alert: TritonGPUMemoryHigh
        expr: |
          nv_gpu_memory_used_bytes / nv_gpu_memory_total_bytes > 0.9
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: 'GPU 内存使用率超过 90% - 存在 OOM 风险'

      # 队列等待时间骤增（批处理瓶颈）
      - alert: TritonQueueBacklog
        expr: |
          rate(nv_inference_queue_duration_us_sum[5m])
          / rate(nv_inference_queue_duration_us_count[5m])
          > 10000
        for: 3m
        labels:
          severity: warning
        annotations:
          summary: '推理队列平均等待时间超过 10ms'

# triton-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: triton-inference-server
  namespace: ml-serving
  labels:
    app: triton
spec:
  replicas: 3
  selector:
    matchLabels:
      app: triton
  template:
    metadata:
      labels:
        app: triton
      annotations:
        prometheus.io/scrape: 'true'
        prometheus.io/port: '8002'
        prometheus.io/path: '/metrics'
    spec:
      containers:
        - name: triton
          image: nvcr.io/nvidia/tritonserver:25.02-py3
          command: ['tritonserver']
          args:
            - '--model-repository=s3://ml-models/triton-repo'
            - '--model-control-mode=poll'
            - '--repository-poll-secs=30'
            - '--strict-model-config=false'
            - '--log-verbose=0'
            - '--exit-on-error=false'
            - '--rate-limiter=execution_count'
            - '--rate-limiter-resource=R1:4:0'
          ports:
            - containerPort: 8000
              name: http
            - containerPort: 8001
              name: grpc
            - containerPort: 8002
              name: metrics
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: '32Gi'
              cpu: '8'
            requests:
              nvidia.com/gpu: 1
              memory: '16Gi'
              cpu: '4'
          readinessProbe:
            httpGet:
              path: /v2/health/ready
              port: 8000
            initialDelaySeconds: 30
            periodSeconds: 10
            failureThreshold: 3
          livenessProbe:
            httpGet:
              path: /v2/health/live
              port: 8000
            initialDelaySeconds: 60
            periodSeconds: 15
            failureThreshold: 5
          volumeMounts:
            - name: model-cache
              mountPath: /models/cache
            - name: shm
              mountPath: /dev/shm
      volumes:
        - name: model-cache
          emptyDir:
            sizeLimit: 50Gi
        - name: shm
          emptyDir:
            medium: Memory
            sizeLimit: 8Gi
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      nodeSelector:
        cloud.google.com/gke-accelerator: nvidia-tesla-a100
---
apiVersion: v1
kind: Service
metadata:
  name: triton-inference-svc
  namespace: ml-serving
spec:
  type: ClusterIP
  ports:
    - port: 8000
      targetPort: 8000
      name: http
    - port: 8001
      targetPort: 8001
      name: grpc
    - port: 8002
      targetPort: 8002
      name: metrics
  selector:
    app: triton
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: triton-hpa
  namespace: ml-serving
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: triton-inference-server
  minReplicas: 2
  maxReplicas: 10
  metrics:
    - type: Pods
      pods:
        metric:
          name: nv_inference_queue_duration_us
        target:
          type: AverageValue
          averageValue: '5000'
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
        - type: Pods
          value: 2
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
        - type: Pods
          value: 1
          periodSeconds: 120

# 1. 检查当前 GPU 内存状态
nvidia-smi --query-gpu=memory.used,memory.total --format=csv

# 2. 对各模型的内存使用量做画像分析
model-analyzer profile \
  --model-repository=/models \
  --profile-models problematic_model \
  --run-config-search-max-instance-count 1

# 3. 修改 config.pbtxt：减少 instance count、缩小 max_batch_size
# instance_group count: 4 -> 2
# max_batch_size: 64 -> 32

# 4. 重新加载模型（无需重启服务器）
curl -X POST localhost:8000/v2/repository/models/problematic_model/unload
curl -X POST localhost:8000/v2/repository/models/problematic_model/load

# 1. 检查 Triton 容器的 TensorRT 版本
docker run --rm nvcr.io/nvidia/tritonserver:25.02-py3 \
  dpkg -l | grep tensorrt

# 2. 在相同版本的 TensorRT 容器中重新构建引擎
docker run --gpus all --rm \
  -v $(pwd):/workspace \
  nvcr.io/nvidia/tritonserver:25.02-py3 \
  trtexec \
    --onnx=/workspace/model.onnx \
    --saveEngine=/workspace/model.plan \
    --fp16

# 3. 检查 GPU 架构
nvidia-smi --query-gpu=gpu_name,compute_cap --format=csv
# A100: compute_cap 8.0 (sm_80)
# A10G: compute_cap 8.6 (sm_86)
# H100: compute_cap 9.0 (sm_90)

# 错误代码（内存泄漏）
class TritonPythonModel:
    def initialize(self, args):
        self.results_cache = []  # 无限增长

    def execute(self, requests):
        for req in requests:
            result = process(req)
            self.results_cache.append(result)  # 永远不会被清理
        return responses

# 正确代码（内存管理得当）
class TritonPythonModel:
    def initialize(self, args):
        self.model = load_model()  # 仅在初始化时加载一次

    def execute(self, requests):
        responses = []
        for req in requests:
            result = self.model.predict(req)
            responses.append(build_response(result))
            # 局部变量在方法结束时自动释放
        return responses

# 1. 检查 GPU 利用率（偏低则需要增加批大小或实例数）
nvidia-smi dmon -s u -d 1

# 2. 在 Triton 指标中检查队列等待时间
curl -s localhost:8002/metrics | grep queue_duration

# 3. 检查各模型的推理时间
curl -s localhost:8002/metrics | grep compute_infer_duration

# 4. 检查批处理效率（inference_count 相对 exec_count 的比例）
# inference_count / exec_count = 平均批大小
curl -s localhost:8002/metrics | grep -E "(nv_inference_count|nv_inference_exec_count)"

# 启用 Rate Limiter（服务器启动时）
tritonserver \
  --model-repository=/models \
  --rate-limiter=execution_count \
  --rate-limiter-resource=GPU_EXEC_SLOTS:8:0
  # GPU 0 上可同时执行的最大槽位数：8

# 让高负载模型消耗更多槽位
# heavy_model/config.pbtxt
name: "heavy_model"
rate_limiter {
  resources [
    {
      name: "GPU_EXEC_SLOTS"
      count: 4  # 该模型占用 8 个中的 4 个槽位
    }
  ]
}

# 轻量模型消耗较少槽位
# light_model/config.pbtxt
name: "light_model"
rate_limiter {
  resources [
    {
      name: "GPU_EXEC_SLOTS"
      count: 1  # 仅占用 1 个槽位
    }
  ]
}

# 启用 Response Cache（64MB 缓存）
tritonserver \
  --model-repository=/models \
  --response-cache-byte-size=67108864

# 仅对特定模型应用缓存
# embedding_model/config.pbtxt
name: "embedding_model"
response_cache { enable: true }