WSL2环境下TorchServe metrics.yaml配置失效及指标异常求助

• 问题1：ts.config中metrics_config指向不存在的文件时，服务仍能正常启动，该参数未生效
• 问题2：注释/移除ts_metrics或model_metrics中的部分指标后，这些指标依然出现在对应日志文件（ts_metrics.log/model_metrics.log）中
• 问题3：通过metrics API只能获取到ts_queue_latency_microseconds和ts_inference_requests_total两个指标，其他ts指标无法获取

相关配置文件 #

ts.config #

models={\
  "doc_model": {\
    "1.0": {\
        "defaultVersion": true,\
        "minWorkers": 1,\
        "maxWorkers": 1,\
        "batchSize": 1\
    }\
  }\
}
inference_address=http://0.0.0.0:8080
management_address=http://0.0.0.0:8081
metrics_address=http://0.0.0.0:8082
metrics_mode=prometheus
metrics_config=./metrics.yaml

number_of_netty_threads=32
job_queue_size=1000
model_store=/home/model-server/model-store
workflow_store=/home/model-server/wf-store

metrics.yaml #

dimensions:
  - &model_name "ModelName"
  - &worker_name "WorkerName"
  - &level "Level"
  - &device_id "DeviceId"
  - &hostname "Hostname"

ts_metrics:
  counter:
    # - name: Requests2XX
    #   unit: Count
    #   dimensions: [*level, *hostname]
    - name: Requests4XX
      unit: Count
      dimensions: [*level, *hostname]
    # - name: Requests5XX
    #   unit: Count
    #   dimensions: [*level, *hostname, *model_name]
    - name: ts_inference_requests_total
      unit: Count
      dimensions: [*level,"model_name", "model_version", "hostname"]
    - name: ts_inference_latency_microseconds
      unit: Microseconds
      dimensions: ["model_name", "model_version", "hostname"]
    - name: ts_queue_latency_microseconds
      unit: Microseconds
      dimensions: ["model_name", "model_version", "hostname"]

  histogram:
    - name: NameOfHistogramMetric
      unit: ms
      dimensions: [*model_name, *level]

  gauge:
    - name: QueueTime
      unit: Milliseconds
      dimensions: [*level, *hostname]
    - name: WorkerThreadTime
      unit: Milliseconds
      dimensions: [*level, *hostname]
    - name: WorkerLoadTime
      unit: Milliseconds
      dimensions: [*worker_name, *level, *hostname]
    - name: CPUUtilization
      unit: Percent
      dimensions: [*level, *hostname]
    - name: MemoryUsed
      unit: Megabytes
      dimensions: [*level, *hostname]
    - name: MemoryAvailable
      unit: Megabytes
      dimensions: [*level, *hostname]
    - name: MemoryUtilization
      unit: Percent
      dimensions: [*level, *hostname]
    - name: DiskUsage
      unit: Gigabytes
      dimensions: [*level, *hostname]
    - name: DiskUtilization
      unit: Percent
      dimensions: [*level, *hostname]
    - name: DiskAvailable
      unit: Gigabytes
      dimensions: [*level, *hostname]
    - name: GPUMemoryUtilization
      unit: Percent
      dimensions: [*level, *device_id, *hostname]
    - name: GPUMemoryUsed
      unit: Megabytes
      dimensions: [*level, *device_id, *hostname]
    - name: GPUUtilization
      unit: Percent
      dimensions: [*level, *device_id, *hostname]

model_metrics:
  # Dimension "Hostname" is automatically added for model metrics in the backend
  gauge:
    - name: HandlerTime
      unit: ms
      dimensions: [*model_name, *level]
    - name: PredictionTime
      unit: ms
      dimensions: [*model_name, *level]

metrics API输出 #

# HELP ts_inference_latency_microseconds Cumulative inference duration in microseconds
# TYPE ts_inference_latency_microseconds counter
ts_inference_latency_microseconds{uuid="57984a8f-c19a-4c93-b9cd-cc7eb8b1fa55",model_name="doc_model",model_version="default",} 4925586.4
# HELP ts_inference_requests_total Total number of inference requests.
# TYPE ts_inference_requests_total counter
ts_inference_requests_total{uuid="57984a8f-c19a-4c93-b9cd-cc7eb8b1fa55",model_name="doc_model",model_version="default",} 3.0
# HELP ts_queue_latency_microseconds Cumulative queue duration in microseconds
# TYPE ts_queue_latency_microseconds counter
ts_queue_latency_microseconds{uuid="57984a8f-c19a-4c93-b9cd-cc7eb8b1fa55",model_name="doc_model",model_version="default",} 291.4

操作命令 #

构建MAR包命令 #

torch-model-archiver \
--model-name doc_model \
--version 1.0 \
--serialized-file model/pytorch_model.bin \
--handler ./src/transformers_vectorizer_handler.py \
--extra-files "./model/config.json,./tokenizer" \
-f
mkdir -p model_store && mv doc_model.mar model_store/

启动TorchServe命令 #

torchserve \
--start \
--model-store model_store \
--models doc_model=doc_model.mar \
--ncs \
--ts-config ./ts.config

内容的提问来源于stack exchange，提问作者feeeper