Skip to content

Commit

Permalink
fix: config files for dcgm exporter (#823)
Browse files Browse the repository at this point in the history
* fix: dcgm exporter config

* refactor: rename dirs for dcgm exporter
  • Loading branch information
kongfei605 authored Mar 8, 2024
1 parent ca97095 commit 0205216
Show file tree
Hide file tree
Showing 25 changed files with 245 additions and 12 deletions.
2 changes: 1 addition & 1 deletion agent/metrics_agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import (
_ "flashcat.cloud/categraf/inputs/conntrack"
_ "flashcat.cloud/categraf/inputs/consul"
_ "flashcat.cloud/categraf/inputs/cpu"
_ "flashcat.cloud/categraf/inputs/dcgm_exporter"
_ "flashcat.cloud/categraf/inputs/dcgm"
_ "flashcat.cloud/categraf/inputs/disk"
_ "flashcat.cloud/categraf/inputs/diskio"
_ "flashcat.cloud/categraf/inputs/dns_query"
Expand Down
66 changes: 66 additions & 0 deletions conf/input.dcgm/1.x-compatibility-metrics.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
# Format
# If line starts with a '#' it is considered a comment
# DCGM FIELD, Prometheus metric type, help message

# Clocks
dcgm_sm_clock, gauge, SM clock frequency (in MHz).
dcgm_memory_clock, gauge, Memory clock frequency (in MHz).

# Temperature
dcgm_memory_temp, gauge, Memory temperature (in C).
dcgm_gpu_temp, gauge, GPU temperature (in C).

# Power
dcgm_power_usage, gauge, Power draw (in W).
dcgm_total_energy_consumption, counter, Total energy consumption since boot (in mJ).

# PCIe
dcgm_pcie_tx_throughput, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML.
dcgm_pcie_rx_throughput, counter, Total number of bytes received through PCIe RX (in KB) via NVML.
dcgm_pcie_replay_counter, counter, Total number of PCIe retries.

# Utilization (the sample period varies depending on the product)
dcgm_gpu_utilization, gauge, GPU utilization (in %).
dcgm_mem_copy_utilization, gauge, Memory utilization (in %).
dcgm_enc_utilization, gauge, Encoder utilization (in %).
dcgm_dec_utilization, gauge, Decoder utilization (in %).

# Errors and violations
dcgm_xid_errors, gauge, Value of the last XID error encountered.
# dcgm_power_violation, counter, Throttling duration due to power constraints (in us).
# dcgm_thermal_violation, counter, Throttling duration due to thermal constraints (in us).
# dcgm_sync_boost_violation, counter, Throttling duration due to sync-boost constraints (in us).
# dcgm_board_limit_violation, counter, Throttling duration due to board limit constraints (in us).
# dcgm_low_util_violation, counter, Throttling duration due to low utilization (in us).
# dcgm_reliability_violation, counter, Throttling duration due to reliability constraints (in us).

# Memory usage
dcgm_fb_free, gauge, Framebuffer memory free (in MiB).
dcgm_fb_used, gauge, Framebuffer memory used (in MiB).

# ECC
# dcgm_ecc_sbe_volatile_total, counter, Total number of single-bit volatile ECC errors.
# dcgm_ecc_dbe_volatile_total, counter, Total number of double-bit volatile ECC errors.
# dcgm_ecc_sbe_aggregate_total, counter, Total number of single-bit persistent ECC errors.
# dcgm_ecc_dbe_aggregate_total, counter, Total number of double-bit persistent ECC errors.

# Retired pages
# dcgm_retired_pages_sbe, counter, Total number of retired pages due to single-bit errors.
# dcgm_retired_pages_dbe, counter, Total number of retired pages due to double-bit errors.
# dcgm_retired_pages_pending, counter, Total number of pages pending retirement.

# NVLink
# dcgm_nvlink_flit_crc_error_count_total, counter, Total number of NVLink flow-control CRC errors.
# dcgm_nvlink_data_crc_error_count_total, counter, Total number of NVLink data CRC errors.
# dcgm_nvlink_replay_error_count_total, counter, Total number of NVLink retries.
# dcgm_nvlink_recovery_error_count_total, counter, Total number of NVLink recovery errors.
dcgm_nvlink_bandwidth_total, counter, Total number of NVLink bandwidth counters for all lanes

# Add DCP metrics
dcgm_fi_prof_gr_engine_active, gauge, Ratio of time the graphics engine is active (in %).
# dcgm_fi_prof_sm_active, gauge, The ratio of cycles an SM has at least 1 warp assigned (in %).
# dcgm_fi_prof_sm_occupancy, gauge, The ratio of number of warps resident on an SM (in %).
dcgm_fi_prof_pipe_tensor_active, gauge, Ratio of cycles the tensor (HMMA) pipe is active (in %).
dcgm_fi_prof_dram_active, gauge, Ratio of cycles the device memory interface is active sending or receiving data (in %).
dcgm_fi_prof_pcie_tx_bytes, counter, The number of bytes of active pcie tx data including both header and payload.
dcgm_fi_prof_pcie_rx_bytes, counter, The number of bytes of active pcie rx data including both header and payload.
90 changes: 90 additions & 0 deletions conf/input.dcgm/dcp-metrics-included.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# Format
# If line starts with a '#' it is considered a comment
# DCGM FIELD, Prometheus metric type, help message

# Clocks
DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz).
DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).

# Temperature
DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C).

# Power
DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W).
DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).

# PCIE
# DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML.
# DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML.
DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.

# Utilization (the sample period varies depending on the product)
DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).
DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %).
DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %).

# Errors and violations
DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.
# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us).
# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us).
# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us).
# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us).
# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).

# Memory usage
DCGM_FI_DEV_FB_FREE, gauge, Framebuffer memory free (in MiB).
DCGM_FI_DEV_FB_USED, gauge, Framebuffer memory used (in MiB).

# ECC
# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.

# Retired pages
# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors.
# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors.
# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.

# NVLink
# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries.
# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes.
# DCGM_FI_DEV_NVLINK_BANDWIDTH_L0, counter, The number of bytes of active NVLink rx or tx data including both header and payload.

# VGPU License status
DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status

# Remapped rows
DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors
DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed

# Static configuration information. These appear as labels on the other metrics
DCGM_FI_DRIVER_VERSION, label, Driver Version
# DCGM_FI_NVML_VERSION, label, NVML Version
# DCGM_FI_DEV_BRAND, label, Device Brand
# DCGM_FI_DEV_SERIAL, label, Device Serial Number
# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version
# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version
# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version
# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version
# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device

# DCP metrics
DCGM_FI_PROF_GR_ENGINE_ACTIVE, gauge, Ratio of time the graphics engine is active (in %).
# DCGM_FI_PROF_SM_ACTIVE, gauge, The ratio of cycles an SM has at least 1 warp assigned (in %).
# DCGM_FI_PROF_SM_OCCUPANCY, gauge, The ratio of number of warps resident on an SM (in %).
DCGM_FI_PROF_PIPE_TENSOR_ACTIVE, gauge, Ratio of cycles the tensor (HMMA) pipe is active (in %).
DCGM_FI_PROF_DRAM_ACTIVE, gauge, Ratio of cycles the device memory interface is active sending or receiving data (in %).
# DCGM_FI_PROF_PIPE_FP64_ACTIVE, gauge, Ratio of cycles the fp64 pipes are active (in %).
# DCGM_FI_PROF_PIPE_FP32_ACTIVE, gauge, Ratio of cycles the fp32 pipes are active (in %).
# DCGM_FI_PROF_PIPE_FP16_ACTIVE, gauge, Ratio of cycles the fp16 pipes are active (in %).
DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second.
DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second.

77 changes: 77 additions & 0 deletions conf/input.dcgm/default-counters.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# Format
# If line starts with a '#' it is considered a comment
# DCGM FIELD, Prometheus metric type, help message

# Clocks
DCGM_FI_DEV_SM_CLOCK, gauge, SM clock frequency (in MHz).
DCGM_FI_DEV_MEM_CLOCK, gauge, Memory clock frequency (in MHz).
# DCGM_EXP_CLOCK_EVENTS_COUNT, gauge, Count of clock events within the user-specified time window (see clock-events-count-window-size param).

# Temperature
DCGM_FI_DEV_MEMORY_TEMP, gauge, Memory temperature (in C).
DCGM_FI_DEV_GPU_TEMP, gauge, GPU temperature (in C).

# Power
DCGM_FI_DEV_POWER_USAGE, gauge, Power draw (in W).
DCGM_FI_DEV_TOTAL_ENERGY_CONSUMPTION, counter, Total energy consumption since boot (in mJ).

# PCIE
DCGM_FI_DEV_PCIE_TX_THROUGHPUT, counter, Total number of bytes transmitted through PCIe TX (in KB) via NVML.
DCGM_FI_DEV_PCIE_RX_THROUGHPUT, counter, Total number of bytes received through PCIe RX (in KB) via NVML.
DCGM_FI_DEV_PCIE_REPLAY_COUNTER, counter, Total number of PCIe retries.

# Utilization (the sample period varies depending on the product)
DCGM_FI_DEV_GPU_UTIL, gauge, GPU utilization (in %).
DCGM_FI_DEV_MEM_COPY_UTIL, gauge, Memory utilization (in %).
DCGM_FI_DEV_ENC_UTIL, gauge, Encoder utilization (in %).
DCGM_FI_DEV_DEC_UTIL , gauge, Decoder utilization (in %).

# Errors and violations
DCGM_FI_DEV_XID_ERRORS, gauge, Value of the last XID error encountered.
# DCGM_FI_DEV_POWER_VIOLATION, counter, Throttling duration due to power constraints (in us).
# DCGM_FI_DEV_THERMAL_VIOLATION, counter, Throttling duration due to thermal constraints (in us).
# DCGM_FI_DEV_SYNC_BOOST_VIOLATION, counter, Throttling duration due to sync-boost constraints (in us).
# DCGM_FI_DEV_BOARD_LIMIT_VIOLATION, counter, Throttling duration due to board limit constraints (in us).
# DCGM_FI_DEV_LOW_UTIL_VIOLATION, counter, Throttling duration due to low utilization (in us).
# DCGM_FI_DEV_RELIABILITY_VIOLATION, counter, Throttling duration due to reliability constraints (in us).
# DCGM_EXP_XID_ERRORS_COUNT, gauge, Count of XID Errors within user-specified time window (see xid-count-window-size param).
# Memory usage
DCGM_FI_DEV_FB_FREE, gauge, Frame buffer memory free (in MB).
DCGM_FI_DEV_FB_USED, gauge, Frame buffer memory used (in MB).

# ECC
# DCGM_FI_DEV_ECC_SBE_VOL_TOTAL, counter, Total number of single-bit volatile ECC errors.
# DCGM_FI_DEV_ECC_DBE_VOL_TOTAL, counter, Total number of double-bit volatile ECC errors.
# DCGM_FI_DEV_ECC_SBE_AGG_TOTAL, counter, Total number of single-bit persistent ECC errors.
# DCGM_FI_DEV_ECC_DBE_AGG_TOTAL, counter, Total number of double-bit persistent ECC errors.

# Retired pages
# DCGM_FI_DEV_RETIRED_SBE, counter, Total number of retired pages due to single-bit errors.
# DCGM_FI_DEV_RETIRED_DBE, counter, Total number of retired pages due to double-bit errors.
# DCGM_FI_DEV_RETIRED_PENDING, counter, Total number of pages pending retirement.

# NVLink
# DCGM_FI_DEV_NVLINK_CRC_FLIT_ERROR_COUNT_TOTAL, counter, Total number of NVLink flow-control CRC errors.
# DCGM_FI_DEV_NVLINK_CRC_DATA_ERROR_COUNT_TOTAL, counter, Total number of NVLink data CRC errors.
# DCGM_FI_DEV_NVLINK_REPLAY_ERROR_COUNT_TOTAL, counter, Total number of NVLink retries.
# DCGM_FI_DEV_NVLINK_RECOVERY_ERROR_COUNT_TOTAL, counter, Total number of NVLink recovery errors.
DCGM_FI_DEV_NVLINK_BANDWIDTH_TOTAL, counter, Total number of NVLink bandwidth counters for all lanes

# VGPU License status
DCGM_FI_DEV_VGPU_LICENSE_STATUS, gauge, vGPU License status

# Remapped rows
DCGM_FI_DEV_UNCORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for uncorrectable errors
DCGM_FI_DEV_CORRECTABLE_REMAPPED_ROWS, counter, Number of remapped rows for correctable errors
DCGM_FI_DEV_ROW_REMAP_FAILURE, gauge, Whether remapping of rows has failed

# Static configuration information. These appear as labels on the other metrics
DCGM_FI_DRIVER_VERSION, label, Driver Version
# DCGM_FI_NVML_VERSION, label, NVML Version
# DCGM_FI_DEV_BRAND, label, Device Brand
# DCGM_FI_DEV_SERIAL, label, Device Serial Number
# DCGM_FI_DEV_OEM_INFOROM_VER, label, OEM inforom version
# DCGM_FI_DEV_ECC_INFOROM_VER, label, ECC inforom version
# DCGM_FI_DEV_POWER_INFOROM_VER, label, Power management object inforom version
# DCGM_FI_DEV_INFOROM_IMAGE_VER, label, Inforom image version
# DCGM_FI_DEV_VBIOS_VERSION, label, VBIOS version of the device
12 changes: 6 additions & 6 deletions conf/input.dcgm/exporter.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#[[instances]]
# path to the file, that contains the DCGM fields to collect
# collectors = "/etc/dcgm-exporter/default-counters.csv"
# collectors = "conf/input.dcgm/default-counters.csv"

# Enable kubernetes mapping metrics to kubernetes pods
# kubernetes=false
Expand All @@ -11,15 +11,15 @@
# Use old 1.x namespace
# use-old-namespace = false

# cpu-devices = "f"
cpu-devices = "f"

# gpu devices
# devices = "f"
devices = "f"

# switch-devices = "f"
switch-devices = "f"

# ConfigMap <NAMESPACE>:<NAME> for metric data
# configmap-data = "none"
configmap-data = "none"

# Connect to remote hostengine at <HOST>:<PORT>
# remote-hostengine-info = "localhost:5555"
Expand All @@ -28,4 +28,4 @@
# fake-gpus = false

# Replaces every blank space in the GPU model name with a dash, ensuring a continuous, space-free identifier.
# replace-blanks-in-model-name = false
# replace-blanks-in-model-name = false
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
8 changes: 4 additions & 4 deletions inputs/dcgm_exporter/exporter.go → inputs/dcgm/exporter.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
//go:build dcgm

package dcgm_exporter
package dcgm

import (
"bytes"
Expand All @@ -15,7 +15,7 @@ import (

"flashcat.cloud/categraf/config"
"flashcat.cloud/categraf/inputs"
"flashcat.cloud/categraf/inputs/dcgm_exporter/dcgmexporter"
"flashcat.cloud/categraf/inputs/dcgm/dcgmexporter"
"flashcat.cloud/categraf/parser/prometheus"
"flashcat.cloud/categraf/types"
)
Expand Down Expand Up @@ -176,7 +176,7 @@ func (ins *Instance) Init() error {
groups, err = dcgm.GetSupportedMetricGroups(0)
if err != nil {
cfg.CollectDCP = false
log.Printf("Not collecting DCP metrics: ", err)
log.Println("Not collecting DCP metrics: ", err)
} else {
log.Println("Collecting DCP Metrics")
cfg.MetricGroups = groups
Expand Down Expand Up @@ -349,7 +349,7 @@ func (ins *Instance) Gather(slist *types.SampleList) {
log.Println("E! dcgm exporter collects error:", err)
return
}
parser := prometheus.NewParser(inputName, labels, http.Header{}, false,
parser := prometheus.NewParser("", labels, http.Header{}, false,
nil, nil)
err = parser.Parse(buf.Bytes(), slist)
if err != nil {
Expand Down
1 change: 1 addition & 0 deletions inputs/dcgm/exporter_none.go
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
package dcgm
1 change: 0 additions & 1 deletion inputs/dcgm_exporter/exporter_none.go

This file was deleted.

0 comments on commit 0205216

Please sign in to comment.