Skip to content

Commit

Permalink
More logging.
Browse files Browse the repository at this point in the history
  • Loading branch information
igorpeshansky committed Jul 26, 2024
1 parent 7e089fd commit 9c9d15a
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 0 deletions.
2 changes: 2 additions & 0 deletions receiver/dcgmreceiver/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -384,9 +384,11 @@ func (client *dcgmClient) collectDeviceMetrics() (map[uint][]dcgmMetric, error)
var err scrapererror.ScrapeErrors
gpuMetrics := make(map[uint][]dcgmMetric)
for _, gpuIndex := range client.deviceIndices {
client.logger.Debugf("Polling DCGM daemon for GPU %d", gpuIndex)
retry := true
for i := 0; retry && i < client.maxRetries; i++ {
fieldValues, pollErr := dcgmGetLatestValuesForFields(gpuIndex, client.enabledFieldIDs)
client.logger.Debugf("Got %d field values", len(fieldValues))
if pollErr == nil {
gpuMetrics[gpuIndex], retry = client.appendMetrics(gpuMetrics[gpuIndex], gpuIndex, fieldValues)
if retry {
Expand Down
7 changes: 7 additions & 0 deletions receiver/dcgmreceiver/scraper.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,14 +180,21 @@ func (s *dcgmScraper) scrape(_ context.Context) (pmetric.Metrics, error) {
return s.mb.Emit(), err
}

s.settings.Logger.Sugar().Info(fmt.Sprintf("Client created: %v; collecting metrics", s.client))
deviceMetrics, err := s.client.collectDeviceMetrics()
if err != nil {
s.settings.Logger.Sugar().Warn(fmt.Sprintf("Metrics not collected; err=%v", err))
return s.mb.Emit(), err
}
s.settings.Logger.Sugar().Info(fmt.Sprintf("Metrics collected: %d", len(deviceMetrics)))

now := pcommon.NewTimestampFromTime(time.Now())
for gpuIndex, gpuMetrics := range deviceMetrics {
metricsByName := make(map[string][]dcgmMetric)
for _, metric := range gpuMetrics {
metricsByName[metric.name] = append(metricsByName[metric.name], metric)
}
s.settings.Logger.Sugar().Warn(fmt.Sprintf("Got %d unique metrics: %v", len(metricsByName), metricsByName))
metrics := make(map[string]dcgmMetric)
for name, points := range metricsByName {
slices.SortStableFunc(points, func(a, b dcgmMetric) int {
Expand Down

0 comments on commit 9c9d15a

Please sign in to comment.