feat: Expose resource stats in traces as separate event (#1973)

Looks like: <img width="922" alt="Screenshot 2024-11-20 at 11 30 09" src="https://github.com/user-attachments/assets/57075f1a-423d-41c3-8aff-4a74349bad82">
cloudquery · Nov 26, 2024 · e74bb27 · e74bb27
1 parent 566409d
commit e74bb27
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 0 deletions.
diff --git a/scheduler/queue/worker.go b/scheduler/queue/worker.go
@@ -80,6 +80,13 @@ func (w *worker) resolveTable(ctx context.Context, table *schema.Table, client s
 		logger.Info().Msg("top level table resolver started")
 	}
 	tableMetrics := w.metrics.TableClient[table.Name][clientName]
+	defer func() {
+		span.AddEvent("sync.finish.stats", trace.WithAttributes(
+			attribute.Key("sync.resources").Int64(int64(atomic.LoadUint64(&tableMetrics.Resources))),
+			attribute.Key("sync.errors").Int64(int64(atomic.LoadUint64(&tableMetrics.Errors))),
+			attribute.Key("sync.panics").Int64(int64(atomic.LoadUint64(&tableMetrics.Panics))),
+		))
+	}()
 	tableMetrics.OtelStartTime(ctx, startTime)
 
 	res := make(chan any)

diff --git a/scheduler/scheduler_dfs.go b/scheduler/scheduler_dfs.go
@@ -91,7 +91,15 @@ func (s *syncClient) resolveTableDfs(ctx context.Context, table *schema.Table, c
 	if parent == nil { // Log only for root tables, otherwise we spam too much.
 		logger.Info().Msg("top level table resolver started")
 	}
+
 	tableMetrics := s.metrics.TableClient[table.Name][clientName]
+	defer func() {
+		span.AddEvent("sync.finish.stats", trace.WithAttributes(
+			attribute.Key("sync.resources").Int64(int64(atomic.LoadUint64(&tableMetrics.Resources))),
+			attribute.Key("sync.errors").Int64(int64(atomic.LoadUint64(&tableMetrics.Errors))),
+			attribute.Key("sync.panics").Int64(int64(atomic.LoadUint64(&tableMetrics.Panics))),
+		))
+	}()
 	tableMetrics.OtelStartTime(ctx, startTime)
 
 	res := make(chan any)