Adding alerting for wait buffer

metrics/prometheus.go

@@ -79,13 +79,19 @@ var (
 		Namespace: "rules",
 		Help:      "etcd rules engine crawler values count",
 	}, []string{"name"})
+	rulesEngineWorkerCount = prometheus.NewGauge(prometheus.GaugeOpts{
+		Name:      "worker_count",
+		Subsystem: "etcd",
+		Namespace: "rules",
+		Help:      "etcd rules engine worker count",
+	})
 )
 
 func init() {
 	prometheus.MustRegister(rulesEngineLockCount, rulesEngineSatisfiedThenNot, rulesEngineEvaluations,
 		rulesEngineWorkerQueueWait, rulesEngineWorkBufferWaitTime, rulesEngineCallbackWaitTime,
 		rulesEngineKeyProcessBufferCap, rulesEngineWatcherErrors, rulesEngineCrawlerQueryTime,
-		rulesEngineCrawlerEvalTime, rulesEngineCrawlerValues)
+		rulesEngineCrawlerEvalTime, rulesEngineCrawlerValues, rulesEngineWorkerCount)
 }
 
 // IncLockMetric increments the lock count.
@@ -145,3 +151,8 @@ func CrawlerEvalTime(name string, startTime time.Time) {
 func CrawlerValuesCount(name string, count int) {
 	rulesEngineCrawlerValues.WithLabelValues(name).Set(float64(count))
 }
+
+// WorkerCount tracks the number of workers
+func WorkersCount(count int) {
+	rulesEngineWorkerCount.Set(float64(count))
+}

metrics/prometheus_test.go

@@ -97,3 +97,8 @@ func TestCrawlerValuesCount(t *testing.T) {
 	CrawlerValuesCount("default", 100)
 	checkMetrics(t, `rules_etcd_crawler_values_count{name="default"} 100`)
 }
+
+func TestWorkersCount(t *testing.T) {
+	WorkersCount(100)
+	checkMetrics(t, `rules_etcd_worker_count 100`)
+}

rules/engine.go

@@ -11,6 +11,7 @@ import (
 	"golang.org/x/net/context"
 
 	"github.com/IBM-Cloud/go-etcd-rules/concurrency"
+	"github.com/IBM-Cloud/go-etcd-rules/metrics"
 	"github.com/IBM-Cloud/go-etcd-rules/rules/lock"
 )
 
@@ -326,6 +327,7 @@ func (e *v3Engine) Run() {
 	go c.run()
 
 	e.logger.Info("Starting workers", zap.Int("count", e.options.concurrency))
+	metrics.WorkersCount(e.options.concurrency)
 	for i := 0; i < e.options.concurrency; i++ {
 		id := fmt.Sprintf("worker%d", i)
 		w, err := newV3Worker(id, e)