fix: serve metrics correctly when there are multiple listeners

core/corehttp/metrics.go

@@ -3,6 +3,7 @@ package corehttp
 import (
 	"net"
 	"net/http"
+	"sync"
 	"time"
 
 	core "github.com/ipfs/kubo/core"
@@ -14,6 +15,19 @@ import (
 	promhttp "github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
+// ocHandlers tracks registration of metrics exporters and instruments used has
+// handlers for serving metrics.
+var ocHandlers = map[string]http.Handler{}
+var ocMutex sync.Mutex
+
+// Names of metrics handlers.
+const (
+	ipfsOC         = "ipfs_oc"
+	ipfsOCDefault  = "ipfs_oc_default"
+	ipfsOCPromMux  = "ipfs_oc_prommux"
+	ipfsOCChildMux = "ipfs_oc_childmux"
+)
+
 // MetricsScrapingOption adds the scraping endpoint which Prometheus uses to fetch metrics.
 func MetricsScrapingOption(path string) ServeOption {
 	return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
@@ -27,25 +41,33 @@ func MetricsOpenCensusCollectionOption() ServeOption {
 	return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
 		log.Info("Init OpenCensus")
 
-		promRegistry := prometheus.NewRegistry()
-		pe, err := ocprom.NewExporter(ocprom.Options{
-			Namespace: "ipfs_oc",
-			Registry:  promRegistry,
-			OnError: func(err error) {
-				log.Errorw("OC ERROR", "error", err)
-			},
-		})
-		if err != nil {
-			return nil, err
-		}
+		ocMutex.Lock()
+		defer ocMutex.Unlock()
+		ocHandler, ok := ocHandlers[ipfsOC]
+		if !ok {
+			promRegistry := prometheus.NewRegistry()
+			pe, err := ocprom.NewExporter(ocprom.Options{
+				Namespace: "ipfs_oc",
+				Registry:  promRegistry,
+				OnError: func(err error) {
+					log.Errorw("OC ERROR", "error", err)
+				},
+			})
+			if err != nil {
+				return nil, err
+			}
 
-		// register prometheus with opencensus
-		view.RegisterExporter(pe)
-		view.SetReportingPeriod(2 * time.Second)
+			// register prometheus with opencensus
+			view.RegisterExporter(pe)
+			view.SetReportingPeriod(2 * time.Second)
+
+			ocHandler = pe
+			ocHandlers[ipfsOC] = ocHandler
+		}
 
 		// Construct the mux
 		zpages.Handle(mux, "/debug/metrics/oc/debugz")
-		mux.Handle("/debug/metrics/oc", pe)
+		mux.Handle("/debug/metrics/oc", ocHandler)
 
 		return mux, nil
 	}
@@ -58,6 +80,13 @@ func MetricsOpenCensusDefaultPrometheusRegistry() ServeOption {
 	return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
 		log.Info("Init OpenCensus with default prometheus registry")
 
+		ocMutex.Lock()
+		defer ocMutex.Unlock()
+		_, ok := ocHandlers[ipfsOCDefault]
+		if ok {
+			return mux, nil
+		}
+
 		pe, err := ocprom.NewExporter(ocprom.Options{
 			Registry: prometheus.DefaultRegisterer.(*prometheus.Registry),
 			OnError: func(err error) {
@@ -71,13 +100,24 @@ func MetricsOpenCensusDefaultPrometheusRegistry() ServeOption {
 		// register prometheus with opencensus
 		view.RegisterExporter(pe)
 
+		ocHandlers[ipfsOCDefault] = pe
+
 		return mux, nil
 	}
 }
 
 // MetricsCollectionOption adds collection of net/http-related metrics.
 func MetricsCollectionOption(handlerName string) ServeOption {
 	return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+		ocMutex.Lock()
+		defer ocMutex.Unlock()
+		promMux, ok := ocHandlers[ipfsOCPromMux]
+		if ok {
+			mux.Handle("/", promMux)
+			childMux := ocHandlers[ipfsOCChildMux]
+			return childMux.(*http.ServeMux), nil
+		}
+
 		// Adapted from github.com/prometheus/client_golang/prometheus/http.go
 		// Work around https://github.com/prometheus/client_golang/pull/311
 		opts := prometheus.SummaryOpts{
@@ -140,13 +180,15 @@ func MetricsCollectionOption(handlerName string) ServeOption {
 
 		// Construct the mux
 		childMux := http.NewServeMux()
-		var promMux http.Handler = childMux
+		promMux = childMux
 		promMux = promhttp.InstrumentHandlerResponseSize(resSz, promMux)
 		promMux = promhttp.InstrumentHandlerRequestSize(reqSz, promMux)
 		promMux = promhttp.InstrumentHandlerDuration(reqDur, promMux)
 		promMux = promhttp.InstrumentHandlerCounter(reqCnt, promMux)
 		mux.Handle("/", promMux)
 
+		ocHandlers[ipfsOCChildMux] = childMux
+		ocHandlers[ipfsOCPromMux] = promMux
 		return childMux, nil
 	}
 }

docs/changelogs/v0.30.md

@@ -13,6 +13,7 @@
   - [`/unix/` socket support in `Addresses.API`](#unix-socket-support-in-addressesapi)
   - [Cleaned Up `ipfs daemon` Startup Log](#cleaned-up-ipfs-daemon-startup-log)
   - [UnixFS 1.5: Mode and Modification Time Support](#unixfs-15-mode-and-modification-time-support)
+  - [Serve Metrics with Multiple Listeners](#serve-metrics-with-multiple-listeners)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -128,6 +129,10 @@ Opt-in support for `mode` and `mtime` was also added to MFS (`ipfs files --help`
 > [!NOTE]
 > Storing `mode` and `mtime` requires root block to be `dag-pb` and disabled `raw-leaves` setting to create envelope for storing the metadata.
 
+#### Serve Metrics with Multiple Listeners
+
+When there are multiple listeners configured for Addresses.API, serving metrics resulted in errors: "xxx was collected before with the same name and label values". This is now fixed by creating and reginstering metrics exporters and instruments once. These are used as handlers when serving metrics once, and now the same once-registered handlers are provided to the mux for every listener.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors