diff --git a/core/corehttp/metrics.go b/core/corehttp/metrics.go index f43362ff755..cb022e37724 100644 --- a/core/corehttp/metrics.go +++ b/core/corehttp/metrics.go @@ -3,6 +3,7 @@ package corehttp import ( "net" "net/http" + "sync" "time" core "github.com/ipfs/kubo/core" @@ -14,6 +15,19 @@ import ( promhttp "github.com/prometheus/client_golang/prometheus/promhttp" ) +// ocHandlers tracks registration of metrics exporters and instruments used has +// handlers for serving metrics. +var ocHandlers = map[string]http.Handler{} +var ocMutex sync.Mutex + +// Names of metrics handlers. +const ( + ipfsOC = "ipfs_oc" + ipfsOCDefault = "ipfs_oc_default" + ipfsOCPromMux = "ipfs_oc_prommux" + ipfsOCChildMux = "ipfs_oc_childmux" +) + // MetricsScrapingOption adds the scraping endpoint which Prometheus uses to fetch metrics. func MetricsScrapingOption(path string) ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { @@ -27,25 +41,33 @@ func MetricsOpenCensusCollectionOption() ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { log.Info("Init OpenCensus") - promRegistry := prometheus.NewRegistry() - pe, err := ocprom.NewExporter(ocprom.Options{ - Namespace: "ipfs_oc", - Registry: promRegistry, - OnError: func(err error) { - log.Errorw("OC ERROR", "error", err) - }, - }) - if err != nil { - return nil, err - } + ocMutex.Lock() + defer ocMutex.Unlock() + ocHandler, ok := ocHandlers[ipfsOC] + if !ok { + promRegistry := prometheus.NewRegistry() + pe, err := ocprom.NewExporter(ocprom.Options{ + Namespace: "ipfs_oc", + Registry: promRegistry, + OnError: func(err error) { + log.Errorw("OC ERROR", "error", err) + }, + }) + if err != nil { + return nil, err + } - // register prometheus with opencensus - view.RegisterExporter(pe) - view.SetReportingPeriod(2 * time.Second) + // register prometheus with opencensus + view.RegisterExporter(pe) + view.SetReportingPeriod(2 * time.Second) + + ocHandler = pe + ocHandlers[ipfsOC] = ocHandler + } // Construct the mux zpages.Handle(mux, "/debug/metrics/oc/debugz") - mux.Handle("/debug/metrics/oc", pe) + mux.Handle("/debug/metrics/oc", ocHandler) return mux, nil } @@ -58,6 +80,13 @@ func MetricsOpenCensusDefaultPrometheusRegistry() ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { log.Info("Init OpenCensus with default prometheus registry") + ocMutex.Lock() + defer ocMutex.Unlock() + _, ok := ocHandlers[ipfsOCDefault] + if ok { + return mux, nil + } + pe, err := ocprom.NewExporter(ocprom.Options{ Registry: prometheus.DefaultRegisterer.(*prometheus.Registry), OnError: func(err error) { @@ -71,6 +100,8 @@ func MetricsOpenCensusDefaultPrometheusRegistry() ServeOption { // register prometheus with opencensus view.RegisterExporter(pe) + ocHandlers[ipfsOCDefault] = pe + return mux, nil } } @@ -78,6 +109,15 @@ func MetricsOpenCensusDefaultPrometheusRegistry() ServeOption { // MetricsCollectionOption adds collection of net/http-related metrics. func MetricsCollectionOption(handlerName string) ServeOption { return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { + ocMutex.Lock() + defer ocMutex.Unlock() + promMux, ok := ocHandlers[ipfsOCPromMux] + if ok { + mux.Handle("/", promMux) + childMux := ocHandlers[ipfsOCChildMux] + return childMux.(*http.ServeMux), nil + } + // Adapted from github.com/prometheus/client_golang/prometheus/http.go // Work around https://github.com/prometheus/client_golang/pull/311 opts := prometheus.SummaryOpts{ @@ -140,13 +180,15 @@ func MetricsCollectionOption(handlerName string) ServeOption { // Construct the mux childMux := http.NewServeMux() - var promMux http.Handler = childMux + promMux = childMux promMux = promhttp.InstrumentHandlerResponseSize(resSz, promMux) promMux = promhttp.InstrumentHandlerRequestSize(reqSz, promMux) promMux = promhttp.InstrumentHandlerDuration(reqDur, promMux) promMux = promhttp.InstrumentHandlerCounter(reqCnt, promMux) mux.Handle("/", promMux) + ocHandlers[ipfsOCChildMux] = childMux + ocHandlers[ipfsOCPromMux] = promMux return childMux, nil } } diff --git a/docs/changelogs/v0.30.md b/docs/changelogs/v0.30.md index 6cc212f7a8b..c5e93db5f8a 100644 --- a/docs/changelogs/v0.30.md +++ b/docs/changelogs/v0.30.md @@ -13,6 +13,7 @@ - [`/unix/` socket support in `Addresses.API`](#unix-socket-support-in-addressesapi) - [Cleaned Up `ipfs daemon` Startup Log](#cleaned-up-ipfs-daemon-startup-log) - [UnixFS 1.5: Mode and Modification Time Support](#unixfs-15-mode-and-modification-time-support) + - [Serve Metrics with Multiple Listeners](#serve-metrics-with-multiple-listeners) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -128,6 +129,10 @@ Opt-in support for `mode` and `mtime` was also added to MFS (`ipfs files --help` > [!NOTE] > Storing `mode` and `mtime` requires root block to be `dag-pb` and disabled `raw-leaves` setting to create envelope for storing the metadata. +#### Serve Metrics with Multiple Listeners + +When there are multiple listeners configured for Addresses.API, serving metrics resulted in errors: "xxx was collected before with the same name and label values". This is now fixed by creating and reginstering metrics exporters and instruments once. These are used as handlers when serving metrics once, and now the same once-registered handlers are provided to the mux for every listener. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors