From b534a1c257121ca36a43d12ade5559e754e9b2b5 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:22:10 +0200 Subject: [PATCH 01/12] sharding: reduce log level for a too spammy message (#3045) --- waku/waku_enr/sharding.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/waku_enr/sharding.nim b/waku/waku_enr/sharding.nim index d1beb2864b..88dc4e200f 100644 --- a/waku/waku_enr/sharding.nim +++ b/waku/waku_enr/sharding.nim @@ -227,7 +227,7 @@ proc containsShard*(r: Record, clusterId, shardId: uint16): bool = return false let record = r.toTyped().valueOr: - debug "invalid ENR record", error = error + trace "invalid ENR record", error = error return false let rs = record.relaySharding().valueOr: From 26a488d5228104686eb2b12641e454e0285931e3 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:23:53 +0200 Subject: [PATCH 02/12] chore: extending store metrics (#3042) * adding query_metrics module * update fleet-dashboard with new store panels for better timing insight --- library/callback.nim | 4 +- library/libwaku.nim | 4 +- metrics/waku-fleet-dashboard.json | 4631 ++++++++++------- waku/common/databases/db_postgres/dbconn.nim | 37 +- .../databases/db_postgres/query_metrics.nim | 7 + waku/waku_store/protocol.nim | 4 +- waku/waku_store/protocol_metrics.nim | 4 +- waku/waku_store_legacy/protocol.nim | 12 +- waku/waku_store_legacy/protocol_metrics.nim | 5 + 9 files changed, 2833 insertions(+), 1875 deletions(-) create mode 100644 waku/common/databases/db_postgres/query_metrics.nim diff --git a/library/callback.nim b/library/callback.nim index 395e8a1d2f..8a8522600c 100644 --- a/library/callback.nim +++ b/library/callback.nim @@ -4,7 +4,9 @@ type WakuCallBack* = proc( callerRet: cint, msg: ptr cchar, len: csize_t, userData: pointer ) {.cdecl, gcsafe, raises: [].} -template checkLibwakuParams*(ctx: ptr WakuContext, callback: WakuCallBack, userData: pointer) = +template checkLibwakuParams*( + ctx: ptr WakuContext, callback: WakuCallBack, userData: pointer +) = ctx[].userData = userData if isNil(callback): diff --git a/library/libwaku.nim b/library/libwaku.nim index cb422b5575..f313d925fe 100644 --- a/library/libwaku.nim +++ b/library/libwaku.nim @@ -487,9 +487,7 @@ proc waku_get_peerids_from_peerstore( let connRes = waku_thread.sendRequestToWakuThread( ctx, RequestType.PEER_MANAGER, - PeerManagementRequest.createShared( - PeerManagementMsgType.GET_ALL_PEER_IDS - ), + PeerManagementRequest.createShared(PeerManagementMsgType.GET_ALL_PEER_IDS), ) if connRes.isErr(): let msg = $connRes.error diff --git a/metrics/waku-fleet-dashboard.json b/metrics/waku-fleet-dashboard.json index ac1f3b3798..23efbd1397 100644 --- a/metrics/waku-fleet-dashboard.json +++ b/metrics/waku-fleet-dashboard.json @@ -822,32 +822,7 @@ }, "unit": "binBps" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "/waku/2/rs/16/32 - net" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 10, @@ -1550,8 +1525,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1566,7 +1540,7 @@ "h": 6, "w": 12, "x": 0, - "y": 2 + "y": 66 }, "id": 48, "options": { @@ -1642,8 +1616,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1658,7 +1631,7 @@ "h": 6, "w": 12, "x": 12, - "y": 2 + "y": 66 }, "id": 50, "options": { @@ -1734,8 +1707,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1750,7 +1722,7 @@ "h": 6, "w": 12, "x": 0, - "y": 8 + "y": 72 }, "id": 60, "options": { @@ -1826,7 +1798,7 @@ "h": 6, "w": 12, "x": 12, - "y": 8 + "y": 72 }, "hiddenSeries": false, "id": 8, @@ -1921,7 +1893,7 @@ "h": 6, "w": 12, "x": 0, - "y": 14 + "y": 78 }, "hiddenSeries": false, "id": 2, @@ -2044,8 +2016,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2061,7 +2032,7 @@ "h": 6, "w": 12, "x": 12, - "y": 14 + "y": 78 }, "id": 83, "options": { @@ -2115,7 +2086,7 @@ "h": 6, "w": 12, "x": 0, - "y": 20 + "y": 84 }, "hiddenSeries": false, "id": 3, @@ -2210,7 +2181,7 @@ "h": 6, "w": 12, "x": 12, - "y": 20 + "y": 84 }, "hiddenSeries": false, "id": 9, @@ -2331,7 +2302,7 @@ "h": 6, "w": 12, "x": 0, - "y": 26 + "y": 90 }, "hiddenSeries": false, "id": 6, @@ -2426,7 +2397,7 @@ "h": 6, "w": 12, "x": 12, - "y": 26 + "y": 90 }, "hiddenSeries": false, "id": 7, @@ -2573,8 +2544,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2590,7 +2560,7 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 96 }, "id": 44, "options": { @@ -2667,7 +2637,7 @@ "h": 6, "w": 12, "x": 12, - "y": 32 + "y": 96 }, "hiddenSeries": false, "id": 10, @@ -2788,8 +2758,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2805,7 +2774,7 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 102 }, "id": 64, "options": { @@ -2857,7 +2826,7 @@ "h": 6, "w": 12, "x": 0, - "y": 40 + "y": 104 }, "hiddenSeries": false, "id": 4, @@ -2952,7 +2921,7 @@ "h": 6, "w": 12, "x": 12, - "y": 46 + "y": 110 }, "hiddenSeries": false, "id": 5, @@ -3094,8 +3063,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3187,8 +3155,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3291,8 +3258,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3383,8 +3349,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3476,8 +3441,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3528,7 +3492,7 @@ "type": "row" }, { - "collapsed": true, + "collapsed": false, "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" @@ -3540,1633 +3504,2577 @@ "y": 67 }, "id": 34, - "panels": [ + "panels": [], + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 4 + "refId": "A" + } + ], + "title": "Store/Archive", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 36, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "exemplar": true, - "expr": "waku_store_peers{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Waku Store Peers", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 2, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 4 + "exemplar": true, + "expr": "waku_store_peers{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Waku Store Peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 38, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 2, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "multi", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "exemplar": true, - "expr": "waku_store_messages{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "hide": false, - "interval": "", - "legendFormat": "{{type}}: {{instance}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "waku_archive_messages{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "hide": false, - "legendFormat": "{{type}}: {{instance}}", - "range": true, - "refId": "B" - } - ], - "title": "Waku Archive Messages", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 38, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 3, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] + "editorMode": "code", + "exemplar": true, + "expr": "waku_store_messages{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", + "hide": false, + "interval": "", + "legendFormat": "{{type}}: {{instance}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 10 + "editorMode": "code", + "expr": "waku_archive_messages{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", + "hide": false, + "legendFormat": "{{type}}: {{instance}}", + "range": true, + "refId": "B" + } + ], + "title": "Waku Archive Messages", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 62, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": true, - "expr": "increase(waku_store_queries{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[1m])", - "interval": "", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Waku Store Queries (1m rate)", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 10 + "editorMode": "code", + "exemplar": true, + "expr": "increase(waku_store_queries{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[1m])", + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Store Queries (1m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 40, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum by (type)(increase(waku_archive_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[1m]))", - "hide": false, - "interval": "", - "legendFormat": "{{type}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "exemplar": true, - "expr": "sum by (type)(increase(waku_store_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[1m]))", - "hide": false, - "interval": "", - "legendFormat": "{{type}}", - "range": true, - "refId": "B" - } - ], - "title": "Waku Archive Errors (1m rate)", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 40, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "description": "", - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_archive_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 77, - "legend": { - "show": false + "editorMode": "code", + "exemplar": true, + "expr": "sum by (type)(increase(waku_store_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "{{type}}", + "range": true, + "refId": "B" + } + ], + "title": "Waku Archive Errors (1m rate)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "maxDataPoints": 60, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Turbo", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "rowsFrame": { - "layout": "auto" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.2.5", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "expr": "sum(increase(waku_archive_query_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "legendFormat": "{{le}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "expr": "sum(increase(waku_store_query_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": true, - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } - ], - "title": "Waku Archive Query Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "yBucketBound": "auto" + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 144, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "tsbuckets", "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "heatmap": {}, - "hideZeroBuckets": true, - "highlightCards": true, - "id": 75, - "legend": { - "show": false + "editorMode": "code", + "exemplar": true, + "expr": "query_time_secs{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", phase=\"sendQuery\"} and deriv(query_time_secs{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", phase=\"sendQuery\"}[1m]) != 0", + "interval": "", + "legendFormat": "{{query}}", + "range": true, + "refId": "A" + } + ], + "title": "Time Send Query To DB (sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "Shows the time spent while waiting for feedback from the database. That time includes the database query time plus the time spent waiting for the response from the database.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "maxDataPoints": 60, - "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Turbo", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" - } - }, - "pluginVersion": "9.2.5", - "reverseYBuckets": false, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(increase(waku_archive_insert_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "B" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" }, - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(increase(waku_store_insert_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "A" + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "exemplar": true, - "expr": "sum(increase(waku_legacy_archive_insert_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{le}}", - "refId": "C" + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Waku Archive Insert Duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true }, - "yAxis": { - "decimals": 0, - "format": "s", - "logBase": 1, - "show": true + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] }, - "yBucketBound": "auto" + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 145, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min", + "stdDev" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 130, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "editorMode": "code", + "exemplar": true, + "expr": "query_time_secs{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", phase=\"waitFinish\"} and deriv(query_time_secs{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", phase=\"waitFinish\"}[45s]) != 0", + "interval": "", + "legendFormat": "{{query}}", + "range": true, + "refId": "A" + } + ], + "title": "Wait Queries To Finish (sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/store-query/3.0.0\", direction=\"in\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 91 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min", + "stdDev" ], - "title": "Store protocol traffic (in)", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 23 + "editorMode": "code", + "exemplar": true, + "expr": "waku_legacy_store_time_seconds{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"} and deriv(waku_legacy_store_time_seconds{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[45s]) != 0", + "interval": "", + "legendFormat": "{{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Store V2 Times (sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 132, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/store-query/3.0.0\", direction=\"out\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 91 + }, + "id": 148, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min", + "stdDev" ], - "title": "Store protocol traffic (out)", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { + "editorMode": "code", + "exemplar": true, + "expr": "waku_store_time_seconds{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"} and deriv(waku_store_time_seconds{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[45s]) != 0", + "interval": "", + "legendFormat": "{{phase}}", + "range": true, + "refId": "A" + } + ], + "title": "Store V3 Times (sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 149, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(query_count_total{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", query!=\"InsertRow\"}[1m]) and rate(query_count_total{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", query!=\"InsertRowMessagesLookup\"}[1m]) ", + "interval": "", + "legendFormat": "{{query}}", + "range": true, + "refId": "A" + } + ], + "title": "Not-Insert Queries Rate (query/sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 3, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "InsertRow" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { "legend": false, "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" + "viz": true } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 147, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(query_count_total{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", query=\"InsertRow\"}[5m])", + "interval": "", + "legendFormat": "{{query}}", + "range": true, + "refId": "A" + } + ], + "title": "Insert Queries Rate (insert/sec)", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 107 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 77, + "legend": { + "show": false + }, + "maxDataPoints": 60, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Turbo", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "expr": "sum(increase(waku_archive_query_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "expr": "sum(increase(waku_store_query_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": true, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Archive Query Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 107 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 75, + "legend": { + "show": false + }, + "maxDataPoints": 60, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Turbo", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_archive_insert_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_store_insert_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(increase(waku_legacy_archive_insert_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "refId": "C" + } + ], + "title": "Waku Archive Insert Duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 142, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_service_network_bytes_total{service=~\"/vac/waku/store/2.*\", direction=\"in\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Store v2 protocol traffic (in)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 130, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/store-query/3.0.0\", direction=\"in\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Store v3 protocol traffic (in)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 127 + }, + "id": 132, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_service_network_bytes_total{service=~\"/vac/waku/store/2.*\", direction=\"out\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Store v2 protocol traffic (out)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 127 + }, + "id": 143, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/store-query/3.0.0\", direction=\"out\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Store v3 protocol traffic (out)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "overrides": [] + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "gridPos": { - "h": 13, - "w": 12, - "x": 0, - "y": 36 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] }, - "id": 128, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 140 + }, + "id": 128, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_service_requests_total{service =~\"/vac/waku/store/2.*\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}} - {{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Store v2 query request rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "rate(waku_service_requests_total{service =~\"/vac/waku/store.*\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "legendFormat": "{{service}} - {{instance}}", - "range": true, - "refId": "A" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 140 + }, + "id": 141, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_service_requests_total{service =~\"/vac/waku/store-query/3.*\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}} - {{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Store v3 query request rates", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 153 + }, + "id": 87, + "panels": [], + "title": "Filter", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 154 + }, + "id": 93, + "options": { + "legend": { + "calcs": [ + "max", + "mean" ], - "title": "Store query request rates", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], + }, "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, + "editorMode": "code", + "expr": "rate(waku_filter_requests{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{type}} : {{instance}}", + "range": true, "refId": "A" } ], - "title": "Store/Archive", - "type": "row" + "title": "Waku Filter Requests", + "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 68 + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" }, - "id": 87, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 5 + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 93, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "expr": "waku_filter_requests{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "legendFormat": "{{type}} : {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Waku Filter Requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 5 - }, - "id": 89, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "waku_filter_subscriptions{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Waku Filter Subscriptions", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 154 + }, + "id": 89, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 5 + "editorMode": "code", + "expr": "waku_filter_subscriptions{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Filter Subscriptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 91, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "waku_filter_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "legendFormat": "{{type}} : {{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Waku Filter Errors", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 154 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 13 - }, - "id": 95, - "options": { - "calculate": false, - "cellGap": 2, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "tooltip": { - "show": true, - "yHistogram": true + "editorMode": "code", + "expr": "rate(waku_filter_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{type}} : {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Filter Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" + "scaleDistribution": { + "type": "linear" } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 162 + }, + "id": 95, + "options": { + "calculate": false, + "cellGap": 2, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "editorMode": "code", - "expr": "sum(increase(waku_filter_request_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" + "editorMode": "code", + "expr": "sum(increase(waku_filter_request_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Filter Request Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" } - ], - "title": "Waku Filter Request Duration", - "type": "heatmap" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 162 + }, + "id": 97, + "options": { + "calculate": false, + "cellGap": 2, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "9.2.5", + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 + "editorMode": "code", + "expr": "sum(increase(waku_filter_handle_message_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Waku Filter Handle Message Duration", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 97, - "options": { - "calculate": false, - "cellGap": 2, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "rowsFrame": { - "layout": "auto" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "tooltip": { - "show": true, - "yHistogram": false + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "yAxis": { - "axisPlacement": "left", - "decimals": 0, - "reverse": false, - "unit": "s" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "9.2.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "sum(increase(waku_filter_handle_message_duration_seconds_bucket{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "legendFormat": "{{le}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "boot-01.ac-cn-hongkong-c.status.prod - rejected", + "boot-01.ac-cn-hongkong-c.status.prod - served" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 170 + }, + "id": 134, + "options": { + "legend": { + "calcs": [ + "max", + "mean" ], - "title": "Waku Filter Handle Message Duration", - "type": "heatmap" + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", - "uid": "P6693426190CB2316" + "uid": "PBFA97CFB590B2093" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" + "editorMode": "code", + "expr": "rate(waku_service_requests_total{service = \"/vac/waku/filter-subscribe/2.0.0-beta1\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}} - {{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Filter subscribe request rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 0, - "y": 21 - }, - "id": 134, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "rate(waku_service_requests_total{service = \"/vac/waku/filter-subscribe/2.0.0-beta1\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "legendFormat": "{{instance}} - {{state}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 170 + }, + "id": 136, + "options": { + "legend": { + "calcs": [ + "max", + "mean" ], - "title": "Filter subscribe request rates", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 12, - "y": 21 - }, - "id": 136, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "uid": "PBFA97CFB590B2093" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/filter-push/2.0.0-beta1\", direction=\"out\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } - ], - "title": "Filter protocol message push traffic (out)", - "type": "timeseries" + "editorMode": "code", + "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/filter-push/2.0.0-beta1\", direction=\"out\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" } ], - "title": "Filter", - "type": "row" + "title": "Filter protocol message push traffic (out)", + "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" @@ -5175,398 +6083,398 @@ "h": 1, "w": 24, "x": 0, - "y": 69 + "y": 183 }, "id": 28, - "panels": [ + "panels": [], + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 6 + "refId": "A" + } + ], + "title": "Lightpush", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "exemplar": true, - "expr": "waku_lightpush_peers{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "title": "Waku Lightpush Peers", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 184 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 6 + "exemplar": true, + "expr": "waku_lightpush_peers{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", + "interval": "", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Waku Lightpush Peers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 32, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "P6693426190CB2316" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "exemplar": true, - "expr": "waku_lightpush_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", - "interval": "", - "legendFormat": "{{type}}: {[instance}}", - "refId": "A" - } - ], - "title": "Waku Lightpush Errors", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 184 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "P6693426190CB2316" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 138, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "exemplar": true, + "expr": "waku_lightpush_errors{instance=~\"[[host]].([[dc:pipe]]).*.([[fleet:pipe]])\"}", + "interval": "", + "legendFormat": "{{type}}: {[instance}}", + "refId": "A" + } + ], + "title": "Waku Lightpush Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "editorMode": "code", - "expr": "rate(waku_service_requests_total{service = \"/vac/waku/lightpush/2.0.0-beta1\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "legendFormat": "{{instance}} . {{state}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 192 + }, + "id": 138, + "options": { + "legend": { + "calcs": [ + "max", + "mean" ], - "title": "Lightpush request rates", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", - "uid": "P6693426190CB2316" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] + "uid": "PBFA97CFB590B2093" }, - "gridPos": { - "h": 12, - "w": 12, - "x": 12, - "y": 14 + "editorMode": "code", + "expr": "rate(waku_service_requests_total{service = \"/vac/waku/lightpush/2.0.0-beta1\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "legendFormat": "{{instance}} . {{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Lightpush request rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "P6693426190CB2316" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "id": 140, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "tooltip": { - "mode": "single", - "sort": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, - "editorMode": "code", - "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/lightpush/2.0.0-beta1\", direction=\"in\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{instance}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 192 + }, + "id": 140, + "options": { + "legend": { + "calcs": [ + "max", + "mean" ], - "title": "Lightpush protocol traffic (in)", - "type": "timeseries" + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], + }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "P6693426190CB2316" + "uid": "PBFA97CFB590B2093" }, + "editorMode": "code", + "expr": "rate(waku_service_network_bytes_total{service=\"/vac/waku/lightpush/2.0.0-beta1\", direction=\"in\", instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{instance}}", + "range": true, "refId": "A" } ], - "title": "Lightpush", - "type": "row" + "title": "Lightpush protocol traffic (in)", + "type": "timeseries" }, { "collapsed": true, @@ -5578,7 +6486,7 @@ "h": 1, "w": 24, "x": 0, - "y": 70 + "y": 204 }, "id": 15, "panels": [ @@ -5887,8 +6795,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5979,8 +6886,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6072,8 +6978,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6135,8 +7040,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6201,8 +7105,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6270,8 +7173,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -6343,7 +7245,7 @@ "h": 1, "w": 24, "x": 0, - "y": 71 + "y": 205 }, "id": 107, "panels": [ @@ -6392,8 +7294,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6453,7 +7354,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": ".*", "value": ".*" }, @@ -6493,10 +7394,10 @@ "current": { "selected": true, "text": [ - "status.prod" + "status.staging" ], "value": [ - "status.prod" + "status.staging" ] }, "datasource": { @@ -6579,6 +7480,6 @@ "timezone": "browser", "title": "Nim-Waku V2", "uid": "qrp_ZCTGz", - "version": 142, + "version": 150, "weekStart": "" -} +} \ No newline at end of file diff --git a/waku/common/databases/db_postgres/dbconn.nim b/waku/common/databases/db_postgres/dbconn.nim index 1c72ea7ca2..54fbc27a1f 100644 --- a/waku/common/databases/db_postgres/dbconn.nim +++ b/waku/common/databases/db_postgres/dbconn.nim @@ -1,4 +1,5 @@ -import std/[times, strutils, asyncnet, os, sequtils], results, chronos +import std/[times, strutils, asyncnet, os, sequtils], results, chronos, metrics, re +import ./query_metrics include db_connector/db_postgres @@ -45,6 +46,9 @@ proc closeDbConn*(db: DbConn) {.raises: [OSError].} = asyncengine.unregister(cast[asyncengine.AsyncFD](fd)) db.close() +proc `$`(self: SqlQuery): string = + return cast[string](self) + proc sendQuery( db: DbConn, query: SqlQuery, args: seq[string] ): Future[Result[void, string]] {.async.} = @@ -152,12 +156,32 @@ proc waitQueryToFinish( proc dbConnQuery*( db: DbConn, query: SqlQuery, args: seq[string], rowCallback: DataProc ): Future[Result[void, string]] {.async, gcsafe.} = + let cleanedQuery = ($query).replace(" ", "").replace("\n", "") + ## remove everything between ' or " all possible sequence of numbers. e.g. rm partition partition + var querySummary = cleanedQuery.replace(re"""(['"]).*?\1""", "") + querySummary = querySummary.replace(re"\d+", "") + querySummary = "query_tag_" & querySummary[0 ..< min(querySummary.len, 200)] + + var queryStartTime = getTime().toUnixFloat() + (await db.sendQuery(query, args)).isOkOr: return err("error in dbConnQuery calling sendQuery: " & $error) + query_time_secs.set( + getTime().toUnixFloat() - queryStartTime, [querySummary, "sendQuery"] + ) + + queryStartTime = getTime().toUnixFloat() + (await db.waitQueryToFinish(rowCallback)).isOkOr: return err("error in dbConnQuery calling waitQueryToFinish: " & $error) + query_time_secs.set( + getTime().toUnixFloat() - queryStartTime, [querySummary, "waitFinish"] + ) + + query_count.inc(labelValues = [querySummary]) + return ok() proc dbConnQueryPrepared*( @@ -168,10 +192,21 @@ proc dbConnQueryPrepared*( paramFormats: seq[int32], rowCallback: DataProc, ): Future[Result[void, string]] {.async, gcsafe.} = + var queryStartTime = getTime().toUnixFloat() db.sendQueryPrepared(stmtName, paramValues, paramLengths, paramFormats).isOkOr: return err("error in dbConnQueryPrepared calling sendQuery: " & $error) + query_time_secs.set(getTime().toUnixFloat() - queryStartTime, [stmtName, "sendQuery"]) + + queryStartTime = getTime().toUnixFloat() + (await db.waitQueryToFinish(rowCallback)).isOkOr: return err("error in dbConnQueryPrepared calling waitQueryToFinish: " & $error) + query_time_secs.set( + getTime().toUnixFloat() - queryStartTime, [stmtName, "waitFinish"] + ) + + query_count.inc(labelValues = [stmtName]) + return ok() diff --git a/waku/common/databases/db_postgres/query_metrics.nim b/waku/common/databases/db_postgres/query_metrics.nim new file mode 100644 index 0000000000..06209cac0a --- /dev/null +++ b/waku/common/databases/db_postgres/query_metrics.nim @@ -0,0 +1,7 @@ +import metrics + +declarePublicGauge query_time_secs, + "query time measured in nanoseconds", labels = ["query", "phase"] + +declarePublicCounter query_count, + "number of times a query is being performed", labels = ["query"] diff --git a/waku/waku_store/protocol.nim b/waku/waku_store/protocol.nim index c4e1cd36c6..5a8a81c132 100644 --- a/waku/waku_store/protocol.nim +++ b/waku/waku_store/protocol.nim @@ -108,7 +108,7 @@ proc initProtocolHandler(self: WakuStore) = resBuf = await self.handleQueryRequest(conn.peerId, reqBuf) let queryDuration = getTime().toUnixFloat() - queryStartTime - waku_store_time_seconds.inc(amount = queryDuration, labelValues = ["query-db"]) + waku_store_time_seconds.set(queryDuration, ["query-db-time"]) successfulQuery = true do: debug "store query request rejected due rate limit exceeded", @@ -127,7 +127,7 @@ proc initProtocolHandler(self: WakuStore) = debug "after sending response", requestId = resBuf.requestId if successfulQuery: let writeDuration = getTime().toUnixFloat() - writeRespStartTime - waku_store_time_seconds.inc(amount = writeDuration, labelValues = ["send-resp"]) + waku_store_time_seconds.set(writeDuration, ["send-store-resp-time"]) waku_service_network_bytes.inc( amount = resBuf.resp.len().int64, labelValues = [WakuStoreCodec, "out"] diff --git a/waku/waku_store/protocol_metrics.nim b/waku/waku_store/protocol_metrics.nim index 851670cdba..b077147a67 100644 --- a/waku/waku_store/protocol_metrics.nim +++ b/waku/waku_store/protocol_metrics.nim @@ -5,8 +5,8 @@ import metrics declarePublicGauge waku_store_errors, "number of store protocol errors", ["type"] declarePublicGauge waku_store_queries, "number of store queries received" -## f.e., we have the "query" phase, where the node performs the query to the database, -## and the "libp2p" phase, where the node writes the store response to the libp2p stream. +## "query-db-time" phase considers the time when node performs the query to the database. +## "send-store-resp-time" phase is the time when node writes the store response to the store-client. declarePublicGauge waku_store_time_seconds, "Time in seconds spent by each store phase", labels = ["phase"] diff --git a/waku/waku_store_legacy/protocol.nim b/waku/waku_store_legacy/protocol.nim index a7886165d9..4e06419b49 100644 --- a/waku/waku_store_legacy/protocol.nim +++ b/waku/waku_store_legacy/protocol.nim @@ -4,7 +4,7 @@ {.push raises: [].} import - std/options, + std/[options, times], results, chronicles, chronos, @@ -102,6 +102,7 @@ proc initProtocolHandler(ws: WakuStore) = ).encode().buffer proc handler(conn: Connection, proto: string) {.async, closure.} = + var successfulQuery = false ## only consider the correct queries in metrics var resBuf: seq[byte] ws.requestRateLimiter.checkUsageLimit(WakuLegacyStoreCodec, conn): let readRes = catch: @@ -115,12 +116,17 @@ proc initProtocolHandler(ws: WakuStore) = amount = reqBuf.len().int64, labelValues = [WakuLegacyStoreCodec, "in"] ) + let queryStartTime = getTime().toUnixFloat() resBuf = await ws.handleLegacyQueryRequest(conn.peerId, reqBuf) + let queryDuration = getTime().toUnixFloat() - queryStartTime + waku_legacy_store_time_seconds.set(queryDuration, ["query-db-time"]) + successfulQuery = true do: debug "Legacy store query request rejected due rate limit exceeded", peerId = conn.peerId, limit = $ws.requestRateLimiter.setting resBuf = rejectResponseBuf + let writeRespStartTime = getTime().toUnixFloat() let writeRes = catch: await conn.writeLp(resBuf) @@ -128,6 +134,10 @@ proc initProtocolHandler(ws: WakuStore) = error "Connection write error", error = writeRes.error.msg return + if successfulQuery: + let writeDuration = getTime().toUnixFloat() - writeRespStartTime + waku_legacy_store_time_seconds.set(writeDuration, ["send-store-resp-time"]) + waku_service_network_bytes.inc( amount = resBuf.len().int64, labelValues = [WakuLegacyStoreCodec, "out"] ) diff --git a/waku/waku_store_legacy/protocol_metrics.nim b/waku/waku_store_legacy/protocol_metrics.nim index 59c7a829a5..53cc71427f 100644 --- a/waku/waku_store_legacy/protocol_metrics.nim +++ b/waku/waku_store_legacy/protocol_metrics.nim @@ -6,6 +6,11 @@ declarePublicGauge waku_legacy_store_errors, "number of legacy store protocol errors", ["type"] declarePublicGauge waku_legacy_store_queries, "number of legacy store queries received" +## "query-db-time" phase considers the time when node performs the query to the database. +## "send-store-resp-time" phase is the time when node writes the store response to the store-client. +declarePublicGauge waku_legacy_store_time_seconds, + "Time in seconds spent by each store phase", labels = ["phase"] + # Error types (metric label values) const dialFailure* = "dial_failure" From bbebbf6d4f72d9c8e8e2e32bd7240cb6a42bd268 Mon Sep 17 00:00:00 2001 From: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:41:27 +0300 Subject: [PATCH 03/12] fix: static linking negentropy in ARM based mac (#3046) --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 8b1b7be5ca..bf6f9b3861 100644 --- a/Makefile +++ b/Makefile @@ -186,6 +186,9 @@ negentropy: | $(LIBNEGENTROPY_FILE) ## Pass libnegentropy and it's deps to linker. $(eval LIBNEGENTROPY_PATH := $(shell if [ -f "$(LIBNEGENTROPY_FILE)" ]; then echo "$(LIBNEGENTROPY_FILE)"; else echo "./$(LIBNEGENTROPY_FILE)"; fi)) $(eval NIM_PARAMS += --passL:$(LIBNEGENTROPY_PATH) --passL:-lcrypto --passL:-lssl --passL:-lstdc++) +ifeq ($(detected_OS),Darwin) + $(eval NIM_PARAMS += --passL:-L/opt/homebrew/lib/) +endif negentropy-clean: $(MAKE) -C $(LIBNEGENTROPY_BUILDDIR) clean && \ From e492c590e9ec06b08ca4ea75a7daf1fa6bbce01c Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:43:56 +0200 Subject: [PATCH 04/12] chore: Better database query logs and logarithmic scale in grafana store panels (#3048) --- metrics/waku-fleet-dashboard.json | 67 +++++++++++++------ waku/common/databases/db_postgres/dbconn.nim | 37 +++++++--- .../postgres_driver/postgres_driver.nim | 48 +------------ .../postgres_driver/postgres_driver.nim | 6 -- 4 files changed, 73 insertions(+), 85 deletions(-) diff --git a/metrics/waku-fleet-dashboard.json b/metrics/waku-fleet-dashboard.json index 23efbd1397..8c7cdd1747 100644 --- a/metrics/waku-fleet-dashboard.json +++ b/metrics/waku-fleet-dashboard.json @@ -3948,7 +3948,8 @@ "lineWidth": 2, "pointSize": 10, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -3973,7 +3974,32 @@ }, "unit": "s" }, - "overrides": [] + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "query_tag_ANALYZEmessages" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] }, "gridPos": { "h": 11, @@ -4048,7 +4074,8 @@ "lineWidth": 2, "pointSize": 10, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -4109,6 +4136,7 @@ "editorMode": "code", "exemplar": true, "expr": "query_time_secs{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", phase=\"waitFinish\"} and deriv(query_time_secs{instance=~\"[[host]].([[dc:pipe]]).([[fleet:pipe]])\", phase=\"waitFinish\"}[45s]) != 0", + "hide": false, "interval": "", "legendFormat": "{{query}}", "range": true, @@ -4147,7 +4175,8 @@ "lineWidth": 2, "pointSize": 10, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -4246,7 +4275,8 @@ "lineWidth": 2, "pointSize": 10, "scaleDistribution": { - "type": "linear" + "log": 10, + "type": "log" }, "showPoints": "auto", "spanNulls": false, @@ -5456,8 +5486,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5551,8 +5580,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5643,8 +5671,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -5897,8 +5924,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -6022,8 +6048,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -7346,7 +7371,7 @@ "type": "row" } ], - "refresh": "30s", + "refresh": false, "schemaVersion": 37, "style": "dark", "tags": [], @@ -7428,10 +7453,10 @@ "current": { "selected": true, "text": [ - "All" + "ac-cn-hongkong-c" ], "value": [ - "$__all" + "ac-cn-hongkong-c" ] }, "datasource": { @@ -7461,8 +7486,8 @@ ] }, "time": { - "from": "now-24h", - "to": "now" + "from": "2024-09-20T12:34:03.849Z", + "to": "2024-09-20T13:52:30.721Z" }, "timepicker": { "refresh_intervals": [ @@ -7480,6 +7505,6 @@ "timezone": "browser", "title": "Nim-Waku V2", "uid": "qrp_ZCTGz", - "version": 150, + "version": 151, "weekStart": "" } \ No newline at end of file diff --git a/waku/common/databases/db_postgres/dbconn.nim b/waku/common/databases/db_postgres/dbconn.nim index 54fbc27a1f..bc5da4ee6c 100644 --- a/waku/common/databases/db_postgres/dbconn.nim +++ b/waku/common/databases/db_postgres/dbconn.nim @@ -1,4 +1,10 @@ -import std/[times, strutils, asyncnet, os, sequtils], results, chronos, metrics, re +import + std/[times, strutils, asyncnet, os, sequtils], + results, + chronos, + metrics, + re, + chronicles import ./query_metrics include db_connector/db_postgres @@ -167,21 +173,26 @@ proc dbConnQuery*( (await db.sendQuery(query, args)).isOkOr: return err("error in dbConnQuery calling sendQuery: " & $error) - query_time_secs.set( - getTime().toUnixFloat() - queryStartTime, [querySummary, "sendQuery"] - ) + let sendDuration = getTime().toUnixFloat() - queryStartTime + query_time_secs.set(sendDuration, [querySummary, "sendQuery"]) queryStartTime = getTime().toUnixFloat() (await db.waitQueryToFinish(rowCallback)).isOkOr: return err("error in dbConnQuery calling waitQueryToFinish: " & $error) - query_time_secs.set( - getTime().toUnixFloat() - queryStartTime, [querySummary, "waitFinish"] - ) + let waitDuration = getTime().toUnixFloat() - queryStartTime + query_time_secs.set(waitDuration, [querySummary, "waitFinish"]) query_count.inc(labelValues = [querySummary]) + if "insert" notin ($query).toLower(): + debug "dbConnQuery", + query = $query, + querySummary, + waitDurationSecs = waitDuration, + sendDurationSecs = sendDuration + return ok() proc dbConnQueryPrepared*( @@ -196,17 +207,21 @@ proc dbConnQueryPrepared*( db.sendQueryPrepared(stmtName, paramValues, paramLengths, paramFormats).isOkOr: return err("error in dbConnQueryPrepared calling sendQuery: " & $error) - query_time_secs.set(getTime().toUnixFloat() - queryStartTime, [stmtName, "sendQuery"]) + let sendDuration = getTime().toUnixFloat() - queryStartTime + query_time_secs.set(sendDuration, [stmtName, "sendQuery"]) queryStartTime = getTime().toUnixFloat() (await db.waitQueryToFinish(rowCallback)).isOkOr: return err("error in dbConnQueryPrepared calling waitQueryToFinish: " & $error) - query_time_secs.set( - getTime().toUnixFloat() - queryStartTime, [stmtName, "waitFinish"] - ) + let waitDuration = getTime().toUnixFloat() - queryStartTime + query_time_secs.set(waitDuration, [stmtName, "waitFinish"]) query_count.inc(labelValues = [stmtName]) + if "insert" notin stmtName.toLower(): + debug "dbConnQueryPrepared", + stmtName, waitDurationSecs = waitDuration, sendDurationSecs = sendDuration + return ok() diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index 6c8ba28d6a..a69fdb9f46 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -346,8 +346,6 @@ method getAllMessages*( s: PostgresDriver ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = ## Retrieve all messages from the store. - debug "beginning of getAllMessages" - var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] proc rowCallback(pqResult: ptr PGresult) = rowCallbackImpl(pqResult, rows) @@ -370,8 +368,6 @@ proc getPartitionsList( ): Future[ArchiveDriverResult[seq[string]]] {.async.} = ## Retrieves the seq of partition table names. ## e.g: @["messages_1708534333_1708534393", "messages_1708534273_1708534333"] - - debug "beginning getPartitionsList" var partitions: seq[string] proc rowCallback(pqResult: ptr PGresult) = for iRow in 0 ..< pqResult.pqNtuples(): @@ -431,7 +427,6 @@ proc getMessagesArbitraryQuery( ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = ## This proc allows to handle atypical queries. We don't use prepared statements for those. - debug "beginning getMessagesArbitraryQuery" var query = SelectClause var statements: seq[string] var args: seq[string] @@ -512,8 +507,6 @@ proc getMessageHashesArbitraryQuery( async .} = ## This proc allows to handle atypical queries. We don't use prepared statements for those. - - debug "beginning of getMessageHashesArbitraryQuery" var query = """SELECT messageHash FROM messages""" var statements: seq[string] @@ -597,7 +590,6 @@ proc getMessagesPreparedStmt( var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] - debug "beginning of getMessagesPreparedStmt" proc rowCallback(pqResult: ptr PGresult) = rowCallbackImpl(pqResult, rows) @@ -689,7 +681,6 @@ proc getMessageHashesPreparedStmt( var rows: seq[(WakuMessageHash, PubsubTopic, WakuMessage)] - debug "beginning of getMessageHashesPreparedStmt" proc rowCallback(pqResult: ptr PGresult) = hashCallbackImpl(pqResult, rows) @@ -775,7 +766,6 @@ proc getMessagesByMessageHashes( ## Retrieves information only filtering by a given messageHashes list. ## This proc levarages on the messages_lookup table to have better query performance ## and only query the desired partitions in the partitioned messages table - debug "beginning of getMessagesByMessageHashes" var query = fmt""" WITH min_timestamp AS ( @@ -814,7 +804,6 @@ proc getMessagesByMessageHashes( ).isOkOr: return err("failed to run query: " & $error) - debug "end of getMessagesByMessageHashes" return ok(rows) proc getMessagesWithinLimits( @@ -894,8 +883,6 @@ method getMessages*( ascendingOrder = true, requestId = "", ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - debug "beginning of getMessages" - let rows = collect(newSeq): for i in countup(0, hashes.len, MaxHashesPerQuery): let stop = min(i + MaxHashesPerQuery, hashes.len) @@ -911,8 +898,6 @@ method getMessages*( for row in subRows: row - debug "end of getMessages" - return ok(rows) proc getStr( @@ -920,8 +905,6 @@ proc getStr( ): Future[ArchiveDriverResult[string]] {.async.} = # Performs a query that is expected to return a single string - debug "beginning of getStr" - var ret: string proc rowCallback(pqResult: ptr PGresult) = if pqResult.pqnfields() != 1: @@ -944,7 +927,6 @@ proc getInt( ): Future[ArchiveDriverResult[int64]] {.async.} = # Performs a query that is expected to return a single numeric value (int64) - debug "beginning of getInt" var retInt = 0'i64 let str = (await s.getStr(query)).valueOr: return err("could not get str in getInt: " & $error) @@ -962,8 +944,6 @@ proc getInt( method getDatabaseSize*( s: PostgresDriver ): Future[ArchiveDriverResult[int64]] {.async.} = - debug "beginning of getDatabaseSize" - let intRes = (await s.getInt("SELECT pg_database_size(current_database())")).valueOr: return err("error in getDatabaseSize: " & error) @@ -973,8 +953,6 @@ method getDatabaseSize*( method getMessagesCount*( s: PostgresDriver ): Future[ArchiveDriverResult[int64]] {.async.} = - debug "beginning of getMessagesCount" - let intRes = await s.getInt("SELECT COUNT(1) FROM messages") if intRes.isErr(): return err("error in getMessagesCount: " & intRes.error) @@ -987,8 +965,6 @@ method getOldestMessageTimestamp*( ## In some cases it could happen that we have ## empty partitions which are older than the current stored rows. ## In those cases we want to consider those older partitions as the oldest considered timestamp. - debug "beginning of getOldestMessageTimestamp" - let oldestPartition = s.partitionMngr.getOldestPartition().valueOr: return err("could not get oldest partition: " & $error) @@ -1004,7 +980,6 @@ method getOldestMessageTimestamp*( method getNewestMessageTimestamp*( s: PostgresDriver ): Future[ArchiveDriverResult[Timestamp]] {.async.} = - debug "beginning of getNewestMessageTimestamp" let intRes = await s.getInt("SELECT MAX(timestamp) FROM messages") if intRes.isErr(): @@ -1015,8 +990,6 @@ method getNewestMessageTimestamp*( method deleteOldestMessagesNotWithinLimit*( s: PostgresDriver, limit: int ): Future[ArchiveDriverResult[void]] {.async.} = - debug "beginning of deleteOldestMessagesNotWithinLimit" - var execRes = await s.writeConnPool.pgQuery( """DELETE FROM messages WHERE messageHash NOT IN ( @@ -1039,12 +1012,9 @@ method deleteOldestMessagesNotWithinLimit*( "error in deleteOldestMessagesNotWithinLimit messages_lookup: " & execRes.error ) - debug "end of deleteOldestMessagesNotWithinLimit" return ok() method close*(s: PostgresDriver): Future[ArchiveDriverResult[void]] {.async.} = - debug "beginning of postgres close" - ## Cancel the partition factory loop s.futLoopPartitionFactory.cancelSoon() @@ -1096,8 +1066,6 @@ proc acquireDatabaseLock*( ## "performWriteQueryWithLock" in the migrations process because we can't nest two PL/SQL ## scripts. - debug "beginning of acquireDatabaseLock", lockId - let locked = ( await s.getStr( fmt""" @@ -1116,7 +1084,6 @@ proc releaseDatabaseLock*( s: PostgresDriver, lockId: int = 841886 ): Future[ArchiveDriverResult[void]] {.async.} = ## Release an advisory lock (useful to avoid more than one application running migrations at the same time) - debug "beginning of releaseDatabaseLock", lockId let unlocked = ( await s.getStr( fmt""" @@ -1143,11 +1110,10 @@ proc performWriteQuery*( const COULD_NOT_ACQUIRE_ADVISORY_LOCK* = "could not acquire advisory lock" -proc performWriteQueryWithLock*( +proc performWriteQueryWithLock( self: PostgresDriver, queryToProtect: string ): Future[ArchiveDriverResult[void]] {.async.} = ## This wraps the original query in a script so that we make sure a pg_advisory lock protects it - debug "performWriteQueryWithLock", queryToProtect let query = fmt""" DO $$ @@ -1210,8 +1176,6 @@ proc addPartition( ## Creates a partition table that will store the messages that fall in the range ## `startTime` <= timestamp < `startTime + duration`. ## `startTime` is measured in seconds since epoch - debug "beginning of addPartition" - let beginning = startTime let `end` = partitions_manager.calcEndPartitionTime(startTime) @@ -1353,8 +1317,6 @@ proc getTableSize*( ): Future[ArchiveDriverResult[string]] {.async.} = ## Returns a human-readable representation of the size for the requested table. ## tableName - table of interest. - debug "beginning of getTableSize" - let tableSize = ( await self.getStr( fmt""" @@ -1442,8 +1404,6 @@ proc removeOldestPartition( self: PostgresDriver, forceRemoval: bool = false, ## To allow cleanup in tests ): Future[ArchiveDriverResult[void]] {.async.} = ## Indirectly called from the retention policy - debug "beginning of removeOldestPartition" - let oldestPartition = self.partitionMngr.getOldestPartition().valueOr: return err("could not remove oldest partition: " & $error) @@ -1465,8 +1425,6 @@ proc containsAnyPartition*(self: PostgresDriver): bool = method decreaseDatabaseSize*( driver: PostgresDriver, targetSizeInBytes: int64, forceRemoval: bool = false ): Future[ArchiveDriverResult[void]] {.async.} = - debug "beginning of decreaseDatabaseSize" - var dbSize = (await driver.getDatabaseSize()).valueOr: return err("decreaseDatabaseSize failed to get database size: " & $error) @@ -1533,8 +1491,6 @@ method existsTable*( proc getCurrentVersion*( s: PostgresDriver ): Future[ArchiveDriverResult[int64]] {.async.} = - debug "beginning of getCurrentVersion" - let existsVersionTable = (await s.existsTable("version")).valueOr: return err("error in getCurrentVersion-existsTable: " & $error) @@ -1551,8 +1507,6 @@ method deleteMessagesOlderThanTimestamp*( ): Future[ArchiveDriverResult[void]] {.async.} = ## First of all, let's remove the older partitions so that we can reduce ## the database size. - debug "beginning of deleteMessagesOlderThanTimestamp" - (await s.removePartitionsOlderThan(tsNanoSec)).isOkOr: return err("error while removing older partitions: " & $error) diff --git a/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim index 4f3532622d..e8eed42386 100644 --- a/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive_legacy/driver/postgres_driver/postgres_driver.nim @@ -259,8 +259,6 @@ method getAllMessages*( s: PostgresDriver ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = ## Retrieve all messages from the store. - debug "beginning of getAllMessages" - var rows: seq[(PubsubTopic, WakuMessage, seq[byte], Timestamp, WakuMessageHash)] proc rowCallback(pqResult: ptr PGresult) = rowCallbackImpl(pqResult, rows) @@ -634,7 +632,6 @@ proc getMessagesByMessageHashes( ## Retrieves information only filtering by a given messageHashes list. ## This proc levarages on the messages_lookup table to have better query performance ## and only query the desired partitions in the partitioned messages table - debug "beginning of getMessagesByMessageHashes" var query = fmt""" WITH min_timestamp AS ( @@ -673,7 +670,6 @@ proc getMessagesByMessageHashes( ).isOkOr: return err("failed to run query: " & $error) - debug "end of getMessagesByMessageHashes" return ok(rows) method getMessages*( @@ -689,8 +685,6 @@ method getMessages*( ascendingOrder = true, requestId = "", ): Future[ArchiveDriverResult[seq[ArchiveRow]]] {.async.} = - debug "beginning of getMessages" - let hexHashes = hashes.mapIt(toHex(it)) if cursor.isNone() and pubsubTopic.isNone() and contentTopicSeq.len == 0 and From 9b445ac49f176f99bd926ea2c0e89e679dcbceef Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:47:52 +0200 Subject: [PATCH 05/12] fix: px protocol decode - do not treat missing response field as error (#3055) * Fix missing response field of PeerExchange RPC treated as error. * Fix PX metrics from gauge to counter for better dashboard stats --- waku/waku_peer_exchange/protocol.nim | 5 +++-- waku/waku_peer_exchange/rpc_codec.nim | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/waku/waku_peer_exchange/protocol.nim b/waku/waku_peer_exchange/protocol.nim index 0374e12772..0d00cfb0f1 100644 --- a/waku/waku_peer_exchange/protocol.nim +++ b/waku/waku_peer_exchange/protocol.nim @@ -20,9 +20,10 @@ declarePublicGauge waku_px_peers_received_total, "number of ENRs received via peer exchange" declarePublicGauge waku_px_peers_received_unknown, "number of previously unknown ENRs received via peer exchange" -declarePublicGauge waku_px_peers_sent, "number of ENRs sent to peer exchange requesters" +declarePublicCounter waku_px_peers_sent, + "number of ENRs sent to peer exchange requesters" declarePublicGauge waku_px_peers_cached, "number of peer exchange peer ENRs cached" -declarePublicGauge waku_px_errors, "number of peer exchange errors", ["type"] +declarePublicCounter waku_px_errors, "number of peer exchange errors", ["type"] logScope: topics = "waku peer_exchange" diff --git a/waku/waku_peer_exchange/rpc_codec.nim b/waku/waku_peer_exchange/rpc_codec.nim index e5e982938f..c0288ae39b 100644 --- a/waku/waku_peer_exchange/rpc_codec.nim +++ b/waku/waku_peer_exchange/rpc_codec.nim @@ -103,8 +103,9 @@ proc decode*(T: type PeerExchangeRpc, buffer: seq[byte]): ProtobufResult[T] = var responseBuffer: seq[byte] if not ?pb.getField(2, responseBuffer): - return err(ProtobufError.missingRequiredField("response")) - - rpc.response = ?PeerExchangeResponse.decode(responseBuffer) + rpc.response = + PeerExchangeResponse(status_code: PeerExchangeResponseStatusCode.UNKNOWN) + else: + rpc.response = ?PeerExchangeResponse.decode(responseBuffer) ok(rpc) From 5afa9b13ea582d8ad7b04a77f31ef0a56072e9af Mon Sep 17 00:00:00 2001 From: NagyZoltanPeter <113987313+NagyZoltanPeter@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:51:50 +0200 Subject: [PATCH 06/12] Fix PeerExchange rpc decode in order not to take response's status_code mandatory - for support old protocol implementation (#3059) --- waku/waku_peer_exchange/protocol.nim | 4 +++- waku/waku_peer_exchange/rpc_codec.nim | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/waku/waku_peer_exchange/protocol.nim b/waku/waku_peer_exchange/protocol.nim index 0d00cfb0f1..e70afe16c7 100644 --- a/waku/waku_peer_exchange/protocol.nim +++ b/waku/waku_peer_exchange/protocol.nim @@ -217,7 +217,9 @@ proc populateEnrCache(wpx: WakuPeerExchange) = proc updatePxEnrCache(wpx: WakuPeerExchange) {.async.} = # try more aggressively to fill the cache at startup - while wpx.enrCache.len < MaxPeersCacheSize: + var attempts = 10 + while wpx.enrCache.len < MaxPeersCacheSize and attempts > 0: + attempts -= 1 wpx.populateEnrCache() await sleepAsync(5.seconds) diff --git a/waku/waku_peer_exchange/rpc_codec.nim b/waku/waku_peer_exchange/rpc_codec.nim index c0288ae39b..b698ffe9f5 100644 --- a/waku/waku_peer_exchange/rpc_codec.nim +++ b/waku/waku_peer_exchange/rpc_codec.nim @@ -71,7 +71,11 @@ proc decode*(T: type PeerExchangeResponse, buffer: seq[byte]): ProtobufResult[T] if ?pb.getField(10, status_code): rpc.status_code = PeerExchangeResponseStatusCode.parse(status_code) else: - return err(ProtobufError.missingRequiredField("status_code")) + # older peers may not support status_code field yet + if rpc.peerInfos.len() > 0: + rpc.status_code = PeerExchangeResponseStatusCode.SUCCESS + else: + rpc.status_code = PeerExchangeResponseStatusCode.SERVICE_UNAVAILABLE var status_desc: string if ?pb.getField(11, status_desc): From 368bb3c199c97a5a683822b81670fa470137ff81 Mon Sep 17 00:00:00 2001 From: Ivan FB <128452529+Ivansete-status@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:52:02 +0200 Subject: [PATCH 07/12] append current version in agentString which is used by the identify protocol (#3057) --- waku/factory/external_config.nim | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/waku/factory/external_config.nim b/waku/factory/external_config.nim index 812cffb768..2446352c10 100644 --- a/waku/factory/external_config.nim +++ b/waku/factory/external_config.nim @@ -26,6 +26,9 @@ include ../waku_core/message/default_values export confTomlDefs, confTomlNet, confEnvvarDefs, confEnvvarNet +# Git version in git describe format (defined at compile time) +const git_version* {.strdefine.} = "n/a" + type ConfResult*[T] = Result[T, string] type EthRpcUrl* = distinct string @@ -157,7 +160,7 @@ type WakuNodeConf* = object .}: uint16 agentString* {. - defaultValue: "nwaku", + defaultValue: "nwaku-" & git_version, desc: "Node agent string which is used as identifier in network", name: "agent-string" .}: string From f8946b8263488378e2ac5ae268121ab189638d55 Mon Sep 17 00:00:00 2001 From: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:35:18 +0300 Subject: [PATCH 08/12] fix: rejecting excess relay connections (#3065) --- waku/node/peer_manager/peer_manager.nim | 107 +++++++++++------------- 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/waku/node/peer_manager/peer_manager.nim b/waku/node/peer_manager/peer_manager.nim index 12c6a084f7..6e5a9c6976 100644 --- a/waku/node/peer_manager/peer_manager.nim +++ b/waku/node/peer_manager/peer_manager.nim @@ -399,6 +399,24 @@ proc onPeerMetadata(pm: PeerManager, peerId: PeerId) {.async.} = asyncSpawn(pm.switch.disconnect(peerId)) pm.peerStore.delete(peerId) +proc connectedPeers*(pm: PeerManager, protocol: string): (seq[PeerId], seq[PeerId]) = + ## Returns the peerIds of physical connections (in and out) + ## containing at least one stream with the given protocol. + + var inPeers: seq[PeerId] + var outPeers: seq[PeerId] + + for peerId, muxers in pm.switch.connManager.getConnections(): + for peerConn in muxers: + let streams = peerConn.getStreams() + if streams.anyIt(it.protocol == protocol): + if peerConn.connection.transportDir == Direction.In: + inPeers.add(peerId) + elif peerConn.connection.transportDir == Direction.Out: + outPeers.add(peerId) + + return (inPeers, outPeers) + # called when a peer i) first connects to us ii) disconnects all connections from us proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = if not pm.wakuMetadata.isNil() and event.kind == PeerEventKind.Joined: @@ -412,6 +430,17 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = direction = if event.initiator: Outbound else: Inbound connectedness = Connected + ## Check max allowed in-relay peers + let inRelayPeers = pm.connectedPeers(WakuRelayCodec)[0] + if inRelayPeers.len > pm.inRelayPeersTarget and + pm.peerStore.hasPeer(peerId, WakuRelayCodec): + debug "disconnecting relay peer because reached max num in-relay peers", + peerId = peerId, + inRelayPeers = inRelayPeers.len, + inRelayPeersTarget = pm.inRelayPeersTarget + await pm.switch.disconnect(peerId) + + ## Apply max ip colocation limit if (let ip = pm.getPeerIp(peerId); ip.isSome()): pm.ipTable.mgetOrPut(ip.get, newSeq[PeerId]()).add(peerId) @@ -494,7 +523,7 @@ proc new*( error "Max backoff time can't be over 1 week", maxBackoff = backoff raise newException(Defect, "Max backoff time can't be over 1 week") - let outRelayPeersTarget = max(maxRelayPeersValue div 3, 10) + let outRelayPeersTarget = maxRelayPeersValue div 3 let pm = PeerManager( switch: switch, @@ -560,46 +589,6 @@ proc addServicePeer*(pm: PeerManager, remotePeerInfo: RemotePeerInfo, proto: str pm.addPeer(remotePeerInfo) -proc reconnectPeers*( - pm: PeerManager, proto: string, backoff: chronos.Duration = chronos.seconds(0) -) {.async.} = - ## Reconnect to peers registered for this protocol. This will update connectedness. - ## Especially useful to resume connections from persistent storage after a restart. - - trace "Reconnecting peers", proto = proto - - # Proto is not persisted, we need to iterate over all peers. - for peerInfo in pm.peerStore.peers(protocolMatcher(proto)): - # Check that the peer can be connected - if peerInfo.connectedness == CannotConnect: - error "Not reconnecting to unreachable or non-existing peer", - peerId = peerInfo.peerId - continue - - # Respect optional backoff period where applicable. - let - # TODO: Add method to peerStore (eg isBackoffExpired()) - disconnectTime = Moment.init(peerInfo.disconnectTime, Second) # Convert - currentTime = Moment.init(getTime().toUnix, Second) - # Current time comparable to persisted value - backoffTime = disconnectTime + backoff - currentTime - # Consider time elapsed since last disconnect - - trace "Respecting backoff", - backoff = backoff, - disconnectTime = disconnectTime, - currentTime = currentTime, - backoffTime = backoffTime - - # TODO: This blocks the whole function. Try to connect to another peer in the meantime. - if backoffTime > ZeroDuration: - trace "Backing off before reconnect...", - peerId = peerInfo.peerId, backoffTime = backoffTime - # We disconnected recently and still need to wait for a backoff period before connecting - await sleepAsync(backoffTime) - - discard await pm.connectRelay(peerInfo) - #################### # Dialer interface # #################### @@ -685,23 +674,29 @@ proc connectToNodes*( # later. await sleepAsync(chronos.seconds(5)) -proc connectedPeers*(pm: PeerManager, protocol: string): (seq[PeerId], seq[PeerId]) = - ## Returns the peerIds of physical connections (in and out) - ## containing at least one stream with the given protocol. +proc reconnectPeers*( + pm: PeerManager, proto: string, backoffTime: chronos.Duration = chronos.seconds(0) +) {.async.} = + ## Reconnect to peers registered for this protocol. This will update connectedness. + ## Especially useful to resume connections from persistent storage after a restart. - var inPeers: seq[PeerId] - var outPeers: seq[PeerId] + debug "Reconnecting peers", proto = proto - for peerId, muxers in pm.switch.connManager.getConnections(): - for peerConn in muxers: - let streams = peerConn.getStreams() - if streams.anyIt(it.protocol == protocol): - if peerConn.connection.transportDir == Direction.In: - inPeers.add(peerId) - elif peerConn.connection.transportDir == Direction.Out: - outPeers.add(peerId) + # Proto is not persisted, we need to iterate over all peers. + for peerInfo in pm.peerStore.peers(protocolMatcher(proto)): + # Check that the peer can be connected + if peerInfo.connectedness == CannotConnect: + error "Not reconnecting to unreachable or non-existing peer", + peerId = peerInfo.peerId + continue - return (inPeers, outPeers) + if backoffTime > ZeroDuration: + debug "Backing off before reconnect", + peerId = peerInfo.peerId, backoffTime = backoffTime + # We disconnected recently and still need to wait for a backoff period before connecting + await sleepAsync(backoffTime) + + await pm.connectToNodes(@[peerInfo]) proc getNumStreams*(pm: PeerManager, protocol: string): (int, int) = var @@ -730,9 +725,7 @@ proc pruneInRelayConns(pm: PeerManager, amount: int) {.async.} = proc connectToRelayPeers*(pm: PeerManager) {.async.} = var (inRelayPeers, outRelayPeers) = pm.connectedPeers(WakuRelayCodec) - let maxConnections = pm.switch.connManager.inSema.size let totalRelayPeers = inRelayPeers.len + outRelayPeers.len - let inPeersTarget = maxConnections - pm.outRelayPeersTarget if inRelayPeers.len > pm.inRelayPeersTarget: await pm.pruneInRelayConns(inRelayPeers.len - pm.inRelayPeersTarget) From 73c1d6074485a33a1c75972ddbfada65b60e3bb8 Mon Sep 17 00:00:00 2001 From: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:26:10 +0300 Subject: [PATCH 09/12] chore: update changelog for v0.33.0 release (#3044) --- CHANGELOG.md | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7a9469180..e1b7e46b5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,67 @@ +## v0.33.0 (2024-09-30) + +#### Notes: + +* The `--pubsub-topic` CLI configuration has been deprecated and support for it will be removed on release v0.35.0. In order to migrate, please use the `--shard` configuration instead. For example, instead of `--pubsub-topic=/waku/2/rs//`, use `--cluster-id=` once and `--shard=` for each subscribed shard +* The `--rest-private` CLI configuration has been removed. Please delete any reference to it when running your nodes +* Introduced the `--reliability` CLI configuration, activating the new experimental StoreV3 message confirmation protocol +* DOS protection configurations of non-relay, req/resp protocols are changed + * `--request-rate-limit` and `--request-rate-period` options are no longer supported. + * `--rate-limit` CLI configuration is now available. + - The new flag can describe various rate-limit requirements for each protocol supported. The setting can be repeated, each instance can define exactly one rate-limit option. + - Format is `:volume/period` + - If protocol is not given, settings will be taken as default for un-set protocols. Ex: 80/2s + - Supported protocols are: lightpush|filter|px|store|storev2|storev3 + - `volume` must be an integer value, representing number of requests over the period of time allowed. + - `period ` must be an integer with defined unit as one of h|m|s|ms + - If not set, no rate limit will be applied to request/response protocols, except for the filter protocol. + + +### Release highlights + +* a new experimental reliability protocol has been implemented, leveraging StoreV3 to confirm message delivery +* Peer Exchange protocol can now be protected by rate-limit boundary checks. +* Fine-grained configuration of DOS protection is available with this release. See, "Notes" above. + +### Bug Fixes + +- rejecting excess relay connections ([#3063](https://github.com/waku-org/nwaku/issues/3063)) ([8b0884c7](https://github.com/waku-org/nwaku/commit/8b0884c7)) +- make Peer Exchange's rpc status_code optional for backward compatibility ([#3059](https://github.com/waku-org/nwaku/pull/3059)) ([5afa9b13](https://github.com/waku-org/nwaku/commit/5afa9b13)) +- px protocol decode - do not treat missing response field as error ([#3054](https://github.com/waku-org/nwaku/issues/3054)) ([9b445ac4](https://github.com/waku-org/nwaku/commit/9b445ac4)) +- setting up node with modified config ([#3036](https://github.com/waku-org/nwaku/issues/3036)) ([8f289925](https://github.com/waku-org/nwaku/commit/8f289925)) +- get back health check for postgres legacy ([#3010](https://github.com/waku-org/nwaku/issues/3010)) ([5a0edff7](https://github.com/waku-org/nwaku/commit/5a0edff7)) +- libnegentropy integration ([#2996](https://github.com/waku-org/nwaku/issues/2996)) ([c3cb06ac](https://github.com/waku-org/nwaku/commit/c3cb06ac)) +- peer-exchange issue ([#2889](https://github.com/waku-org/nwaku/issues/2889)) ([43157102](https://github.com/waku-org/nwaku/commit/43157102)) + +### Changes + +- append current version in agentString which is used by the identify protocol ([#3057](https://github.com/waku-org/nwaku/pull/3057)) ([368bb3c1](https://github.com/waku-org/nwaku/commit/368bb3c1)) +- rate limit peer exchange protocol, enhanced response status in RPC ([#3035](https://github.com/waku-org/nwaku/issues/3035)) ([0a7f16a3](https://github.com/waku-org/nwaku/commit/0a7f16a3)) +- Switch libnegentropy library build from shared to static linkage ([#3041](https://github.com/waku-org/nwaku/issues/3041)) ([83f25c3e](https://github.com/waku-org/nwaku/commit/83f25c3e)) +- libwaku reduce repetitive code by adding a template handling resp returns ([#3032](https://github.com/waku-org/nwaku/issues/3032)) ([1713f562](https://github.com/waku-org/nwaku/commit/1713f562)) +- libwaku - extending the library with peer_manager and peer_exchange features ([#3026](https://github.com/waku-org/nwaku/issues/3026)) ([5ea1cf0c](https://github.com/waku-org/nwaku/commit/5ea1cf0c)) +- use submodule nph in CI to check lint ([#3027](https://github.com/waku-org/nwaku/issues/3027)) ([ce9a8c46](https://github.com/waku-org/nwaku/commit/ce9a8c46)) +- deprecating pubsub topic ([#2997](https://github.com/waku-org/nwaku/issues/2997)) ([a3cd2a1a](https://github.com/waku-org/nwaku/commit/a3cd2a1a)) +- lightpush - error metric less variable by only setting a fixed string ([#3020](https://github.com/waku-org/nwaku/issues/3020)) ([d3e6717a](https://github.com/waku-org/nwaku/commit/d3e6717a)) +- Bump dependencies for v0.33 ([#3017](https://github.com/waku-org/nwaku/issues/3017)) ([6c890de4](https://github.com/waku-org/nwaku/commit/6c890de4)) +- enhance libpq management ([#3015](https://github.com/waku-org/nwaku/issues/3015)) ([45319f09](https://github.com/waku-org/nwaku/commit/45319f09)) +- per limit split of PostgreSQL queries ([#3008](https://github.com/waku-org/nwaku/issues/3008)) ([e1e05afb](https://github.com/waku-org/nwaku/commit/e1e05afb)) +- Added metrics to liteprotocoltester ([#3002](https://github.com/waku-org/nwaku/issues/3002)) ([8baf627f](https://github.com/waku-org/nwaku/commit/8baf627f)) +- extending store metrics ([#2995](https://github.com/waku-org/nwaku/issues/2995)) ([fd83b42f](https://github.com/waku-org/nwaku/commit/fd83b42f)) +- Better timing and requestId detail for slower store db queries ([#2994](https://github.com/waku-org/nwaku/issues/2994)) ([e8a49b76](https://github.com/waku-org/nwaku/commit/e8a49b76)) +- remove unused setting from external_config.nim ([#3004](https://github.com/waku-org/nwaku/issues/3004)) ([fd84363e](https://github.com/waku-org/nwaku/commit/fd84363e)) +- delivery monitor for store v3 reliability protocol ([#2977](https://github.com/waku-org/nwaku/issues/2977)) ([0f68274c](https://github.com/waku-org/nwaku/commit/0f68274c)) + +This release supports the following [libp2p protocols](https://docs.libp2p.io/concepts/protocols/): +| Protocol | Spec status | Protocol id | +| ---: | :---: | :--- | +| [`11/WAKU2-RELAY`](https://github.com/vacp2p/rfc-index/blob/main/waku/standards/core/11/relay.md) | `stable` | `/vac/waku/relay/2.0.0` | +| [`12/WAKU2-FILTER`](https://github.com/vacp2p/rfc-index/blob/main/waku/standards/core/12/filter.md) | `draft` | `/vac/waku/filter/2.0.0-beta1`
`/vac/waku/filter-subscribe/2.0.0-beta1`
`/vac/waku/filter-push/2.0.0-beta1` | +| [`13/WAKU2-STORE`](https://github.com/vacp2p/rfc-index/blob/main/waku/standards/core/13/store.md) | `draft` | `/vac/waku/store/2.0.0-beta4` | +| [`19/WAKU2-LIGHTPUSH`](https://github.com/vacp2p/rfc-index/blob/main/waku/standards/core/19/lightpush.md) | `draft` | `/vac/waku/lightpush/2.0.0-beta1` | +| [`66/WAKU2-METADATA`](https://github.com/waku-org/specs/blob/master/standards/core/metadata.md) | `raw` | `/vac/waku/metadata/1.0.0` | +| [`WAKU-SYNC`](https://github.com/waku-org/specs/blob/feat--waku-sync/standards/core/sync.md) | `draft` | `/vac/waku/sync/1.0.0` | + ## v0.32.0 (2024-08-30) #### Notes: From 973b20e36891221369bb1bfdde1d484c50df4b42 Mon Sep 17 00:00:00 2001 From: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Date: Mon, 30 Sep 2024 21:04:32 +0300 Subject: [PATCH 10/12] Update waku-fleet-dashboard.json --- metrics/waku-fleet-dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/waku-fleet-dashboard.json b/metrics/waku-fleet-dashboard.json index d15d246331..0916f14d49 100644 --- a/metrics/waku-fleet-dashboard.json +++ b/metrics/waku-fleet-dashboard.json @@ -7902,4 +7902,4 @@ "uid": "qrp_ZCTGz", "version": 154, "weekStart": "" -} \ No newline at end of file +} From 45fec1a1d9d970c2b4d364f9bb0c84edb22b9e36 Mon Sep 17 00:00:00 2001 From: gabrielmer <101006718+gabrielmer@users.noreply.github.com> Date: Mon, 30 Sep 2024 21:05:03 +0300 Subject: [PATCH 11/12] Update peer_manager.nim --- waku/node/peer_manager/peer_manager.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/waku/node/peer_manager/peer_manager.nim b/waku/node/peer_manager/peer_manager.nim index 33e1bf3168..07a091e864 100644 --- a/waku/node/peer_manager/peer_manager.nim +++ b/waku/node/peer_manager/peer_manager.nim @@ -1021,4 +1021,4 @@ proc start*(pm: PeerManager) = asyncSpawn pm.logAndMetrics() proc stop*(pm: PeerManager) = - pm.started = false \ No newline at end of file + pm.started = false From 7bc42b5dc549ca982b5cfdfc9958894759af8553 Mon Sep 17 00:00:00 2001 From: Gabriel mermelstein Date: Wed, 2 Oct 2024 10:31:45 +0300 Subject: [PATCH 12/12] remove dependency bumping from changelog --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1b7e46b5f..0cd2b70c0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,6 @@ - use submodule nph in CI to check lint ([#3027](https://github.com/waku-org/nwaku/issues/3027)) ([ce9a8c46](https://github.com/waku-org/nwaku/commit/ce9a8c46)) - deprecating pubsub topic ([#2997](https://github.com/waku-org/nwaku/issues/2997)) ([a3cd2a1a](https://github.com/waku-org/nwaku/commit/a3cd2a1a)) - lightpush - error metric less variable by only setting a fixed string ([#3020](https://github.com/waku-org/nwaku/issues/3020)) ([d3e6717a](https://github.com/waku-org/nwaku/commit/d3e6717a)) -- Bump dependencies for v0.33 ([#3017](https://github.com/waku-org/nwaku/issues/3017)) ([6c890de4](https://github.com/waku-org/nwaku/commit/6c890de4)) - enhance libpq management ([#3015](https://github.com/waku-org/nwaku/issues/3015)) ([45319f09](https://github.com/waku-org/nwaku/commit/45319f09)) - per limit split of PostgreSQL queries ([#3008](https://github.com/waku-org/nwaku/issues/3008)) ([e1e05afb](https://github.com/waku-org/nwaku/commit/e1e05afb)) - Added metrics to liteprotocoltester ([#3002](https://github.com/waku-org/nwaku/issues/3002)) ([8baf627f](https://github.com/waku-org/nwaku/commit/8baf627f))