diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index b0698f5c32..52e2164c59 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -21,3 +21,6 @@ Add any other context or screenshots about the feature request here. ### Acceptance criteria A list of tasks that need to be done for the issue to be considered resolved. + +### Epic +Epic title and link the feature refers to. diff --git a/Makefile b/Makefile index 98517e3aa3..d0c21a6ef4 100644 --- a/Makefile +++ b/Makefile @@ -130,6 +130,10 @@ ifeq ($(POSTGRES), 1) NIM_PARAMS := $(NIM_PARAMS) -d:postgres -d:nimDebugDlOpen endif +ifeq ($(DEBUG_DISCV5), 1) +NIM_PARAMS := $(NIM_PARAMS) -d:debugDiscv5 +endif + clean: | clean-libbacktrace diff --git a/apps/chat2/chat2.nim b/apps/chat2/chat2.nim index 42e2461b1a..20a6929036 100644 --- a/apps/chat2/chat2.nim +++ b/apps/chat2/chat2.nim @@ -4,10 +4,7 @@ when not (compileOption("threads")): {.fatal: "Please, compile this program with the --threads:on option!".} -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[strformat, strutils, times, options, random] import diff --git a/apps/chat2bridge/chat2bridge.nim b/apps/chat2bridge/chat2bridge.nim index 6a0aaef6cd..2e30aaaa2f 100644 --- a/apps/chat2bridge/chat2bridge.nim +++ b/apps/chat2bridge/chat2bridge.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[tables, times, strutils, hashes, sequtils], diff --git a/apps/liteprotocoltester/liteprotocoltester.nim b/apps/liteprotocoltester/liteprotocoltester.nim index 3f09eb31db..d63003be33 100644 --- a/apps/liteprotocoltester/liteprotocoltester.nim +++ b/apps/liteprotocoltester/liteprotocoltester.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, strutils, os, sequtils, net], diff --git a/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json b/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json index 2b024e32fc..f269b0aeb8 100644 --- a/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json +++ b/apps/liteprotocoltester/monitoring/configuration/dashboards/nwaku-monitoring.json @@ -88,7 +88,7 @@ "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -155,10 +155,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -218,10 +219,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -285,10 +287,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -494,6 +497,196 @@ "title": "Average msg size (kBytes)", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 0, + "y": 16 + }, + "id": 147, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_relay_network_bytes_total{direction=\"in\"}[$__rate_interval])", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Relay traffic per shard (in)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 9, + "y": 16 + }, + "id": 148, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(waku_relay_network_bytes_total{direction=\"out\"}[$__rate_interval])", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Relay traffic per shard (out)", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -532,7 +725,7 @@ "h": 5, "w": 3, "x": 0, - "y": 16 + "y": 25 }, "id": 2, "options": { @@ -547,7 +740,7 @@ }, "showHeader": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -610,7 +803,7 @@ "h": 5, "w": 3, "x": 3, - "y": 16 + "y": 25 }, "id": 22, "options": { @@ -625,10 +818,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -688,7 +882,7 @@ "h": 5, "w": 3, "x": 6, - "y": 16 + "y": 25 }, "id": 32, "options": { @@ -703,10 +897,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -754,7 +949,7 @@ "h": 5, "w": 3, "x": 9, - "y": 16 + "y": 25 }, "id": 33, "options": { @@ -769,10 +964,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -833,7 +1029,7 @@ "h": 5, "w": 3, "x": 12, - "y": 16 + "y": 25 }, "id": 25, "options": { @@ -848,10 +1044,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -899,7 +1096,7 @@ "h": 5, "w": 3, "x": 15, - "y": 16 + "y": 25 }, "id": 28, "options": { @@ -914,10 +1111,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -962,7 +1160,7 @@ "h": 5, "w": 3, "x": 0, - "y": 21 + "y": 30 }, "id": 10, "options": { @@ -977,10 +1175,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -1028,7 +1227,7 @@ "h": 10, "w": 15, "x": 3, - "y": 21 + "y": 30 }, "id": 44, "options": { @@ -1043,10 +1242,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -1061,7 +1261,6 @@ } ], "title": "Connected Peers (Direction/Protocol)", - "transformations": [], "type": "stat" }, { @@ -1091,7 +1290,7 @@ "h": 5, "w": 3, "x": 0, - "y": 26 + "y": 35 }, "id": 36, "options": { @@ -1106,25 +1305,311 @@ "fields": "", "values": false }, - "textMode": "auto", - "wideLayout": true + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "waku_peer_store_size", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Peer Store Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 0, + "y": 40 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(libp2p_network_bytes_total{direction=\"in\"}[$__rate_interval])", + "legendFormat": "traffic_{{direction}}", + "range": true, + "refId": "A" + } + ], + "title": "libp2p traffic (in)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 9, + "y": 40 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(libp2p_network_bytes_total{direction=\"out\"}[$__rate_interval])", + "legendFormat": "traffic_{{direction}}", + "range": true, + "refId": "A" + } + ], + "title": "libp2p traffic (out)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 49 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "10.2.3", + "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "waku_peer_store_size", - "legendFormat": "__auto", + "editorMode": "builder", + "expr": "libp2p_peers", + "legendFormat": "{{__name__}}", "range": true, "refId": "A" } ], - "title": "Peer Store Size", - "type": "stat" + "title": "Connected Peers", + "type": "timeseries" }, { "datasource": { @@ -1181,17 +1666,18 @@ "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 6, - "x": 0, - "y": 31 + "h": 10, + "w": 12, + "x": 6, + "y": 49 }, - "id": 4, + "id": 149, "options": { "legend": { "calcs": [], @@ -1204,21 +1690,20 @@ "sort": "none" } }, - "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "builder", - "expr": "libp2p_peers", - "legendFormat": "{{__name__}}", + "editorMode": "code", + "expr": "rate(waku_service_requests_total[$__rate_interval])", + "legendFormat": "{{proto}}/{{state}}", "range": true, "refId": "A" } ], - "title": "Connected Peers", + "title": "Protocol request rates", "type": "timeseries" }, { @@ -1283,11 +1768,11 @@ }, "gridPos": { "h": 9, - "w": 6, - "x": 6, - "y": 31 + "w": 9, + "x": 0, + "y": 59 }, - "id": 8, + "id": 150, "options": { "legend": { "calcs": [], @@ -1307,13 +1792,13 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(libp2p_network_bytes_total{direction=\"in\"}[$__rate_interval])", - "legendFormat": "traffic_{{direction}}", + "expr": "rate(waku_service_network_bytes_total{direction=\"in\"}[$__rate_interval])", + "legendFormat": "{{service}}", "range": true, "refId": "A" } ], - "title": "libp2p traffic (in)", + "title": "Non relay protocol traffic (in)", "type": "timeseries" }, { @@ -1378,11 +1863,11 @@ }, "gridPos": { "h": 9, - "w": 6, - "x": 12, - "y": 31 + "w": 9, + "x": 9, + "y": 59 }, - "id": 29, + "id": 151, "options": { "legend": { "calcs": [], @@ -1402,13 +1887,13 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(libp2p_network_bytes_total{direction=\"out\"}[$__rate_interval])", - "legendFormat": "traffic_{{direction}}", + "expr": "rate(waku_service_network_bytes_total{direction=\"out\"}[$__rate_interval])", + "legendFormat": "{{service}}", "range": true, "refId": "A" } ], - "title": "libp2p traffic (out)", + "title": "Non relay protocol traffic (out)", "type": "timeseries" }, { @@ -1475,7 +1960,7 @@ "h": 8, "w": 6, "x": 0, - "y": 40 + "y": 68 }, "id": 20, "options": { @@ -1570,7 +2055,7 @@ "h": 8, "w": 6, "x": 6, - "y": 40 + "y": 68 }, "id": 18, "options": { @@ -1662,12 +2147,12 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 6, "x": 12, - "y": 40 + "y": 68 }, - "id": 128, + "id": 135, "options": { "legend": { "calcs": [], @@ -1688,7 +2173,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "waku_rln_number_registered_memberships", + "expr": "waku_rln_membership_insertion_duration_seconds", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1698,7 +2183,7 @@ "useBackend": false } ], - "title": "RLN Registered Memberships", + "title": "RLN Membership Insertion (seconds)", "type": "timeseries" }, { @@ -1761,12 +2246,12 @@ "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 6, - "x": 0, - "y": 48 + "x": 12, + "y": 74 }, - "id": 127, + "id": 128, "options": { "legend": { "calcs": [], @@ -1787,7 +2272,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "waku_rln_proof_generation_duration_seconds", + "expr": "waku_rln_number_registered_memberships", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1797,7 +2282,7 @@ "useBackend": false } ], - "title": "RLN Proof Generation (seconds)", + "title": "RLN Registered Memberships", "type": "timeseries" }, { @@ -1862,10 +2347,10 @@ "gridPos": { "h": 6, "w": 6, - "x": 6, - "y": 48 + "x": 0, + "y": 76 }, - "id": 126, + "id": 127, "options": { "legend": { "calcs": [], @@ -1886,7 +2371,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "waku_rln_proof_verification_duration_seconds", + "expr": "waku_rln_proof_generation_duration_seconds", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1896,7 +2381,7 @@ "useBackend": false } ], - "title": "RLN Proof Verification (seconds)", + "title": "RLN Proof Generation (seconds)", "type": "timeseries" }, { @@ -1961,10 +2446,10 @@ "gridPos": { "h": 6, "w": 6, - "x": 12, - "y": 48 + "x": 6, + "y": 76 }, - "id": 135, + "id": 126, "options": { "legend": { "calcs": [], @@ -1985,7 +2470,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "waku_rln_membership_insertion_duration_seconds", + "expr": "waku_rln_proof_verification_duration_seconds", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1995,7 +2480,7 @@ "useBackend": false } ], - "title": "RLN Membership Insertion (seconds)", + "title": "RLN Proof Verification (seconds)", "type": "timeseries" }, { @@ -2061,7 +2546,7 @@ "h": 6, "w": 6, "x": 0, - "y": 54 + "y": 82 }, "id": 134, "options": { @@ -2160,7 +2645,7 @@ "h": 6, "w": 6, "x": 6, - "y": 54 + "y": 82 }, "id": 137, "options": { @@ -2259,7 +2744,7 @@ "h": 6, "w": 6, "x": 12, - "y": 54 + "y": 82 }, "id": 136, "options": { @@ -2358,7 +2843,7 @@ "h": 6, "w": 6, "x": 0, - "y": 60 + "y": 88 }, "id": 133, "options": { @@ -2457,7 +2942,7 @@ "h": 6, "w": 6, "x": 6, - "y": 60 + "y": 88 }, "id": 130, "options": { @@ -2556,7 +3041,7 @@ "h": 6, "w": 6, "x": 12, - "y": 60 + "y": 88 }, "id": 138, "options": { @@ -2657,7 +3142,7 @@ "h": 11, "w": 9, "x": 0, - "y": 66 + "y": 94 }, "id": 141, "options": { @@ -2794,7 +3279,7 @@ "h": 11, "w": 9, "x": 9, - "y": 66 + "y": 94 }, "id": 144, "options": { @@ -2812,7 +3297,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -2964,31 +3449,209 @@ } ] }, - "unit": "none" + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 105 + }, + "id": 146, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pg_tb_messages_count{}", + "instant": false, + "interval": "", + "legendFormat": "messages", + "range": true, + "refId": "A" + } + ], + "title": "Unique stored messages (Postgres)", + "type": "timeseries" + }, + { + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Clients executing Statements.\n\nSource: pg_stat_activity", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 9, + "y": 105 + }, + "id": 23, + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(pg_stat_activity_count{state=\"active\",instance=\"$Instance\"})", + "refId": "A" + } + ], + "thresholds": "", + "title": "Active clients (Postgres)", + "type": "stat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dateTimeAsIso" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 9, - "x": 0, - "y": 77 + "h": 3, + "w": 4, + "x": 13, + "y": 105 }, - "id": 146, + "id": 125, "options": { - "legend": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": [ - "last" + "lastNotNull" ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "fields": "", + "values": false }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, + "pluginVersion": "10.4.2", "targets": [ { "datasource": { @@ -2996,17 +3659,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "exemplar": false, - "expr": "pg_tb_messages_count{}", - "instant": false, - "interval": "", - "legendFormat": "messages", + "expr": "pg_postmaster_start_time_seconds*1000", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Unique stored messages (Postgres)", - "type": "timeseries" + "title": "Postgres start time", + "type": "stat" }, { "collapsed": false, @@ -3014,7 +3674,7 @@ "h": 1, "w": 24, "x": 0, - "y": 84 + "y": 112 }, "id": 46, "panels": [], @@ -3033,8 +3693,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -3053,10 +3712,9 @@ "h": 3, "w": 4, "x": 0, - "y": 85 + "y": 113 }, "id": 11, - "links": [], "mappingType": 1, "mappingTypes": [ { @@ -3082,10 +3740,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3141,8 +3800,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3165,11 +3823,10 @@ "h": 3, "w": 4, "x": 4, - "y": 85 + "y": 113 }, "id": 14, "interval": "", - "links": [], "mappingType": 1, "mappingTypes": [ { @@ -3195,10 +3852,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3255,8 +3913,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3279,10 +3936,9 @@ "h": 3, "w": 4, "x": 8, - "y": 85 + "y": 113 }, "id": 93, - "links": [], "mappingType": 1, "mappingTypes": [ { @@ -3308,10 +3964,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3368,8 +4025,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3392,10 +4048,9 @@ "h": 3, "w": 4, "x": 12, - "y": 85 + "y": 113 }, "id": 102, - "links": [], "mappingType": 1, "mappingTypes": [ { @@ -3421,10 +4076,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3482,8 +4138,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] }, @@ -3503,10 +4158,9 @@ "h": 3, "w": 4, "x": 16, - "y": 85 + "y": 113 }, "id": 37, - "links": [], "mappingType": 1, "mappingTypes": [ { @@ -3532,10 +4186,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3590,8 +4245,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3614,10 +4268,9 @@ "h": 3, "w": 4, "x": 20, - "y": 85 + "y": 113 }, "id": 84, - "links": [], "mappingType": 1, "mappingTypes": [ { @@ -3643,10 +4296,11 @@ "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "postfix": "", "postfixFontSize": "50%", "prefix": "", @@ -3727,10 +4381,9 @@ "h": 7, "w": 3, "x": 0, - "y": 88 + "y": 116 }, "id": 16, - "links": [], "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -3746,7 +4399,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "expr": "sum(pg_stat_database_blks_hit{instance=~\"$Instance\"})/(sum(pg_stat_database_blks_hit{instance=~\"$Instance\"})+sum(pg_stat_database_blks_read{instance=~\"$Instance\"}))*100", @@ -3777,8 +4430,7 @@ "mode": "absolute", "steps": [ { - "color": "semi-dark-green", - "value": null + "color": "semi-dark-green" }, { "color": "semi-dark-yellow", @@ -3798,10 +4450,9 @@ "h": 7, "w": 3, "x": 3, - "y": 88 + "y": 116 }, "id": 9, - "links": [], "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -3817,7 +4468,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "expr": "sum(pg_stat_database_numbackends)/max(pg_settings_max_connections)", @@ -3848,8 +4499,7 @@ "mode": "absolute", "steps": [ { - "color": "semi-dark-red", - "value": null + "color": "semi-dark-red" }, { "color": "#EAB839", @@ -3869,10 +4519,9 @@ "h": 7, "w": 3, "x": 6, - "y": 88 + "y": 116 }, "id": 15, - "links": [], "options": { "minVizHeight": 75, "minVizWidth": 75, @@ -3888,7 +4537,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "targets": [ { "expr": "sum(pg_stat_database_xact_commit{instance=\"$Instance\"})/(sum(pg_stat_database_xact_commit{instance=\"$Instance\"}) + sum(pg_stat_database_xact_rollback{instance=\"$Instance\"}))", @@ -3898,180 +4547,6 @@ "title": "Commit Ratio (Postgres)", "type": "gauge" }, - { - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "description": "Clients executing Statements.\n\nSource: pg_stat_activity", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 88 - }, - "id": 23, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(pg_stat_activity_count{state=\"active\",instance=\"$Instance\"})", - "refId": "A" - } - ], - "thresholds": "", - "title": "Active clients (Postgres)", - "type": "stat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "dateTimeAsIso" - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 88 - }, - "id": 125, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.3", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "pg_postmaster_start_time_seconds*1000", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Postgres start time", - "type": "stat" - }, { "datasource": { "type": "prometheus", @@ -4120,8 +4595,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4137,7 +4611,7 @@ "h": 8, "w": 12, "x": 12, - "y": 91 + "y": 116 }, "id": 142, "options": { @@ -4213,7 +4687,7 @@ "h": 9, "w": 12, "x": 0, - "y": 95 + "y": 123 }, "hiddenSeries": false, "id": 24, @@ -4245,7 +4719,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4312,7 +4786,7 @@ "h": 9, "w": 12, "x": 12, - "y": 99 + "y": 124 }, "hiddenSeries": false, "id": 121, @@ -4344,7 +4818,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4411,7 +4885,7 @@ "h": 9, "w": 12, "x": 0, - "y": 104 + "y": 132 }, "hiddenSeries": false, "id": 122, @@ -4436,7 +4910,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4508,7 +4982,7 @@ "h": 9, "w": 12, "x": 12, - "y": 108 + "y": 133 }, "hiddenSeries": false, "id": 27, @@ -4530,7 +5004,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4609,7 +5083,7 @@ "h": 9, "w": 12, "x": 0, - "y": 113 + "y": 141 }, "hiddenSeries": false, "id": 26, @@ -4630,7 +5104,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4703,7 +5177,7 @@ "h": 9, "w": 12, "x": 12, - "y": 117 + "y": 142 }, "hiddenSeries": false, "id": 111, @@ -4725,7 +5199,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4798,7 +5272,7 @@ "h": 10, "w": 12, "x": 0, - "y": 122 + "y": 150 }, "hiddenSeries": false, "id": 123, @@ -4825,7 +5299,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4889,7 +5363,7 @@ "h": 5, "w": 12, "x": 12, - "y": 126 + "y": 151 }, "hiddenSeries": false, "id": 30, @@ -4918,7 +5392,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -4985,7 +5459,7 @@ "h": 5, "w": 12, "x": 12, - "y": 131 + "y": 156 }, "hiddenSeries": false, "id": 31, @@ -5014,7 +5488,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -5081,7 +5555,7 @@ "h": 10, "w": 12, "x": 0, - "y": 132 + "y": 160 }, "hiddenSeries": false, "id": 120, @@ -5098,13 +5572,12 @@ }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, - "pluginVersion": "10.2.3", + "pluginVersion": "10.4.2", "pointradius": 2, "points": false, "renderer": "flot", @@ -5154,7 +5627,6 @@ } } ], - "refresh": "1m", "revision": 1, "schemaVersion": 39, "tags": [], @@ -5162,9 +5634,10 @@ "list": [ { "current": { + "isNone": true, "selected": false, - "text": "postgres-exporter:9187", - "value": "postgres-exporter:9187" + "text": "None", + "value": "" }, "datasource": { "type": "prometheus", @@ -5281,8 +5754,8 @@ ] }, "time": { - "from": "now-15m", - "to": "now" + "from": "2024-06-26T02:42:06.763Z", + "to": "2024-06-26T04:42:06.771Z" }, "timepicker": { "refresh_intervals": [ diff --git a/apps/liteprotocoltester/statistics.nim b/apps/liteprotocoltester/statistics.nim index 4c4f40709b..13e749193c 100644 --- a/apps/liteprotocoltester/statistics.nim +++ b/apps/liteprotocoltester/statistics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, tables, strutils, sequtils, options, strformat], diff --git a/apps/liteprotocoltester/tester_message.nim b/apps/liteprotocoltester/tester_message.nim index 635a31ab40..a34cb4b75b 100644 --- a/apps/liteprotocoltester/tester_message.nim +++ b/apps/liteprotocoltester/tester_message.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, diff --git a/apps/networkmonitor/networkmonitor.nim b/apps/networkmonitor/networkmonitor.nim index 1bfaee0623..fa21c231be 100644 --- a/apps/networkmonitor/networkmonitor.nim +++ b/apps/networkmonitor/networkmonitor.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[tables, strutils, times, sequtils, random], diff --git a/apps/networkmonitor/networkmonitor_metrics.nim b/apps/networkmonitor/networkmonitor_metrics.nim index 70b10cdc41..e8bcb7097d 100644 --- a/apps/networkmonitor/networkmonitor_metrics.nim +++ b/apps/networkmonitor/networkmonitor_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[json, tables, sequtils], diff --git a/apps/networkmonitor/networkmonitor_utils.nim b/apps/networkmonitor/networkmonitor_utils.nim index 34cd3093a8..eef6f66811 100644 --- a/apps/networkmonitor/networkmonitor_utils.nim +++ b/apps/networkmonitor/networkmonitor_utils.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/json, diff --git a/apps/wakucanary/README.md b/apps/wakucanary/README.md index 2f8e5275ec..6ae4ca3e9f 100644 --- a/apps/wakucanary/README.md +++ b/apps/wakucanary/README.md @@ -15,9 +15,11 @@ The following options are available: -p, --protocol Protocol required to be supported: store,relay,lightpush,filter (can be used multiple times). -l, --log-level Sets the log level [=LogLevel.DEBUG]. - -np, --node-port Listening port for waku node [=60000]. + -np, --node-port Listening port for waku node [=60000]. --websocket-secure-key-path Secure websocket key path: '/path/to/key.txt' . --websocket-secure-cert-path Secure websocket Certificate path: '/path/to/cert.txt' . + -c, --cluster-id Cluster ID of the fleet node to check status [Default=1] + -s, --shard Shards index to subscribe to topics [ Argument may be repeated ] ``` diff --git a/apps/wakucanary/wakucanary.nim b/apps/wakucanary/wakucanary.nim index 175c962ed2..8aa75affd0 100644 --- a/apps/wakucanary/wakucanary.nim +++ b/apps/wakucanary/wakucanary.nim @@ -84,6 +84,21 @@ type WakuCanaryConf* = object desc: "Ping the peer node to measure latency", defaultValue: true, name: "ping" .}: bool + shards* {. + desc: "Shards index to subscribe to [0..MAX_SHARDS-1]. Argument may be repeated.", + defaultValue: @[], + name: "shard", + abbr: "s" + .}: seq[uint16] + + clusterId* {. + desc: + "Cluster id that the node is running in. Node in a different cluster id is disconnected.", + defaultValue: 1, + name: "cluster-id", + abbr: "c" + .}: uint16 + proc parseCmdArg*(T: type chronos.Duration, p: string): T = try: result = chronos.seconds(parseInt(p)) @@ -190,6 +205,13 @@ proc main(rng: ref HmacDrbgContext): Future[int] {.async.} = var enrBuilder = EnrBuilder.init(nodeKey) + let relayShards = RelayShards.init(conf.clusterId, conf.shards).valueOr: + error "Relay shards initialization failed", error = error + return 1 + enrBuilder.withWakuRelaySharding(relayShards).isOkOr: + error "Building ENR with relay sharding failed", error = error + return 1 + let recordRes = enrBuilder.build() let record = if recordRes.isErr(): @@ -214,6 +236,8 @@ proc main(rng: ref HmacDrbgContext): Future[int] {.async.} = ) let node = builder.build().tryGet() + node.mountMetadata(conf.clusterId).isOkOr: + error "failed to mount waku metadata protocol: ", err = error if conf.ping: try: diff --git a/apps/wakunode2/wakunode2.nim b/apps/wakunode2/wakunode2.nim index 474eef1f7a..b5f165c0d4 100644 --- a/apps/wakunode2/wakunode2.nim +++ b/apps/wakunode2/wakunode2.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, strutils, sequtils, net], diff --git a/ci/Jenkinsfile.release b/ci/Jenkinsfile.release index 476b09ae22..5ea0751e13 100644 --- a/ci/Jenkinsfile.release +++ b/ci/Jenkinsfile.release @@ -1,5 +1,5 @@ #!/usr/bin/env groovy -library 'status-jenkins-lib@v1.8.9' +library 'status-jenkins-lib@v1.8.17' pipeline { agent { label 'linux' } @@ -68,7 +68,7 @@ pipeline { "${params.IMAGE_NAME}:${params.IMAGE_TAG ?: env.GIT_COMMIT.take(8)}", "--label=build='${env.BUILD_URL}' " + "--label=commit='${git.commit()}' " + - "--label=version='${git.describe()}' " + + "--label=version='${git.describe('--tags')}' " + "--build-arg=MAKE_TARGET='${params.MAKE_TARGET}' " + "--build-arg=NIMFLAGS='${params.NIMFLAGS} -d:postgres ' " + "--build-arg=LOG_LEVEL='${params.LOWEST_LOG_LEVEL_ALLOWED}' " + diff --git a/examples/wakustealthcommitments/erc_5564_interface.nim b/examples/wakustealthcommitments/erc_5564_interface.nim index a2bd53b6e9..89753815db 100644 --- a/examples/wakustealthcommitments/erc_5564_interface.nim +++ b/examples/wakustealthcommitments/erc_5564_interface.nim @@ -1,8 +1,5 @@ ## Nim wrappers for the functions defined in librln -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results diff --git a/examples/wakustealthcommitments/node_spec.nim b/examples/wakustealthcommitments/node_spec.nim index a080b5107e..85289b32a8 100644 --- a/examples/wakustealthcommitments/node_spec.nim +++ b/examples/wakustealthcommitments/node_spec.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ../../waku/common/logging, ../../waku/factory/[waku, networks_config, external_config] diff --git a/examples/wakustealthcommitments/stealth_commitment_protocol.nim b/examples/wakustealthcommitments/stealth_commitment_protocol.nim index 1b1c9cd468..3a50297a99 100644 --- a/examples/wakustealthcommitments/stealth_commitment_protocol.nim +++ b/examples/wakustealthcommitments/stealth_commitment_protocol.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, diff --git a/examples/wakustealthcommitments/wakustealthcommitments.nim b/examples/wakustealthcommitments/wakustealthcommitments.nim index a6cf4a5627..7f17bd01bc 100644 --- a/examples/wakustealthcommitments/wakustealthcommitments.nim +++ b/examples/wakustealthcommitments/wakustealthcommitments.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, chronicles, ./node_spec as Waku, ./stealth_commitment_protocol as SCP diff --git a/tests/node/test_wakunode_peer_manager.nim b/tests/node/test_wakunode_peer_manager.nim index c4c930797b..1857b1e247 100644 --- a/tests/node/test_wakunode_peer_manager.nim +++ b/tests/node/test_wakunode_peer_manager.nim @@ -351,7 +351,7 @@ suite "Peer Manager": await server2.stop() suite "Tracked Peer Metadata": - xasyncTest "Metadata Recording": + asyncTest "Metadata Recording": # When adding a peer other than self to the peer store serverRemotePeerInfo.enr = some(server.enr) client.peerManager.addPeer(serverRemotePeerInfo) diff --git a/tests/waku_archive/test_waku_archive.nim b/tests/waku_archive/test_waku_archive.nim index 8408d3f3de..cebb807aef 100644 --- a/tests/waku_archive/test_waku_archive.nim +++ b/tests/waku_archive/test_waku_archive.nim @@ -233,13 +233,13 @@ procSuite "Waku Archive - find messages": response.messages.anyIt(it == msg1) response.messages.anyIt(it == msg3) - test "handle query with more than 10 content filters": + test "handle query with more than 100 content filters": ## Setup let driver = newSqliteArchiveDriver() archive = newWakuArchive(driver) - let queryTopics = toSeq(1 .. 15).mapIt(ContentTopic($it)) + let queryTopics = toSeq(1 .. 150).mapIt(ContentTopic($it)) ## Given let req = ArchiveQuery(contentTopics: queryTopics) diff --git a/tests/waku_core/test_namespaced_topics.nim b/tests/waku_core/test_namespaced_topics.nim index 3344b56316..a0cf6ae5bb 100644 --- a/tests/waku_core/test_namespaced_topics.nim +++ b/tests/waku_core/test_namespaced_topics.nim @@ -69,7 +69,7 @@ suite "Waku Message - Content topics namespacing": let err = ns.tryError() check: err.kind == ParsingErrorKind.InvalidFormat - err.cause == "topic must start with slash" + err.cause == "content-topic '" & topic & "' must start with slash" test "Parse content topic string - Invalid string: not namespaced": ## Given diff --git a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim index f87e6c7022..b839a2f7a5 100644 --- a/tests/waku_rln_relay/test_rln_group_manager_onchain.nim +++ b/tests/waku_rln_relay/test_rln_group_manager_onchain.nim @@ -1,9 +1,6 @@ {.used.} -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, os, osproc, sequtils, deques, streams, strutils, tempfiles, strformat], diff --git a/tests/waku_rln_relay/test_rln_group_manager_static.nim b/tests/waku_rln_relay/test_rln_group_manager_static.nim index a9f2a71e4c..e0617df7e2 100644 --- a/tests/waku_rln_relay/test_rln_group_manager_static.nim +++ b/tests/waku_rln_relay/test_rln_group_manager_static.nim @@ -1,9 +1,6 @@ {.used.} -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import testutils/unittests, diff --git a/tests/waku_rln_relay/test_rln_serde.nim b/tests/waku_rln_relay/test_rln_serde.nim index b1b57f53c0..4517d86ddc 100644 --- a/tests/waku_rln_relay/test_rln_serde.nim +++ b/tests/waku_rln_relay/test_rln_serde.nim @@ -1,9 +1,6 @@ {.used.} -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ./rln/waku_rln_relay_utils, diff --git a/tests/wakunode_rest/test_rest_admin.nim b/tests/wakunode_rest/test_rest_admin.nim index 5cece660d4..d53b949b65 100644 --- a/tests/wakunode_rest/test_rest_admin.nim +++ b/tests/wakunode_rest/test_rest_admin.nim @@ -37,17 +37,18 @@ suite "Waku v2 Rest API - Admin": asyncSetup: node1 = newTestWakuNode(generateSecp256k1Key(), parseIpAddress("127.0.0.1"), Port(60600)) - peerInfo1 = node1.switch.peerInfo node2 = newTestWakuNode(generateSecp256k1Key(), parseIpAddress("127.0.0.1"), Port(60602)) - peerInfo2 = node2.switch.peerInfo node3 = newTestWakuNode(generateSecp256k1Key(), parseIpAddress("127.0.0.1"), Port(60604)) - peerInfo3 = node3.switch.peerInfo await allFutures(node1.start(), node2.start(), node3.start()) await allFutures(node1.mountRelay(), node2.mountRelay(), node3.mountRelay()) + peerInfo1 = node1.switch.peerInfo + peerInfo2 = node2.switch.peerInfo + peerInfo3 = node3.switch.peerInfo + var restPort = Port(0) let restAddress = parseIpAddress("127.0.0.1") restServer = WakuRestServerRef.init(restAddress, restPort).tryGet() @@ -165,3 +166,29 @@ suite "Waku v2 Rest API - Admin": check: getRes.status == 400 getRes.data == "Error: Filter Protocol is not mounted to the node" + + asyncTest "Get peer origin": + # Adding peers to the Peer Store + node1.peerManager.addPeer(peerInfo2, Discv5) + node1.peerManager.addPeer(peerInfo3, PeerExchange) + + # Connecting to both peers + let conn2 = await node1.peerManager.connectRelay(peerInfo2) + let conn3 = await node1.peerManager.connectRelay(peerInfo3) + + # Check successful connections + check: + conn2 == true + conn3 == true + + # Query peers REST endpoint + let getRes = await client.getPeers() + + check: + getRes.status == 200 + $getRes.contentType == $MIMETYPE_JSON + getRes.data.len() == 2 + # Check peer 2 + getRes.data.anyIt(it.origin == Discv5) + # Check peer 3 + getRes.data.anyIt(it.origin == PeerExchange) diff --git a/tests/wakunode_rest/test_rest_filter.nim b/tests/wakunode_rest/test_rest_filter.nim index e3893ec081..1127a9cc98 100644 --- a/tests/wakunode_rest/test_rest_filter.nim +++ b/tests/wakunode_rest/test_rest_filter.nim @@ -1,6 +1,8 @@ {.used.} import + std/os, + chronos/timer, stew/byteutils, stew/shims/net, testutils/unittests, @@ -52,7 +54,7 @@ proc init(T: type RestFilterTest): Future[T] {.async.} = await allFutures(testSetup.serviceNode.start(), testSetup.subscriberNode.start()) await testSetup.serviceNode.mountRelay() - await testSetup.serviceNode.mountFilter() + await testSetup.serviceNode.mountFilter(messageCacheTTL = 1.seconds) await testSetup.subscriberNode.mountFilterClient() testSetup.subscriberNode.peerManager.addServicePeer( @@ -315,3 +317,147 @@ suite "Waku v2 Rest API - Filter V2": messages == @[testMessage] await restFilterTest.shutdown() + + asyncTest "duplicate message push to filter subscriber": + # setup filter service and client node + let restFilterTest = await RestFilterTest.init() + let subPeerId = restFilterTest.subscriberNode.peerInfo.toRemotePeerInfo().peerId + restFilterTest.serviceNode.subscribe((kind: PubsubSub, topic: DefaultPubsubTopic)) + + let requestBody = FilterSubscribeRequest( + requestId: "1001", + contentFilters: @[DefaultContentTopic], + pubsubTopic: some(DefaultPubsubTopic), + ) + let response = await restFilterTest.client.filterPostSubscriptions(requestBody) + + # subscribe fiter service + let subscribedPeer = restFilterTest.serviceNode.wakuFilter.subscriptions.findSubscribedPeers( + DefaultPubsubTopic, DefaultContentTopic + ) + + check: + response.status == 200 + $response.contentType == $MIMETYPE_JSON + response.data.requestId == "1001" + subscribedPeer.len() == 1 + + # ping subscriber node + restFilterTest.messageCache.pubsubSubscribe(DefaultPubsubTopic) + + let pingResponse = await restFilterTest.client.filterSubscriberPing("1002") + + check: + pingResponse.status == 200 + pingResponse.data.requestId == "1002" + pingResponse.data.statusDesc == "OK" + + # first - message push from service node to subscriber client + let testMessage = WakuMessage( + payload: "TEST-PAYLOAD-MUST-RECEIVE".toBytes(), + contentTopic: DefaultContentTopic, + timestamp: int64(2022), + meta: "test-meta".toBytes(), + ) + + let postMsgResponse1 = await restFilterTest.clientTwdServiceNode.relayPostMessagesV1( + DefaultPubsubTopic, toRelayWakuMessage(testMessage) + ) + + # check messages received client side or not + let messages1 = await restFilterTest.client.filterGetMessagesV1(DefaultContentTopic) + + check: + postMsgResponse1.status == 200 + $postMsgResponse1.contentType == $MIMETYPE_TEXT + postMsgResponse1.data == "OK" + len(messages1.data) == 1 + + # second - message push from service node to subscriber client + let postMsgResponse2 = await restFilterTest.clientTwdServiceNode.relayPostMessagesV1( + DefaultPubsubTopic, toRelayWakuMessage(testMessage) + ) + + # check message received client side or not + let messages2 = await restFilterTest.client.filterGetMessagesV1(DefaultContentTopic) + + check: + postMsgResponse2.status == 200 + $postMsgResponse2.contentType == $MIMETYPE_TEXT + postMsgResponse2.data == "OK" + len(messages2.data) == 0 + + await restFilterTest.shutdown() + + asyncTest "duplicate message push to filter subscriber ( sleep in between )": + # setup filter service and client node + let restFilterTest = await RestFilterTest.init() + let subPeerId = restFilterTest.subscriberNode.peerInfo.toRemotePeerInfo().peerId + restFilterTest.serviceNode.subscribe((kind: PubsubSub, topic: DefaultPubsubTopic)) + + let requestBody = FilterSubscribeRequest( + requestId: "1001", + contentFilters: @[DefaultContentTopic], + pubsubTopic: some(DefaultPubsubTopic), + ) + let response = await restFilterTest.client.filterPostSubscriptions(requestBody) + + # subscribe fiter service + let subscribedPeer = restFilterTest.serviceNode.wakuFilter.subscriptions.findSubscribedPeers( + DefaultPubsubTopic, DefaultContentTopic + ) + + check: + response.status == 200 + $response.contentType == $MIMETYPE_JSON + response.data.requestId == "1001" + subscribedPeer.len() == 1 + + # ping subscriber node + restFilterTest.messageCache.pubsubSubscribe(DefaultPubsubTopic) + + let pingResponse = await restFilterTest.client.filterSubscriberPing("1002") + + check: + pingResponse.status == 200 + pingResponse.data.requestId == "1002" + pingResponse.data.statusDesc == "OK" + + # first - message push from service node to subscriber client + let testMessage = WakuMessage( + payload: "TEST-PAYLOAD-MUST-RECEIVE".toBytes(), + contentTopic: DefaultContentTopic, + timestamp: int64(2022), + meta: "test-meta".toBytes(), + ) + + let postMsgResponse1 = await restFilterTest.clientTwdServiceNode.relayPostMessagesV1( + DefaultPubsubTopic, toRelayWakuMessage(testMessage) + ) + + # check messages received client side or not + let messages1 = await restFilterTest.client.filterGetMessagesV1(DefaultContentTopic) + + check: + postMsgResponse1.status == 200 + $postMsgResponse1.contentType == $MIMETYPE_TEXT + postMsgResponse1.data == "OK" + len(messages1.data) == 1 + + # Pause execution for 1 seconds to test TimeCache functionality of service node + await sleepAsync(1.seconds) + + # second - message push from service node to subscriber client + let postMsgResponse2 = await restFilterTest.clientTwdServiceNode.relayPostMessagesV1( + DefaultPubsubTopic, toRelayWakuMessage(testMessage) + ) + + # check message received client side or not + let messages2 = await restFilterTest.client.filterGetMessagesV1(DefaultContentTopic) + + check: + postMsgResponse2.status == 200 + $postMsgResponse2.contentType == $MIMETYPE_TEXT + postMsgResponse2.data == "OK" + len(messages2.data) == 1 + await restFilterTest.shutdown() diff --git a/tests/wakunode_rest/test_rest_relay.nim b/tests/wakunode_rest/test_rest_relay.nim index c672ee6e4f..d146521f53 100644 --- a/tests/wakunode_rest/test_rest_relay.nim +++ b/tests/wakunode_rest/test_rest_relay.nim @@ -511,11 +511,12 @@ suite "Waku v2 Rest API - Relay": let client = newRestHttpClient(initTAddress(restAddress, restPort)) + let invalidContentTopic = "invalidContentTopic" # When let response = await client.relayPostAutoMessagesV1( RelayWakuMessage( payload: base64.encode("TEST-PAYLOAD"), - contentTopic: some("invalidContentTopic"), + contentTopic: some(invalidContentTopic), timestamp: some(int64(2022)), ) ) @@ -525,7 +526,8 @@ suite "Waku v2 Rest API - Relay": response.status == 400 $response.contentType == $MIMETYPE_TEXT response.data == - "Failed to publish. Autosharding error: invalid format: topic must start with slash" + "Failed to publish. Autosharding error: invalid format: content-topic '" & + invalidContentTopic & "' must start with slash" await restServer.stop() await restServer.closeWait() diff --git a/vendor/nim-chronos b/vendor/nim-chronos index 8a306763ce..4ad38079de 160000 --- a/vendor/nim-chronos +++ b/vendor/nim-chronos @@ -1 +1 @@ -Subproject commit 8a306763cec8105fa83574b56734b0f66823f844 +Subproject commit 4ad38079dec8407c396ebaaf6ba60e5e94e3fce5 diff --git a/waku/common/base64.nim b/waku/common/base64.nim index 46fc6ec5a8..65e74cd8f5 100644 --- a/waku/common/base64.nim +++ b/waku/common/base64.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/[results, byteutils, base64] diff --git a/waku/common/confutils/envvar/defs.nim b/waku/common/confutils/envvar/defs.nim index 2c23df4afb..4f71bd88fa 100644 --- a/waku/common/confutils/envvar/defs.nim +++ b/waku/common/confutils/envvar/defs.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import confutils/defs as confutilsDefs import ../../envvar_serialization diff --git a/waku/common/confutils/envvar/std/net.nim b/waku/common/confutils/envvar/std/net.nim index 4f46fa9e1e..4264373a61 100644 --- a/waku/common/confutils/envvar/std/net.nim +++ b/waku/common/confutils/envvar/std/net.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[strutils, net] import ../../../envvar_serialization diff --git a/waku/common/databases/db_postgres/pgasyncpool.nim b/waku/common/databases/db_postgres/pgasyncpool.nim index bf20625dc8..0b08b9c312 100644 --- a/waku/common/databases/db_postgres/pgasyncpool.nim +++ b/waku/common/databases/db_postgres/pgasyncpool.nim @@ -1,9 +1,6 @@ # Simple async pool driver for postgress. # Inspired by: https://github.com/treeform/pg/ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sequtils, nre, strformat, sets], stew/results, chronos import ./dbconn, ../common diff --git a/waku/common/enr/builder.nim b/waku/common/enr/builder.nim index 84b89ca025..5795e37f58 100644 --- a/waku/common/enr/builder.nim +++ b/waku/common/enr/builder.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, net], diff --git a/waku/common/enr/typed_record.nim b/waku/common/enr/typed_record.nim index ea2c542786..babf9cc76f 100644 --- a/waku/common/enr/typed_record.nim +++ b/waku/common/enr/typed_record.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, eth/keys as eth_keys, libp2p/crypto/crypto as libp2p_crypto diff --git a/waku/common/envvar_serialization.nim b/waku/common/envvar_serialization.nim index 71607ed26e..53314ef093 100644 --- a/waku/common/envvar_serialization.nim +++ b/waku/common/envvar_serialization.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/shims/macros, serialization import ./envvar_serialization/reader, ./envvar_serialization/writer diff --git a/waku/common/envvar_serialization/reader.nim b/waku/common/envvar_serialization/reader.nim index a4f94e9edd..66869e3b4a 100644 --- a/waku/common/envvar_serialization/reader.nim +++ b/waku/common/envvar_serialization/reader.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[tables, typetraits, options, os], diff --git a/waku/common/envvar_serialization/utils.nim b/waku/common/envvar_serialization/utils.nim index c0a9762a06..cb2c6283c4 100644 --- a/waku/common/envvar_serialization/utils.nim +++ b/waku/common/envvar_serialization/utils.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[os, strutils], stew/byteutils, stew/ptrops diff --git a/waku/common/hexstrings.nim b/waku/common/hexstrings.nim index 0b1d963d21..fd6b590bc9 100644 --- a/waku/common/hexstrings.nim +++ b/waku/common/hexstrings.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} type HexDataStr* = distinct string diff --git a/waku/common/logging.nim b/waku/common/logging.nim index a955d3d684..e8394e89f4 100644 --- a/waku/common/logging.nim +++ b/waku/common/logging.nim @@ -8,10 +8,7 @@ import export chronicles.LogLevel -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} type LogFormat* = enum TEXT diff --git a/waku/common/nimchronos.nim b/waku/common/nimchronos.nim index 8f3d400fd1..dc425c4754 100644 --- a/waku/common/nimchronos.nim +++ b/waku/common/nimchronos.nim @@ -1,8 +1,5 @@ ## An extension wrapper around nim-chronos -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronos, chronicles diff --git a/waku/common/protobuf.nim b/waku/common/protobuf.nim index cafd83e650..20c29ab835 100644 --- a/waku/common/protobuf.nim +++ b/waku/common/protobuf.nim @@ -1,9 +1,6 @@ # Extensions for libp2p's protobuf library implementation -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, libp2p/protobuf/minprotobuf, libp2p/varint diff --git a/waku/common/ratelimit.nim b/waku/common/ratelimit.nim index 8fb91219a7..1c40cd42ab 100644 --- a/waku/common/ratelimit.nim +++ b/waku/common/ratelimit.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, chronos/timer, libp2p/stream/connection @@ -43,10 +40,10 @@ template checkUsageLimit*( bodyWithinLimit, bodyRejected: untyped, ) = if t.checkUsage(proto, conn): - waku_service_requests.inc(labelValues = [proto]) + waku_service_requests.inc(labelValues = [proto, "served"]) bodyWithinLimit else: - waku_service_requests_rejected.inc(labelValues = [proto]) + waku_service_requests.inc(labelValues = [proto, "rejected"]) bodyRejected func `$`*(ob: Option[TokenBucket]): string {.inline.} = diff --git a/waku/common/tokenbucket.nim b/waku/common/tokenbucket.nim index 2223e0bbfc..a35939f7ab 100644 --- a/waku/common/tokenbucket.nim +++ b/waku/common/tokenbucket.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronos diff --git a/waku/common/utils/matterbridge_client.nim b/waku/common/utils/matterbridge_client.nim index 634a583680..c04d2280ff 100644 --- a/waku/common/utils/matterbridge_client.nim +++ b/waku/common/utils/matterbridge_client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[httpclient, json, uri, options], stew/results diff --git a/waku/common/utils/nat.nim b/waku/common/utils/nat.nim index 5835a8e7f0..8af1ec2ad6 100644 --- a/waku/common/utils/nat.nim +++ b/waku/common/utils/nat.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, strutils, net] import chronicles, eth/net/nat, stew/results, nativesockets diff --git a/waku/common/utils/sequence.nim b/waku/common/utils/sequence.nim index 7af75c32b1..ad243d82c8 100644 --- a/waku/common/utils/sequence.nim +++ b/waku/common/utils/sequence.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} proc flatten*[T](a: seq[seq[T]]): seq[T] = var aFlat = newSeq[T](0) diff --git a/waku/common/waku_service_metrics.nim b/waku/common/waku_service_metrics.nim index a58c48d1b8..fe0bbf05c8 100644 --- a/waku/common/waku_service_metrics.nim +++ b/waku/common/waku_service_metrics.nim @@ -1,18 +1,9 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics declarePublicCounter waku_service_requests, - "number of non-relay service requests received", ["service"] -declarePublicCounter waku_service_requests_rejected, - "number of non-relay service requests received being rejected due to limit overdue", - ["service"] + "number of non-relay service requests received", ["service", "state"] -declarePublicCounter waku_service_inbound_network_bytes, - "total incoming traffic of specific waku services", labels = ["service"] - -declarePublicCounter waku_service_outbound_network_bytes, - "total outgoing traffic of specific waku services", labels = ["service"] +declarePublicCounter waku_service_network_bytes, + "total incoming traffic of specific waku services", labels = ["service", "direction"] diff --git a/waku/discovery/waku_discv5.nim b/waku/discovery/waku_discv5.nim index b39e05a07b..c6dc5553d7 100644 --- a/waku/discovery/waku_discv5.nim +++ b/waku/discovery/waku_discv5.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sequtils, strutils, options, sets, net, json], @@ -172,6 +169,31 @@ proc updateENRShards( return ok() +proc logDiscv5FoundPeers(discoveredRecords: seq[waku_enr.Record]) = + for record in discoveredRecords: + let recordUri = record.toURI() + let capabilities = record.getCapabilities() + + let typedRecord = record.toTyped().valueOr: + warn "Could not parse to typed record", error = error, enr = recordUri + continue + + let peerInfo = record.toRemotePeerInfo().valueOr: + warn "Could not generate remote peer info", error = error, enr = recordUri + continue + + let addrs = peerInfo.constructMultiaddrStr() + + let rs = typedRecord.relaySharding() + let shardsStr = + if rs.isSome(): + $rs.get() + else: + "no shards found" + + notice "Received discv5 node", + addrs = addrs, enr = recordUri, capabilities = capabilities, shards = shardsStr + proc findRandomPeers*( wd: WakuDiscoveryV5, overridePred = none(WakuDiscv5Predicate) ): Future[seq[waku_enr.Record]] {.async.} = @@ -180,6 +202,9 @@ proc findRandomPeers*( var discoveredRecords = discoveredNodes.mapIt(it.record) + when defined(debugDiscv5): + logDiscv5FoundPeers(discoveredRecords) + # Filter out nodes that do not match the predicate if overridePred.isSome(): discoveredRecords = discoveredRecords.filter(overridePred.get()) diff --git a/waku/discovery/waku_dnsdisc.nim b/waku/discovery/waku_dnsdisc.nim index 9d1b76b6a3..b31de2cf5f 100644 --- a/waku/discovery/waku_dnsdisc.nim +++ b/waku/discovery/waku_dnsdisc.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} ## A set of utilities to integrate EIP-1459 DNS-based discovery ## for Waku v2 nodes. diff --git a/waku/factory/builder.nim b/waku/factory/builder.nim index 3d64f58559..f06d28331f 100644 --- a/waku/factory/builder.nim +++ b/waku/factory/builder.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, net], diff --git a/waku/factory/networks_config.nim b/waku/factory/networks_config.nim index 1c5ab00970..49c6dfd9a7 100644 --- a/waku/factory/networks_config.nim +++ b/waku/factory/networks_config.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} type ClusterConf* = object maxMessageSize*: string diff --git a/waku/factory/node_factory.nim b/waku/factory/node_factory.nim index b74d1a3e74..ad9719ec94 100644 --- a/waku/factory/node_factory.nim +++ b/waku/factory/node_factory.nim @@ -135,17 +135,21 @@ proc setupProtocols( # only peers with populated records .mapIt(toRemotePeerInfo(it.record.get())) - debug "connecting to exchanged peers", + debug "adding exchanged peers", src = peer, topic = topic, numPeers = exchangedPeers.len - # asyncSpawn, as we don't want to block here - asyncSpawn node.connectToNodes(exchangedPeers, "peer exchange") + for peer in exchangedPeers: + # Peers added are filtered by the peer manager + node.peerManager.addPeer(peer, PeerOrigin.PeerExchange) peerExchangeHandler = some(handlePeerExchange) + let shards = + conf.contentTopics.mapIt(node.wakuSharding.getShard(it).expect("Valid Shard")) + debug "Shards created from content topics", + contentTopics = conf.contentTopics, shards = shards + if conf.relay: - let shards = - conf.contentTopics.mapIt(node.wakuSharding.getShard(it).expect("Valid Shard")) let pubsubTopics = conf.pubsubTopics & shards let parsedMaxMsgSize = parseMsgSize(conf.maxMessageSize).valueOr: diff --git a/waku/factory/validator_signed.nim b/waku/factory/validator_signed.nim index 5686ed1f8a..f4a9253adb 100644 --- a/waku/factory/validator_signed.nim +++ b/waku/factory/validator_signed.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, diff --git a/waku/factory/waku.nim b/waku/factory/waku.nim index dc3b013149..709be62c27 100644 --- a/waku/factory/waku.nim +++ b/waku/factory/waku.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils], diff --git a/waku/node/config.nim b/waku/node/config.nim index d3f2a3dfd7..2ebe2a0076 100644 --- a/waku/node/config.nim +++ b/waku/node/config.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils, strutils, net], diff --git a/waku/node/health_monitor.nim b/waku/node/health_monitor.nim index 444a46b557..b3fe9b227e 100644 --- a/waku/node/health_monitor.nim +++ b/waku/node/health_monitor.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options], chronos diff --git a/waku/node/peer_manager/peer_manager.nim b/waku/node/peer_manager/peer_manager.nim index 679041580f..8b166adf4f 100644 --- a/waku/node/peer_manager/peer_manager.nim +++ b/waku/node/peer_manager/peer_manager.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sets, sequtils, times, strutils, math, random], @@ -128,9 +125,16 @@ proc addPeer*(pm: PeerManager, remotePeerInfo: RemotePeerInfo, origin = UnknownO if pm.peerStore[AddressBook][remotePeerInfo.peerId] == remotePeerInfo.addrs and pm.peerStore[KeyBook][remotePeerInfo.peerId] == remotePeerInfo.publicKey and pm.peerStore[ENRBook][remotePeerInfo.peerId].raw.len > 0: - trace "peer already managed and ENR info is already saved", - remote_peer_id = $remotePeerInfo.peerId - return + let incomingEnr = remotePeerInfo.enr.valueOr: + trace "peer already managed and incoming ENR is empty", + remote_peer_id = $remotePeerInfo.peerId + return + + if pm.peerStore[ENRBook][remotePeerInfo.peerId].raw == incomingEnr.raw or + pm.peerStore[ENRBook][remotePeerInfo.peerId].seqNum > incomingEnr.seqNum: + trace "peer already managed and ENR info is already saved", + remote_peer_id = $remotePeerInfo.peerId + return trace "Adding peer to manager", peerId = remotePeerInfo.peerId, addresses = remotePeerInfo.addrs @@ -138,6 +142,8 @@ proc addPeer*(pm: PeerManager, remotePeerInfo: RemotePeerInfo, origin = UnknownO pm.peerStore[AddressBook][remotePeerInfo.peerId] = remotePeerInfo.addrs pm.peerStore[KeyBook][remotePeerInfo.peerId] = remotePeerInfo.publicKey pm.peerStore[SourceBook][remotePeerInfo.peerId] = origin + pm.peerStore[ProtoVersionBook][remotePeerInfo.peerId] = remotePeerInfo.protoVersion + pm.peerStore[AgentBook][remotePeerInfo.peerId] = remotePeerInfo.agent if remotePeerInfo.protocols.len > 0: pm.peerStore[ProtoBook][remotePeerInfo.peerId] = remotePeerInfo.protocols @@ -380,8 +386,8 @@ proc onPeerMetadata(pm: PeerManager, peerId: PeerId) {.async.} = pm.peerStore.hasPeer(peerId, WakuRelayCodec) and not metadata.shards.anyIt(pm.wakuMetadata.shards.contains(it)) ): - let myShardsString = "[ " & toSeq(pm.wakuMetadata.shards).join(", ") & "]" - let otherShardsString = "[ " & metadata.shards.join(", ") & "]" + let myShardsString = "[ " & toSeq(pm.wakuMetadata.shards).join(", ") & " ]" + let otherShardsString = "[ " & metadata.shards.join(", ") & " ]" reason = "no shards in common: my_shards = " & myShardsString & " others_shards = " & otherShardsString @@ -435,7 +441,9 @@ proc onPeerEvent(pm: PeerManager, peerId: PeerId, event: PeerEvent) {.async.} = if not pm.storage.isNil: var remotePeerInfo = pm.peerStore.get(peerId) - remotePeerInfo.disconnectTime = getTime().toUnix + + if event.kind == PeerEventKind.Left: + remotePeerInfo.disconnectTime = getTime().toUnix pm.storage.insertOrReplace(remotePeerInfo) @@ -640,18 +648,29 @@ proc connectToNodes*( info "Dialing multiple peers", numOfPeers = nodes.len var futConns: seq[Future[bool]] + var connectedPeers: seq[RemotePeerInfo] for node in nodes: let node = parsePeerInfo(node) if node.isOk(): futConns.add(pm.connectRelay(node.value)) + connectedPeers.add(node.value) else: error "Couldn't parse node info", error = node.error await allFutures(futConns) - let successfulConns = futConns.mapIt(it.read()).countIt(it == true) + + # Filtering successful connectedPeers based on futConns + let combined = zip(connectedPeers, futConns) + connectedPeers = combined.filterIt(it[1].read() == true).mapIt(it[0]) + + when defined(debugDiscv5): + let peerIds = connectedPeers.mapIt(it.peerId) + let origin = connectedPeers.mapIt(it.origin) + notice "established connections with found peers", + peerIds = peerIds, origin = origin info "Finished dialing multiple peers", - successfulConns = successfulConns, attempted = nodes.len + successfulConns = connectedPeers.len, attempted = nodes.len # The issue seems to be around peers not being fully connected when # trying to subscribe. So what we do is sleep to guarantee nodes are @@ -717,15 +736,15 @@ proc connectToRelayPeers*(pm: PeerManager) {.async.} = if outRelayPeers.len >= pm.outRelayPeersTarget: return - let notConnectedPeers = - pm.peerStore.getNotConnectedPeers().mapIt(RemotePeerInfo.init(it.peerId, it.addrs)) + let notConnectedPeers = pm.peerStore.getNotConnectedPeers() var outsideBackoffPeers = notConnectedPeers.filterIt(pm.canBeConnected(it.peerId)) shuffle(outsideBackoffPeers) var index = 0 - var numPendingConnReqs = outsideBackoffPeers.len + var numPendingConnReqs = + min(outsideBackoffPeers.len, pm.outRelayPeersTarget - outRelayPeers.len) ## number of outstanding connection requests while numPendingConnReqs > 0 and outRelayPeers.len < pm.outRelayPeersTarget: diff --git a/waku/node/peer_manager/peer_store/migrations.nim b/waku/node/peer_manager/peer_store/migrations.nim index abb628d194..dc0214cfee 100644 --- a/waku/node/peer_manager/peer_store/migrations.nim +++ b/waku/node/peer_manager/peer_store/migrations.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[tables, strutils, os], stew/results, chronicles import ../../../common/databases/db_sqlite, ../../../common/databases/common diff --git a/waku/node/peer_manager/peer_store/peer_storage.nim b/waku/node/peer_manager/peer_store/peer_storage.nim index c3b377f514..ded0762996 100644 --- a/waku/node/peer_manager/peer_store/peer_storage.nim +++ b/waku/node/peer_manager/peer_store/peer_storage.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results import ../../../waku_core, ../waku_peer_store diff --git a/waku/node/peer_manager/peer_store/waku_peer_storage.nim b/waku/node/peer_manager/peer_store/waku_peer_storage.nim index 2ba3006ede..56544e2f51 100644 --- a/waku/node/peer_manager/peer_store/waku_peer_storage.nim +++ b/waku/node/peer_manager/peer_store/waku_peer_storage.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, options], diff --git a/waku/node/peer_manager/waku_peer_store.nim b/waku/node/peer_manager/waku_peer_store.nim index e8b99f85cb..a7db829f29 100644 --- a/waku/node/peer_manager/waku_peer_store.nim +++ b/waku/node/peer_manager/waku_peer_store.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[tables, sequtils, sets, options, strutils], diff --git a/waku/node/waku_metrics.nim b/waku/node/waku_metrics.nim index 9ee19c9d30..c349f08495 100644 --- a/waku/node/waku_metrics.nim +++ b/waku/node/waku_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, chronos, metrics, metrics/chronos_httpserver import diff --git a/waku/node/waku_node.nim b/waku/node/waku_node.nim index 8d1ff8d790..51abfd3420 100644 --- a/waku/node/waku_node.nim +++ b/waku/node/waku_node.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[hashes, options, sugar, tables, strutils, sequtils, os, net], @@ -437,12 +434,14 @@ proc mountFilter*( filter_subscriptions.DefaultSubscriptionTimeToLiveSec, maxFilterPeers: uint32 = filter_subscriptions.MaxFilterPeers, maxFilterCriteriaPerPeer: uint32 = filter_subscriptions.MaxFilterCriteriaPerPeer, + messageCacheTTL: Duration = filter_subscriptions.MessageCacheTTL, ) {.async: (raises: []).} = ## Mounting filter v2 protocol info "mounting filter protocol" node.wakuFilter = WakuFilter.new( - node.peerManager, subscriptionTimeout, maxFilterPeers, maxFilterCriteriaPerPeer + node.peerManager, subscriptionTimeout, maxFilterPeers, maxFilterCriteriaPerPeer, + messageCacheTTL, ) if node.started: diff --git a/waku/node/waku_switch.nim b/waku/node/waku_switch.nim index d062911051..65e7c19992 100644 --- a/waku/node/waku_switch.nim +++ b/waku/node/waku_switch.nim @@ -1,8 +1,5 @@ # Waku Switch utils. -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, diff --git a/waku/utils/collector.nim b/waku/utils/collector.nim index 32ade16c55..de6411ae3e 100644 --- a/waku/utils/collector.nim +++ b/waku/utils/collector.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics diff --git a/waku/utils/noise.nim b/waku/utils/noise.nim index 77ea86b81c..80b5371970 100644 --- a/waku/utils/noise.nim +++ b/waku/utils/noise.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results import ../waku_core, ../waku_noise/noise_types, ../waku_noise/noise_utils diff --git a/waku/utils/requests.nim b/waku/utils/requests.nim index 40ee6773b0..5e5b9d9602 100644 --- a/waku/utils/requests.nim +++ b/waku/utils/requests.nim @@ -1,9 +1,6 @@ # Request utils. -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import bearssl/rand, stew/byteutils diff --git a/waku/waku_api/handlers.nim b/waku/waku_api/handlers.nim index 0b99021fbd..2b6997e15b 100644 --- a/waku/waku_api/handlers.nim +++ b/waku/waku_api/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronos, std/[options, sequtils], stew/results import ../discovery/waku_discv5, ../waku_relay, ../waku_core, ./message_cache diff --git a/waku/waku_api/message_cache.nim b/waku/waku_api/message_cache.nim index 66b9c62d91..ef9ebb44c0 100644 --- a/waku/waku_api/message_cache.nim +++ b/waku/waku_api/message_cache.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sequtils, sugar, algorithm, options], diff --git a/waku/waku_api/rest/admin/client.nim b/waku/waku_api/rest/admin/client.nim index 6c9d92472f..1fd9fdfc8c 100644 --- a/waku/waku_api/rest/admin/client.nim +++ b/waku/waku_api/rest/admin/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, diff --git a/waku/waku_api/rest/admin/handlers.nim b/waku/waku_api/rest/admin/handlers.nim index 70c0630dbf..1ac7a79d59 100644 --- a/waku/waku_api/rest/admin/handlers.nim +++ b/waku/waku_api/rest/admin/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[strformat, sequtils, tables], @@ -32,11 +29,12 @@ logScope: const ROUTE_ADMIN_V1_PEERS* = "/admin/v1/peers" const ROUTE_ADMIN_V1_FILTER_SUBS* = "/admin/v1/filter/subscriptions" -type PeerProtocolTuple = tuple[multiaddr: string, protocol: string, connected: bool] +type PeerProtocolTuple = + tuple[multiaddr: string, protocol: string, connected: bool, origin: PeerOrigin] proc tuplesToWakuPeers(peers: var WakuPeers, peersTup: seq[PeerProtocolTuple]) = for peer in peersTup: - peers.add(peer.multiaddr, peer.protocol, peer.connected) + peers.add(peer.multiaddr, peer.protocol, peer.connected, peer.origin) proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = router.api(MethodGet, ROUTE_ADMIN_V1_PEERS) do() -> RestApiResponse: @@ -48,6 +46,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = multiaddr: constructMultiaddrStr(it), protocol: WakuRelayCodec, connected: it.connectedness == Connectedness.Connected, + origin: it.origin, ) ) tuplesToWakuPeers(peers, relayPeers) @@ -60,6 +59,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = multiaddr: constructMultiaddrStr(it), protocol: WakuFilterSubscribeCodec, connected: it.connectedness == Connectedness.Connected, + origin: it.origin, ) ) tuplesToWakuPeers(peers, filterV2Peers) @@ -70,6 +70,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = multiaddr: constructMultiaddrStr(it), protocol: WakuStoreCodec, connected: it.connectedness == Connectedness.Connected, + origin: it.origin, ) ) tuplesToWakuPeers(peers, storePeers) @@ -82,6 +83,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = multiaddr: constructMultiaddrStr(it), protocol: WakuLegacyStoreCodec, connected: it.connectedness == Connectedness.Connected, + origin: it.origin, ) ) tuplesToWakuPeers(peers, legacyStorePeers) @@ -93,6 +95,7 @@ proc installAdminV1GetPeersHandler(router: var RestRouter, node: WakuNode) = multiaddr: constructMultiaddrStr(it), protocol: WakuLightPushCodec, connected: it.connectedness == Connectedness.Connected, + origin: it.origin, ) ) tuplesToWakuPeers(peers, lightpushPeers) diff --git a/waku/waku_api/rest/admin/types.nim b/waku/waku_api/rest/admin/types.nim index 6ee1ed86ae..fc6470658e 100644 --- a/waku/waku_api/rest/admin/types.nim +++ b/waku/waku_api/rest/admin/types.nim @@ -1,14 +1,11 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options, json_serialization/lexer -import ../serdes +import ../serdes, ../../../waku_core #### Types @@ -19,6 +16,7 @@ type ProtocolState* = object type WakuPeer* = object multiaddr*: string protocols*: seq[ProtocolState] + origin*: PeerOrigin type WakuPeers* = seq[WakuPeer] @@ -46,6 +44,7 @@ proc writeValue*( writer.beginRecord() writer.writeField("multiaddr", value.multiaddr) writer.writeField("protocols", value.protocols) + writer.writeField("origin", value.origin) writer.endRecord() proc writeValue*( @@ -100,6 +99,7 @@ proc readValue*( var multiaddr: Option[string] protocols: Option[seq[ProtocolState]] + origin: Option[PeerOrigin] for fieldName in readObjectFields(reader): case fieldName @@ -111,6 +111,10 @@ proc readValue*( if protocols.isSome(): reader.raiseUnexpectedField("Multiple `protocols` fields found", "WakuPeer") protocols = some(reader.readValue(seq[ProtocolState])) + of "origin": + if origin.isSome(): + reader.raiseUnexpectedField("Multiple `origin` fields found", "WakuPeer") + origin = some(reader.readValue(PeerOrigin)) else: unrecognizedFieldWarning() @@ -120,7 +124,12 @@ proc readValue*( if protocols.isNone(): reader.raiseUnexpectedValue("Field `protocols` are missing") - value = WakuPeer(multiaddr: multiaddr.get(), protocols: protocols.get()) + if origin.isNone(): + reader.raiseUnexpectedValue("Field `origin` is missing") + + value = WakuPeer( + multiaddr: multiaddr.get(), protocols: protocols.get(), origin: origin.get() + ) proc readValue*( reader: var JsonReader[RestJson], value: var FilterTopic @@ -196,10 +205,17 @@ func `==`*(a: ProtocolState, b: string): bool {.inline.} = func `==`*(a, b: WakuPeer): bool {.inline.} = return a.multiaddr == b.multiaddr -proc add*(peers: var WakuPeers, multiaddr: string, protocol: string, connected: bool) = +proc add*( + peers: var WakuPeers, + multiaddr: string, + protocol: string, + connected: bool, + origin: PeerOrigin, +) = var peer: WakuPeer = WakuPeer( multiaddr: multiaddr, protocols: @[ProtocolState(protocol: protocol, connected: connected)], + origin: origin, ) let idx = peers.find(peer) diff --git a/waku/waku_api/rest/builder.nim b/waku/waku_api/rest/builder.nim index eda835b6db..08ef998a7b 100644 --- a/waku/waku_api/rest/builder.nim +++ b/waku/waku_api/rest/builder.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import net, tables import presto diff --git a/waku/waku_api/rest/client.nim b/waku/waku_api/rest/client.nim index 7693e8878d..2f61eaaa8d 100644 --- a/waku/waku_api/rest/client.nim +++ b/waku/waku_api/rest/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import presto/client diff --git a/waku/waku_api/rest/debug/client.nim b/waku/waku_api/rest/debug/client.nim index 1929b28bf4..cc8c0883bf 100644 --- a/waku/waku_api/rest/debug/client.nim +++ b/waku/waku_api/rest/debug/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options, presto/[route, client] diff --git a/waku/waku_api/rest/debug/handlers.nim b/waku/waku_api/rest/debug/handlers.nim index 52cd60152f..fa55f0372a 100644 --- a/waku/waku_api/rest/debug/handlers.nim +++ b/waku/waku_api/rest/debug/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, presto/route import ../../../waku_node, ../responses, ../serdes, ./types diff --git a/waku/waku_api/rest/debug/types.nim b/waku/waku_api/rest/debug/types.nim index 1f5949f75e..0ca52c4a55 100644 --- a/waku/waku_api/rest/debug/types.nim +++ b/waku/waku_api/rest/debug/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options import ../../../waku_node, ../serdes diff --git a/waku/waku_api/rest/filter/client.nim b/waku/waku_api/rest/filter/client.nim index abd530c41d..b674bc5945 100644 --- a/waku/waku_api/rest/filter/client.nim +++ b/waku/waku_api/rest/filter/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import json, diff --git a/waku/waku_api/rest/filter/handlers.nim b/waku/waku_api/rest/filter/handlers.nim index a5308b8538..29c5d7a264 100644 --- a/waku/waku_api/rest/filter/handlers.nim +++ b/waku/waku_api/rest/filter/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/strformat, diff --git a/waku/waku_api/rest/filter/types.nim b/waku/waku_api/rest/filter/types.nim index 19b9e71af7..a4b69521fc 100644 --- a/waku/waku_api/rest/filter/types.nim +++ b/waku/waku_api/rest/filter/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, strformat], diff --git a/waku/waku_api/rest/health/client.nim b/waku/waku_api/rest/health/client.nim index 834230dcc5..14215ebbb5 100644 --- a/waku/waku_api/rest/health/client.nim +++ b/waku/waku_api/rest/health/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options, presto/[route, client] diff --git a/waku/waku_api/rest/health/handlers.nim b/waku/waku_api/rest/health/handlers.nim index d82e46fc80..48dad92762 100644 --- a/waku/waku_api/rest/health/handlers.nim +++ b/waku/waku_api/rest/health/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, presto/route import ../../../waku_node, ../responses, ../serdes, ./types diff --git a/waku/waku_api/rest/health/types.nim b/waku/waku_api/rest/health/types.nim index a4f966467c..db76f5b8c5 100644 --- a/waku/waku_api/rest/health/types.nim +++ b/waku/waku_api/rest/health/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options import ../../../waku_node, ../serdes diff --git a/waku/waku_api/rest/legacy_store/client.nim b/waku/waku_api/rest/legacy_store/client.nim index fa85b9b2eb..24ad38d9aa 100644 --- a/waku/waku_api/rest/legacy_store/client.nim +++ b/waku/waku_api/rest/legacy_store/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options, presto/[route, client] diff --git a/waku/waku_api/rest/legacy_store/handlers.nim b/waku/waku_api/rest/legacy_store/handlers.nim index b05ece356e..680e8c782a 100644 --- a/waku/waku_api/rest/legacy_store/handlers.nim +++ b/waku/waku_api/rest/legacy_store/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/strformat, stew/results, chronicles, uri, json_serialization, presto/route import diff --git a/waku/waku_api/rest/legacy_store/types.nim b/waku/waku_api/rest/legacy_store/types.nim index c05335e9cb..0ae36e05e5 100644 --- a/waku/waku_api/rest/legacy_store/types.nim +++ b/waku/waku_api/rest/legacy_store/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, strformat, uri], diff --git a/waku/waku_api/rest/lightpush/client.nim b/waku/waku_api/rest/lightpush/client.nim index 4c18175a42..3e7f85524f 100644 --- a/waku/waku_api/rest/lightpush/client.nim +++ b/waku/waku_api/rest/lightpush/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import json, diff --git a/waku/waku_api/rest/lightpush/handlers.nim b/waku/waku_api/rest/lightpush/handlers.nim index 83b15c098f..3692d118e0 100644 --- a/waku/waku_api/rest/lightpush/handlers.nim +++ b/waku/waku_api/rest/lightpush/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/strformat, diff --git a/waku/waku_api/rest/lightpush/types.nim b/waku/waku_api/rest/lightpush/types.nim index e25a41c598..3d8d0715fe 100644 --- a/waku/waku_api/rest/lightpush/types.nim +++ b/waku/waku_api/rest/lightpush/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, strformat], diff --git a/waku/waku_api/rest/origin_handler.nim b/waku/waku_api/rest/origin_handler.nim index 1825b030d2..3cedfa26dc 100644 --- a/waku/waku_api/rest/origin_handler.nim +++ b/waku/waku_api/rest/origin_handler.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, strutils, re, net], diff --git a/waku/waku_api/rest/relay/client.nim b/waku/waku_api/rest/relay/client.nim index c6f58fa959..5e72bb609c 100644 --- a/waku/waku_api/rest/relay/client.nim +++ b/waku/waku_api/rest/relay/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/sets, diff --git a/waku/waku_api/rest/relay/handlers.nim b/waku/waku_api/rest/relay/handlers.nim index a3abfe4310..1f8563ab69 100644 --- a/waku/waku_api/rest/relay/handlers.nim +++ b/waku/waku_api/rest/relay/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/sequtils, diff --git a/waku/waku_api/rest/relay/types.nim b/waku/waku_api/rest/relay/types.nim index c39ea683ae..98957c747e 100644 --- a/waku/waku_api/rest/relay/types.nim +++ b/waku/waku_api/rest/relay/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, strformat, times], diff --git a/waku/waku_api/rest/responses.nim b/waku/waku_api/rest/responses.nim index 105c7423dc..ace6cd2d42 100644 --- a/waku/waku_api/rest/responses.nim +++ b/waku/waku_api/rest/responses.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/typetraits, stew/results, chronicles, presto/common import ./serdes diff --git a/waku/waku_api/rest/rest_serdes.nim b/waku/waku_api/rest/rest_serdes.nim index 0e2ee77a90..a68144b6ed 100644 --- a/waku/waku_api/rest/rest_serdes.nim +++ b/waku/waku_api/rest/rest_serdes.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/typetraits, diff --git a/waku/waku_api/rest/serdes.nim b/waku/waku_api/rest/serdes.nim index 54be180d5a..bb75fbfde8 100644 --- a/waku/waku_api/rest/serdes.nim +++ b/waku/waku_api/rest/serdes.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/typetraits, diff --git a/waku/waku_api/rest/server.nim b/waku/waku_api/rest/server.nim index 7e27c0cd50..b2e797e2a0 100644 --- a/waku/waku_api/rest/server.nim +++ b/waku/waku_api/rest/server.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/net import diff --git a/waku/waku_api/rest/store/client.nim b/waku/waku_api/rest/store/client.nim index 90fab2f57d..887279cef3 100644 --- a/waku/waku_api/rest/store/client.nim +++ b/waku/waku_api/rest/store/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, json_serialization, json_serialization/std/options, presto/[route, client] diff --git a/waku/waku_api/rest/store/handlers.nim b/waku/waku_api/rest/store/handlers.nim index 23817118b9..b37bc691da 100644 --- a/waku/waku_api/rest/store/handlers.nim +++ b/waku/waku_api/rest/store/handlers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/strformat, stew/results, chronicles, uri, json_serialization, presto/route import diff --git a/waku/waku_api/rest/store/types.nim b/waku/waku_api/rest/store/types.nim index 872d0ce596..a3db8fbaa1 100644 --- a/waku/waku_api/rest/store/types.nim +++ b/waku/waku_api/rest/store/types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, strformat, uri, options, sequtils], diff --git a/waku/waku_archive/archive.nim b/waku/waku_archive/archive.nim index f7b05b748e..70caf78cc9 100644 --- a/waku/waku_archive/archive.nim +++ b/waku/waku_archive/archive.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[times, options, sequtils, strutils, algorithm], @@ -141,7 +138,7 @@ proc findMessages*( let isAscendingOrder = query.direction.into() - if query.contentTopics.len > 10: + if query.contentTopics.len > 100: return err(ArchiveError.invalidQuery("too many content topics")) if query.cursor.isSome() and query.cursor.get().hash.len != 32: @@ -229,7 +226,7 @@ proc findMessagesV2*( let isAscendingOrder = query.direction.into() - if query.contentTopics.len > 10: + if query.contentTopics.len > 100: return err(ArchiveError.invalidQuery("too many content topics")) let queryStartTime = getTime().toUnixFloat() diff --git a/waku/waku_archive/archive_metrics.nim b/waku/waku_archive/archive_metrics.nim index 08b15b0a60..fd39f923e8 100644 --- a/waku/waku_archive/archive_metrics.nim +++ b/waku/waku_archive/archive_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics diff --git a/waku/waku_archive/common.nim b/waku/waku_archive/common.nim index 5b14fb111c..427a30c516 100644 --- a/waku/waku_archive/common.nim +++ b/waku/waku_archive/common.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, stew/byteutils, stew/arrayops, nimcrypto/sha2 import ../waku_core, ../common/paging diff --git a/waku/waku_archive/driver.nim b/waku/waku_archive/driver.nim index 235c4b8a7c..c17b58d2c7 100644 --- a/waku/waku_archive/driver.nim +++ b/waku/waku_archive/driver.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, chronos import ../waku_core, ./common diff --git a/waku/waku_archive/driver/builder.nim b/waku/waku_archive/driver/builder.nim index 1768774d24..b165789528 100644 --- a/waku/waku_archive/driver/builder.nim +++ b/waku/waku_archive/driver/builder.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, chronicles, chronos import diff --git a/waku/waku_archive/driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver.nim index a106eb2c40..1da5156d55 100644 --- a/waku/waku_archive/driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ./postgres_driver/postgres_driver, diff --git a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim index 516d8d70e3..ffa33faaf9 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_driver.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_driver.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[nre, options, sequtils, strutils, strformat, times], diff --git a/waku/waku_archive/driver/postgres_driver/postgres_healthcheck.nim b/waku/waku_archive/driver/postgres_driver/postgres_healthcheck.nim index cb918f7fdc..b6a59a3adf 100644 --- a/waku/waku_archive/driver/postgres_driver/postgres_healthcheck.nim +++ b/waku/waku_archive/driver/postgres_driver/postgres_healthcheck.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronos, stew/results import ../../../common/databases/db_postgres, ../../../common/error_handling diff --git a/waku/waku_archive/driver/queue_driver.nim b/waku/waku_archive/driver/queue_driver.nim index 1ea8a29d3a..df75737802 100644 --- a/waku/waku_archive/driver/queue_driver.nim +++ b/waku/waku_archive/driver/queue_driver.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ./queue_driver/queue_driver, ./queue_driver/index diff --git a/waku/waku_archive/driver/queue_driver/index.nim b/waku/waku_archive/driver/queue_driver/index.nim index d34b550c85..22e612aab7 100644 --- a/waku/waku_archive/driver/queue_driver/index.nim +++ b/waku/waku_archive/driver/queue_driver/index.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/byteutils, nimcrypto/sha2 import ../../../waku_core, ../../common diff --git a/waku/waku_archive/driver/queue_driver/queue_driver.nim b/waku/waku_archive/driver/queue_driver/queue_driver.nim index dcc45f9700..8340ad8500 100644 --- a/waku/waku_archive/driver/queue_driver/queue_driver.nim +++ b/waku/waku_archive/driver/queue_driver/queue_driver.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, stew/sorted_set, chronicles, chronos import ../../../waku_core, ../../common, ../../driver, ./index diff --git a/waku/waku_archive/driver/sqlite_driver.nim b/waku/waku_archive/driver/sqlite_driver.nim index 027e00488c..da46f6c841 100644 --- a/waku/waku_archive/driver/sqlite_driver.nim +++ b/waku/waku_archive/driver/sqlite_driver.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ./sqlite_driver/sqlite_driver diff --git a/waku/waku_archive/driver/sqlite_driver/cursor.nim b/waku/waku_archive/driver/sqlite_driver/cursor.nim index 9729f0ff79..ada14cc24d 100644 --- a/waku/waku_archive/driver/sqlite_driver/cursor.nim +++ b/waku/waku_archive/driver/sqlite_driver/cursor.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ../../../waku_core, ../../common diff --git a/waku/waku_archive/driver/sqlite_driver/queries.nim b/waku/waku_archive/driver/sqlite_driver/queries.nim index 94f323b2de..037dddd807 100644 --- a/waku/waku_archive/driver/sqlite_driver/queries.nim +++ b/waku/waku_archive/driver/sqlite_driver/queries.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils], stew/[results, byteutils], sqlite3_abi import diff --git a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim index e9072ceefa..e9902fcca7 100644 --- a/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim +++ b/waku/waku_archive/driver/sqlite_driver/sqlite_driver.nim @@ -1,9 +1,6 @@ # The code in this file is an adaptation of the Sqlite KV Store found in nim-eth. # https://github.com/status-im/nim-eth/blob/master/eth/db/kvstore_sqlite3.nim -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/[byteutils, results], chronicles, chronos import diff --git a/waku/waku_archive/retention_policy.nim b/waku/waku_archive/retention_policy.nim index dde2fede4f..0c92caba39 100644 --- a/waku/waku_archive/retention_policy.nim +++ b/waku/waku_archive/retention_policy.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, chronos import ./driver diff --git a/waku/waku_archive/retention_policy/builder.nim b/waku/waku_archive/retention_policy/builder.nim index 98d9cf1a5e..6ed50a4ac1 100644 --- a/waku/waku_archive/retention_policy/builder.nim +++ b/waku/waku_archive/retention_policy/builder.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[strutils, options], regex, stew/results import diff --git a/waku/waku_archive/retention_policy/retention_policy_capacity.nim b/waku/waku_archive/retention_policy/retention_policy_capacity.nim index 0e50662567..b29ca6531d 100644 --- a/waku/waku_archive/retention_policy/retention_policy_capacity.nim +++ b/waku/waku_archive/retention_policy/retention_policy_capacity.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, chronicles, chronos import ../driver, ../retention_policy diff --git a/waku/waku_archive/retention_policy/retention_policy_size.nim b/waku/waku_archive/retention_policy/retention_policy_size.nim index a01ebfcf0b..2e09e197fd 100644 --- a/waku/waku_archive/retention_policy/retention_policy_size.nim +++ b/waku/waku_archive/retention_policy/retention_policy_size.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, chronicles, chronos import ../driver, ../retention_policy diff --git a/waku/waku_archive/retention_policy/retention_policy_time.nim b/waku/waku_archive/retention_policy/retention_policy_time.nim index a63a040e98..2a3d1de4b3 100644 --- a/waku/waku_archive/retention_policy/retention_policy_time.nim +++ b/waku/waku_archive/retention_policy/retention_policy_time.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/times, stew/results, chronicles, chronos import ../../waku_core, ../driver, ../retention_policy diff --git a/waku/waku_core/message/codec.nim b/waku/waku_core/message/codec.nim index 7b71d95382..9b01cf667f 100644 --- a/waku/waku_core/message/codec.nim +++ b/waku/waku_core/message/codec.nim @@ -2,10 +2,7 @@ # See: # - RFC 14: https://rfc.vac.dev/spec/14/ # - Proto definition: https://github.com/vacp2p/waku/blob/main/waku/message/v1/message.proto -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ../../common/protobuf, ../topics, ../time, ./message diff --git a/waku/waku_core/message/digest.nim b/waku/waku_core/message/digest.nim index 67e8d81c28..1d4d122816 100644 --- a/waku/waku_core/message/digest.nim +++ b/waku/waku_core/message/digest.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/sequtils, stew/[byteutils, endians2, arrayops], nimcrypto/sha2 import ../topics, ./message diff --git a/waku/waku_core/message/message.nim b/waku/waku_core/message/message.nim index d96ac4996d..acd7055a04 100644 --- a/waku/waku_core/message/message.nim +++ b/waku/waku_core/message/message.nim @@ -3,10 +3,7 @@ ## See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-message.md ## for spec. -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ../topics, ../time diff --git a/waku/waku_core/multiaddrstr.nim b/waku/waku_core/multiaddrstr.nim index 46654cbc06..cd0caf2aad 100644 --- a/waku/waku_core/multiaddrstr.nim +++ b/waku/waku_core/multiaddrstr.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import libp2p/[peerinfo, switch] diff --git a/waku/waku_core/peers.nim b/waku/waku_core/peers.nim index d44e89cd9f..ead73014bb 100644 --- a/waku/waku_core/peers.nim +++ b/waku/waku_core/peers.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils, strutils, uri, net], diff --git a/waku/waku_core/subscription/push_handler.nim b/waku/waku_core/subscription/push_handler.nim index a33668152c..139f13445a 100644 --- a/waku/waku_core/subscription/push_handler.nim +++ b/waku/waku_core/subscription/push_handler.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronos diff --git a/waku/waku_core/subscription/subscription_manager.nim b/waku/waku_core/subscription/subscription_manager.nim index fe1f0a096b..18af7fd95e 100644 --- a/waku/waku_core/subscription/subscription_manager.nim +++ b/waku/waku_core/subscription/subscription_manager.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/tables, stew/results, chronicles, chronos diff --git a/waku/waku_core/time.nim b/waku/waku_core/time.nim index 10b937ab36..c8aa563558 100644 --- a/waku/waku_core/time.nim +++ b/waku/waku_core/time.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/times, metrics diff --git a/waku/waku_core/topics/content_topic.nim b/waku/waku_core/topics/content_topic.nim index 092c973b3e..bd7b5782a6 100644 --- a/waku/waku_core/topics/content_topic.nim +++ b/waku/waku_core/topics/content_topic.nim @@ -2,10 +2,7 @@ ## ## See 23/WAKU2-TOPICS RFC: https://rfc.vac.dev/spec/23/ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, std/strutils, stew/results import ./parsing @@ -68,7 +65,9 @@ proc parse*( ## Autosharding adds 1 optional prefix `/ if not topic.startsWith("/"): - return err(ParsingError.invalidFormat("topic must start with slash")) + return err( + ParsingError.invalidFormat("content-topic '" & topic & "' must start with slash") + ) let parts = topic[1 ..< topic.len].split("/") diff --git a/waku/waku_core/topics/parsing.nim b/waku/waku_core/topics/parsing.nim index 300037ea74..47a8979a8f 100644 --- a/waku/waku_core/topics/parsing.nim +++ b/waku/waku_core/topics/parsing.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results diff --git a/waku/waku_core/topics/pubsub_topic.nim b/waku/waku_core/topics/pubsub_topic.nim index 553b9c9baf..516d093bf8 100644 --- a/waku/waku_core/topics/pubsub_topic.nim +++ b/waku/waku_core/topics/pubsub_topic.nim @@ -2,10 +2,7 @@ ## ## See 23/WAKU2-TOPICS RFC: https://rfc.vac.dev/spec/23/ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/strutils, stew/[results, base10] import ./parsing diff --git a/waku/waku_core/topics/sharding.nim b/waku/waku_core/topics/sharding.nim index 5517af9ffe..c2bf633014 100644 --- a/waku/waku_core/topics/sharding.nim +++ b/waku/waku_core/topics/sharding.nim @@ -2,10 +2,7 @@ ## ## See 51/WAKU2-RELAY-SHARDING RFC: https://rfc.vac.dev/spec/51/#automatic-sharding -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import nimcrypto, std/options, std/tables, stew/endians2, stew/results, stew/byteutils diff --git a/waku/waku_enr/capabilities.nim b/waku/waku_enr/capabilities.nim index 171bce18f1..fb434e8cec 100644 --- a/waku/waku_enr/capabilities.nim +++ b/waku/waku_enr/capabilities.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, bitops, sequtils, net], stew/results, eth/keys, libp2p/crypto/crypto diff --git a/waku/waku_enr/multiaddr.nim b/waku/waku_enr/multiaddr.nim index a779faf53e..4c815a22b3 100644 --- a/waku/waku_enr/multiaddr.nim +++ b/waku/waku_enr/multiaddr.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils, net], @@ -55,7 +52,8 @@ func decodeMultiaddrs(buffer: seq[byte]): EnrResult[seq[MultiAddress]] = return err("malformed multiaddr field: invalid length") let addrRaw = ?readBytes(buffer, addrLen.int, pos) - let address = MultiAddress.init(addrRaw).get() + let address = MultiAddress.init(addrRaw).valueOr: + continue # Not a valid multiaddress multiaddrs.add(address) diff --git a/waku/waku_enr/sharding.nim b/waku/waku_enr/sharding.nim index 3f0774a4e1..fd7ab939a6 100644 --- a/waku/waku_enr/sharding.nim +++ b/waku/waku_enr/sharding.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, bitops, sequtils, net], diff --git a/waku/waku_filter_v2/client.nim b/waku/waku_filter_v2/client.nim index e20aeaef37..6823972b23 100644 --- a/waku/waku_filter_v2/client.nim +++ b/waku/waku_filter_v2/client.nim @@ -1,9 +1,6 @@ ## Waku Filter client for subscribing and receiving filtered messages -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, chronicles, chronos, libp2p/protocols/protocol, bearssl/rand import diff --git a/waku/waku_filter_v2/common.nim b/waku/waku_filter_v2/common.nim index ad5c7f21dd..7d2a4ad0a7 100644 --- a/waku/waku_filter_v2/common.nim +++ b/waku/waku_filter_v2/common.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results diff --git a/waku/waku_filter_v2/protocol.nim b/waku/waku_filter_v2/protocol.nim index eff3d0990d..5b87d61485 100644 --- a/waku/waku_filter_v2/protocol.nim +++ b/waku/waku_filter_v2/protocol.nim @@ -1,9 +1,6 @@ ## Waku Filter protocol for subscribing and filtering messages -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils, sets, strutils, tables], @@ -11,7 +8,8 @@ import chronicles, chronos, libp2p/peerid, - libp2p/protocols/protocol + libp2p/protocols/protocol, + libp2p/protocols/pubsub/timedcache import ../node/peer_manager, ../waku_core, @@ -31,6 +29,7 @@ type WakuFilter* = ref object of LPProtocol # a mapping of peer ids to a sequence of filter criteria peerManager: PeerManager maintenanceTask: TimerCallback + messageCache: TimedCache[string] proc pingSubscriber(wf: WakuFilter, peerId: PeerID): FilterSubscribeResult = trace "pinging subscriber", peerId = peerId @@ -176,20 +175,27 @@ proc pushToPeers( let msgHash = messagePush.pubsubTopic.computeMessageHash(messagePush.wakuMessage).to0xHex() - notice "pushing message to subscribed peers", - pubsubTopic = messagePush.pubsubTopic, - contentTopic = messagePush.wakuMessage.contentTopic, - target_peer_ids = targetPeerIds, - msg_hash = msgHash - - let bufferToPublish = messagePush.encode().buffer + ## it's also refresh expire of msghash, that's why update cache every time, even if it has a value. + if wf.messageCache.put(msgHash, Moment.now()): + notice "duplicate message found, not-pushing message to subscribed peers", + pubsubTopic = messagePush.pubsubTopic, + contentTopic = messagePush.wakuMessage.contentTopic, + target_peer_ids = targetPeerIds, + msg_hash = msgHash + else: + notice "pushing message to subscribed peers", + pubsubTopic = messagePush.pubsubTopic, + contentTopic = messagePush.wakuMessage.contentTopic, + target_peer_ids = targetPeerIds, + msg_hash = msgHash - var pushFuts: seq[Future[void]] - for peerId in peers: - let pushFut = wf.pushToPeer(peerId, bufferToPublish) - pushFuts.add(pushFut) + let bufferToPublish = messagePush.encode().buffer + var pushFuts: seq[Future[void]] - await allFutures(pushFuts) + for peerId in peers: + let pushFut = wf.pushToPeer(peerId, bufferToPublish) + pushFuts.add(pushFut) + await allFutures(pushFuts) proc maintainSubscriptions*(wf: WakuFilter) = trace "maintaining subscriptions" @@ -289,12 +295,14 @@ proc new*( subscriptionTimeout: Duration = DefaultSubscriptionTimeToLiveSec, maxFilterPeers: uint32 = MaxFilterPeers, maxFilterCriteriaPerPeer: uint32 = MaxFilterCriteriaPerPeer, + messageCacheTTL: Duration = MessageCacheTTL, ): T = let wf = WakuFilter( subscriptions: FilterSubscriptions.init( subscriptionTimeout, maxFilterPeers, maxFilterCriteriaPerPeer ), peerManager: peerManager, + messageCache: init(TimedCache[string], messageCacheTTL), ) wf.initProtocolHandler() diff --git a/waku/waku_filter_v2/protocol_metrics.nim b/waku/waku_filter_v2/protocol_metrics.nim index bb30d51a19..b19f612f31 100644 --- a/waku/waku_filter_v2/protocol_metrics.nim +++ b/waku/waku_filter_v2/protocol_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics diff --git a/waku/waku_filter_v2/rpc.nim b/waku/waku_filter_v2/rpc.nim index 1878bffa25..e3ee458805 100644 --- a/waku/waku_filter_v2/rpc.nim +++ b/waku/waku_filter_v2/rpc.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import json_serialization, std/options import ../waku_core diff --git a/waku/waku_filter_v2/rpc_codec.nim b/waku/waku_filter_v2/rpc_codec.nim index 0705e6bc86..94bdb367c2 100644 --- a/waku/waku_filter_v2/rpc_codec.nim +++ b/waku/waku_filter_v2/rpc_codec.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options import ../common/protobuf, ../waku_core, ./rpc diff --git a/waku/waku_filter_v2/subscriptions.nim b/waku/waku_filter_v2/subscriptions.nim index 37348025d6..8a5c5bc91a 100644 --- a/waku/waku_filter_v2/subscriptions.nim +++ b/waku/waku_filter_v2/subscriptions.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sets, tables], chronicles, chronos, libp2p/peerid, stew/shims/sets import ../waku_core, ../utils/tableutils @@ -13,6 +10,7 @@ const MaxFilterPeers* = 1000 MaxFilterCriteriaPerPeer* = 1000 DefaultSubscriptionTimeToLiveSec* = 5.minutes + MessageCacheTTL* = 2.minutes type # a single filter criterion is fully defined by a pubsub topic and content topic diff --git a/waku/waku_keystore/conversion_utils.nim b/waku/waku_keystore/conversion_utils.nim index 160404651b..4aaa6ec3e5 100644 --- a/waku/waku_keystore/conversion_utils.nim +++ b/waku/waku_keystore/conversion_utils.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import json, stew/[results, byteutils], ./protocol_types diff --git a/waku/waku_keystore/keyfile.nim b/waku/waku_keystore/keyfile.nim index 58a7ce32ae..5ea1aceb91 100644 --- a/waku/waku_keystore/keyfile.nim +++ b/waku/waku_keystore/keyfile.nim @@ -3,10 +3,7 @@ # - allow storage of multiple keyfiles (encrypted with different passwords) in same file and iteration among successful decryptions # - enable/disable at compilation time the keyfile id and version fields -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[os, strutils, json, sequtils], diff --git a/waku/waku_keystore/keystore.nim b/waku/waku_keystore/keystore.nim index 65371c5a72..23dd43e239 100644 --- a/waku/waku_keystore/keystore.nim +++ b/waku/waku_keystore/keystore.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import options, json, strutils, sequtils, std/[tables, os] diff --git a/waku/waku_keystore/protocol_types.nim b/waku/waku_keystore/protocol_types.nim index 2fab64e6f4..02934fa78f 100644 --- a/waku/waku_keystore/protocol_types.nim +++ b/waku/waku_keystore/protocol_types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sequtils, tables], stew/[results, endians2], nimcrypto, stint diff --git a/waku/waku_keystore/utils.nim b/waku/waku_keystore/utils.nim index 8bfa104bac..422b96106c 100644 --- a/waku/waku_keystore/utils.nim +++ b/waku/waku_keystore/utils.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import json, std/[os, sequtils] diff --git a/waku/waku_lightpush/callbacks.nim b/waku/waku_lightpush/callbacks.nim index 6d4bff3d29..5cc14eb0f8 100644 --- a/waku/waku_lightpush/callbacks.nim +++ b/waku/waku_lightpush/callbacks.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ../waku_core, diff --git a/waku/waku_lightpush/client.nim b/waku/waku_lightpush/client.nim index 45c09353fa..c419e35e6b 100644 --- a/waku/waku_lightpush/client.nim +++ b/waku/waku_lightpush/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, chronicles, chronos, metrics, bearssl/rand import diff --git a/waku/waku_lightpush/common.nim b/waku/waku_lightpush/common.nim index 4e376b1aed..dc1ec05782 100644 --- a/waku/waku_lightpush/common.nim +++ b/waku/waku_lightpush/common.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, chronos, libp2p/peerid import ../waku_core diff --git a/waku/waku_lightpush/protocol.nim b/waku/waku_lightpush/protocol.nim index 1e270ccc7a..f352bb2761 100644 --- a/waku/waku_lightpush/protocol.nim +++ b/waku/waku_lightpush/protocol.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, stew/byteutils, chronicles, chronos, metrics, bearssl/rand @@ -73,8 +70,8 @@ proc initProtocolHandler(wl: WakuLightPush) = wl.requestRateLimiter.checkUsageLimit(WakuLightPushCodec, conn): let buffer = await conn.readLp(DefaultMaxRpcSize) - waku_service_inbound_network_bytes.inc( - amount = buffer.len().int64, labelValues = [WakuLightPushCodec] + waku_service_network_bytes.inc( + amount = buffer.len().int64, labelValues = [WakuLightPushCodec, "in"] ) rpc = await handleRequest(wl, conn.peerId, buffer) @@ -115,4 +112,4 @@ proc new*( requestRateLimiter: newTokenBucket(rateLimitSetting), ) wl.initProtocolHandler() - return wl \ No newline at end of file + return wl diff --git a/waku/waku_lightpush/protocol_metrics.nim b/waku/waku_lightpush/protocol_metrics.nim index 8e3b58f9da..7a30f9e705 100644 --- a/waku/waku_lightpush/protocol_metrics.nim +++ b/waku/waku_lightpush/protocol_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics diff --git a/waku/waku_lightpush/rpc.nim b/waku/waku_lightpush/rpc.nim index da1f123f6e..33ba3f5e3b 100644 --- a/waku/waku_lightpush/rpc.nim +++ b/waku/waku_lightpush/rpc.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options import ../waku_core diff --git a/waku/waku_lightpush/rpc_codec.nim b/waku/waku_lightpush/rpc_codec.nim index d2e37e4958..25d2bd210d 100644 --- a/waku/waku_lightpush/rpc_codec.nim +++ b/waku/waku_lightpush/rpc_codec.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options import ../common/protobuf, ../waku_core, ./rpc diff --git a/waku/waku_lightpush/self_req_handler.nim b/waku/waku_lightpush/self_req_handler.nim index b3094f23c7..900e1796da 100644 --- a/waku/waku_lightpush/self_req_handler.nim +++ b/waku/waku_lightpush/self_req_handler.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} ## Notice that the REST /lightpush requests normally assume that the node ## is acting as a lightpush-client that will trigger the service provider node diff --git a/waku/waku_metadata.nim b/waku/waku_metadata.nim index 823dae4faf..96380baf2e 100644 --- a/waku/waku_metadata.nim +++ b/waku/waku_metadata.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ./waku_metadata/protocol diff --git a/waku/waku_metadata/protocol.nim b/waku/waku_metadata/protocol.nim index 0608be6d59..8ecfc3cbf2 100644 --- a/waku/waku_metadata/protocol.nim +++ b/waku/waku_metadata/protocol.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils, sets], diff --git a/waku/waku_metadata/rpc.nim b/waku/waku_metadata/rpc.nim index 12ab873ac9..fcb11e57dd 100644 --- a/waku/waku_metadata/rpc.nim +++ b/waku/waku_metadata/rpc.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options diff --git a/waku/waku_noise/noise.nim b/waku/waku_noise/noise.nim index dc0434500c..b464b0edb3 100644 --- a/waku/waku_noise/noise.nim +++ b/waku/waku_noise/noise.nim @@ -6,10 +6,7 @@ ## Implementation partially inspired by noise-libp2p: ## https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/noise.nim -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, strutils] import stew/byteutils diff --git a/waku/waku_noise/noise_handshake_processing.nim b/waku/waku_noise/noise_handshake_processing.nim index a5b0dbc634..328d45c66e 100644 --- a/waku/waku_noise/noise_handshake_processing.nim +++ b/waku/waku_noise/noise_handshake_processing.nim @@ -2,10 +2,7 @@ ## See spec for more details: ## https://github.com/vacp2p/rfc/tree/master/content/docs/rfcs/35 -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, strutils, tables] import chronos diff --git a/waku/waku_noise/noise_types.nim b/waku/waku_noise/noise_types.nim index 33e8a2db03..3d288a2425 100644 --- a/waku/waku_noise/noise_types.nim +++ b/waku/waku_noise/noise_types.nim @@ -5,10 +5,7 @@ ## Implementation partially inspired by noise-libp2p: ## https://github.com/status-im/nim-libp2p/blob/master/libp2p/protocols/secure/noise.nim -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, tables] import chronos diff --git a/waku/waku_noise/noise_utils.nim b/waku/waku_noise/noise_utils.nim index 23201f965a..a612c07285 100644 --- a/waku/waku_noise/noise_utils.nim +++ b/waku/waku_noise/noise_utils.nim @@ -3,10 +3,7 @@ ## See spec for more details: ## https://github.com/vacp2p/rfc/tree/master/content/docs/rfcs/35 -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[algorithm, base64, oids, options, strutils, tables, sequtils] import chronos diff --git a/waku/waku_peer_exchange.nim b/waku/waku_peer_exchange.nim index b28263a5ee..ca707f1621 100644 --- a/waku/waku_peer_exchange.nim +++ b/waku/waku_peer_exchange.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ./waku_peer_exchange/protocol diff --git a/waku/waku_peer_exchange/rpc_codec.nim b/waku/waku_peer_exchange/rpc_codec.nim index 8d4086106d..92ebd70842 100644 --- a/waku/waku_peer_exchange/rpc_codec.nim +++ b/waku/waku_peer_exchange/rpc_codec.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import ../common/protobuf, ./rpc diff --git a/waku/waku_relay/message_id.nim b/waku/waku_relay/message_id.nim index 83a70624d4..2b86411caf 100644 --- a/waku/waku_relay/message_id.nim +++ b/waku/waku_relay/message_id.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import stew/results, diff --git a/waku/waku_relay/protocol.nim b/waku/waku_relay/protocol.nim index 03d5b596ec..5baf2ce8c6 100644 --- a/waku/waku_relay/protocol.nim +++ b/waku/waku_relay/protocol.nim @@ -2,10 +2,7 @@ ## ## See https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-relay.md ## for spec. -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/strformat, @@ -59,6 +56,9 @@ const TopicParameters = TopicParams( invalidMessageDeliveriesDecay: 0.5, ) +declareCounter waku_relay_network_bytes, + "total traffic per topic", labels = ["topic", "direction"] + # see: https://rfc.vac.dev/spec/29/#gossipsub-v10-parameters const GossipsubParameters = GossipSubParams.init( pruneBackoff = chronos.minutes(1), @@ -150,6 +150,78 @@ proc initProtocolHandler(w: WakuRelay) = w.handler = handler w.codec = WakuRelayCodec +proc initRelayMetricObserver(w: WakuRelay) = + proc decodeRpcMessageInfo( + peer: PubSubPeer, msg: Message + ): Result[ + tuple[msgId: string, topic: string, wakuMessage: WakuMessage, msgSize: int], void + ] = + let msg_id = w.msgIdProvider(msg).valueOr: + warn "Error generating message id", + my_peer_id = w.switch.peerInfo.peerId, + from_peer_id = peer.peerId, + pubsub_topic = msg.topic, + error = $error + return err() + + let msg_id_short = shortLog(msg_id) + + let wakuMessage = WakuMessage.decode(msg.data).valueOr: + warn "Error decoding to Waku Message", + my_peer_id = w.switch.peerInfo.peerId, + msg_id = msg_id_short, + from_peer_id = peer.peerId, + pubsub_topic = msg.topic, + error = $error + return err() + + let msgSize = msg.data.len + msg.topic.len + return ok((msg_id_short, msg.topic, wakuMessage, msgSize)) + + proc logMessageInfo( + peer: PubSubPeer, topic: string, msg_id_short: string, msg: WakuMessage + ) = + let msg_hash = computeMessageHash(topic, msg).to0xHex() + + notice "sent relay message", + my_peer_id = w.switch.peerInfo.peerId, + msg_hash = msg_hash, + msg_id = msg_id_short, + to_peer_id = peer.peerId, + topic = topic, + sentTime = getNowInNanosecondTime(), + payloadSizeBytes = msg.payload.len + + proc updateMetrics( + peer: PubSubPeer, + pubsub_topic: string, + msg: WakuMessage, + msgSize: int, + onRecv: bool, + ) = + waku_relay_network_bytes.inc( + msgSize.int64, labelValues = [pubsub_topic, if onRecv: "in" else: "out"] + ) + + proc onRecv(peer: PubSubPeer, msgs: var RPCMsg) = + for msg in msgs.messages: + let (msg_id_short, topic, wakuMessage, msgSize) = decodeRpcMessageInfo(peer, msg).valueOr: + continue + # message receive log happens in treaceHandler as this one is called before checks + updateMetrics(peer, topic, wakuMessage, msgSize, onRecv = true) + discard + + proc onSend(peer: PubSubPeer, msgs: var RPCMsg) = + for msg in msgs.messages: + let (msg_id_short, topic, wakuMessage, msgSize) = decodeRpcMessageInfo(peer, msg).valueOr: + continue + logMessageInfo(peer, topic, msg_id_short, wakuMessage) + updateMetrics(peer, topic, wakuMessage, msgSize, onRecv = false) + + let administrativeObserver = PubSubObserver(onRecv: onRecv, onSend: onSend) + + w.addObserver(administrativeObserver) + proc new*( T: type WakuRelay, switch: Switch, maxMessageSize = int(DefaultMaxWakuMessageSize) ): WakuRelayResult[T] = @@ -170,6 +242,7 @@ proc new*( procCall GossipSub(w).initPubSub() w.initProtocolHandler() + w.initRelayMetricObserver() except InitializationError: return err("initialization error: " & getCurrentExceptionMsg()) @@ -180,6 +253,9 @@ proc addValidator*( ) {.gcsafe.} = w.wakuValidators.add((handler, errorMessage)) +proc addObserver*(w: WakuRelay, observer: PubSubObserver) {.gcsafe.} = + procCall GossipSub(w).addObserver(observer) + method start*(w: WakuRelay) {.async, base.} = debug "start" await procCall GossipSub(w).start() @@ -311,4 +387,6 @@ proc publish*( let msgHash = computeMessageHash(pubsubTopic, message).to0xHex() notice "start publish Waku message", msg_hash = msgHash, pubsubTopic = pubsubTopic - return await procCall GossipSub(w).publish(pubsubTopic, data) + let relayedPeerCount = await procCall GossipSub(w).publish(pubsubTopic, data) + + return relayedPeerCount diff --git a/waku/waku_rln_relay/conversion_utils.nim b/waku/waku_rln_relay/conversion_utils.nim index afdae614d7..f6ce81bdae 100644 --- a/waku/waku_rln_relay/conversion_utils.nim +++ b/waku/waku_rln_relay/conversion_utils.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sequtils, strutils, algorithm], diff --git a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim index 637bd13543..5cc89a6f73 100644 --- a/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim +++ b/waku/waku_rln_relay/group_manager/on_chain/group_manager.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import os, diff --git a/waku/waku_rln_relay/nonce_manager.nim b/waku/waku_rln_relay/nonce_manager.nim index 6fe463ce1c..490713d20c 100644 --- a/waku/waku_rln_relay/nonce_manager.nim +++ b/waku/waku_rln_relay/nonce_manager.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronos, stew/results, times import ./constants diff --git a/waku/waku_rln_relay/protocol_metrics.nim b/waku/waku_rln_relay/protocol_metrics.nim index d53caad2a6..6afb448f08 100644 --- a/waku/waku_rln_relay/protocol_metrics.nim +++ b/waku/waku_rln_relay/protocol_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import chronicles, diff --git a/waku/waku_rln_relay/protocol_types.nim b/waku/waku_rln_relay/protocol_types.nim index 7fa392247f..17db65d90e 100644 --- a/waku/waku_rln_relay/protocol_types.nim +++ b/waku/waku_rln_relay/protocol_types.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, tables, deques], stew/arrayops, stint, chronos, web3, eth/keys import ../waku_core, ../waku_keystore, ../common/protobuf diff --git a/waku/waku_rln_relay/rln/rln_interface.nim b/waku/waku_rln_relay/rln/rln_interface.nim index 54af6a3b61..cc468b124f 100644 --- a/waku/waku_rln_relay/rln/rln_interface.nim +++ b/waku/waku_rln_relay/rln/rln_interface.nim @@ -1,10 +1,7 @@ ## Nim wrappers for the functions defined in librln import ../protocol_types -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} ## Buffer struct is taken from # https://github.com/celo-org/celo-threshold-bls-rs/blob/master/crates/threshold-bls-ffi/src/ffi.rs diff --git a/waku/waku_rln_relay/rln_relay.nim b/waku/waku_rln_relay/rln_relay.nim index c7c320c275..fc5dc44bde 100644 --- a/waku/waku_rln_relay/rln_relay.nim +++ b/waku/waku_rln_relay/rln_relay.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[sequtils, tables, times, deques], diff --git a/waku/waku_store/client.nim b/waku/waku_store/client.nim index 4f378ddbcf..10abbb8e0b 100644 --- a/waku/waku_store/client.nim +++ b/waku/waku_store/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, chronicles, chronos, metrics, bearssl/rand import diff --git a/waku/waku_store/common.nim b/waku/waku_store/common.nim index f1e6a78bc0..bd6ccb3376 100644 --- a/waku/waku_store/common.nim +++ b/waku/waku_store/common.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options], stew/results import ../waku_core, ../common/paging diff --git a/waku/waku_store/protocol.nim b/waku/waku_store/protocol.nim index 357d303b25..e91acb207a 100644 --- a/waku/waku_store/protocol.nim +++ b/waku/waku_store/protocol.nim @@ -1,10 +1,7 @@ ## Waku Store protocol for historical messaging support. ## See spec for more details: ## https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, @@ -103,8 +100,8 @@ proc initProtocolHandler(self: WakuStore) = error "Connection read error", error = error.msg return - waku_service_inbound_network_bytes.inc( - amount = reqBuf.len().int64, labelValues = [WakuStoreCodec] + waku_service_network_bytes.inc( + amount = reqBuf.len().int64, labelValues = [WakuStoreCodec, "in"] ) resBuf = await self.handleQueryRequest(conn.peerId, reqBuf) @@ -120,8 +117,8 @@ proc initProtocolHandler(self: WakuStore) = error "Connection write error", error = writeRes.error.msg return - waku_service_outbound_network_bytes.inc( - amount = resBuf.len().int64, labelValues = [WakuStoreCodec] + waku_service_network_bytes.inc( + amount = resBuf.len().int64, labelValues = [WakuStoreCodec, "out"] ) self.handler = handler diff --git a/waku/waku_store/protocol_metrics.nim b/waku/waku_store/protocol_metrics.nim index 1566841f9c..d413c0a678 100644 --- a/waku/waku_store/protocol_metrics.nim +++ b/waku/waku_store/protocol_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics diff --git a/waku/waku_store/rpc_codec.nim b/waku/waku_store/rpc_codec.nim index 5f3f65dfff..a5e5e2d0b1 100644 --- a/waku/waku_store/rpc_codec.nim +++ b/waku/waku_store/rpc_codec.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/arrayops import ../common/[protobuf, paging], ../waku_core, ./common diff --git a/waku/waku_store_legacy/client.nim b/waku/waku_store_legacy/client.nim index 9f72ed3fc4..f8eea5ceb9 100644 --- a/waku/waku_store_legacy/client.nim +++ b/waku/waku_store_legacy/client.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, stew/results, chronicles, chronos, metrics, bearssl/rand import diff --git a/waku/waku_store_legacy/common.nim b/waku/waku_store_legacy/common.nim index 0de36fe192..0048555841 100644 --- a/waku/waku_store_legacy/common.nim +++ b/waku/waku_store_legacy/common.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils], stew/results, stew/byteutils, nimcrypto/sha2 import ../waku_core, ../common/paging diff --git a/waku/waku_store_legacy/protocol.nim b/waku/waku_store_legacy/protocol.nim index c0ede7b85f..2c552634b1 100644 --- a/waku/waku_store_legacy/protocol.nim +++ b/waku/waku_store_legacy/protocol.nim @@ -1,10 +1,7 @@ ## Waku Store protocol for historical messaging support. ## See spec for more details: ## https://github.com/vacp2p/specs/blob/master/specs/waku/v2/waku-store.md -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, @@ -118,8 +115,8 @@ proc initProtocolHandler(ws: WakuStore) = error "Connection read error", error = error.msg return - waku_service_inbound_network_bytes.inc( - amount = reqBuf.len().int64, labelValues = [WakuLegacyStoreCodec] + waku_service_network_bytes.inc( + amount = reqBuf.len().int64, labelValues = [WakuLegacyStoreCodec, "in"] ) resBuf = await ws.handleLegacyQueryRequest(conn.peerId, reqBuf) @@ -135,8 +132,8 @@ proc initProtocolHandler(ws: WakuStore) = error "Connection write error", error = writeRes.error.msg return - waku_service_outbound_network_bytes.inc( - amount = resBuf.len().int64, labelValues = [WakuLegacyStoreCodec] + waku_service_network_bytes.inc( + amount = resBuf.len().int64, labelValues = [WakuLegacyStoreCodec, "out"] ) ws.handler = handler diff --git a/waku/waku_store_legacy/protocol_metrics.nim b/waku/waku_store_legacy/protocol_metrics.nim index d854150232..59c7a829a5 100644 --- a/waku/waku_store_legacy/protocol_metrics.nim +++ b/waku/waku_store_legacy/protocol_metrics.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import metrics diff --git a/waku/waku_store_legacy/rpc.nim b/waku/waku_store_legacy/rpc.nim index 9a8887aeef..fad177d3f2 100644 --- a/waku/waku_store_legacy/rpc.nim +++ b/waku/waku_store_legacy/rpc.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/[options, sequtils], stew/results import ../waku_core, ../common/paging, ./common diff --git a/waku/waku_store_legacy/rpc_codec.nim b/waku/waku_store_legacy/rpc_codec.nim index 2d5867e00b..f9c518e83b 100644 --- a/waku/waku_store_legacy/rpc_codec.nim +++ b/waku/waku_store_legacy/rpc_codec.nim @@ -1,7 +1,4 @@ -when (NimMajor, NimMinor) < (1, 4): - {.push raises: [Defect].} -else: - {.push raises: [].} +{.push raises: [].} import std/options, nimcrypto/hash import ../common/[protobuf, paging], ../waku_core, ./common, ./rpc