From eb19d418de373d72a34cd8b44bb5e5b1576b99ee Mon Sep 17 00:00:00 2001 From: Guillaume Michel Date: Thu, 10 Oct 2024 14:05:14 +0200 Subject: [PATCH] remove inaccurate dht client data (#135) --- config/plotdefs/ipfs-servers-vs-clients.yaml | 29 ++++++++----------- config/plotdefs/ipfs-unique-agents.yaml | 15 ++++++++-- content.en/ipfs/kpi/_index.md | 7 +++++ .../plots/2024/10/07/ipfs-unique-agents.json | 2 +- .../plots/latest/ipfs-servers-vs-clients.json | 2 +- .../plots/latest/ipfs-unique-agents.json | 2 +- 6 files changed, 34 insertions(+), 23 deletions(-) diff --git a/config/plotdefs/ipfs-servers-vs-clients.yaml b/config/plotdefs/ipfs-servers-vs-clients.yaml index 44cb78f7e1..6085b9fd0a 100644 --- a/config/plotdefs/ipfs-servers-vs-clients.yaml +++ b/config/plotdefs/ipfs-servers-vs-clients.yaml @@ -2,24 +2,19 @@ frequency: daily datasets: # unique peers is calculated from total unique peers seen connecting to preload and bootstrap servers # caracol started collecting this data automatically from week ending 20 Mar 2023 + # between 2024-09-09 and 2024-09-30 the IPFS team had an outage and we didn't collect data - name: "unique peers" source: "caracol" query: > - select * from (values - ('2023-01-09 00:00:00 Z'::timestamptz, 317798), - ('2023-01-16 00:00:00 Z'::timestamptz, 420280), - ('2023-01-23 00:00:00 Z'::timestamptz, 487695), - ('2023-01-30 00:00:00 Z'::timestamptz, 458967), - ('2023-02-06 00:00:00 Z'::timestamptz, 490907), - ('2023-02-13 00:00:00 Z'::timestamptz, 397445), - ('2023-02-20 00:00:00 Z'::timestamptz, 342812), - ('2023-02-27 00:00:00 Z'::timestamptz, 327501), - ('2023-03-06 00:00:00 Z'::timestamptz, 401014), - ('2023-03-13 00:00:00 Z'::timestamptz, 304623) - ) as t (date,value) - union - select date, value from get_collected_values(3, '2023-03-20 00:00:00 Z'::timestamptz, {{ .Now | timestamptz }}) - order by date; + SELECT date, value + FROM get_collected_values(3, {{ .StartOfWeek | timestamptz }}-'14 week'::interval, {{ .Now | timestamptz }}) + WHERE date NOT IN ( + '2024-09-09 00:00:00 Z'::timestamptz, + '2024-09-16 00:00:00 Z'::timestamptz, + '2024-09-23 00:00:00 Z'::timestamptz, + '2024-09-30 00:00:00 Z'::timestamptz + ) + ORDER BY date; - name: "dht servers" source: "nebula_ipfs" @@ -127,7 +122,6 @@ series: color: "black" marker: "circle" - layout: barmode: stack title: @@ -144,4 +138,5 @@ layout: autorange: true rangemode: "tozero" title: - text: "Number of unique peers" \ No newline at end of file + text: "Number of unique peers" + diff --git a/config/plotdefs/ipfs-unique-agents.yaml b/config/plotdefs/ipfs-unique-agents.yaml index 85f0ca3691..8b064c158c 100644 --- a/config/plotdefs/ipfs-unique-agents.yaml +++ b/config/plotdefs/ipfs-unique-agents.yaml @@ -5,8 +5,16 @@ datasets: - name: "preload unique agents" source: "caracol" query: > - select date, value from get_collected_values(17, {{ .StartOfWeek | timestamptz }}-'14 week'::interval, {{ .Now | timestamptz }}) - order by date; + SELECT date, value + FROM get_collected_values(17, {{ .StartOfWeek | timestamptz }}-'14 week'::interval, {{ .Now | timestamptz }}) + WHERE date NOT IN ( + '2024-09-09 00:00:00 Z'::timestamptz, + '2024-09-16 00:00:00 Z'::timestamptz, + '2024-09-23 00:00:00 Z'::timestamptz, + '2024-09-30 00:00:00 Z'::timestamptz + ) + ORDER BY date; + - name: "nebula unique agents" source: "nebula_ipfs" query: > @@ -82,4 +90,5 @@ layout: autorange: true rangemode: "tozero" title: - text: "Number of unique agents" \ No newline at end of file + text: "Number of unique agents" + diff --git a/content.en/ipfs/kpi/_index.md b/content.en/ipfs/kpi/_index.md index d74967995a..043cbf5b77 100644 --- a/content.en/ipfs/kpi/_index.md +++ b/content.en/ipfs/kpi/_index.md @@ -12,6 +12,13 @@ Our KPIs are currently focusing primarily on the public [IPFS DHT](https://docs. ## Network Size & Stability + +{{< hint info >}} +⚠️ Due to a migration, data from preload servers was not collected between +`2024-09-07` and `2024-09-26`. As a result, the DHT clients count and bootstrap +unique agents for this period are unavailable. +{{< /hint >}} + ### Client vs Server Node Estimate The total number of peers in the network is estimated using the number of unique [Peer IDs](https://docs.ipfs.tech/concepts/glossary/#peer-id) seen by Protocol Labs' [bootstrap nodes](https://docs.ipfs.tech/concepts/glossary/#bootstrap-node). The number of unique [DHT Server](https://docs.ipfs.tech/concepts/dht/#routing-tables) peer IDs identified by the [Nebula crawler](/tools/nebula/) is then subtracted from the total number of peers (seen by the bootstrappers) to estimate the number of peers that exclusively function as DHT Clients. diff --git a/content.en/plots/2024/10/07/ipfs-unique-agents.json b/content.en/plots/2024/10/07/ipfs-unique-agents.json index 939b00cfd9..b0a7a05134 100644 --- a/content.en/plots/2024/10/07/ipfs-unique-agents.json +++ b/content.en/plots/2024/10/07/ipfs-unique-agents.json @@ -1 +1 @@ -{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,519,797,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null} +{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null} diff --git a/content.en/plots/latest/ipfs-servers-vs-clients.json b/content.en/plots/latest/ipfs-servers-vs-clients.json index deba8c4fcc..a36919fde5 100644 --- a/content.en/plots/latest/ipfs-servers-vs-clients.json +++ b/content.en/plots/latest/ipfs-servers-vs-clients.json @@ -1 +1 @@ -{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Servers","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[46765,46216,45090,46471,71321,71548,70538,70621,70258,65908,63746,63683,64576,54748,44456],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"DHT Clients","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[150778,167309,158079,157676,197012,224680,231106,295128,245549,210690,61465,79951,179234],"yaxis":""},{"type":"scatter","marker":{"color":"black","symbol":"circle"},"mode":"lines+markers","name":"DHT Server IP Addresses","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[64565,60568,60247,61050,142903,144477,136871,138850,134683,120864,108473,112514,102854,63695,43577],"yaxis":""}],"layout":{"annotations":null,"barmode":"stack","legend":{"orientation":"h"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique peers"}}},"params":null,"dynamicLayout":null,"config":null} +{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Servers","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[46765,46216,45090,46471,71321,71548,70538,70621,70258,65908,63746,63683,64576,54748,44456],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"DHT Clients","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[150778,167309,158079,157676,197012,224680,231106,295128,245549,210690,179234],"yaxis":""},{"type":"scatter","marker":{"color":"black","symbol":"circle"},"mode":"lines+markers","name":"DHT Server IP Addresses","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[64565,60568,60247,61050,142903,144477,136871,138850,134683,120864,108473,112514,102854,63695,43577],"yaxis":""}],"layout":{"annotations":null,"barmode":"stack","legend":{"orientation":"h"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique peers"}}},"params":null,"dynamicLayout":null,"config":null} diff --git a/content.en/plots/latest/ipfs-unique-agents.json b/content.en/plots/latest/ipfs-unique-agents.json index 939b00cfd9..b0a7a05134 100644 --- a/content.en/plots/latest/ipfs-unique-agents.json +++ b/content.en/plots/latest/ipfs-unique-agents.json @@ -1 +1 @@ -{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,519,797,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null} +{"data":[{"type":"bar","hovertemplate":"","marker":{"color":"EE99AA"},"name":"Bootstrap Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-10-07T00:00:00Z"],"y":[948,1101,1019,940,949,1019,997,1034,1046,1148,1445],"yaxis":""},{"type":"bar","hovertemplate":"","marker":{"color":"6699cc"},"name":"DHT Unique Agents","orientation":"v","visible":true,"x":["2024-07-01T00:00:00Z","2024-07-08T00:00:00Z","2024-07-15T00:00:00Z","2024-07-22T00:00:00Z","2024-07-29T00:00:00Z","2024-08-05T00:00:00Z","2024-08-12T00:00:00Z","2024-08-19T00:00:00Z","2024-08-26T00:00:00Z","2024-09-02T00:00:00Z","2024-09-09T00:00:00Z","2024-09-16T00:00:00Z","2024-09-23T00:00:00Z","2024-09-30T00:00:00Z","2024-10-07T00:00:00Z"],"y":[327,320,315,310,322,315,320,327,330,381,424,453,467,456,475],"yaxis":""}],"layout":{"annotations":null,"barmode":"group","legend":{"orientation":"h","y":-0.2,"yanchor":"top"},"title":{"font":{"size":10},"text":"Data: 30 Jun 2024 to 6 Oct 2024. Sources: Bootstrap+preload server logs; Nebula.","x":0.99,"xanchor":"right","y":0.01,"yanchor":"bottom"},"xaxis":{"dtick":604800000,"tick0":"2024-07-01T00:00:00Z","tickmode":"linear","type":"date"},"yaxis":{"autorange":true,"rangemode":"tozero","title":{"text":"Number of unique agents"}}},"params":null,"dynamicLayout":null,"config":null}