Added net performance graphs to MPClientList UI.

This is a spinoff from RigsOfRods#3055
Requires updated rorserver.

New features:
- Added send timestamp to `RoRnet::Header` - all packets now track their time shift between remote client and local client. Note that the existing truck time shift tracking made by Ulteq is unchanged for now.
- Added queue timestamp to `RoRnet::Header` - all packets now track the delay between creating the packet and transmitting it to server.
- Added recv queue timestamp to `NetRecvPacket` - all packets now track local delay between receiving packet and processing it.
- Added 3 new graphs to the MPClientList UI: 1. timeshift (should be stable), 2. local recv lag, 3. remote send lag.

Motivation:
- to smoothen out character movement the same way truck movement is smoothed. That requires tracking the timeshift, and Ulteq's truck smoothing code is very much tied to `Actor` and `ActorManager`.
- To clean up network timers - there are 3 separate timers (`Character`, `Actor`, `ActorManager` - the pattern is not clear).
- To better understand and tweak network traffic.
- I like diagnostic views.

Time spent: 6.5hours, not counting rorserver.

source/main/CMakeLists.txt

@@ -141,6 +141,7 @@ set(SOURCE_FILES
         gui/panels/GUI_VehicleDescription.{h,cpp}
         gui/panels/GUI_VehicleButtons.{h,cpp}
         network/DiscordRpc.{h,cpp}
+        network/NetStats.{h,cpp}
         network/Network.{h,cpp}
         network/OutGauge.{h,cpp}
         network/RoRnet.h

source/main/gui/panels/GUI_MultiplayerClientList.cpp

@@ -65,7 +65,7 @@ void MpClientList::Draw()
 
     ImGuiWindowFlags flags = ImGuiWindowFlags_NoCollapse |
         ImGuiWindowFlags_NoMove | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_NoScrollbar;
-    const float content_width = 200.f;
+    const float content_width = 300.f;
     ImGui::SetNextWindowContentWidth(content_width);
     ImGui::SetNextWindowPos(ImVec2(
         ImGui::GetIO().DisplaySize.x - (content_width + (2*ImGui::GetStyle().WindowPadding.x) + theme.screen_edge_padding.x),
@@ -85,6 +85,8 @@ void MpClientList::Draw()
     const RoRnet::UserInfo& local_user = m_users[0]; // See `UpdateClients()`
     for (RoRnet::UserInfo const& user: m_users)
     {
+        ImGui::PushID(user.uniqueid);
+
         // Icon sizes: flag(16x11), auth(16x16), up(16x16), down(16x16)
         bool hovered = false;
         Ogre::TexturePtr flag_tex;
@@ -113,8 +115,18 @@ void MpClientList::Draw()
             default:;
             }
         }
+
+        // net graphs
+        NetClientStats stats;
+        App::GetNetwork()->GetUserStats(user.uniqueid, /*out*/stats);
+
+        hovered |= this->DrawPlotSmall("##Time", "", stats.client_time_offset, 40.f);
+
         // Always invoke to keep usernames aligned
         hovered |= this->DrawIcon(down_tex, ImVec2(8.f, ImGui::GetTextLineHeight()));
+        hovered |= this->DrawPlotSmall("##Send", "", stats.remote_queue_delay, 25.f);
+        hovered |= this->DrawPlotSmall("##Recv", "", stats.local_queue_delay, 25.f);
+        // Always invoke to keep usernames aligned
         hovered |= this->DrawIcon(up_tex, ImVec2(8.f, ImGui::GetTextLineHeight()));
 
         // Auth icon
@@ -214,10 +226,18 @@ void MpClientList::Draw()
                 case 2: ImGui::Text("%s", _LC("MultiplayerClientList", "No Trucks loaded")); break;
                 default:; // never happens
                 }
+
+                // Stats
+                ImGui::Separator();
+                this->DrawPlotBig("Time shift (ms)", stats.client_time_offset);
+                this->DrawPlotBig("Send lag (ms)", stats.remote_queue_delay);
+                this->DrawPlotBig("Recv lag (ms)", stats.local_queue_delay);
             }
 
             ImGui::EndTooltip();
         }
+
+        ImGui::PopID(); //user.uniqueid
     }
 
     if (App::GetNetwork()->GetNetQuality() != 0)
@@ -266,3 +286,26 @@ void MpClientList::CacheIcons()
 
     m_icons_cached = true;
 }
+
+bool MpClientList::DrawPlotSmall(const char* label, const char* overlay_text, NetGraphData& graphdata, float width)
+{
+    graphdata.ImPlotLines(label, overlay_text, ImVec2(width, ImGui::GetTextLineHeight()));
+    ImGui::SameLine();
+    return ImGui::IsItemHovered();
+}
+
+void MpClientList::DrawPlotBig(const char* label, NetGraphData& graphdata)
+{
+    const float PLOT_WIDTH = 125.f;
+
+    // The plot widget
+    const ImVec2 cursor_start = ImGui::GetCursorPos();
+    graphdata.ImPlotLines(label, nullptr, ImVec2(PLOT_WIDTH, 35.f));
+    const ImVec2 cursor_end = ImGui::GetCursorPos();
+
+    // The plot scale text
+    const ImVec2 scale_pos = cursor_start + ImVec2(PLOT_WIDTH + 5.f, ImGui::GetTextLineHeight() + 2.f);
+    ImGui::SetCursorPos(scale_pos);
+    ImGui::TextDisabled("%5.2f - %5.2f", graphdata.plotline_smoothmin, graphdata.plotline_smoothmax);
+    ImGui::SetCursorPos(cursor_end);
+}

source/main/gui/panels/GUI_MultiplayerClientList.h

@@ -28,6 +28,7 @@
 #pragma once
 
 #include "RoRnet.h"
+#include "Network.h"
 
 #include <Ogre.h>
 #include <imgui.h>
@@ -43,6 +44,8 @@ class MpClientList
 
 private:
     bool DrawIcon(Ogre::TexturePtr tex, ImVec2 reference_box); // Returns true if hovered
+    bool DrawPlotSmall(const char* label, const char* overlay_text, NetGraphData& graphdata, float width);
+    void DrawPlotBig(const char* label, NetGraphData& graphdata);
     void CacheIcons();
 
     std::vector<RoRnet::UserInfo> m_users; // only updated on demand to reduce mutex locking and vector allocating overhead.

source/main/network/NetStats.cpp

@@ -0,0 +1,98 @@
+/*
+    This source file is part of Rigs of Rods
+    Copyright 2005-2012 Pierre-Michel Ricordel
+    Copyright 2007-2012 Thomas Fischer
+    Copyright 2013-2023 Petr Ohlidal
+
+    For more information, see http://www.rigsofrods.org/
+
+    Rigs of Rods is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License version 3, as
+    published by the Free Software Foundation.
+
+    Rigs of Rods is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Rigs of Rods. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifdef USE_SOCKETW
+
+#include "Network.h"
+
+#include "Application.h"
+#include "ChatSystem.h"
+#include "Console.h"
+#include "ErrorUtils.h"
+#include "GameContext.h"
+#include "GUIManager.h"
+#include "GUI_TopMenubar.h"
+#include "Language.h"
+#include "RoRVersion.h"
+#include "ScriptEngine.h"
+#include "Utils.h"
+
+#include <Ogre.h>
+#include <SocketW.h>
+
+#include <algorithm>
+#include <chrono>
+#include <cstring>
+
+using namespace RoR;
+
+NetGraphData::NetGraphData(size_t size)
+{
+    plotline.resize(size, RoRnet::NetTime32_t(0));
+}
+
+void NetGraphData::AddSample(RoRnet::NetTime32_t sample)
+{
+    plotline.erase(plotline.begin());
+    plotline.push_back(sample);
+    const auto minmax = std::minmax_element(plotline.begin(), plotline.end());
+    plotline_min = *minmax.first;
+    plotline_max = *minmax.second;
+
+    // smooth scale
+    const float SMOOTH_UP = 0.4f;
+    const float SMOOTH_DOWN = 0.05f;
+
+    const float fmin = static_cast<float>(*minmax.first);
+    if (fmin < plotline_smoothmin)
+        plotline_smoothmin = fmin * SMOOTH_UP + plotline_smoothmin * (1.f - SMOOTH_UP);
+    else
+        plotline_smoothmin = fmin * SMOOTH_DOWN + plotline_smoothmin * (1.f - SMOOTH_DOWN);
+
+    const float fmax = static_cast<float>(*minmax.second);
+    if (fmax > plotline_smoothmax)
+        plotline_smoothmax = fmax * SMOOTH_UP + plotline_smoothmax * (1.f - SMOOTH_UP);
+    else
+        plotline_smoothmax = fmax * SMOOTH_DOWN + plotline_smoothmax * (1.f - SMOOTH_DOWN);
+}
+
+// static
+float NetGraphData::ImPlotGetSample(void* data, int idx)
+{
+    NetGraphPlotline* plotline = static_cast<NetGraphPlotline*>(data);
+    return static_cast<float>(plotline->at(idx));
+}
+
+void NetGraphData::ImPlotLines(const char* label, const char* overlay_text, ImVec2 size)
+{
+    ImGui::PlotLines(
+        label,
+        NetGraphData::ImPlotGetSample,
+        static_cast<void*>(&plotline),
+        static_cast<int>(plotline.size()),
+        /*values_offset:*/0,
+        overlay_text,
+        plotline_smoothmin,
+        plotline_smoothmax,
+        size);
+}
+
+#endif // USE_SOCKETW

source/main/network/NetStats.h

@@ -0,0 +1,63 @@
+/*
+    This source file is part of Rigs of Rods
+    Copyright 2005-2012 Pierre-Michel Ricordel
+    Copyright 2007-2012 Thomas Fischer
+    Copyright 2013-2023 Petr Ohlidal
+
+    For more information, see http://www.rigsofrods.org/
+
+    Rigs of Rods is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License version 3, as
+    published by the Free Software Foundation.
+
+    Rigs of Rods is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with Rigs of Rods. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#ifdef USE_SOCKETW
+
+#include "Application.h"
+#include "OgreImGui.h"
+#include "RoRnet.h"
+
+namespace RoR {
+
+/// @addtogroup Network
+/// @{
+
+typedef std::vector<RoRnet::NetTime32_t> NetGraphPlotline;
+
+struct NetGraphData
+{
+    NetGraphPlotline plotline;
+    RoRnet::NetTime32_t plotline_max = 0;
+    RoRnet::NetTime32_t plotline_min = 0;
+    float plotline_smoothmax = 0.f;
+    float plotline_smoothmin = 0.f;
+
+    NetGraphData(size_t size);
+
+    void         AddSample(RoRnet::NetTime32_t sample);
+    static float ImPlotGetSample(void* data, int idx);
+    void         ImPlotLines(const char* label, const char* overlay_text, ImVec2 size);
+};
+
+struct NetClientStats
+{
+    NetGraphData remote_queue_delay = NetGraphData(20); //!< How long it takes remote client to transmit outgoing packet.
+    NetGraphData local_queue_delay = NetGraphData(20);  //!< How long it takes us to process received packet.
+    NetGraphData client_time_offset = NetGraphData(40); //!< Difference between client timers; we monitor it's stability.
+};
+
+/// @}   //addtogroup Network
+
+} // namespace RoR
+
+#endif // USE_SOCKETW

source/main/network/Network.cpp

@@ -2,7 +2,7 @@
     This source file is part of Rigs of Rods
     Copyright 2005-2012 Pierre-Michel Ricordel
     Copyright 2007-2012 Thomas Fischer
-    Copyright 2013-2016 Petr Ohlidal
+    Copyright 2013-2023 Petr Ohlidal
 
     For more information, see http://www.rigsofrods.org/
 
@@ -167,7 +167,8 @@ void Network::QueueStreamData(RoRnet::Header &header, char *buffer, size_t buffe
     memcpy(packet.buffer, buffer, std::min(buffer_len, size_t(RORNET_MAX_MESSAGE_LENGTH)));
 
     std::lock_guard<std::mutex> lock(m_recv_packetqueue_mutex);
-    m_recv_packet_buffer.push_back(packet);
+    packet.recv_queue_time = m_recv_packet_timer.getMilliseconds();
+    m_recv_packet_buffer.push_back(packet);    
 }
 
 int Network::ReceiveMessage(RoRnet::Header *head, char* content, int bufferlen)
@@ -228,6 +229,7 @@ void Network::SendThread()
                 break;
             }
             packet = m_send_packet_buffer.front();
+            packet.GetHeader()->source_send_time = m_send_packet_timer.getMilliseconds();
             m_send_packet_buffer.pop_front();
         }
         SendMessageRaw(packet.buffer, packet.size);
@@ -552,6 +554,8 @@ bool Network::ConnectThread()
     m_shutdown = false;
 
     LOG("[RoR|Networking] Connect(): Creating Send/Recv threads");
+    m_send_packet_timer.reset();
+    m_recv_packet_timer.reset();
     m_send_thread = std::thread(&Network::SendThread, this);
     m_recv_thread = std::thread(&Network::RecvThread, this);
     PushNetMessage(MSG_NET_CONNECT_SUCCESS, "");
@@ -614,25 +618,23 @@ void Network::AddPacket(int streamid, int type, int len, const char *content)
     }
 
     NetSendPacket packet;
-    memset(&packet, 0, sizeof(NetSendPacket));
-
-    char *buffer = (char*)(packet.buffer);
-
-    RoRnet::Header *head = (RoRnet::Header *)buffer;
-    head->command     = type;
-    head->source      = m_uid;
-    head->size        = len;
-    head->streamid    = streamid;
+
+    // fill in the header
+    packet.GetHeader()->command     = type;
+    packet.GetHeader()->source      = m_uid;
+    packet.GetHeader()->size        = len;
+    packet.GetHeader()->streamid    = streamid;
 
     // then copy the contents
-    char *bufferContent = (char *)(buffer + sizeof(RoRnet::Header));
+    char *bufferContent = (char *)(packet.buffer + sizeof(RoRnet::Header));
     memcpy(bufferContent, content, len);
 
     // record the packet size
     packet.size = len + sizeof(RoRnet::Header);
 
     { // Lock scope
         std::lock_guard<std::mutex> lock(m_send_packetqueue_mutex);
+
         if (type == MSG2_STREAM_DATA_DISCARDABLE)
         {
             if (m_send_packet_buffer.size() > m_packet_buffer_size)
@@ -651,6 +653,7 @@ void Network::AddPacket(int streamid, int type, int len, const char *content)
             }
         }
         //DebugPacket("send", head, buffer);
+        packet.GetHeader()->source_queue_time = m_send_packet_timer.getMilliseconds();
         m_send_packet_buffer.push_back(packet);
     }
 
@@ -674,6 +677,19 @@ std::vector<NetRecvPacket> Network::GetIncomingStreamData()
     std::lock_guard<std::mutex> lock(m_recv_packetqueue_mutex);
     std::vector<NetRecvPacket> buf_copy = m_recv_packet_buffer;
     m_recv_packet_buffer.clear();
+
+    // update stats (graphs)
+    // TODO: this should be done _after_ the packets get pruned in `HandleActorStreamData()` 
+    //      - that logic should be moved here so that characters benefit too.
+    for (const NetRecvPacket& packet: buf_copy)
+    {
+        const RoRnet::NetTime32_t cur_time = m_recv_packet_timer.getMilliseconds();
+        NetClientStats& stats = m_recv_client_stats[packet.header.source];
+        stats.remote_queue_delay.AddSample(packet.header.source_send_time - packet.header.source_queue_time);
+        stats.local_queue_delay.AddSample(cur_time - packet.recv_queue_time);
+        stats.client_time_offset.AddSample(cur_time - packet.header.source_queue_time);
+    }
+
     return buf_copy;
 }
 
@@ -770,6 +786,18 @@ bool Network::FindUserInfo(std::string const& username, RoRnet::UserInfo &result
     return false;
 }
 
+bool Network::GetUserStats(int uid, NetClientStats& result)
+{
+    std::lock_guard<std::mutex> lock(m_recv_packetqueue_mutex);
+    const auto it = m_recv_client_stats.find(uid);
+    const bool found = it != m_recv_client_stats.end();
+    if (found)
+    {
+        result = it->second;
+    }
+    return found;
+}
+
 void Network::BroadcastChatMsg(const char* msg)
 {
     AddPacket(m_stream_id, RoRnet::MSG2_UTF8_CHAT, (int)std::strlen(msg), msg);