Stability improvements: Lossy UI updates, websocket send thread aware

General - sendDataWs: one function (remove overloads) - setValueLossy instead of setValueV in loops - add queueLength in WebSocket library to implement lossy data SysModModel - add setValueLossy function SysModUI - send loopfun frames lossy (only if no queue) SysModWeb - add queueLength in UI table, update UI table every second (temp) - sendDataWs: exit function if already running (parallel exec between loopTask and async, wip) - sendDataWs: one function (remove overloads) UserModInstances - udp payload 1460 instead of 1472 ?!?!
ewowi · Oct 12, 2023 · 8c1ffc2 · 8c1ffc2
1 parent d758f26
commit 8c1ffc2
Show file tree

Hide file tree

Showing 11 changed files with 84 additions and 49 deletions.
diff --git a/src/App/AppModLeds.h b/src/App/AppModLeds.h
@@ -188,7 +188,7 @@ class AppModLeds:public Module {
     //update ui
     if (millis() - secondMillis >= 1000) {
       secondMillis = millis();
-      mdl->setValueV("realFps", "%lu /s", frameCounter);
+      mdl->setValueLossy("realFps", "%lu /s", frameCounter);
       frameCounter = 0;
     }
 

diff --git a/src/Sys/SysModFiles.cpp b/src/Sys/SysModFiles.cpp
@@ -77,7 +77,7 @@ void SysModFiles::loop(){
   if (millis() - secondMillis >= 1000) {
     secondMillis = millis();
 
-    mdl->setValueV("drsize", "%d / %d B", usedBytes(), totalBytes());
+    mdl->setValueLossy("drsize", "%d / %d B", usedBytes(), totalBytes());
 
         // if something changed in fileTbl
     if (filesChanged) {

diff --git a/src/Sys/SysModModel.cpp b/src/Sys/SysModModel.cpp
@@ -30,7 +30,7 @@ SysModModel::SysModModel() :Module("Model") {
   USER_PRINTF("Reading model from /model.json... (deserializeConfigFromFS)\n");
   if (files->readObjectFromFile("/model.json", model)) {//not part of success...
     print->printJson("Read model", *model);
-    web->sendDataWs(nullptr, false); //send new data: all clients, no def, no ws here yet!!!
+    web->sendDataWs(*model);
   }
 
   USER_PRINT_FUNCTION("%s %s %s\n", __PRETTY_FUNCTION__, name, success?"success":"failed");
@@ -99,7 +99,7 @@ void SysModModel::setup() {
 
   if (millis() - secondMillis >= 1000) {
     secondMillis = millis();
-    setValueV("mSize", "%d / %d B", model->memoryUsage(), model->capacity());
+    setValueLossy("mSize", "%d / %d B", model->memoryUsage(), model->capacity());
   }
 
   if (model->memoryUsage() / model->capacity() > 0.95) {
@@ -148,6 +148,8 @@ JsonObject SysModModel::setValueC(const char * id, const char * value) {
       // USER_PRINTF("setValue changed %s %s->%s\n", id, var["value"].as<String>().c_str(), value);
       if (var["ro"]) { // do not update var["value"]
         ui->setChFunAndWs(var, value); //value: bypass var["value"]
+        //now only used for ro not lossy
+        USER_PRINTF("setValueC: RO non lossy %s %s\n", id, value);
       } else {
         var["value"] = (char *)value; //(char *) forces a copy (https://arduinojson.org/v6/api/jsonvariant/subscript/) (otherwise crash!!)
         ui->setChFunAndWs(var);
@@ -240,6 +242,34 @@ JsonObject SysModModel::setValueP(const char * id, const char * format, ...) {
   return setValueC(id, value);
 }
 
+void SysModModel::setValueLossy(const char * id, const char * format, ...) {
+
+  va_list args;
+  va_start(args, format);
+
+  // size_t len = vprintf(format, args);
+  char value[128];
+  vsnprintf(value, sizeof(value)-1, format, args);
+
+  va_end(args);
+
+  JsonDocument *responseDoc = web->getResponseDoc();
+  responseDoc->clear(); //needed for deserializeJson?
+  JsonVariant responseVariant = responseDoc->as<JsonVariant>();
+
+  web->addResponse(id, "value", value);
+
+  bool isOk = true;
+  for (auto client:SysModWeb::ws->getClients()) {
+      if (client->status() != WS_CONNECTED || client->queueIsFull() || client->queueLength()>1) //lossy
+        isOk = false;
+  }
+  if (isOk)
+    web->sendDataWs(responseVariant);
+  else
+    USER_PRINTF(".");
+}
+
 JsonVariant SysModModel::getValue(const char * id) {
   JsonObject var = findVar(id);
   if (!var.isNull())

diff --git a/src/Sys/SysModModel.h b/src/Sys/SysModModel.h
@@ -42,6 +42,9 @@ class SysModModel:public Module {
   //Set value with argument list and print
   JsonObject setValueP(const char * id, const char * format, ...);
 
+  //Send value directly to ws (tbd: no model function but web?)
+  void setValueLossy(const char * id, const char * format, ...);
+
   //tbd
   // template <typename Type>
   // static JsonObject setValue(const char * id, Type value) {

diff --git a/src/Sys/SysModPrint.cpp b/src/Sys/SysModPrint.cpp
@@ -72,7 +72,7 @@ size_t SysModPrint::print(const char * format, ...) {
           Serial.print(va_arg(args, unsigned int));
           break;
         case 'c':
-          Serial.print(va_arg(args, char));
+          Serial.print(va_arg(args, int));
           break;
         case 'd':
           Serial.print(va_arg(args, int));

diff --git a/src/Sys/SysModPrint.h b/src/Sys/SysModPrint.h
@@ -14,7 +14,7 @@
 #define USER_PRINTF(x...) print->print(x)
 #define USER_PRINT_FUNCTION(x...) //print->print(x)
 #define USER_PRINT_NOT(x...) //print->print(x)
-#define USER_PRINT_Async(x...) //print->print(x)
+#define USER_PRINT_Async(x...) print->print(x)
 
 class SysModPrint:public Module {
 

diff --git a/src/Sys/SysModSystem.cpp b/src/Sys/SysModSystem.cpp
@@ -84,10 +84,10 @@ void SysModSystem::loop() {
   if (millis() - secondMillis >= 1000) {
     secondMillis = millis();
 
-    mdl->setValueV("upTime", "%lu s", millis()/1000);
-    mdl->setValueV("loops", "%lu /s", loopCounter);
-    mdl->setValueV("heap", "%d / %d (%d) B", ESP.getFreeHeap(), ESP.getHeapSize(), ESP.getMaxAllocHeap());
-    mdl->setValueV("stack", "%d B", uxTaskGetStackHighWaterMark(NULL));
+    mdl->setValueLossy("upTime", "%lu s", millis()/1000);
+    mdl->setValueLossy("loops", "%lu /s", loopCounter);
+    mdl->setValueLossy("heap", "%d / %d (%d) B", ESP.getFreeHeap(), ESP.getHeapSize(), ESP.getMaxAllocHeap());
+    mdl->setValueLossy("stack", "%d B", uxTaskGetStackHighWaterMark(NULL));
 
     loopCounter = 0;
   }

diff --git a/src/Sys/SysModUI.cpp b/src/Sys/SysModUI.cpp
@@ -103,11 +103,11 @@ void SysModUI::loop() {
           // USER_PRINTF("interval2 %u %d %d %d %d %d %d\n", millis(), varLoop->interval, varLoop->bufSize, buffer[0], buffer[1], buffer[2], buffer[3]);
 
           for (auto client:SysModWeb::ws->getClients()) {
-            if (client->status() == WS_CONNECTED && !client->queueIsFull()) 
+            if (client->status() == WS_CONNECTED && !client->queueIsFull() && client->queueLength()<=1) //lossy
               client->binary(wsBuf);
             else {
               // web->clientsChanged = true; tbd: changed also if full status changes
-              web->printClient("loopFun client full or not connected", client);
+              web->printClient("loopFun skip frame", client);
             }
           }
           wsBuf->unlock();

diff --git a/src/Sys/SysModWeb.cpp b/src/Sys/SysModWeb.cpp
@@ -32,6 +32,7 @@ bool SysModWeb::clientsChanged = false;
 
 unsigned long SysModWeb::wsSendBytesCounter = 0;
 unsigned long SysModWeb::wsSendJsonCounter = 0;
+unsigned long SysModWeb::wsSendDataWsCounter = 0;
 
 SysModWeb::SysModWeb() :Module("Web") {
   ws = new AsyncWebSocket("/ws");
@@ -71,6 +72,9 @@ void SysModWeb::setup() {
     select.add("Connected"); //1
     select.add("Disconnecting"); //2
   });
+  ui->initNumber(tableVar, "clLength", -1, true, [](JsonObject var) { //uiFun
+    web->addResponse(var["id"], "label", "Length");
+  });
 
   ui->initText(parentVar, "wsSendBytes");
   ui->initText(parentVar, "wsSendJson");
@@ -86,7 +90,8 @@ void SysModWeb::loop() {
 
   //currently not used as each variable is send individually
   if (this->modelUpdated) {
-    sendDataWs(nullptr, false); //send new data, all clients, no def
+    sendDataWs(*SysModModel::model); //send new data, all clients, no def
+
     this->modelUpdated = false;
   }
 
@@ -96,20 +101,21 @@ void SysModWeb::loop() {
     // if something changed in clTbl
     if (clientsChanged) {
       clientsChanged = false;
-
-      ui->processUiFun("clTbl");
     }
 
-    uint8_t rowNr = 0;
-    for (auto client:SysModWeb::ws->getClients()) {
-      mdl->setValueB("clIsFull", client->queueIsFull(), rowNr);
-      mdl->setValueI("clStatus", client->status());
-      rowNr++;
-    }
+    ui->processUiFun("clTbl"); //every second (temp)
 
-    mdl->setValueV("wsSendBytes", "%lu /s", wsSendBytesCounter);
+    // uint8_t rowNr = 0;
+    // for (auto client:SysModWeb::ws->getClients()) {
+    //   mdl->setValueB("clIsFull", client->queueIsFull(), rowNr);
+    //   mdl->setValueI("clStatus", client->status(), rowNr);
+    //   mdl->setValueI("clLength", client->queueLength(), rowNr);
+    //   rowNr++;
+    // }
+
+    mdl->setValueLossy("wsSendBytes", "%lu /s", wsSendBytesCounter);
     wsSendBytesCounter = 0;
-    mdl->setValueV("wsSendJson", "%lu /s", wsSendJsonCounter);
+    mdl->setValueLossy("wsSendJson", "%lu /s", wsSendJsonCounter);
     wsSendJsonCounter = 0;
   }
 }
@@ -142,7 +148,7 @@ void SysModWeb::wsEvent(AsyncWebSocket * server, AsyncWebSocketClient * client,
   // }
   if (type == WS_EVT_CONNECT) {
     printClient("WS client connected", client);
-    sendDataWs(client, true); //send definition to client
+    sendDataWs(*SysModModel::model, client); //send definition to client
     clientsChanged = true;
   } else if (type == WS_EVT_DISCONNECT) {
     printClient("WS Client disconnected", client);
@@ -186,7 +192,7 @@ void SysModWeb::wsEvent(AsyncWebSocket * server, AsyncWebSocketClient * client,
               if (responseVariant["uiFun"].isNull())
                 sendDataWs(responseVariant);
               else
-                sendDataWs(client, responseVariant);
+                sendDataWs(responseVariant, client);
             }
             else {
               USER_PRINT_Async("WS_EVT_DATA no responseDoc\n");
@@ -345,16 +351,23 @@ void SysModWeb::wsEvent2(AsyncWebSocket * server, AsyncWebSocketClient * client,
 
 
 void SysModWeb::printClient(const char * text, AsyncWebSocketClient * client) {
-  USER_PRINT_Async("%s client: %d %s q:%d s:%d (#:%d)\n", text, client?client->id():-1, client?client->remoteIP().toString().c_str():"No", client->queueIsFull(), client->status(), ws->count());
+  USER_PRINT_Async("%s client: %d %s q:%d l:%d s:%d (#:%d)\n", text, client?client->id():-1, client?client->remoteIP().toString().c_str():"No", client->queueIsFull(), client->queueLength(), client->status(), ws->count());
   //status: { WS_DISCONNECTED, WS_CONNECTED, WS_DISCONNECTING }
 }
 
-void SysModWeb::sendDataWs(AsyncWebSocketClient * client, JsonVariant json) {
+void SysModWeb::sendDataWs(JsonVariant json, AsyncWebSocketClient * client) {
   if (!ws) {
     USER_PRINT_Async("sendDataWs no ws\n");
     return;
   }
-  // return;
+
+  wsSendDataWsCounter++;
+  if (wsSendDataWsCounter > 1) {
+    USER_PRINT_Async("sendDataWs parallel %d %s\n", wsSendDataWsCounter, pcTaskGetTaskName(NULL));
+    wsSendDataWsCounter--;
+    return;
+  }
+
   wsSendJsonCounter++;
   ws->cleanupClients();
 
@@ -392,16 +405,7 @@ void SysModWeb::sendDataWs(AsyncWebSocketClient * client, JsonVariant json) {
       ws->_cleanBuffers();
     }
   }
-}
-
-//specific json data send to all clients
-void SysModWeb::sendDataWs(JsonVariant json) {
-  sendDataWs(nullptr, json);
-}
-
-void SysModWeb::sendDataWs(AsyncWebSocketClient * client, bool inclDef) {
-  //tbd: remove inclDef paramater (now only used to type overload sendDataWS)
-  sendDataWs(client, *SysModModel::model);
+  wsSendDataWsCounter--;
 }
 
 //add an url to the webserver to listen to
@@ -574,6 +578,7 @@ void SysModWeb::clientsToJson(JsonArray array, bool nameOnly, const char * filte
       row.add((char *)client->remoteIP().toString().c_str()); //create a copy!
       row.add(client->queueIsFull());
       row.add(client->status());
+      row.add(client->queueLength());
     }
   }
 }

diff --git a/src/Sys/SysModWeb.h b/src/Sys/SysModWeb.h
@@ -32,13 +32,7 @@ class SysModWeb:public Module {
   static void printClient(const char * text, AsyncWebSocketClient * client);
 
   //send json to client or all clients
-  static void sendDataWs(AsyncWebSocketClient * client = nullptr, JsonVariant json = JsonVariant());
-
-  //specific json data send to all clients
-  static void sendDataWs(JsonVariant json);
-
-  //complete document send to client or all clients
-  static void sendDataWs(AsyncWebSocketClient * client = nullptr, bool inclDef = false);
+  static void sendDataWs(JsonVariant json = JsonVariant(), AsyncWebSocketClient * client = nullptr);
 
   //add an url to the webserver to listen to
   bool addURL(const char * uri, const char * contentType, const char * path = nullptr, const uint8_t * content = nullptr, size_t len = -1);
@@ -75,6 +69,7 @@ class SysModWeb:public Module {
 
   static unsigned long wsSendBytesCounter;
   static unsigned long wsSendJsonCounter;
+  static unsigned long wsSendDataWsCounter;
 private:
   bool modelUpdated = false;
 

diff --git a/src/User/UserModInstances.h b/src/User/UserModInstances.h
@@ -35,7 +35,7 @@ struct UDPWLEDMessage {
 //compatible with WLED nodes as it only interprets first 44 bytes
 struct UDPStarModMessage {
   UDPWLEDMessage header;
-  char body[1428]; //1472 - 44, one udp frame
+  char body[1416]; //1460 - 44, one udp frame should be 1460, not 1472 (then split by network in 1460 and 12)
 };
 
 class UserModInstances:public Module {
@@ -93,7 +93,7 @@ class UserModInstances:public Module {
       // USER_PRINTF("udpMessage size %d = %d + %d + %d + ...\n", sizeof(UDPWLEDMessage), sizeof(udpMessage.ip0), sizeof(udpMessage.version), sizeof(udpMessage.name));
       success = false;
     }
-    if (sizeof(UDPStarModMessage) != 1472) { //size of UDP Packet
+    if (sizeof(UDPStarModMessage) != 1460) { //size of UDP Packet
       USER_PRINTF("Program error: Size of UDP message is not 44: %d\n", sizeof(UDPStarModMessage));
       // USER_PRINTF("udpMessage size %d = %d + %d + %d + ...\n", sizeof(UDPWLEDMessage), sizeof(udpMessage.ip0), sizeof(udpMessage.version), sizeof(udpMessage.name));
       success = false;
@@ -128,13 +128,15 @@ class UserModInstances:public Module {
   {
     if(!SysModModules::isConnected) return;
 
+    instanceUDP.flush(); //tbd: test if needed
+
     int packetSize = instanceUDP.parsePacket();
 
     if (packetSize) {
       IPAddress remoteIp = instanceUDP.remoteIP();
       // TODO: actually look at the contents of the packet to fetch version, name etc
 
-      USER_PRINTF("UDPStarModMessage %d check %d or %d\n", packetSize, sizeof(UDPWLEDMessage), sizeof(UDPStarModMessage));
+      USER_PRINTF("UDPStarModMessage %s %d check %d or %d\n", remoteIp.toString().c_str(), packetSize, sizeof(UDPWLEDMessage), sizeof(UDPStarModMessage));
 
       if (packetSize == sizeof(UDPWLEDMessage)) { //WLED instance
         UDPWLEDMessage udpMessage;
@@ -200,7 +202,7 @@ class UserModInstances:public Module {
     starModMessage.header.type = 32; //esp32 tbd: CONFIG_IDF_TARGET_ESP32S3 etc
     starModMessage.header.nodeId = ip[3];
     starModMessage.header.version = 2309280; //tbd
-    strncpy(starModMessage.body, "Effect x, Projection y, Leds begin..end", 1428-1);
+    strncpy(starModMessage.body, "Effect x, Projection y, Leds begin..end", 1416-1);
 
     // const char *bump = reinterpret_cast<const char*>(starModMessage.body);
     // bump = "SM";