Migrate from ArduinoJson 6 to 7 (.0.3)

General:
- change .createNestedArray("value") to ["value"].to<JsonArray>()
- change .createNestedArray() to .add<JsonArray>()
- same for JsonObject
- change BasicJsonDocument to JsonDocument

AppFixture: projectAndMap: check on indexV not too high and adding extra physMaps

AppModLeds: fxSize: f_ValueFun: small bugfix

SysModModel:
- RAM_Allocator to AJ v7
- remove initProgress and loop10s updates
- remove auto garbageCollect (bad idea anyway)

SysModPrint
- simplify printJDocInfo

SysModWeb
- serveJson: AsyncJsonResponse: no need to specify size anymore

platformio.ini

@@ -34,7 +34,7 @@ build_flags =
   ${ESPAsyncWebServer.build_flags}
 lib_deps = 
   ${ESPAsyncWebServer.lib_deps}
-  https://github.com/bblanchon/ArduinoJson.git#v6.21.4
+  https://github.com/bblanchon/ArduinoJson.git#v7.0.3
   ; https://github.com/Jason2866/ESP32_Show_Info.git
 
 

src/App/AppFixture.cpp

@@ -179,8 +179,12 @@
                   }
                   break;
               }
+              leds->nrOfLeds = leds->size.x * leds->size.y * leds->size.z;
 
-              if (indexV != UINT16_MAX) {
+              if (indexV > leds->nrOfLeds) {
+                USER_PRINTF("indexV too high %d>=%d (p:%d) p:%d,%d,%d\n", indexV, leds->nrOfLeds, indexP, pixel.x, pixel.y, pixel.z);
+              }
+              else if (indexV != UINT16_MAX) {
                 //post processing: inverse mapping
                 switch(leds->projectionNr) {
                 case p_DistanceFromCenter:
@@ -293,8 +297,9 @@
 
           if (leds->projectionNr > p_Random) {
 
-            for (int i = leds->mappingTable.size(); i<leds->size.x * leds->size.y * leds->size.z;i++) {
-              USER_PRINTF("mapping add extra physMap %d %d\n", i, leds->mappingTable.size());
+            if (leds->mappingTable.size() < leds->size.x * leds->size.y * leds->size.z)
+              USER_PRINTF("mapping add extra physMap %d of %d %d,%d,%d\n", leds->mappingTable.size(), leds->size.x * leds->size.y * leds->size.z, leds->size.x, leds->size.y, leds->size.z);
+            for (int i = leds->mappingTable.size(); i<min(leds->size.x * leds->size.y * leds->size.z, 2048);i++) {
               std::vector<uint16_t> physMap;
               leds->mappingTable.push_back(physMap);
             }

src/App/AppModLeds.h

@@ -205,11 +205,13 @@ class AppModLeds:public SysModule {
 
     ui->initText(tableVar, "fxSize", nullptr, 32, true, [this](JsonObject var, uint8_t rowNr, uint8_t funType) { switch (funType) { //varFun
       case f_ValueFun:
-        for (uint8_t rowNr = 0; rowNr < fixture.ledsList.size(); rowNr++) {
-          Leds leds = fixture.ledsList[rowNr];
+        for (std::vector<Leds>::iterator leds=fixture.ledsList.begin(); leds!=fixture.ledsList.end(); ++leds) {
+        // for (uint8_t rowNr = 0; rowNr < fixture.ledsList.size(); rowNr++) {
+          USER_PRINTF("fxSize valueFun %d %d %d\n", fixture.ledsList.size(), rowNr, fixture.ledsList[rowNr].rowNr);
+          // Leds leds = fixture.ledsList[rowNr];
           char message[32];
-          print->fFormat(message, sizeof(message)-1, "%d x %d x %d = %d", leds.size.x, leds.size.y, leds.size.z, leds.nrOfLeds);
-          mdl->setValue(var, JsonString(message, JsonString::Copied), rowNr);
+          print->fFormat(message, sizeof(message)-1, "%d x %d x %d = %d", leds->size.x, leds->size.y, leds->size.z, leds->nrOfLeds);
+          mdl->setValue(var, JsonString(message, JsonString::Copied), leds - fixture.ledsList.begin()); //rowNr
         }
         return true;
       case f_UIFun:

src/Sys/SysModFiles.cpp

@@ -151,7 +151,7 @@ void SysModFiles::dirToJson(JsonArray array, bool nameOnly, const char * filter)
         array.add(JsonString(file.name(), JsonString::Copied));
       }
       else {
-        JsonArray row = array.createNestedArray();
+        JsonArray row = array.add<JsonArray>();
         row.add(JsonString(file.name(), JsonString::Copied));
         row.add(file.size());
         char urlString[32] = "file/";

src/Sys/SysModModel.cpp

@@ -15,11 +15,11 @@
 #include "SysJsonRDWS.h"
 #include "SysModUI.h"
 
-BasicJsonDocument<RAM_Allocator> * SysModModel::model = nullptr;
+JsonDocument * SysModModel::model = nullptr;
 JsonObject SysModModel::modelParentVar;
 
 SysModModel::SysModModel() :SysModule("Model") {
-  model = new BasicJsonDocument<RAM_Allocator>(24576);
+  model = new JsonDocument(&allocator);
 
   JsonArray root = model->to<JsonArray>(); //create
 
@@ -38,20 +38,6 @@ void SysModModel::setup() {
   parentVar = ui->initSysMod(parentVar, name);
   if (parentVar["o"] > -1000) parentVar["o"] = -4000; //set default order. Don't use auto generated order as order can be changed in the ui (WIP)
 
-  ui->initProgress(parentVar, "mSize", UINT16_MAX, 0, model->capacity(), true, [](JsonObject var, uint8_t rowNr, uint8_t funType) { switch (funType) { //varFun
-    case f_ValueFun:
-      mdl->setValue(var, model->memoryUsage());
-      return true;
-    case f_UIFun:
-      ui->setLabel(var, "Size");
-      return true;
-    case f_ChangeFun:
-      var["max"] = model->capacity(); //makes sense?
-      web->addResponseV(var["id"], "comment", "%d / %d B", model->memoryUsage(), model->capacity());
-      return true;
-    default: return false;
-  }});
-
   ui->initButton(parentVar, "saveModel", false, [this](JsonObject var, uint8_t rowNr, uint8_t funType) { switch (funType) { //varFun
     case f_UIFun:
       ui->setComment(var, "Write to model.json");
@@ -104,16 +90,6 @@ void SysModModel::setup() {
 
     doWriteModel = false;
   }
-
-  if (model->memoryUsage() / model->capacity() > 0.95) {
-    print->printJDocInfo("model", *model);
-    size_t memBefore = model->memoryUsage();
-    model->garbageCollect();
-    print->printJDocInfo("garbageCollect", *model);
-  }
-}
-void SysModModel::loop10s() {
-  ui->callVarFun(mdl->findVar("mSize"));
 }
 
 void SysModModel::cleanUpModel(JsonArray vars, bool oPos, bool ro) {
@@ -201,13 +177,13 @@ void SysModModel::findVars(const char * property, bool value, FindFun fun, JsonA
 void SysModModel::varToValues(JsonObject var, JsonArray row) {
 
     //add value for each child
-    // JsonArray row = rows.createNestedArray();
+    // JsonArray row = rows.add<JsonArray>();
     for (JsonObject childVar : var["n"].as<JsonArray>()) {
       print->printJson("fxTbl childs", childVar);
       row.add(childVar["value"]);
 
       if (!childVar["n"].isNull()) {
-        varToValues(childVar, row.createNestedArray());
+        varToValues(childVar, row.add<JsonArray>());
       }
     }
 }

src/Sys/SysModModel.h

@@ -102,18 +102,18 @@ namespace ArduinoJson {
   };
 }
 
-// https://arduinojson.org/v6/api/basicjsondocument/
-struct RAM_Allocator {
-  void* allocate(size_t size) {
+// https://arduinojson.org/v7/api/jsondocument/
+struct RAM_Allocator: ArduinoJson::Allocator {
+  void* allocate(size_t size) override {
     if (psramFound()) return ps_malloc(size); // use PSRAM if it exists
     else              return malloc(size);    // fallback
     // return heap_caps_malloc(size, MALLOC_CAP_SPIRAM);
   }
-  void deallocate(void* pointer) {
+  void deallocate(void* pointer) override {
     free(pointer);
     // heap_caps_free(pointer);
   }
-  void* reallocate(void* ptr, size_t new_size) {
+  void* reallocate(void* ptr, size_t new_size) override {
     if (psramFound()) return ps_realloc(ptr, new_size); // use PSRAM if it exists
     else              return realloc(ptr, new_size);    // fallback
     // return heap_caps_realloc(ptr, new_size, MALLOC_CAP_SPIRAM);
@@ -127,8 +127,9 @@ class SysModModel:public SysModule {
 public:
 
   // StaticJsonDocument<24576> model; //not static as that blows up the stack. Use extern??
-  // static BasicJsonDocument<DefaultAllocator> *model; //needs to be static as loopTask and asyncTask is using it...
-  static BasicJsonDocument<RAM_Allocator> *model; //needs to be static as loopTask and asyncTask is using it...
+  // static JsonDocument<DefaultAllocator> *model; //needs to be static as loopTask and asyncTask is using it...
+  RAM_Allocator allocator;
+  static JsonDocument *model; //needs to be static as loopTask and asyncTask is using it...
 
   static JsonObject modelParentVar;
 
@@ -137,8 +138,7 @@ class SysModModel:public SysModule {
   SysModModel();
   void setup();
   void loop();
-  void loop10s();
-
+
   //scan all vars in the model and remove vars where var["o"] is negative or positive, if ro then remove ro values
   void cleanUpModel(JsonArray vars, bool oPos = true, bool ro = false);
 
@@ -187,7 +187,7 @@ class SysModModel:public SysModule {
       if (var["value"].isNull() || !var["value"].is<JsonArray>()) {
         USER_PRINTF("setValue var %s[%d] value %s not array, creating\n", jsonToChar(var, "id"), rowNr, var["value"].as<String>().c_str());
         // print->printJson("setValueB var %s value %s not array, creating", id, var["value"].as<String>().c_str());
-        var.createNestedArray("value");
+        var["value"].to<JsonArray>();
       }
 
       if (var["value"].is<JsonArray>()) {

src/Sys/SysModPrint.cpp

@@ -180,11 +180,6 @@ size_t SysModPrint::fFormat(char * buf, size_t size, const char * format, ...) {
   return len;
 }
 
-void SysModPrint::printJDocInfo(const char * text, DynamicJsonDocument source) {
-  uint8_t percentage;
-  if (source.capacity() == 0)
-    percentage = 0;
-  else
-    percentage = 100 * source.memoryUsage() / source.capacity();
-  print("%s  %u / %u (%u%%) (%u %u %u)\n", text, source.memoryUsage(), source.capacity(), percentage, source.size(), source.overflowed(), source.nesting());
+void SysModPrint::printJDocInfo(const char * text, JsonDocument source) {
+  print("%s (%u %u %u)\n", text, source.size(), source.overflowed(), source.nesting());
 }

src/Sys/SysModPrint.h

@@ -36,7 +36,7 @@ class SysModPrint:public SysModule {
 
   size_t fFormat(char * buf, size_t size, const char * format, ...);
 
-  void printJDocInfo(const char * text, DynamicJsonDocument source);
+  void printJDocInfo(const char * text, JsonDocument source);
 
 private:
   bool setupsDone = false;

src/Sys/SysModUI.cpp

@@ -99,10 +99,10 @@ JsonObject SysModUI::initVar(JsonObject parent, const char * id, const char * ty
     USER_PRINTF("initVar create new %s var: %s->%s\n", type, parentId, id);
     if (parent.isNull()) {
       JsonArray vars = mdl->model->as<JsonArray>();
-      var = vars.createNestedObject();
+      var = vars.add<JsonObject>();
     } else {
-      if (parent["n"].isNull()) parent.createNestedArray("n"); //if parent exist and no "n" array, create it
-      var = parent["n"].createNestedObject();
+      if (parent["n"].isNull()) parent["n"].to<JsonArray>(); //if parent exist and no "n" array, create it
+      var = parent["n"].add<JsonObject>();
       // serializeJson(model, Serial);Serial.println();
     }
     var["id"] = JsonString(id, JsonString::Copied);

src/Sys/SysModWeb.cpp

@@ -30,8 +30,8 @@
 WebServer * SysModWeb::server = nullptr;
 WebSocket * SysModWeb::ws = nullptr;
 
-DynamicJsonDocument * SysModWeb::responseDocLoopTask = nullptr;
-DynamicJsonDocument * SysModWeb::responseDocAsyncTCP = nullptr;
+JsonDocument * SysModWeb::responseDocLoopTask = nullptr;
+JsonDocument * SysModWeb::responseDocAsyncTCP = nullptr;
 bool SysModWeb::clientsChanged = false;
 
 unsigned long SysModWeb::sendDataWsCounter = 0;
@@ -52,8 +52,8 @@ SysModWeb::SysModWeb() :SysModule("Web") {
   DefaultHeaders::Instance().addHeader(F("Access-Control-Allow-Methods"), "*");
   DefaultHeaders::Instance().addHeader(F("Access-Control-Allow-Headers"), "*");
 
-  responseDocLoopTask = new DynamicJsonDocument(2048); responseDocLoopTask->to<JsonObject>();
-  responseDocAsyncTCP = new DynamicJsonDocument(3072); responseDocAsyncTCP->to<JsonObject>();
+  responseDocLoopTask = new JsonDocument; responseDocLoopTask->to<JsonObject>();
+  responseDocAsyncTCP = new JsonDocument; responseDocAsyncTCP->to<JsonObject>();
 };
 
 void SysModWeb::setup() {
@@ -552,7 +552,7 @@ void SysModWeb::clientsToJson(JsonArray array, bool nameOnly, const char * filte
       array.add(JsonString(client->remoteIP().toString().c_str(), JsonString::Copied));
     } else {
       // USER_PRINTF("Client %d %d ...%d\n", client->id(), client->queueIsFull(), client->remoteIP()[3]);
-      JsonArray row = array.createNestedArray();
+      JsonArray row = array.add<JsonArray>();
       row.add(client->id());
       array.add(JsonString(client->remoteIP().toString().c_str(), JsonString::Copied));
       row.add(client->queueIsFull());
@@ -590,9 +590,9 @@ void SysModWeb::serveJson(WebRequest *request) {
   // return model.json
   if (request->url().indexOf("mdl") > 0) {
     JsonArray model = mdl->model->as<JsonArray>();
-    USER_PRINTF("serveJson model ...%d, %s %d %d %d %d\n", request->client()->remoteIP()[3], request->url().c_str(), model.size(),  measureJson(model), model.memoryUsage(), mdl->model->capacity());
+    USER_PRINTF("serveJson model ...%d, %s %d %d\n", request->client()->remoteIP()[3], request->url().c_str(), model.size(),  measureJson(model));
 
-    response = new AsyncJsonResponse(true,  model.memoryUsage()); //array tbd: here copy is mode, see WLED for using reference
+    response = new AsyncJsonResponse(true); //array, removed size as ArduinoJson v7 doesnt care (tbd: here copy is mode, see WLED for using reference)
     JsonArray root = response->getRoot();
 
     // root = model does not work? so add each element individually
@@ -613,12 +613,12 @@ void SysModWeb::serveJson(WebRequest *request) {
   // }
   else { //WLED compatible
     USER_PRINTF("serveJson ...%d, %s\n", request->client()->remoteIP()[3], request->url().c_str());
-    response = new AsyncJsonResponse(false, 5000); //object
+    response = new AsyncJsonResponse(false); //object. removed size as ArduinoJson v7 doesnt care
     JsonObject root = response->getRoot();
 
     //temporary set all WLED variables (as otherwise WLED-native does not show the instance): tbd: clean up (state still needed, info not)
     const char* jsonState = "{\"transition\":7,\"ps\":9,\"pl\":-1,\"nl\":{\"on\":false,\"dur\":60,\"mode\":1,\"tbri\":0,\"rem\":-1},\"udpn\":{\"send\":false,\"recv\":true},\"lor\":0,\"mainseg\":0,\"seg\":[{\"id\":0,\"start\":0,\"stop\":144,\"len\":144,\"grp\":1,\"spc\":0,\"of\":0,\"on\":true,\"frz\":false,\"bri\":255,\"cct\":127,\"col\":[[182,15,98,0],[0,0,0,0],[255,224,160,0]],\"fx\":0,\"sx\":128,\"ix\":128,\"pal\":11,\"c1\":8,\"c2\":20,\"c3\":31,\"sel\":true,\"rev\":false,\"mi\":false,\"o1\":false,\"o2\":false,\"o3\":false,\"ssim\":0,\"mp12\":1}]}";
-    StaticJsonDocument<5000> docState;
+    JsonDocument docState;
     deserializeJson(docState, jsonState);
     root["state"] = docState;
 

src/Sys/SysModWeb.h

@@ -109,25 +109,25 @@ class SysModWeb:public SysModule {
   template <typename Type>
   void addResponse(const char * id, const char * key, Type value, uint8_t rowNr = UINT8_MAX) {
     JsonObject responseObject = getResponseObject();
-    // if (responseObject[id].isNull()) responseObject.createNestedObject(id);
+    // if (responseObject[id].isNull()) responseObject[id].to<JsonObject>();;
     if (rowNr == UINT8_MAX)
       responseObject[id][key] = value;
     else {
       if (!responseObject[id][key].is<JsonArray>())
-        responseObject[id].createNestedArray(key);
+        responseObject[id][key].to<JsonArray>();
       responseObject[id][key][rowNr] = value;
     }
   }
 
   JsonArray addResponseA(const char * id, const char * key) {
     JsonObject responseObject = getResponseObject();
-    // if (responseObject[id].isNull()) responseObject.createNestedObject(id);
-    return responseObject[id].createNestedArray(key);
+    // if (responseObject[id].isNull()) responseObject[id].to<JsonObject>();;
+    return responseObject[id][key].to<JsonArray>();
   }
 
   void addResponseV(const char * id, const char * key, const char * format, ...) {
     JsonObject responseObject = getResponseObject();
-    // if (responseObject[id].isNull()) responseObject.createNestedObject(id);
+    // if (responseObject[id].isNull()) responseObject[id].to<JsonObject>();;
 
     va_list args;
     va_start(args, format);
@@ -160,8 +160,8 @@ class SysModWeb:public SysModule {
 
   static WebServer *server;
 
-  static DynamicJsonDocument *responseDocLoopTask;
-  static DynamicJsonDocument *responseDocAsyncTCP;
+  static JsonDocument *responseDocLoopTask;
+  static JsonDocument *responseDocAsyncTCP;
 
   static unsigned long sendDataWsCounter;
 };

src/User/UserModArtNet.h

@@ -37,7 +37,7 @@ class UserModArtNet:public SysModule {
         ui->setLabel(var, "Instance");
         ui->setComment(var, "Instance to send data");
         JsonArray options = ui->setOptions(var);
-        JsonArray instanceObject = options.createNestedArray();
+        JsonArray instanceObject = options.add<JsonArray>();
         instanceObject.add(0);
         instanceObject.add("no sync");
         for (auto node=instances->instances.begin(); node!=instances->instances.end(); ++node) {
@@ -46,7 +46,7 @@ class UserModArtNet:public SysModule {
             strncpy(option, node->ip.toString().c_str(), sizeof(option)-1);
             strncat(option, " ", sizeof(option)-1);
             strncat(option, node->name, sizeof(option)-1);
-            instanceObject = options.createNestedArray();
+            instanceObject = options.add<JsonArray>();
             instanceObject.add(node->ip[3]);
             instanceObject.add(option);
           }

src/User/UserModDDP.h

@@ -54,7 +54,7 @@ class UserModDDP:public SysModule {
         ui->setLabel(var, "Instance");
         ui->setComment(var, "Instance to send data");
         JsonArray options = ui->setOptions(var);
-        JsonArray instanceObject = options.createNestedArray();
+        JsonArray instanceObject = options.add<JsonArray>();
         instanceObject.add(0);
         instanceObject.add("no sync");
         for (auto node=instances->instances.begin(); node!=instances->instances.end(); ++node) {
@@ -63,7 +63,7 @@ class UserModDDP:public SysModule {
             strncpy(option, node->ip.toString().c_str(), sizeof(option)-1);
             strncat(option, " ", sizeof(option)-1);
             strncat(option, node->name, sizeof(option)-1);
-            instanceObject = options.createNestedArray();
+            instanceObject = options.add<JsonArray>();
             instanceObject.add(node->ip[3]);
             instanceObject.add(option);
           }

src/User/UserModInstances.h

@@ -256,15 +256,15 @@ class UserModInstances:public SysModule {
         ui->setLabel(var, "Sync Master");
         ui->setComment(var, "Instance to sync from");
         JsonArray options = ui->setOptions(var);
-        JsonArray instanceObject = options.createNestedArray();
+        JsonArray instanceObject = options.add<JsonArray>();
         instanceObject.add(0);
         instanceObject.add("no sync");
         for (auto instance=instances.begin(); instance!=instances.end(); ++instance) {
           char option[32] = { 0 };
           strncpy(option, instance->ip.toString().c_str(), sizeof(option)-1);
           strncat(option, " ", sizeof(option)-1);
           strncat(option, instance->name, sizeof(option)-1);
-          instanceObject = options.createNestedArray();
+          instanceObject = options.add<JsonArray>();
           instanceObject.add(instance->ip[3]);
           instanceObject.add(option);
         }
@@ -649,7 +649,7 @@ class UserModInstances:public SysModule {
 
           // responseObject["updRow"]["id"] = "insTbl";
           // responseObject["updRow"]["rowNr"] = rowNr;
-          // responseObject["updRow"].createNestedArray("value");
+          // responseObject["updRow"]["value"].to<JsonArray>();
           // addTblRow(responseObject["updRow"]["value"], instance);
 
           // web->sendResponseObject();

src/main.cpp

@@ -84,7 +84,7 @@ void setup() {
     wledAudioMod = new UserModWLEDAudio();
   #endif
 
-  //Reorder with care! If changed make sure mdlEnabled.chFun executes var.createNestedArray("value"); and saveModel! 
+  //Reorder with care! If changed make sure mdlEnabled.chFun executes var["value"].to<JsonArray>(); and saveModel! 
   //Default: add below, not in between
   #ifdef STARMOD_APPMOD_LEDS
     mdls->add(fix);