End oappend v2

tools/all_xml.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Pull all settings pages for comparison
+HOST=$1
+TGT_PATH=$2
+CURL_ARGS="--compressed"
+
+# Replicate one target many times
+function replicate() {
+  for i in {0..10}
+  do  
+    echo -n " http://${HOST}/settings.js?p=$i -o ${TGT_PATH}/$i.xml"
+  done
+}
+read -a TARGETS <<< $(replicate)
+
+mkdir -p ${TGT_PATH}
+curl ${CURL_ARGS} ${TARGETS[@]}

wled00/fcn_declare.h

@@ -302,7 +302,7 @@ class Usermod {
     virtual bool handleButton(uint8_t b) { return false; }                   // button overrides are possible here
     virtual bool getUMData(um_data_t **data) { if (data) *data = nullptr; return false; }; // usermod data exchange [see examples for audio effects]
     virtual void connected() {}                                              // called when WiFi is (re)connected
-    virtual void appendConfigData() {}                                       // helper function called from usermod settings page to add metadata for entry fields
+    virtual void appendConfigData(Print& settingsScript);                    // helper function called from usermod settings page to add metadata for entry fields
     virtual void addToJsonState(JsonObject& obj) {}                          // add JSON objects for WLED state
     virtual void addToJsonInfo(JsonObject& obj) {}                           // add JSON objects for UI Info page
     virtual void readFromJsonState(JsonObject& obj) {}                       // process JSON messages received from web server
@@ -314,6 +314,16 @@ class Usermod {
     virtual void onUpdateBegin(bool) {}                                      // fired prior to and after unsuccessful firmware update
     virtual void onStateChange(uint8_t mode) {}                              // fired upon WLED state change
     virtual uint16_t getId() {return USERMOD_ID_UNSPECIFIED;}
+
+  // API shims
+  private:
+    static Print* oappend_shim;
+    // old form of appendConfigData; called by default appendConfigData(Print&) with oappend_shim set up
+    // private so it is not accidentally invoked except via Usermod::appendConfigData(Print&)
+    virtual void appendConfigData() {}    
+  protected:
+    // Shim for oappend(), which used to exist in utils.cpp
+    template<typename T> static inline void oappend(const T& t) { oappend_shim->print(t); };
 };
 
 class UsermodManager {
@@ -328,7 +338,7 @@ class UsermodManager {
     static bool getUMData(um_data_t **um_data, uint8_t mod_id = USERMOD_ID_RESERVED); // USERMOD_ID_RESERVED will poll all usermods
     static void setup();
     static void connected();
-    static void appendConfigData();
+    static void appendConfigData(Print&);
     static void addToJsonState(JsonObject& obj);
     static void addToJsonInfo(JsonObject& obj);
     static void readFromJsonState(JsonObject& obj);
@@ -362,10 +372,11 @@ void parseNumber(const char* str, byte* val, byte minv=0, byte maxv=255);
 bool getVal(JsonVariant elem, byte* val, byte minv=0, byte maxv=255);
 bool getBoolVal(JsonVariant elem, bool dflt);
 bool updateVal(const char* req, const char* key, byte* val, byte minv=0, byte maxv=255);
-bool oappend(const char* txt); // append new c string to temp buffer efficiently
-bool oappendi(int i);          // append new number to temp buffer efficiently
-void sappend(char stype, const char* key, int val);
-void sappends(char stype, const char* key, char* val);
+size_t printSetFormCheckbox(Print& settingsScript, const char* key, int val);
+size_t printSetFormValue(Print& settingsScript, const char* key, int val);
+size_t printSetFormValue(Print& settingsScript, const char* key, const char* val);
+size_t printSetFormIndex(Print& settingsScript, const char* key, int index);
+size_t printSetClassElementHTML(Print& settingsScript, const char* key, const int index, const char* val);
 void prepareHostname(char* hostname);
 bool isAsterisksOnly(const char* str, byte maxLen);
 bool requestJSONBufferLock(uint8_t module=255);
@@ -444,7 +455,7 @@ void wsEvent(AsyncWebSocket * server, AsyncWebSocketClient * client, AwsEventTyp
 void sendDataWs(AsyncWebSocketClient * client = nullptr);
 
 //xml.cpp
-void XML_response(AsyncWebServerRequest *request, char* dest = nullptr);
-void getSettingsJS(byte subPage, char* dest);
+void XML_response(Print& dest);
+void getSettingsJS(byte subPage, Print& dest);
 
 #endif

wled00/file.cpp

@@ -433,7 +433,7 @@ bool handleFileRead(AsyncWebServerRequest* request, String path){
   }
   #endif
   if(WLED_FS.exists(path) || WLED_FS.exists(path + ".gz")) {
-    request->send(WLED_FS, path, String(), request->hasArg(F("download")));
+    request->send(request->beginResponse(WLED_FS, path, {}, request->hasArg(F("download")), {}));
     return true;
   }
   return false;

wled00/mqtt.cpp

@@ -124,6 +124,32 @@ static void onMqttMessage(char* topic, char* payload, AsyncMqttClientMessageProp
   payloadStr = nullptr;
 }
 
+// Print adapter for flat buffers
+namespace { 
+class bufferPrint : public Print {
+  char* _buf;
+  size_t _size, _offset;
+  public:
+
+  bufferPrint(char* buf, size_t size) : _buf(buf), _size(size), _offset(0) {};
+
+  size_t write(const uint8_t *buffer, size_t size) {
+    size = std::min(size, _size - _offset);
+    memcpy(_buf + _offset, buffer, size);
+    _offset += size;
+    return size;
+  }
+
+  size_t write(uint8_t c) {
+    return this->write(&c, 1);
+  }
+
+  char* data() const { return _buf; }
+  size_t size() const { return _offset; }
+  size_t capacity() const { return _size; }
+};
+}; // anonymous namespace
+
 
 void publishMqtt()
 {
@@ -148,11 +174,13 @@ void publishMqtt()
   strcat_P(subuf, PSTR("/status"));
   mqtt->publish(subuf, 0, true, "online");          // retain message for a LWT
 
-  char apires[1024];                                // allocating 1024 bytes from stack can be risky
-  XML_response(nullptr, apires);
+  // TODO: use a DynamicBufferList.  Requires a list-read-capable MQTT client API.
+  DynamicBuffer buf(1024);
+  bufferPrint pbuf(buf.data(), buf.size());
+  XML_response(pbuf);
   strlcpy(subuf, mqttDeviceTopic, 33);
   strcat_P(subuf, PSTR("/v"));
-  mqtt->publish(subuf, 0, retainMqttMsg, apires);   // optionally retain message (#2263)
+  mqtt->publish(subuf, 0, retainMqttMsg, buf.data(), pbuf.size());   // optionally retain message (#2263)
   #endif
 }
 

wled00/set.cpp

@@ -1191,7 +1191,11 @@ bool handleSet(AsyncWebServerRequest *request, const String& req, bool apply)
 
   // internal call, does not send XML response
   pos = req.indexOf(F("IN"));
-  if (pos < 1) XML_response(request);
+  if (pos < 1) {
+    auto response = request->beginResponseStream("text/xml");
+    XML_response(*response);
+    request->send(response);
+  }
 
   return true;
 }

wled00/um_manager.cpp

@@ -8,7 +8,7 @@ void UsermodManager::setup()             { for (unsigned i = 0; i < numMods; i++
 void UsermodManager::connected()         { for (unsigned i = 0; i < numMods; i++) ums[i]->connected(); }
 void UsermodManager::loop()              { for (unsigned i = 0; i < numMods; i++) ums[i]->loop();  }
 void UsermodManager::handleOverlayDraw() { for (unsigned i = 0; i < numMods; i++) ums[i]->handleOverlayDraw(); }
-void UsermodManager::appendConfigData()  { for (unsigned i = 0; i < numMods; i++) ums[i]->appendConfigData(); }
+void UsermodManager::appendConfigData(Print& dest)  { for (unsigned i = 0; i < numMods; i++) ums[i]->appendConfigData(dest); }
 bool UsermodManager::handleButton(uint8_t b) {
   bool overrideIO = false;
   for (unsigned i = 0; i < numMods; i++) {
@@ -71,3 +71,13 @@ bool UsermodManager::add(Usermod* um)
 
 Usermod* UsermodManager::ums[WLED_MAX_USERMODS] = {nullptr};
 byte UsermodManager::numMods = 0;
+
+/* Usermod v2 interface shim for oappend */
+Print* Usermod::oappend_shim = nullptr;
+
+void Usermod::appendConfigData(Print& settingsScript) {
+  assert(!oappend_shim);
+  oappend_shim = &settingsScript;
+  this->appendConfigData();
+  oappend_shim = nullptr;
+}

wled00/util.cpp

@@ -87,90 +87,30 @@ bool updateVal(const char* req, const char* key, byte* val, byte minv, byte maxv
   return true;
 }
 
-
-//append a numeric setting to string buffer
-void sappend(char stype, const char* key, int val)
-{
-  char ds[] = "d.Sf.";
-
-  switch(stype)
-  {
-    case 'c': //checkbox
-      oappend(ds);
-      oappend(key);
-      oappend(".checked=");
-      oappendi(val);
-      oappend(";");
-      break;
-    case 'v': //numeric
-      oappend(ds);
-      oappend(key);
-      oappend(".value=");
-      oappendi(val);
-      oappend(";");
-      break;
-    case 'i': //selectedIndex
-      oappend(ds);
-      oappend(key);
-      oappend(SET_F(".selectedIndex="));
-      oappendi(val);
-      oappend(";");
-      break;
-  }
+static size_t printSetFormInput(Print& settingsScript, const char* key, const char* selector, int value) {
+  return settingsScript.printf_P(PSTR("d.Sf.%s.%s=%d;"), key, selector, value);
 }
 
-
-//append a string setting to buffer
-void sappends(char stype, const char* key, char* val)
-{
-  switch(stype)
-  {
-    case 's': {//string (we can interpret val as char*)
-      String buf = val;
-      //convert "%" to "%%" to make EspAsyncWebServer happy
-      //buf.replace("%","%%");
-      oappend("d.Sf.");
-      oappend(key);
-      oappend(".value=\"");
-      oappend(buf.c_str());
-      oappend("\";");
-      break;}
-    case 'm': //message
-      oappend(SET_F("d.getElementsByClassName"));
-      oappend(key);
-      oappend(SET_F(".innerHTML=\""));
-      oappend(val);
-      oappend("\";");
-      break;
-  }
+size_t printSetFormCheckbox(Print& settingsScript, const char* key, int val) {
+  return printSetFormInput(settingsScript, key, PSTR("checked"), val);
 }
-
-
-bool oappendi(int i)
-{
-  char s[12]; // 32bit signed number can have 10 digits plus - sign
-  sprintf(s, "%d", i);
-  return oappend(s);
+size_t printSetFormValue(Print& settingsScript, const char* key, int val) {
+  return printSetFormInput(settingsScript, key, PSTR("value"), val);
+}
+size_t printSetFormIndex(Print& settingsScript, const char* key, int index) {
+  return printSetFormInput(settingsScript, key, PSTR("selectedIndex"), index);
 }
 
+size_t printSetFormValue(Print& settingsScript, const char* key, const char* val) {
+  return settingsScript.printf_P(PSTR("d.Sf.%s.value=\"%s\";"),key,val);
+}
 
-bool oappend(const char* txt)
-{
-  unsigned len = strlen(txt);
-  if ((obuf == nullptr) || (olen + len >= SETTINGS_STACK_BUF_SIZE)) { // sanity checks
-#ifdef WLED_DEBUG
-    DEBUG_PRINT(F("oappend() buffer overflow. Cannot append "));
-    DEBUG_PRINT(len); DEBUG_PRINT(F(" bytes \t\""));
-    DEBUG_PRINT(txt); DEBUG_PRINTLN(F("\""));
-#endif
-    return false;        // buffer full
-  }
-  strcpy(obuf + olen, txt);
-  olen += len;
-  return true;
+size_t printSetClassElementHTML(Print& settingsScript, const char* key, const int index, const char* val) {
+  return settingsScript.printf_P(PSTR("d.getElementsByClassName(\"%s\")[%d].innerHTML=\"%s\";"), key, index, val);
 }
 
 
+
 void prepareHostname(char* hostname)
 {
   sprintf_P(hostname, PSTR("wled-%*s"), 6, escapedMac.c_str() + 6);

wled00/wled.h

@@ -839,10 +839,6 @@ WLED_GLOBAL time_t sunrise _INIT(0);
 WLED_GLOBAL time_t sunset _INIT(0);
 WLED_GLOBAL Toki toki _INIT(Toki());
 
-// Temp buffer
-WLED_GLOBAL char* obuf;
-WLED_GLOBAL uint16_t olen _INIT(0);
-
 // General filesystem
 WLED_GLOBAL size_t fsBytesUsed _INIT(0);
 WLED_GLOBAL size_t fsBytesTotal _INIT(0);

wled00/wled_server.cpp

@@ -520,27 +520,23 @@ void serveSettingsJS(AsyncWebServerRequest* request)
     handleStaticContent(request, FPSTR(_common_js), 200, FPSTR(CONTENT_TYPE_JAVASCRIPT), JS_common, JS_common_length);
     return;
   }
-  char buf[SETTINGS_STACK_BUF_SIZE+37];
-  buf[0] = 0;
   byte subPage = request->arg(F("p")).toInt();
   if (subPage > 10) {
-    strcpy_P(buf, PSTR("alert('Settings for this request are not implemented.');"));
-    request->send(501, FPSTR(CONTENT_TYPE_JAVASCRIPT), buf);
+    request->send_P(501, FPSTR(CONTENT_TYPE_JAVASCRIPT), PSTR("alert('Settings for this request are not implemented.');"));
     return;
   }
   if (subPage > 0 && !correctPIN && strlen(settingsPIN)>0) {
-    strcpy_P(buf, PSTR("alert('PIN incorrect.');"));
-    request->send(401, FPSTR(CONTENT_TYPE_JAVASCRIPT), buf);
+    request->send_P(401, FPSTR(CONTENT_TYPE_JAVASCRIPT), PSTR("alert('PIN incorrect.');"));
     return;
   }
-  strcat_P(buf,PSTR("function GetV(){var d=document;"));
-  getSettingsJS(subPage, buf+strlen(buf));  // this may overflow by 35bytes!!!
-  strcat_P(buf,PSTR("}"));
 
-  AsyncWebServerResponse *response;
-  response = request->beginResponse(200, FPSTR(CONTENT_TYPE_JAVASCRIPT), buf);
+  AsyncResponseStream *response = request->beginResponseStream(FPSTR(CONTENT_TYPE_JAVASCRIPT));
   response->addHeader(F("Cache-Control"), F("no-store"));
   response->addHeader(F("Expires"), F("0"));
+
+  response->print(F("function GetV(){var d=document;"));
+  getSettingsJS(subPage, *response);
+  response->print(F("}"));
   request->send(response);
 }