From ffe2ce7859fa1ddf0b75dd544d30d005dfcc6bb0 Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Mon, 12 Feb 2024 09:25:39 -0500 Subject: [PATCH 01/10] Fixed default ssid and pasphrase. --- pio_examples/gui/src/gui.ino | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pio_examples/gui/src/gui.ino b/pio_examples/gui/src/gui.ino index 8953a0d..f60ffdc 100644 --- a/pio_examples/gui/src/gui.ino +++ b/pio_examples/gui/src/gui.ino @@ -11,8 +11,8 @@ DNSServer dnsServer; #include #endif -const char* ssid = "MaRtInG"; -const char* password = "martinshomenetwork"; +const char* ssid = "YourNetworkName"; +const char* password = "YourNetworkPassphrase"; const char* hostname = "espui"; From f472dc1158bb29392fdb28fe3d4a2a1b14a5d06c Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Thu, 29 Feb 2024 16:32:04 -0500 Subject: [PATCH 02/10] Added mechanism to estimate the marshaled size of a control. --- src/ESPUIcontrol.cpp | 12 ++++++++++-- src/ESPUIcontrol.h | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/ESPUIcontrol.cpp b/src/ESPUIcontrol.cpp index 44002ac..554e7d5 100644 --- a/src/ESPUIcontrol.cpp +++ b/src/ESPUIcontrol.cpp @@ -19,6 +19,7 @@ Control::Control(ControlType type, const char* label, std::function Date: Thu, 29 Feb 2024 16:34:20 -0500 Subject: [PATCH 03/10] Arduino 7 does not report an overflow util heap is exhausted. Added a mechanism to estimate how much memory the doc is using so we can limit the number of entries in the messages --- src/ESPUIclient.cpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ESPUIclient.cpp b/src/ESPUIclient.cpp index a0b6cc3..4670f1c 100644 --- a/src/ESPUIclient.cpp +++ b/src/ESPUIclient.cpp @@ -269,6 +269,8 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // Serial.println(String("prepareJSONChunk: Start. InUpdateMode: ") + String(InUpdateMode)); int elementcount = 0; + uint32_t MaxEstimatedMarshaledJsonSize = (!InUpdateMode) ? ESPUI.jsonInitialDocumentSize: ESPUI.jsonUpdateDocumentSize; + uint32_t TotalEstimatedMarshaledJsonSize = 0; do // once { @@ -386,11 +388,16 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, } } + TotalEstimatedMarshaledJsonSize += control->GetEstimatedMarshaledSize(); + bool DocWillOverflow = TotalEstimatedMarshaledJsonSize >= MaxEstimatedMarshaledJsonSize; JsonObject item = items.createNestedObject(); elementcount++; - control->MarshalControl(item, InUpdateMode, DataOffset); - - if (rootDoc.overflowed() || (ESPUI.jsonChunkNumberMax > 0 && (elementcount % ESPUI.jsonChunkNumberMax) == 0)) + if(!DocWillOverflow) + { + control->MarshalControl(item, InUpdateMode, DataOffset); + } + + if (DocWillOverflow || (ESPUI.jsonChunkNumberMax > 0 && (elementcount % ESPUI.jsonChunkNumberMax) == 0)) { // String("prepareJSONChunk: too much data in the message. Remove the last entry"); if (1 == elementcount) From 09aede269baa33a33fb1feea87464db79c857e57 Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Tue, 5 Mar 2024 13:33:48 -0500 Subject: [PATCH 04/10] Added some debug output --- data/js/controls.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/data/js/controls.js b/data/js/controls.js index 908dd1a..6d9fff0 100644 --- a/data/js/controls.js +++ b/data/js/controls.js @@ -225,7 +225,7 @@ function handleVisibilityChange() { function start() { let location = window.location.hostname; let port = window.location.port; -// let location = "192.168.10.219"; +// let location = "192.168.10.198"; // let port = ""; document.addEventListener("visibilitychange", handleVisibilityChange, false); @@ -655,6 +655,7 @@ function start() { break; case UI_FRAGMENT: + // console.info("Starting Fragment Processing"); let FragmentLen = data.length; let FragementOffset = data.offset; let NextFragmentOffset = FragementOffset + FragmentLen; @@ -671,6 +672,7 @@ function start() { if (!data.hasOwnProperty('control')) { console.error("UI_FRAGMENT:Missing control record, skipping control"); + // console.info("Done Fragment Processing"); break; } let control = data.control; @@ -686,7 +688,8 @@ function start() { StartFragmentAssemblyTimer(control.id); let TotalRequest = JSON.stringify({ 'id' : control.id, 'offset' : NextFragmentOffset }); websock.send("uifragmentok:" + 0 + ": " + TotalRequest + ":"); - // console.info("asked for fragment 2"); + // console.info("asked for fragment " + TotalRequest); + // console.info("Done Fragment Processing"); break; } @@ -698,7 +701,8 @@ function start() { StartFragmentAssemblyTimer(control.id); let TotalRequest = JSON.stringify({ 'id' : control.id, 'offset' : 0 }); websock.send("uifragmentok:" + 0 + ": " + TotalRequest + ":"); - // console.info("asked for fragment 1"); + // console.info("asked for fragment " + TotalRequest); + // console.info("Done Fragment Processing"); break; } @@ -709,7 +713,8 @@ function start() { StartFragmentAssemblyTimer(control.id); let TotalRequest = JSON.stringify({ 'id' : control.id, 'offset' : controlAssemblyArray[control.id].length + controlAssemblyArray[control.id].offset }); websock.send("uifragmentok:" + 0 + ": " + TotalRequest + ":"); - // console.info("asked for the expected fragment"); + // console.info("asked for the expected fragment: " + TotalRequest); + // console.info("Done Fragment Processing"); break; } @@ -733,7 +738,9 @@ function start() { StartFragmentAssemblyTimer(control.id); let TotalRequest = JSON.stringify({ 'id' : control.id, 'offset' : NextFragmentOffset}); websock.send("uifragmentok:" + 0 + ": " + TotalRequest + ":"); + // console.info("asked for the next fragment: " + TotalRequest); } + // console.info("Done Fragment Processing"); break; default: From e6e4d6b09cd5d54f66c41dc133ea6d4553a1402e Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Tue, 5 Mar 2024 13:36:18 -0500 Subject: [PATCH 05/10] Added a fragmentation test object Made compilation of some of the other corner case controls optional --- pio_examples/gui/src/gui.ino | 41 ++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/pio_examples/gui/src/gui.ino b/pio_examples/gui/src/gui.ino index f60ffdc..18fcc1d 100644 --- a/pio_examples/gui/src/gui.ino +++ b/pio_examples/gui/src/gui.ino @@ -16,12 +16,22 @@ const char* password = "YourNetworkPassphrase"; const char* hostname = "espui"; +#ifdef TEST_FILEDISPLAY String DisplayTestFileName = "/FileName.txt"; +int fileDisplayId = Control::noParent; +#endif // def TEST_FILEDISPLAY + int statusLabelId = Control::noParent; + +#ifdef TEST_GRAPH int graphId = Control::noParent; +#endif // def TEST_GRAPH int millisLabelId = Control::noParent; int testSwitchId = Control::noParent; -int fileDisplayId = Control::noParent; + +#ifdef TEST_HUGE_TEXT +char HugeText[1025]; +#endif // def TEST_HUGE_TEXT void numberCall(Control* sender, int type) { @@ -166,6 +176,11 @@ void setup(void) ESPUI.setVerbosity(Verbosity::VerboseJSON); Serial.begin(115200); +#ifdef TEST_HUGE_TEXT + memset(HugeText, 0x0, sizeof(HugeText)); + memset(HugeText, 'a', sizeof(HugeText)-1); +#endif // def TEST_HUGE_TEXT + #if defined(ESP32) WiFi.setHostname(hostname); #else @@ -239,10 +254,20 @@ void setup(void) ESPUI.slider("Slider one", &slider, ControlColor::Alizarin, 30, 0, 30); ESPUI.slider("Slider two", &slider, ControlColor::None, 100); ESPUI.text("Text Test:", &textCall, ControlColor::Alizarin, "a Text Field"); - ESPUI.number("Numbertest", &numberCall, ControlColor::Alizarin, 5, 0, 10); - fileDisplayId = ESPUI.fileDisplay("Filetest", ControlColor::Turquoise, DisplayTestFileName); +#ifdef TEST_HUGE_TEXT + ESPUI.text("Huge Text Test:", &textCall, ControlColor::Alizarin, HugeText); +#endif // def TEST_HUGE_TEXT + + ESPUI.number("Numbertest", &numberCall, ControlColor::Alizarin, 5, 0, 10); + +#ifdef TEST_FILEDISPLAY + fileDisplayId = ESPUI.fileDisplay("Filetest", ControlColor::Turquoise, DisplayTestFileName); +#endif // def TEST_FILEDISPLAY + +#ifdef TEST_GRAPH graphId = ESPUI.graph("Graph Test", ControlColor::Wetasphalt); +#endif // def TEST_GRAPH /* * .begin loads and serves all files from PROGMEM directly. @@ -261,8 +286,9 @@ void setup(void) * password, for example begin("ESPUI Control", "username", "password") */ ESPUI.sliderContinuous = true; + +#ifdef TEST_FILEDISPLAY ESPUI.prepareFileSystem(); - ESPUI.beginLITTLEFS("ESPUI Control"); // create a text file ESPUI.writeFile("/DisplayFile.txt", "Test Line\n"); @@ -270,6 +296,9 @@ void setup(void) // these files are used by browsers to auto config a connection. ESPUI.writeFile("/wpad.dat", " "); ESPUI.writeFile("/connecttest.txt", " "); +#endif // def TEST_FILEDISPLAY + + ESPUI.beginLITTLEFS("ESPUI Control"); } void loop(void) @@ -284,11 +313,14 @@ void loop(void) { ESPUI.print(millisLabelId, String(millis())); +#ifdef TEST_GRAPH ESPUI.addGraphPoint(graphId, random(1, 50)); +#endif // def TEST_GRAPH testSwitchState = !testSwitchState; ESPUI.updateSwitcher(testSwitchId, testSwitchState); +#ifdef TEST_FILEDISPLAY // update the file Display file. File testFile = ESPUI.EspuiLittleFS.open(String("/") + DisplayTestFileName, "a"); uint32_t filesize = testFile.size(); @@ -303,6 +335,7 @@ void loop(void) // Serial.println(TestLine); } testFile.close(); +#endif // def TEST_FILEDISPLAY oldTime = millis(); } From f2146309fe9829b945f08e48e1d407d7f4cc69ab Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Tue, 5 Mar 2024 13:40:01 -0500 Subject: [PATCH 06/10] Reworked control marshaling code to be aware of and enforce size limits by starting to fragment sooner. --- pio_examples/gui/platformio.ini | 3 +- src/ESPUIclient.cpp | 46 ++++++++++----- src/ESPUIcontrol.cpp | 99 ++++++++++++++++++++++++--------- src/ESPUIcontrol.h | 15 ++--- 4 files changed, 111 insertions(+), 52 deletions(-) diff --git a/pio_examples/gui/platformio.ini b/pio_examples/gui/platformio.ini index fda3d38..ae409ea 100644 --- a/pio_examples/gui/platformio.ini +++ b/pio_examples/gui/platformio.ini @@ -41,7 +41,8 @@ platform = espressif32 board = esp32dev monitor_filters = esp32_exception_decoder board_build.flash_mode = dout -;build_flags = +build_flags = + -D TEST_HUGE_TEXT ; -D DEBUG_ESPUI lib_deps = ${env.lib_deps} diff --git a/src/ESPUIclient.cpp b/src/ESPUIclient.cpp index 4670f1c..df4c077 100644 --- a/src/ESPUIclient.cpp +++ b/src/ESPUIclient.cpp @@ -267,10 +267,12 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, xSemaphoreTake(ESPUI.ControlsSemaphore, portMAX_DELAY); #endif // def ESP32 - // Serial.println(String("prepareJSONChunk: Start. InUpdateMode: ") + String(InUpdateMode)); + // Serial.println(String("prepareJSONChunk: Start. InUpdateMode: ") + String(InUpdateMode)); + // Serial.println(String("prepareJSONChunk: Start. startindex: ") + String(startindex)); + // Serial.println(String("prepareJSONChunk: Start. FragmentRequestString: '") + FragmentRequestString + "'"); int elementcount = 0; - uint32_t MaxEstimatedMarshaledJsonSize = (!InUpdateMode) ? ESPUI.jsonInitialDocumentSize: ESPUI.jsonUpdateDocumentSize; - uint32_t TotalEstimatedMarshaledJsonSize = 0; + uint32_t MaxMarshaledJsonSize = (!InUpdateMode) ? ESPUI.jsonInitialDocumentSize: ESPUI.jsonUpdateDocumentSize; + uint32_t EstimatedUsedMarshaledJsonSize = 0; do // once { @@ -388,21 +390,32 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, } } - TotalEstimatedMarshaledJsonSize += control->GetEstimatedMarshaledSize(); - bool DocWillOverflow = TotalEstimatedMarshaledJsonSize >= MaxEstimatedMarshaledJsonSize; + // Serial.println(String(F("prepareJSONChunk: MaxMarshaledJsonSize: ")) + String(MaxMarshaledJsonSize)); + // Serial.println(String(F("prepareJSONChunk: Cur EstimatedUsedMarshaledJsonSize: ")) + String(EstimatedUsedMarshaledJsonSize)); + JsonObject item = items.createNestedObject(); elementcount++; - if(!DocWillOverflow) - { - control->MarshalControl(item, InUpdateMode, DataOffset); - } + uint32_t RemainingSpace = (MaxMarshaledJsonSize - EstimatedUsedMarshaledJsonSize) - 100; + // Serial.println(String(F("prepareJSONChunk: RemainingSpace: ")) + String(RemainingSpace)); + uint32_t SpaceUsedByMarshaledControl = 0; + bool ControlIsFragmented = control->MarshalControl(item, + InUpdateMode, + DataOffset, + RemainingSpace, + SpaceUsedByMarshaledControl); + // Serial.println(String(F("prepareJSONChunk: SpaceUsedByMarshaledControl: ")) + String(SpaceUsedByMarshaledControl)); + EstimatedUsedMarshaledJsonSize += SpaceUsedByMarshaledControl; + // Serial.println(String(F("prepareJSONChunk: New EstimatedUsedMarshaledJsonSize: ")) + String(EstimatedUsedMarshaledJsonSize)); + // Serial.println(String(F("prepareJSONChunk: ControlIsFragmented: ")) + String(ControlIsFragmented)); - if (DocWillOverflow || (ESPUI.jsonChunkNumberMax > 0 && (elementcount % ESPUI.jsonChunkNumberMax) == 0)) + // did the control get added to the doc? + if (0 == SpaceUsedByMarshaledControl || + (ESPUI.jsonChunkNumberMax > 0 && (elementcount % ESPUI.jsonChunkNumberMax) == 0)) { - // String("prepareJSONChunk: too much data in the message. Remove the last entry"); + // Serial.println( String("prepareJSONChunk: too much data in the message. Remove the last entry")); if (1 == elementcount) { - Serial.println(String(F("ERROR: prepareJSONChunk: Control ")) + String(control->id) + F(" is too large to be sent to the browser.")); + // Serial.println(String(F("prepareJSONChunk: Control ")) + String(control->id) + F(" is too large to be sent to the browser.")); // Serial.println(String(F("ERROR: prepareJSONChunk: value: ")) + control->value); rootDoc.clear(); item = items.createNestedObject(); @@ -420,13 +433,16 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // exit the loop control = nullptr; } - else if (SingleControl) + else if ((SingleControl) || + (ControlIsFragmented) || + (MaxMarshaledJsonSize < (EstimatedUsedMarshaledJsonSize + 100))) { - // Serial.println("prepareJSONChunk: exit loop"); + // Serial.println("prepareJSONChunk: Doc is Full, Fragmented Control or Single Control. exit loop"); control = nullptr; } else { + // Serial.println("prepareJSONChunk: Next Control"); control = control->next; } } // end while (control != nullptr) @@ -437,7 +453,7 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, xSemaphoreGive(ESPUI.ControlsSemaphore); #endif // def ESP32 - // Serial.println(String("prepareJSONChunk: elementcount: ") + String(elementcount)); + // Serial.println(String("prepareJSONChunk: END: elementcount: ") + String(elementcount)); return elementcount; } diff --git a/src/ESPUIcontrol.cpp b/src/ESPUIcontrol.cpp index 554e7d5..a6d8f41 100644 --- a/src/ESPUIcontrol.cpp +++ b/src/ESPUIcontrol.cpp @@ -19,7 +19,6 @@ Control::Control(ControlType type, const char* label, std::function maxLength) || StartingOffset) - { - /* - Serial.println(String("MarshalControl:Start Fragment Processing")); - Serial.println(String("MarshalControl:id: ") + String(id)); - Serial.println(String("MarshalControl:length: ") + String(length)); - Serial.println(String("MarshalControl:StartingOffset: ") + String(StartingOffset)); - Serial.println(String("MarshalControl:maxLength: ") + String(maxLength)); + // this code assumes MaxMarshaledLength > JsonMarshalingRatio + // Serial.println(String("MarshalControl: StartingOffset: ") + String(StartingOffset)); + // Serial.println(String("MarshalControl: AvailMarshaledLength: ") + String(AvailMarshaledLength)); + // Serial.println(String("MarshalControl: Control ID: ") + String(id)); + bool ControlIsFragmented = false; + // create a new item in the response document + JsonObject & item = _item; + + // how much space do we expect to use? + uint32_t ValueMarshaledLength = (value.length() - StartingOffset) * JsonMarshalingRatio; + uint32_t LabelMarshaledLength = strlen(label) * JsonMarshalingRatio; + uint32_t MinimumMarshaledLength = LabelMarshaledLength + JsonMarshaledOverhead; + uint32_t MaximumMarshaledLength = ValueMarshaledLength + MinimumMarshaledLength; + uint32_t SpaceForMarshaledValue = AvailMarshaledLength - MinimumMarshaledLength; + // Serial.println(String("MarshalControl: value.length(): ") + String(value.length())); + // Serial.println(String("MarshalControl: ValueMarshaledLength: ") + String(ValueMarshaledLength)); + // Serial.println(String("MarshalControl: LabelMarshaledLength: ") + String(LabelMarshaledLength)); + // Serial.println(String("MarshalControl: MaximumMarshaledLength: ") + String(MaximumMarshaledLength)); + // Serial.println(String("MarshalControl: MinimumMarshaledLength: ") + String(MinimumMarshaledLength)); + // Serial.println(String("MarshalControl: SpaceForMarshaledValue: ") + String(SpaceForMarshaledValue)); + + // will the item fit in the remaining space? Fragment if not + if (AvailMarshaledLength < MinimumMarshaledLength) + { + // Serial.println(String("MarshalControl: Cannot Marshal control. Not enough space for basic headers.")); + EstimatedMarshaledLength = 0; + return false; + } + + uint32_t MaxValueLength = (SpaceForMarshaledValue / JsonMarshalingRatio); + // Serial.println(String("MarshalControl: MaxValueLength: ") + String(MaxValueLength)); + + uint32_t ValueLenToSend = min((value.length() - StartingOffset), MaxValueLength); + // Serial.println(String("MarshalControl: ValueLenToSend: ") + String(ValueLenToSend)); + + uint32_t AdjustedMarshaledLength = (ValueLenToSend * JsonMarshalingRatio) + MinimumMarshaledLength; + // Serial.println(String("MarshalControl: AdjustedMarshaledLength: ") + String(AdjustedMarshaledLength)); + + bool NeedToFragment = (ValueLenToSend < value.length()); + // Serial.println(String("MarshalControl: NeedToFragment: ") + String(NeedToFragment)); + + if ((AdjustedMarshaledLength > AvailMarshaledLength) && (0 != ValueLenToSend)) + { + // Serial.println(String("MarshalControl: Cannot Marshal control. Not enough space for marshaled control.")); + EstimatedMarshaledLength = 0; + return false; + } + + EstimatedMarshaledLength = AdjustedMarshaledLength; + + // are we fragmenting? + if(NeedToFragment || StartingOffset) + { + // Serial.println(String("MarshalControl:Start Fragment Processing")); + // Serial.println(String("MarshalControl:id: ") + String(id)); + // Serial.println(String("MarshalControl:StartingOffset: ") + String(StartingOffset)); +/* if(0 == StartingOffset) { Serial.println(String("MarshalControl: New control to fragement. ID: ") + String(id)); @@ -72,17 +120,18 @@ void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t Starting { Serial.println(String("MarshalControl: Next fragement. ID: ") + String(id)); } - */ - +*/ // indicate that no additional controls should be sent + ControlIsFragmented = true; + + // fill in the fragment header _item[F("type")] = uint32_t(ControlType::Fragment); _item[F("id")] = id; - length = min((length - StartingOffset), maxLength); // Serial.println(String("MarshalControl:Final length: ") + String(length)); _item[F("offset")] = StartingOffset; - _item[F("length")] = length; + _item[F("length")] = ValueLenToSend; _item[F("total")] = value.length(); item = _item.createNestedObject(F("control")); } @@ -99,7 +148,7 @@ void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t Starting } item[F("label")] = label; - item[F ("value")] = (ControlType::Password == type) ? F ("--------") : value.substring(StartingOffset, length + StartingOffset); + item[F ("value")] = (ControlType::Password == type) ? F ("--------") : value.substring(StartingOffset, StartingOffset + ValueLenToSend); item[F("visible")] = visible; item[F("color")] = (int)color; item[F("enabled")] = enabled; @@ -132,6 +181,9 @@ void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t Starting item[F("selected")] = ""; } } + + // Serial.println(String("MarshalControl:Done")); + return ControlIsFragmented; } void Control::MarshalErrorMessage(JsonObject & item) @@ -282,10 +334,3 @@ void Control::onWsEvent(String & cmd, String& data) } } while (false); } - -void Control::EstimateMarshaledSize() -{ - EstimatedMarshaledSize = MarshalingOverhead + (JsonMarshalingRatio * (strlen(label) + value.length())); - -} // EstimateSerializedSize - diff --git a/src/ESPUIcontrol.h b/src/ESPUIcontrol.h index a84d094..1701ae0 100644 --- a/src/ESPUIcontrol.h +++ b/src/ESPUIcontrol.h @@ -83,26 +83,23 @@ public: void SendCallback(int type); bool HasCallback() { return (nullptr != callback); } - void MarshalControl(ArduinoJson::JsonObject& item, bool refresh, uint32_t DataOffset); + bool MarshalControl(ArduinoJson::JsonObject& item, bool refresh, uint32_t DataOffset, uint32_t MaxLength, uint32_t & EstimmatedUsedSpace); void MarshalErrorMessage(ArduinoJson::JsonObject& item); void DeleteControl(); void onWsEvent(String& cmd, String& data); inline bool ToBeDeleted() { return _ToBeDeleted; } inline bool NeedsSync(uint32_t lastControlChangeID) {return (false == _ToBeDeleted) && (lastControlChangeID < ControlChangeID);} - void SetControlChangedId(uint32_t value) {ControlChangeID = value; EstimateMarshaledSize();} - uint32_t GetEstimatedMarshaledSize() {return EstimatedMarshaledSize;} + void SetControlChangedId(uint32_t value) {ControlChangeID = value;} private: bool _ToBeDeleted = false; uint32_t ControlChangeID = 0; - uint32_t EstimatedMarshaledSize = 0; String OldValue = emptyString; - // multiplier for converting a typical controller label and value to a Json object - #define JsonMarshalingRatio 6 - // estimated number of bytes for the fixed portion of a control rendered as Json - #define MarshalingOverhead 90 - void EstimateMarshaledSize(); + // multiplier for converting a typical controller label or value to a Json object + #define JsonMarshalingRatio 3 + // Marshaed Control overhead length + #define JsonMarshaledOverhead 64 }; #define UI_TITLE ControlType::Title From 7dc51d1274ea7d4e51990a35df5d5895723b4994 Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Tue, 26 Mar 2024 16:04:55 -0400 Subject: [PATCH 07/10] Fixes to the startup sequence to fix file system errors. --- pio_examples/gui/src/gui.ino | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pio_examples/gui/src/gui.ino b/pio_examples/gui/src/gui.ino index 18fcc1d..bb9aadc 100644 --- a/pio_examples/gui/src/gui.ino +++ b/pio_examples/gui/src/gui.ino @@ -287,18 +287,18 @@ void setup(void) */ ESPUI.sliderContinuous = true; -#ifdef TEST_FILEDISPLAY ESPUI.prepareFileSystem(); - // create a text file - ESPUI.writeFile("/DisplayFile.txt", "Test Line\n"); + ESPUI.beginLITTLEFS("ESPUI Control"); // these files are used by browsers to auto config a connection. ESPUI.writeFile("/wpad.dat", " "); ESPUI.writeFile("/connecttest.txt", " "); -#endif // def TEST_FILEDISPLAY - ESPUI.beginLITTLEFS("ESPUI Control"); +#ifdef TEST_FILEDISPLAY + // create a text file + ESPUI.writeFile("/DisplayFile.txt", "Test Line\n"); +#endif // def TEST_FILEDISPLAY } void loop(void) From 75bd3dc378c7a8066c12d5ca6439a774e334d765 Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Tue, 26 Mar 2024 16:06:23 -0400 Subject: [PATCH 08/10] Changes to support ArduinoJson 6 & 7 --- pio_examples/gui/platformio.ini | 5 ++++- src/ESPUI.cpp | 6 +++--- src/ESPUI.h | 15 ++++++++++++++- src/ESPUIclient.cpp | 23 +++++++++++++---------- src/ESPUIclient.h | 6 +++--- src/ESPUIcontrol.cpp | 2 +- 6 files changed, 38 insertions(+), 19 deletions(-) diff --git a/pio_examples/gui/platformio.ini b/pio_examples/gui/platformio.ini index ae409ea..d9afbbc 100644 --- a/pio_examples/gui/platformio.ini +++ b/pio_examples/gui/platformio.ini @@ -17,7 +17,8 @@ framework = arduino board_build.filesystem = littlefs lib_extra_dirs = ../../ lib_deps = - bblanchon/ArduinoJson @ ^6.18.5 +; bblanchon/ArduinoJson @ ^6.18.5 + bblanchon/ArduinoJson @ ^7.0.4 https://github.com/bmedici/ESPAsyncWebServer ; Use a fork of the library that has a bugfix for the compile.... https://github.com/esphome/ESPAsyncWebServer/pull/17 lib_ignore = @@ -43,7 +44,9 @@ monitor_filters = esp32_exception_decoder board_build.flash_mode = dout build_flags = -D TEST_HUGE_TEXT + -D TEST_FILEDISPLAY ; -D DEBUG_ESPUI + lib_deps = ${env.lib_deps} me-no-dev/AsyncTCP diff --git a/src/ESPUI.cpp b/src/ESPUI.cpp index 34bff21..92e657f 100644 --- a/src/ESPUI.cpp +++ b/src/ESPUI.cpp @@ -927,7 +927,7 @@ void ESPUIClass::clearGraph(uint16_t id, int clientId) break; } - DynamicJsonDocument document(jsonUpdateDocumentSize); + AllocateJsonDocument(document, jsonUpdateDocumentSize); JsonObject root = document.to(); root[F("type")] = (int)ControlType::Graph + UpdateOffset; @@ -949,7 +949,7 @@ void ESPUIClass::addGraphPoint(uint16_t id, int nValue, int clientId) break; } - DynamicJsonDocument document(jsonUpdateDocumentSize); + AllocateJsonDocument(document, jsonUpdateDocumentSize); JsonObject root = document.to(); root[F("type")] = (int)ControlType::GraphPoint; @@ -961,7 +961,7 @@ void ESPUIClass::addGraphPoint(uint16_t id, int nValue, int clientId) } while (false); } -bool ESPUIClass::SendJsonDocToWebSocket(ArduinoJson::DynamicJsonDocument& document, uint16_t clientId) +bool ESPUIClass::SendJsonDocToWebSocket(ArduinoJson::JsonDocument& document, uint16_t clientId) { bool Response = false; diff --git a/src/ESPUI.h b/src/ESPUI.h index d02659a..0e70a70 100644 --- a/src/ESPUI.h +++ b/src/ESPUI.h @@ -5,7 +5,20 @@ #define WS_AUTHENTICATION false #include + #include +#if ARDUINOJSON_VERSION_MAJOR > 6 + #define AllocateJsonDocument(name, size) JsonDocument name + #define AllocateJsonArray(doc, name) doc[name].to() + #define AllocateJsonObject(doc) doc.add() + #define AllocateNamedJsonObject(t, s, n) t[n] = s +#else + #define AllocateJsonDocument(name, size) DynamicJsonDocument name(size) + #define AllocateJsonArray(doc, name) doc.createNestedArray(name) + #define AllocateJsonObject(doc) doc.createNestedObject() + #define AllocateNamedJsonObject(t, s, n) t = s.createNestedObject(n) +#endif + #include #ifdef ESP32 #if (ESP_IDF_VERSION_MAJOR == 4 && ESP_IDF_VERSION_MINOR >= 4) || ESP_IDF_VERSION_MAJOR > 4 @@ -271,7 +284,7 @@ protected: void NotifyClients(ClientUpdateType_t newState); void NotifyClient(uint32_t WsClientId, ClientUpdateType_t newState); - bool SendJsonDocToWebSocket(ArduinoJson::DynamicJsonDocument& document, uint16_t clientId); + bool SendJsonDocToWebSocket(ArduinoJson::JsonDocument& document, uint16_t clientId); std::map MapOfClients; diff --git a/src/ESPUIclient.cpp b/src/ESPUIclient.cpp index df4c077..0becacf 100644 --- a/src/ESPUIclient.cpp +++ b/src/ESPUIclient.cpp @@ -74,14 +74,14 @@ bool ESPUIclient::CanSend() return Response; } -void ESPUIclient::FillInHeader(DynamicJsonDocument& document) +void ESPUIclient::FillInHeader(JsonDocument& document) { document[F("type")] = UI_EXTEND_GUI; document[F("sliderContinuous")] = ESPUI.sliderContinuous; document[F("startindex")] = 0; document[F("totalcontrols")] = ESPUI.controlCount; - JsonArray items = document.createNestedArray(F("controls")); - JsonObject titleItem = items.createNestedObject(); + JsonArray items = AllocateJsonArray(document, F("controls")); + JsonObject titleItem = AllocateJsonObject(items); titleItem[F("type")] = (int)UI_TITLE; titleItem[F("label")] = ESPUI.ui_title; } @@ -104,7 +104,7 @@ bool ESPUIclient::SendClientNotification(ClientUpdateType_t value) break; } - DynamicJsonDocument document(ESPUI.jsonUpdateDocumentSize); + AllocateJsonDocument(document, ESPUI.jsonUpdateDocumentSize); FillInHeader(document); if(ClientUpdateType_t::ReloadNeeded == value) { @@ -259,7 +259,7 @@ number this will represent the entire UI. More likely, it will represent a small client will acknowledge receipt by requesting the next chunk. */ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, - DynamicJsonDocument & rootDoc, + JsonDocument & rootDoc, bool InUpdateMode, String FragmentRequestString) { @@ -293,12 +293,15 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // this is actually a fragment or directed update request // parse the string we got from the UI and try to update that specific // control. - DynamicJsonDocument FragmentRequest(FragmentRequestString.length() * 3); + AllocateJsonDocument(FragmentRequest, FragmentRequestString.length() * 3); +/* + ArduinoJson::detail::sizeofObject(N); if(0 >= FragmentRequest.capacity()) { Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Could not allocate memory for a fragmentation request. Skipping Response")); break; } +*/ size_t FragmentRequestStartOffset = FragmentRequestString.indexOf("{"); DeserializationError error = deserializeJson(FragmentRequest, FragmentRequestString.substring(FragmentRequestStartOffset)); if(DeserializationError::Ok != error) @@ -393,7 +396,7 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // Serial.println(String(F("prepareJSONChunk: MaxMarshaledJsonSize: ")) + String(MaxMarshaledJsonSize)); // Serial.println(String(F("prepareJSONChunk: Cur EstimatedUsedMarshaledJsonSize: ")) + String(EstimatedUsedMarshaledJsonSize)); - JsonObject item = items.createNestedObject(); + JsonObject item = AllocateJsonObject(items); elementcount++; uint32_t RemainingSpace = (MaxMarshaledJsonSize - EstimatedUsedMarshaledJsonSize) - 100; // Serial.println(String(F("prepareJSONChunk: RemainingSpace: ")) + String(RemainingSpace)); @@ -418,7 +421,7 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // Serial.println(String(F("prepareJSONChunk: Control ")) + String(control->id) + F(" is too large to be sent to the browser.")); // Serial.println(String(F("ERROR: prepareJSONChunk: value: ")) + control->value); rootDoc.clear(); - item = items.createNestedObject(); + item = AllocateJsonObject(items); control->MarshalErrorMessage(item); elementcount = 0; } @@ -496,7 +499,7 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx, ClientUpdateType_t Tra break; } - DynamicJsonDocument document(ESPUI.jsonInitialDocumentSize); + AllocateJsonDocument(document, ESPUI.jsonInitialDocumentSize); FillInHeader(document); document[F("startindex")] = startidx; document[F("totalcontrols")] = uint16_t(-1); // ESPUI.controlCount; @@ -544,7 +547,7 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx, ClientUpdateType_t Tra return Response; } -bool ESPUIclient::SendJsonDocToWebSocket(DynamicJsonDocument& document) +bool ESPUIclient::SendJsonDocToWebSocket(JsonDocument& document) { bool Response = true; diff --git a/src/ESPUIclient.h b/src/ESPUIclient.h index 444725b..b013f5e 100644 --- a/src/ESPUIclient.h +++ b/src/ESPUIclient.h @@ -43,8 +43,8 @@ protected: // bool NeedsNotification() { return pCurrentFsmState != &fsm_EspuiClient_state_Idle_imp; } bool CanSend(); - void FillInHeader(ArduinoJson::DynamicJsonDocument& document); - uint32_t prepareJSONChunk(uint16_t startindex, DynamicJsonDocument& rootDoc, bool InUpdateMode, String value); + void FillInHeader(ArduinoJson::JsonDocument& document); + uint32_t prepareJSONChunk(uint16_t startindex, JsonDocument& rootDoc, bool InUpdateMode, String value); bool SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode, String FragmentRequest); bool SendClientNotification(ClientUpdateType_t value); @@ -62,6 +62,6 @@ public: bool IsSyncronized(); uint32_t id() { return client->id(); } void SetState(ClientUpdateType_t value); - bool SendJsonDocToWebSocket(ArduinoJson::DynamicJsonDocument& document); + bool SendJsonDocToWebSocket(ArduinoJson::JsonDocument& document); }; diff --git a/src/ESPUIcontrol.cpp b/src/ESPUIcontrol.cpp index a6d8f41..36647bf 100644 --- a/src/ESPUIcontrol.cpp +++ b/src/ESPUIcontrol.cpp @@ -133,7 +133,7 @@ bool Control::MarshalControl(JsonObject & _item, _item[F("offset")] = StartingOffset; _item[F("length")] = ValueLenToSend; _item[F("total")] = value.length(); - item = _item.createNestedObject(F("control")); + AllocateNamedJsonObject(item, _item, F("control")); } item[F("id")] = id; From 7e6390ba0aa48d6f33c92997ddf3a3fb2b7fbd28 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 7 Apr 2024 16:27:04 -0400 Subject: [PATCH 09/10] Moved FILE_WRITTING into the common code section. --- src/ESPUI.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ESPUI.h b/src/ESPUI.h index 610d815..0e7bcc4 100644 --- a/src/ESPUI.h +++ b/src/ESPUI.h @@ -47,10 +47,10 @@ #include #include -#define FILE_WRITING "w" - #endif +#define FILE_WRITING "w" + // Message Types (and control types) enum MessageTypes : uint8_t From 47db2ad6d83878918fd13d539e79d7782acecc85 Mon Sep 17 00:00:00 2001 From: Martin Date: Sun, 14 Jul 2024 13:08:15 -0400 Subject: [PATCH 10/10] Removed conditional flags for the large tests. --- pio_examples/gui/platformio.ini | 2 -- pio_examples/gui/src/gui.ino | 18 ++---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/pio_examples/gui/platformio.ini b/pio_examples/gui/platformio.ini index d9afbbc..5c8dc03 100644 --- a/pio_examples/gui/platformio.ini +++ b/pio_examples/gui/platformio.ini @@ -43,8 +43,6 @@ board = esp32dev monitor_filters = esp32_exception_decoder board_build.flash_mode = dout build_flags = - -D TEST_HUGE_TEXT - -D TEST_FILEDISPLAY ; -D DEBUG_ESPUI lib_deps = diff --git a/pio_examples/gui/src/gui.ino b/pio_examples/gui/src/gui.ino index bb9aadc..c479db6 100644 --- a/pio_examples/gui/src/gui.ino +++ b/pio_examples/gui/src/gui.ino @@ -16,10 +16,8 @@ const char* password = "YourNetworkPassphrase"; const char* hostname = "espui"; -#ifdef TEST_FILEDISPLAY String DisplayTestFileName = "/FileName.txt"; int fileDisplayId = Control::noParent; -#endif // def TEST_FILEDISPLAY int statusLabelId = Control::noParent; @@ -29,9 +27,7 @@ int graphId = Control::noParent; int millisLabelId = Control::noParent; int testSwitchId = Control::noParent; -#ifdef TEST_HUGE_TEXT char HugeText[1025]; -#endif // def TEST_HUGE_TEXT void numberCall(Control* sender, int type) { @@ -176,11 +172,9 @@ void setup(void) ESPUI.setVerbosity(Verbosity::VerboseJSON); Serial.begin(115200); -#ifdef TEST_HUGE_TEXT memset(HugeText, 0x0, sizeof(HugeText)); memset(HugeText, 'a', sizeof(HugeText)-1); -#endif // def TEST_HUGE_TEXT - + #if defined(ESP32) WiFi.setHostname(hostname); #else @@ -255,15 +249,11 @@ void setup(void) ESPUI.slider("Slider two", &slider, ControlColor::None, 100); ESPUI.text("Text Test:", &textCall, ControlColor::Alizarin, "a Text Field"); -#ifdef TEST_HUGE_TEXT ESPUI.text("Huge Text Test:", &textCall, ControlColor::Alizarin, HugeText); -#endif // def TEST_HUGE_TEXT ESPUI.number("Numbertest", &numberCall, ControlColor::Alizarin, 5, 0, 10); -#ifdef TEST_FILEDISPLAY fileDisplayId = ESPUI.fileDisplay("Filetest", ControlColor::Turquoise, DisplayTestFileName); -#endif // def TEST_FILEDISPLAY #ifdef TEST_GRAPH graphId = ESPUI.graph("Graph Test", ControlColor::Wetasphalt); @@ -295,10 +285,8 @@ void setup(void) ESPUI.writeFile("/wpad.dat", " "); ESPUI.writeFile("/connecttest.txt", " "); -#ifdef TEST_FILEDISPLAY // create a text file ESPUI.writeFile("/DisplayFile.txt", "Test Line\n"); -#endif // def TEST_FILEDISPLAY } void loop(void) @@ -320,7 +308,6 @@ void loop(void) testSwitchState = !testSwitchState; ESPUI.updateSwitcher(testSwitchId, testSwitchState); -#ifdef TEST_FILEDISPLAY // update the file Display file. File testFile = ESPUI.EspuiLittleFS.open(String("/") + DisplayTestFileName, "a"); uint32_t filesize = testFile.size(); @@ -330,12 +317,11 @@ void loop(void) { testFile.write((const uint8_t*)TestLine.c_str(), TestLine.length()); ESPUI.updateControl(fileDisplayId); - + TestLine += String("filesize: ") + String(filesize); // Serial.println(TestLine); } testFile.close(); -#endif // def TEST_FILEDISPLAY oldTime = millis(); }