From f2146309fe9829b945f08e48e1d407d7f4cc69ab Mon Sep 17 00:00:00 2001 From: MartinMueller2003 Date: Tue, 5 Mar 2024 13:40:01 -0500 Subject: [PATCH] Reworked control marshaling code to be aware of and enforce size limits by starting to fragment sooner. --- pio_examples/gui/platformio.ini | 3 +- src/ESPUIclient.cpp | 46 ++++++++++----- src/ESPUIcontrol.cpp | 99 ++++++++++++++++++++++++--------- src/ESPUIcontrol.h | 15 ++--- 4 files changed, 111 insertions(+), 52 deletions(-) diff --git a/pio_examples/gui/platformio.ini b/pio_examples/gui/platformio.ini index fda3d38..ae409ea 100644 --- a/pio_examples/gui/platformio.ini +++ b/pio_examples/gui/platformio.ini @@ -41,7 +41,8 @@ platform = espressif32 board = esp32dev monitor_filters = esp32_exception_decoder board_build.flash_mode = dout -;build_flags = +build_flags = + -D TEST_HUGE_TEXT ; -D DEBUG_ESPUI lib_deps = ${env.lib_deps} diff --git a/src/ESPUIclient.cpp b/src/ESPUIclient.cpp index 4670f1c..df4c077 100644 --- a/src/ESPUIclient.cpp +++ b/src/ESPUIclient.cpp @@ -267,10 +267,12 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, xSemaphoreTake(ESPUI.ControlsSemaphore, portMAX_DELAY); #endif // def ESP32 - // Serial.println(String("prepareJSONChunk: Start. InUpdateMode: ") + String(InUpdateMode)); + // Serial.println(String("prepareJSONChunk: Start. InUpdateMode: ") + String(InUpdateMode)); + // Serial.println(String("prepareJSONChunk: Start. startindex: ") + String(startindex)); + // Serial.println(String("prepareJSONChunk: Start. FragmentRequestString: '") + FragmentRequestString + "'"); int elementcount = 0; - uint32_t MaxEstimatedMarshaledJsonSize = (!InUpdateMode) ? ESPUI.jsonInitialDocumentSize: ESPUI.jsonUpdateDocumentSize; - uint32_t TotalEstimatedMarshaledJsonSize = 0; + uint32_t MaxMarshaledJsonSize = (!InUpdateMode) ? ESPUI.jsonInitialDocumentSize: ESPUI.jsonUpdateDocumentSize; + uint32_t EstimatedUsedMarshaledJsonSize = 0; do // once { @@ -388,21 +390,32 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, } } - TotalEstimatedMarshaledJsonSize += control->GetEstimatedMarshaledSize(); - bool DocWillOverflow = TotalEstimatedMarshaledJsonSize >= MaxEstimatedMarshaledJsonSize; + // Serial.println(String(F("prepareJSONChunk: MaxMarshaledJsonSize: ")) + String(MaxMarshaledJsonSize)); + // Serial.println(String(F("prepareJSONChunk: Cur EstimatedUsedMarshaledJsonSize: ")) + String(EstimatedUsedMarshaledJsonSize)); + JsonObject item = items.createNestedObject(); elementcount++; - if(!DocWillOverflow) - { - control->MarshalControl(item, InUpdateMode, DataOffset); - } + uint32_t RemainingSpace = (MaxMarshaledJsonSize - EstimatedUsedMarshaledJsonSize) - 100; + // Serial.println(String(F("prepareJSONChunk: RemainingSpace: ")) + String(RemainingSpace)); + uint32_t SpaceUsedByMarshaledControl = 0; + bool ControlIsFragmented = control->MarshalControl(item, + InUpdateMode, + DataOffset, + RemainingSpace, + SpaceUsedByMarshaledControl); + // Serial.println(String(F("prepareJSONChunk: SpaceUsedByMarshaledControl: ")) + String(SpaceUsedByMarshaledControl)); + EstimatedUsedMarshaledJsonSize += SpaceUsedByMarshaledControl; + // Serial.println(String(F("prepareJSONChunk: New EstimatedUsedMarshaledJsonSize: ")) + String(EstimatedUsedMarshaledJsonSize)); + // Serial.println(String(F("prepareJSONChunk: ControlIsFragmented: ")) + String(ControlIsFragmented)); - if (DocWillOverflow || (ESPUI.jsonChunkNumberMax > 0 && (elementcount % ESPUI.jsonChunkNumberMax) == 0)) + // did the control get added to the doc? + if (0 == SpaceUsedByMarshaledControl || + (ESPUI.jsonChunkNumberMax > 0 && (elementcount % ESPUI.jsonChunkNumberMax) == 0)) { - // String("prepareJSONChunk: too much data in the message. Remove the last entry"); + // Serial.println( String("prepareJSONChunk: too much data in the message. Remove the last entry")); if (1 == elementcount) { - Serial.println(String(F("ERROR: prepareJSONChunk: Control ")) + String(control->id) + F(" is too large to be sent to the browser.")); + // Serial.println(String(F("prepareJSONChunk: Control ")) + String(control->id) + F(" is too large to be sent to the browser.")); // Serial.println(String(F("ERROR: prepareJSONChunk: value: ")) + control->value); rootDoc.clear(); item = items.createNestedObject(); @@ -420,13 +433,16 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // exit the loop control = nullptr; } - else if (SingleControl) + else if ((SingleControl) || + (ControlIsFragmented) || + (MaxMarshaledJsonSize < (EstimatedUsedMarshaledJsonSize + 100))) { - // Serial.println("prepareJSONChunk: exit loop"); + // Serial.println("prepareJSONChunk: Doc is Full, Fragmented Control or Single Control. exit loop"); control = nullptr; } else { + // Serial.println("prepareJSONChunk: Next Control"); control = control->next; } } // end while (control != nullptr) @@ -437,7 +453,7 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, xSemaphoreGive(ESPUI.ControlsSemaphore); #endif // def ESP32 - // Serial.println(String("prepareJSONChunk: elementcount: ") + String(elementcount)); + // Serial.println(String("prepareJSONChunk: END: elementcount: ") + String(elementcount)); return elementcount; } diff --git a/src/ESPUIcontrol.cpp b/src/ESPUIcontrol.cpp index 554e7d5..a6d8f41 100644 --- a/src/ESPUIcontrol.cpp +++ b/src/ESPUIcontrol.cpp @@ -19,7 +19,6 @@ Control::Control(ControlType type, const char* label, std::function maxLength) || StartingOffset) - { - /* - Serial.println(String("MarshalControl:Start Fragment Processing")); - Serial.println(String("MarshalControl:id: ") + String(id)); - Serial.println(String("MarshalControl:length: ") + String(length)); - Serial.println(String("MarshalControl:StartingOffset: ") + String(StartingOffset)); - Serial.println(String("MarshalControl:maxLength: ") + String(maxLength)); + // this code assumes MaxMarshaledLength > JsonMarshalingRatio + // Serial.println(String("MarshalControl: StartingOffset: ") + String(StartingOffset)); + // Serial.println(String("MarshalControl: AvailMarshaledLength: ") + String(AvailMarshaledLength)); + // Serial.println(String("MarshalControl: Control ID: ") + String(id)); + bool ControlIsFragmented = false; + // create a new item in the response document + JsonObject & item = _item; + + // how much space do we expect to use? + uint32_t ValueMarshaledLength = (value.length() - StartingOffset) * JsonMarshalingRatio; + uint32_t LabelMarshaledLength = strlen(label) * JsonMarshalingRatio; + uint32_t MinimumMarshaledLength = LabelMarshaledLength + JsonMarshaledOverhead; + uint32_t MaximumMarshaledLength = ValueMarshaledLength + MinimumMarshaledLength; + uint32_t SpaceForMarshaledValue = AvailMarshaledLength - MinimumMarshaledLength; + // Serial.println(String("MarshalControl: value.length(): ") + String(value.length())); + // Serial.println(String("MarshalControl: ValueMarshaledLength: ") + String(ValueMarshaledLength)); + // Serial.println(String("MarshalControl: LabelMarshaledLength: ") + String(LabelMarshaledLength)); + // Serial.println(String("MarshalControl: MaximumMarshaledLength: ") + String(MaximumMarshaledLength)); + // Serial.println(String("MarshalControl: MinimumMarshaledLength: ") + String(MinimumMarshaledLength)); + // Serial.println(String("MarshalControl: SpaceForMarshaledValue: ") + String(SpaceForMarshaledValue)); + + // will the item fit in the remaining space? Fragment if not + if (AvailMarshaledLength < MinimumMarshaledLength) + { + // Serial.println(String("MarshalControl: Cannot Marshal control. Not enough space for basic headers.")); + EstimatedMarshaledLength = 0; + return false; + } + + uint32_t MaxValueLength = (SpaceForMarshaledValue / JsonMarshalingRatio); + // Serial.println(String("MarshalControl: MaxValueLength: ") + String(MaxValueLength)); + + uint32_t ValueLenToSend = min((value.length() - StartingOffset), MaxValueLength); + // Serial.println(String("MarshalControl: ValueLenToSend: ") + String(ValueLenToSend)); + + uint32_t AdjustedMarshaledLength = (ValueLenToSend * JsonMarshalingRatio) + MinimumMarshaledLength; + // Serial.println(String("MarshalControl: AdjustedMarshaledLength: ") + String(AdjustedMarshaledLength)); + + bool NeedToFragment = (ValueLenToSend < value.length()); + // Serial.println(String("MarshalControl: NeedToFragment: ") + String(NeedToFragment)); + + if ((AdjustedMarshaledLength > AvailMarshaledLength) && (0 != ValueLenToSend)) + { + // Serial.println(String("MarshalControl: Cannot Marshal control. Not enough space for marshaled control.")); + EstimatedMarshaledLength = 0; + return false; + } + + EstimatedMarshaledLength = AdjustedMarshaledLength; + + // are we fragmenting? + if(NeedToFragment || StartingOffset) + { + // Serial.println(String("MarshalControl:Start Fragment Processing")); + // Serial.println(String("MarshalControl:id: ") + String(id)); + // Serial.println(String("MarshalControl:StartingOffset: ") + String(StartingOffset)); +/* if(0 == StartingOffset) { Serial.println(String("MarshalControl: New control to fragement. ID: ") + String(id)); @@ -72,17 +120,18 @@ void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t Starting { Serial.println(String("MarshalControl: Next fragement. ID: ") + String(id)); } - */ - +*/ // indicate that no additional controls should be sent + ControlIsFragmented = true; + + // fill in the fragment header _item[F("type")] = uint32_t(ControlType::Fragment); _item[F("id")] = id; - length = min((length - StartingOffset), maxLength); // Serial.println(String("MarshalControl:Final length: ") + String(length)); _item[F("offset")] = StartingOffset; - _item[F("length")] = length; + _item[F("length")] = ValueLenToSend; _item[F("total")] = value.length(); item = _item.createNestedObject(F("control")); } @@ -99,7 +148,7 @@ void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t Starting } item[F("label")] = label; - item[F ("value")] = (ControlType::Password == type) ? F ("--------") : value.substring(StartingOffset, length + StartingOffset); + item[F ("value")] = (ControlType::Password == type) ? F ("--------") : value.substring(StartingOffset, StartingOffset + ValueLenToSend); item[F("visible")] = visible; item[F("color")] = (int)color; item[F("enabled")] = enabled; @@ -132,6 +181,9 @@ void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t Starting item[F("selected")] = ""; } } + + // Serial.println(String("MarshalControl:Done")); + return ControlIsFragmented; } void Control::MarshalErrorMessage(JsonObject & item) @@ -282,10 +334,3 @@ void Control::onWsEvent(String & cmd, String& data) } } while (false); } - -void Control::EstimateMarshaledSize() -{ - EstimatedMarshaledSize = MarshalingOverhead + (JsonMarshalingRatio * (strlen(label) + value.length())); - -} // EstimateSerializedSize - diff --git a/src/ESPUIcontrol.h b/src/ESPUIcontrol.h index a84d094..1701ae0 100644 --- a/src/ESPUIcontrol.h +++ b/src/ESPUIcontrol.h @@ -83,26 +83,23 @@ public: void SendCallback(int type); bool HasCallback() { return (nullptr != callback); } - void MarshalControl(ArduinoJson::JsonObject& item, bool refresh, uint32_t DataOffset); + bool MarshalControl(ArduinoJson::JsonObject& item, bool refresh, uint32_t DataOffset, uint32_t MaxLength, uint32_t & EstimmatedUsedSpace); void MarshalErrorMessage(ArduinoJson::JsonObject& item); void DeleteControl(); void onWsEvent(String& cmd, String& data); inline bool ToBeDeleted() { return _ToBeDeleted; } inline bool NeedsSync(uint32_t lastControlChangeID) {return (false == _ToBeDeleted) && (lastControlChangeID < ControlChangeID);} - void SetControlChangedId(uint32_t value) {ControlChangeID = value; EstimateMarshaledSize();} - uint32_t GetEstimatedMarshaledSize() {return EstimatedMarshaledSize;} + void SetControlChangedId(uint32_t value) {ControlChangeID = value;} private: bool _ToBeDeleted = false; uint32_t ControlChangeID = 0; - uint32_t EstimatedMarshaledSize = 0; String OldValue = emptyString; - // multiplier for converting a typical controller label and value to a Json object - #define JsonMarshalingRatio 6 - // estimated number of bytes for the fixed portion of a control rendered as Json - #define MarshalingOverhead 90 - void EstimateMarshaledSize(); + // multiplier for converting a typical controller label or value to a Json object + #define JsonMarshalingRatio 3 + // Marshaed Control overhead length + #define JsonMarshaledOverhead 64 }; #define UI_TITLE ControlType::Title