From 488a6cb252d274aa73e009889c052827112c3283 Mon Sep 17 00:00:00 2001 From: Martin Date: Sat, 9 Sep 2023 15:58:12 -0400 Subject: [PATCH] first working version of the fragmentation code. --- src/ESPUIclient.cpp | 95 ++++++++++++++++++++++++++++++++++++------ src/ESPUIclient.h | 11 +---- src/ESPUIclientFsm.cpp | 46 ++++++++++++++++---- src/ESPUIclientFsm.h | 10 ++--- src/ESPUIcontrol.cpp | 40 +++++++++++++++++- src/ESPUIcontrol.h | 3 +- 6 files changed, 166 insertions(+), 39 deletions(-) diff --git a/src/ESPUIclient.cpp b/src/ESPUIclient.cpp index 62644a8..19fd264 100644 --- a/src/ESPUIclient.cpp +++ b/src/ESPUIclient.cpp @@ -193,7 +193,21 @@ void ESPUIclient::onWsEvent(AwsEventType type, void* arg, uint8_t* data, size_t if (cmd.equals(F("uiok"))) { // Serial.println(F("ESPUIclient::OnWsEvent:WS_EVT_DATA:uiok:ProcessAck")); - pCurrentFsmState->ProcessAck(id); + pCurrentFsmState->ProcessAck(id, emptyString); + break; + } + + if (cmd.equals(F("uifragmentok"))) + { + if(!emptyString.equals(value)) + { + Serial.println(String(F("ESPUIclient::OnWsEvent:WS_EVT_DATA:uifragmentok:ProcessAck:value: '")) + value + "'"); + pCurrentFsmState->ProcessAck(uint16_t(-1), value); + } + else + { + // Serial.println(F("ERROR:ESPUIclient::OnWsEvent:WS_EVT_DATA:uifragmentok:ProcessAck:Fragment Header is missing")); + } break; } @@ -233,7 +247,8 @@ client will acknowledge receipt by requesting the next chunk. */ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, DynamicJsonDocument & rootDoc, - bool InUpdateMode) + bool InUpdateMode, + String FragmentRequestString) { #ifdef ESP32 xSemaphoreTake(ESPUI.ControlsSemaphore, portMAX_DELAY); @@ -247,8 +262,61 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, // Follow the list until control points to the startindex'th node Control* control = ESPUI.controls; uint32_t currentIndex = 0; + uint32_t DataOffset = 0; JsonArray items = rootDoc[F("controls")]; + bool SingleControl = false; + if(!emptyString.equals(FragmentRequestString)) + { + // Serial.println(F("prepareJSONChunk:Fragmentation:Got Header (1)")); + // Serial.println(String("prepareJSONChunk:startindex: ") + String(startindex)); + // Serial.println(String("prepareJSONChunk:currentIndex: ") + String(currentIndex)); + // Serial.println(String("prepareJSONChunk:FragmentRequestString: '") + FragmentRequestString + "'"); + + // this is actually a fragment or directed update request + // parse the string we got from the UI and try to update that specific + // control. + DynamicJsonDocument FragmentRequest(FragmentRequestString.length() * 3); + if(0 >= FragmentRequest.capacity()) + { + Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Could not allocate memory for a fragmentation request. Skipping Response")); + break; + } + size_t FragmentRequestStartOffset = FragmentRequestString.indexOf("{"); + DeserializationError error = deserializeJson(FragmentRequest, FragmentRequestString.substring(FragmentRequestStartOffset)); + if(DeserializationError::Ok != error) + { + Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Could not extract json from the fragment request")); + break; + } + + if(!FragmentRequest.containsKey(F("id"))) + { + Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Request does not contain a control ID")); + break; + } + uint16_t ControlId = uint16_t(FragmentRequest[F("id")]); + + if(!FragmentRequest.containsKey(F("offset"))) + { + Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Request does not contain a starting offset")); + break; + } + DataOffset = uint16_t(FragmentRequest[F("offset")]); + control = ESPUI.getControlNoLock(ControlId, false); + if(nullptr == control) + { + Serial.println(String(F("ERROR:prepareJSONChunk:Fragmentation:Requested control: ")) + String(ControlId) + F(" does not exist")); + break; + } + + // Serial.println(F("prepareJSONChunk:Fragmentation:disable the control search operation")); + currentIndex = 1; + startindex = 0; + SingleControl = true; + } + + // find a control to send while ((startindex > currentIndex) && (nullptr != control)) { // only count active controls @@ -284,14 +352,14 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, while (nullptr != control) { // skip deleted controls or controls that have not been updated - if (control->ToBeDeleted()) + if (control->ToBeDeleted() && !SingleControl) { // Serial.println(String("prepareJSONChunk: Ignoring Deleted control: ") + String(control->id)); control = control->next; continue; } - if(InUpdateMode) + if(InUpdateMode && !SingleControl) { if(control->IsUpdated()) { @@ -307,7 +375,7 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, JsonObject item = items.createNestedObject(); elementcount++; - control->MarshalControl(item, InUpdateMode, ClientTransferContext); + control->MarshalControl(item, InUpdateMode, DataOffset); if (rootDoc.overflowed()) { @@ -325,13 +393,17 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex, { // Serial.println(String("prepareJSONChunk: Defering control: ") + String(control->id)); // Serial.println(String("prepareJSONChunk: elementcount: ") + String(elementcount)); - items.remove(elementcount); --elementcount; } // exit the loop control = nullptr; } + else if (SingleControl) + { + // Serial.println("prepareJSONChunk: exit loop"); + control = nullptr; + } else { control = control->next; @@ -368,8 +440,7 @@ CLIENT: controls.js:handleEvent() etc. Returns true if all controls have been sent (aka: Done) */ -bool ESPUIclient::SendControlsToClient(uint16_t startidx, - ClientUpdateType_t TransferMode) +bool ESPUIclient::SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode, String FragmentRequest) { bool Response = false; // Serial.println(String("ESPUIclient:SendControlsToClient:startidx: ") + String(startidx)); @@ -381,9 +452,9 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx, break; } - if (startidx >= ESPUI.controlCount) + else if ((startidx >= ESPUI.controlCount) && (emptyString.equals(FragmentRequest))) { - // Serial.println("ESPUIclient:SendControlsToClient: No more controls to send."); + // Serial.println(F("ERROR:ESPUIclient:SendControlsToClient: No more controls to send.")); Response = true; break; } @@ -391,7 +462,7 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx, DynamicJsonDocument document(ESPUI.jsonInitialDocumentSize); FillInHeader(document); document[F("startindex")] = startidx; - document[F("totalcontrols")] = 65534; // ESPUI.controlCount; + document[F("totalcontrols")] = uint16_t(-1); // ESPUI.controlCount; if(0 == startidx) { @@ -401,7 +472,7 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx, // Serial.println(String("ESPUIclient:SendControlsToClient:type: ") + String((uint32_t)document["type"])); // Serial.println("ESPUIclient:SendControlsToClient: Build Controls."); - if(prepareJSONChunk(startidx, document, ClientUpdateType_t::UpdateNeeded == TransferMode)) + if(prepareJSONChunk(startidx, document, ClientUpdateType_t::UpdateNeeded == TransferMode, FragmentRequest)) { #if defined(DEBUG_ESPUI) if (ESPUI.verbosity >= Verbosity::VerboseJSON) diff --git a/src/ESPUIclient.h b/src/ESPUIclient.h index 6a67fda..72c53f8 100644 --- a/src/ESPUIclient.h +++ b/src/ESPUIclient.h @@ -17,12 +17,6 @@ public: ReloadNeeded = 3, }; - struct ClientTransferContext_t - { - void *control = nullptr; - uint16_t Offset = 0; - }; - protected: // bool HasBeenNotified = false; // Set when a notification has been sent and we are waiting for a reply // bool DelayedNotification = false; // set if a delayed notification is needed @@ -45,14 +39,13 @@ protected: fsm_EspuiClient_state* pCurrentFsmState = &fsm_EspuiClient_state_Idle_imp; time_t EspuiClientEndTime = 0; - ClientTransferContext_t ClientTransferContext; // bool NeedsNotification() { return pCurrentFsmState != &fsm_EspuiClient_state_Idle_imp; } bool CanSend(); void FillInHeader(ArduinoJson::DynamicJsonDocument& document); - uint32_t prepareJSONChunk(uint16_t startindex, DynamicJsonDocument& rootDoc, bool InUpdateMode); - bool SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode); + uint32_t prepareJSONChunk(uint16_t startindex, DynamicJsonDocument& rootDoc, bool InUpdateMode, String value); + bool SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode, String FragmentRequest); bool SendClientNotification(ClientUpdateType_t value); diff --git a/src/ESPUIclientFsm.cpp b/src/ESPUIclientFsm.cpp index 1bd628b..e068691 100644 --- a/src/ESPUIclientFsm.cpp +++ b/src/ESPUIclientFsm.cpp @@ -58,12 +58,20 @@ bool fsm_EspuiClient_state_Idle::NotifyClient() return Response; } -void fsm_EspuiClient_state_Idle::ProcessAck(uint16_t) +void fsm_EspuiClient_state_Idle::ProcessAck(uint16_t ControlIndex, String FragmentRequestString) { - // This is an unexpected request for control data from the browser - // treat it as if it was a rebuild operation - // Serial.println(F("fsm_EspuiClient_state_Idle: ProcessAck")); - Parent->NotifyClient(ClientUpdateType_t::RebuildNeeded); + if(!emptyString.equals(FragmentRequestString)) + { + // Serial.println(F("fsm_EspuiClient_state_Idle::ProcessAck:Fragmentation:Got fragment Header")); + Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded, FragmentRequestString); + } + else + { + // This is an unexpected request for control data from the browser + // treat it as if it was a rebuild operation + // Serial.println(F("fsm_EspuiClient_state_Idle: ProcessAck")); + Parent->NotifyClient(ClientUpdateType_t::RebuildNeeded); + } } //---------------------------------------------- @@ -75,10 +83,14 @@ bool fsm_EspuiClient_state_SendingUpdate::NotifyClient() return true; /* Ignore request */ } -void fsm_EspuiClient_state_SendingUpdate::ProcessAck(uint16_t ControlIndex) +void fsm_EspuiClient_state_SendingUpdate::ProcessAck(uint16_t ControlIndex, String FragmentRequest) { + if(!emptyString.equals(FragmentRequest)) + { + // Serial.println(F("fsm_EspuiClient_state_SendingUpdate: ProcessAck")); + } // Serial.println(F("fsm_EspuiClient_state_SendingUpdate: ProcessAck")); - if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded)) + if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded, FragmentRequest)) { // No more data to send. Go back to idle or start next request Parent->fsm_EspuiClient_state_Idle_imp.Init(); @@ -95,13 +107,29 @@ bool fsm_EspuiClient_state_Rebuilding::NotifyClient() return true; /* Ignore request */ } -void fsm_EspuiClient_state_Rebuilding::ProcessAck(uint16_t ControlIndex) +void fsm_EspuiClient_state_Rebuilding::ProcessAck(uint16_t ControlIndex, String FragmentRequest) { + if(!emptyString.equals(FragmentRequest)) + { + // Serial.println(F("fsm_EspuiClient_state_Rebuilding: ProcessAck")); + } // Serial.println(F("fsm_EspuiClient_state_Rebuilding: ProcessAck")); - if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::RebuildNeeded)) + if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::RebuildNeeded, FragmentRequest)) { // No more data to send. Go back to idle or start next request Parent->fsm_EspuiClient_state_Idle_imp.Init(); Parent->fsm_EspuiClient_state_Idle_imp.NotifyClient(); } } + +//---------------------------------------------- +//---------------------------------------------- +//---------------------------------------------- +void fsm_EspuiClient_state_Reloading::ProcessAck(uint16_t ControlIndex, String FragmentRequestString) +{ + if(!emptyString.equals(FragmentRequestString)) + { + // Serial.println(F("fsm_EspuiClient_state_Reloading::ProcessAck:Fragmentation:Got fragment Header")); + Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded, FragmentRequestString); + } +} diff --git a/src/ESPUIclientFsm.h b/src/ESPUIclientFsm.h index b04bb54..0b794c1 100644 --- a/src/ESPUIclientFsm.h +++ b/src/ESPUIclientFsm.h @@ -20,7 +20,7 @@ public: void Init(); virtual bool NotifyClient() = 0; - virtual void ProcessAck(uint16_t id) = 0; + virtual void ProcessAck(uint16_t id, String FragmentRequest) = 0; virtual String GetStateName () = 0; void SetParent(ESPUIclient * value) { Parent = value; } @@ -36,7 +36,7 @@ public: virtual ~fsm_EspuiClient_state_Idle() {} virtual bool NotifyClient(); - virtual void ProcessAck(uint16_t id); + virtual void ProcessAck(uint16_t id, String FragmentRequest); String GetStateName() { return String(F("Idle")); } }; // fsm_EspuiClient_state_Idle @@ -48,7 +48,7 @@ public: virtual ~fsm_EspuiClient_state_SendingUpdate() {} virtual bool NotifyClient(); - virtual void ProcessAck(uint16_t id); + virtual void ProcessAck(uint16_t id, String FragmentRequest); String GetStateName() { return String(F("Sending Update")); } }; // fsm_EspuiClient_state_SendingUpdate @@ -60,7 +60,7 @@ public: virtual ~fsm_EspuiClient_state_Rebuilding() {} virtual bool NotifyClient(); - virtual void ProcessAck(uint16_t id); + virtual void ProcessAck(uint16_t id, String FragmentRequest); String GetStateName() { return String(F("Sending Rebuild")); } }; // fsm_EspuiClient_state_Rebuilding @@ -72,7 +72,7 @@ public: virtual ~fsm_EspuiClient_state_Reloading() {} virtual bool NotifyClient() { return false; } - virtual void ProcessAck(uint16_t) {} + virtual void ProcessAck(uint16_t id, String FragmentRequest); String GetStateName() { return String(F("Reloading")); } }; // fsm_EspuiClient_state_Reloading diff --git a/src/ESPUIcontrol.cpp b/src/ESPUIcontrol.cpp index f276c68..a913292 100644 --- a/src/ESPUIcontrol.cpp +++ b/src/ESPUIcontrol.cpp @@ -56,8 +56,43 @@ void Control::DeleteControl() callback = nullptr; } -void Control::MarshalControl(JsonObject & item, bool refresh) +void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t StartingOffset) { + JsonObject & item = _item; + uint32_t length = value.length(); + uint32_t maxLength = uint32_t(double(ESPUI.jsonInitialDocumentSize) * 0.75); + if((length > maxLength) || StartingOffset) + { + /* + Serial.println(String("MarshalControl:Start Fragment Processing")); + Serial.println(String("MarshalControl:id: ") + String(id)); + Serial.println(String("MarshalControl:length: ") + String(length)); + Serial.println(String("MarshalControl:StartingOffset: ") + String(StartingOffset)); + Serial.println(String("MarshalControl:maxLength: ") + String(maxLength)); + */ + // indicate that no additional controls should be sent + + if(0 == StartingOffset) + { + Serial.println(String("MarshalControl: New control to fragement. ID: ") + String(id)); + } + else + { + Serial.println(String("MarshalControl: Next fragement. ID: ") + String(id)); + } + + _item[F("type")] = uint32_t(ControlType::Fragment); + _item[F("id")] = id; + + length = min((length - StartingOffset), maxLength); + Serial.println(String("MarshalControl:Final length: ") + String(length)); + + _item[F("offset")] = StartingOffset; + _item[F("length")] = length; + _item[F("total")] = value.length(); + item = _item.createNestedObject(F("control")); + } + item[F("id")] = id; ControlType TempType = (ControlType::Password == type) ? ControlType::Text : type; if(refresh) @@ -68,8 +103,9 @@ void Control::MarshalControl(JsonObject & item, bool refresh) { item[F("type")] = uint32_t(TempType); } + item[F("label")] = label; - item[F ("value")] = (ControlType::Password == type) ? F ("--------") : value; + item[F ("value")] = (ControlType::Password == type) ? F ("--------") : value.substring(StartingOffset, length + StartingOffset); item[F("visible")] = visible; item[F("color")] = (int)color; item[F("enabled")] = enabled; diff --git a/src/ESPUIcontrol.h b/src/ESPUIcontrol.h index 6e80c06..12fc0ba 100644 --- a/src/ESPUIcontrol.h +++ b/src/ESPUIcontrol.h @@ -32,7 +32,6 @@ enum ControlType : uint8_t Time, Fragment, - Password = 99, UpdateOffset = 100, }; @@ -85,7 +84,7 @@ public: void SendCallback(int type); bool HasCallback() { return ((nullptr != callback) || (nullptr != extendedCallback)); } - void MarshalControl(ArduinoJson::JsonObject& item, bool refresh, ESPUIclient::ClientTransferContext_t & ClientTransferContext); + void MarshalControl(ArduinoJson::JsonObject& item, bool refresh, uint32_t StartingOffset); void MarshalErrorMessage(ArduinoJson::JsonObject& item); bool ToBeDeleted() { return (ControlSyncState_t::deleted == ControlSyncState); } void DeleteControl();