first working version of the fragmentation code.

src/ESPUIclient.cpp

@@ -193,7 +193,21 @@ void ESPUIclient::onWsEvent(AwsEventType type, void* arg, uint8_t* data, size_t
             if (cmd.equals(F("uiok")))
             {
                 // Serial.println(F("ESPUIclient::OnWsEvent:WS_EVT_DATA:uiok:ProcessAck"));
-                pCurrentFsmState->ProcessAck(id);
+                pCurrentFsmState->ProcessAck(id, emptyString);
+                break;
+            }
+
+            if (cmd.equals(F("uifragmentok")))
+            {
+                if(!emptyString.equals(value))
+                {
+                    Serial.println(String(F("ESPUIclient::OnWsEvent:WS_EVT_DATA:uifragmentok:ProcessAck:value: '")) + value + "'");
+                    pCurrentFsmState->ProcessAck(uint16_t(-1), value);
+                }
+                else
+                {
+                    // Serial.println(F("ERROR:ESPUIclient::OnWsEvent:WS_EVT_DATA:uifragmentok:ProcessAck:Fragment Header is missing"));
+                }
                 break;
             }
 
@@ -233,7 +247,8 @@ client will acknowledge receipt by requesting the next chunk.
  */
 uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex,
                                       DynamicJsonDocument & rootDoc,
-                                      bool InUpdateMode)
+                                      bool InUpdateMode,
+                                      String FragmentRequestString)
 {
 #ifdef ESP32
     xSemaphoreTake(ESPUI.ControlsSemaphore, portMAX_DELAY);
@@ -247,8 +262,61 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex,
         // Follow the list until control points to the startindex'th node
         Control* control = ESPUI.controls;
         uint32_t currentIndex = 0;
+        uint32_t DataOffset = 0;
         JsonArray items = rootDoc[F("controls")];
+        bool SingleControl = false;
 
+        if(!emptyString.equals(FragmentRequestString))
+        {
+            // Serial.println(F("prepareJSONChunk:Fragmentation:Got Header (1)"));
+            // Serial.println(String("prepareJSONChunk:startindex:                  ") + String(startindex));
+            // Serial.println(String("prepareJSONChunk:currentIndex:                ") + String(currentIndex));
+            // Serial.println(String("prepareJSONChunk:FragmentRequestString:      '") + FragmentRequestString + "'");
+
+            // this is actually a fragment or directed update request
+            // parse the string we got from the UI and try to update that specific 
+            // control.
+            DynamicJsonDocument FragmentRequest(FragmentRequestString.length() * 3);
+            if(0 >= FragmentRequest.capacity())
+            {
+                Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Could not allocate memory for a fragmentation request. Skipping Response"));
+                break;
+            }
+            size_t FragmentRequestStartOffset = FragmentRequestString.indexOf("{");
+            DeserializationError error = deserializeJson(FragmentRequest, FragmentRequestString.substring(FragmentRequestStartOffset));
+            if(DeserializationError::Ok != error)
+            {
+                Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Could not extract json from the fragment request"));
+                break;
+            }
+
+            if(!FragmentRequest.containsKey(F("id")))
+            {
+                Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Request does not contain a control ID"));
+                break;
+            }
+            uint16_t ControlId = uint16_t(FragmentRequest[F("id")]);
+
+            if(!FragmentRequest.containsKey(F("offset")))
+            {
+                Serial.println(F("ERROR:prepareJSONChunk:Fragmentation:Request does not contain a starting offset"));
+                break;
+            }
+            DataOffset = uint16_t(FragmentRequest[F("offset")]);
+            control = ESPUI.getControlNoLock(ControlId, false);
+            if(nullptr == control)
+            {
+                Serial.println(String(F("ERROR:prepareJSONChunk:Fragmentation:Requested control: ")) + String(ControlId) + F(" does not exist"));
+                break;
+            }
+
+            // Serial.println(F("prepareJSONChunk:Fragmentation:disable the control search operation"));
+            currentIndex = 1;
+            startindex = 0;
+            SingleControl = true;
+        }
+
+        // find a control to send
         while ((startindex > currentIndex) && (nullptr != control))
         {
             // only count active controls
@@ -284,14 +352,14 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex,
         while (nullptr != control)
         {
             // skip deleted controls or controls that have not been updated
-            if (control->ToBeDeleted())
+            if (control->ToBeDeleted() && !SingleControl)
             {
                 // Serial.println(String("prepareJSONChunk: Ignoring Deleted control: ") + String(control->id));
                 control = control->next;
                 continue;
             }
 
-            if(InUpdateMode)
+            if(InUpdateMode && !SingleControl)
             {
                 if(control->IsUpdated())
                 {
@@ -307,7 +375,7 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex,
 
             JsonObject item = items.createNestedObject();
             elementcount++;
-            control->MarshalControl(item, InUpdateMode, ClientTransferContext);
+            control->MarshalControl(item, InUpdateMode, DataOffset);
             
             if (rootDoc.overflowed())
             {
@@ -325,13 +393,17 @@ uint32_t ESPUIclient::prepareJSONChunk(uint16_t startindex,
                 {
                     // Serial.println(String("prepareJSONChunk: Defering control: ") + String(control->id));
                     // Serial.println(String("prepareJSONChunk: elementcount: ") + String(elementcount));
-
                     items.remove(elementcount);
                     --elementcount;
                 }
                 // exit the loop
                 control = nullptr;
             }
+            else if (SingleControl)
+            {
+                // Serial.println("prepareJSONChunk: exit loop");
+                control = nullptr;
+            }
             else
             {
                 control = control->next;
@@ -368,8 +440,7 @@ CLIENT: controls.js:handleEvent()
 etc.
     Returns true if all controls have been sent (aka: Done)
 */
-bool ESPUIclient::SendControlsToClient(uint16_t startidx,
+bool ESPUIclient::SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode, String FragmentRequest)
-                                       ClientUpdateType_t TransferMode)
 {
     bool Response = false;
     // Serial.println(String("ESPUIclient:SendControlsToClient:startidx: ") + String(startidx));
@@ -381,9 +452,9 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx,
             break;
         }
 
-        if (startidx >= ESPUI.controlCount)
+        else if ((startidx >= ESPUI.controlCount) && (emptyString.equals(FragmentRequest)))
         {
-            // Serial.println("ESPUIclient:SendControlsToClient: No more controls to send.");
+            // Serial.println(F("ERROR:ESPUIclient:SendControlsToClient: No more controls to send."));
             Response = true;
             break;
         }
@@ -391,7 +462,7 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx,
         DynamicJsonDocument document(ESPUI.jsonInitialDocumentSize);
         FillInHeader(document);
         document[F("startindex")] = startidx;
-        document[F("totalcontrols")] = 65534; // ESPUI.controlCount;
+        document[F("totalcontrols")] = uint16_t(-1); // ESPUI.controlCount;
 
         if(0 == startidx)
         {
@@ -401,7 +472,7 @@ bool ESPUIclient::SendControlsToClient(uint16_t startidx,
         // Serial.println(String("ESPUIclient:SendControlsToClient:type: ") + String((uint32_t)document["type"]));
 
         // Serial.println("ESPUIclient:SendControlsToClient: Build Controls.");
-        if(prepareJSONChunk(startidx, document, ClientUpdateType_t::UpdateNeeded == TransferMode))
+        if(prepareJSONChunk(startidx, document, ClientUpdateType_t::UpdateNeeded == TransferMode, FragmentRequest))
         {
             #if defined(DEBUG_ESPUI)
                 if (ESPUI.verbosity >= Verbosity::VerboseJSON)

src/ESPUIclient.h

@@ -17,12 +17,6 @@ public:
         ReloadNeeded    = 3,
     };
 
-    struct ClientTransferContext_t
-    {
-        void       *control = nullptr;
-        uint16_t    Offset = 0;
-    };
-
 protected:
     // bool HasBeenNotified      = false;  // Set when a notification has been sent and we are waiting for a reply
     // bool DelayedNotification  = false;  // set if a delayed notification is needed
@@ -45,14 +39,13 @@ protected:
     fsm_EspuiClient_state* pCurrentFsmState = &fsm_EspuiClient_state_Idle_imp;
 
     time_t      EspuiClientEndTime = 0;
-    ClientTransferContext_t ClientTransferContext;
     
     // bool        NeedsNotification() { return pCurrentFsmState != &fsm_EspuiClient_state_Idle_imp; }
 
     bool        CanSend();
     void        FillInHeader(ArduinoJson::DynamicJsonDocument& document);
-    uint32_t    prepareJSONChunk(uint16_t startindex, DynamicJsonDocument& rootDoc, bool InUpdateMode);
+    uint32_t    prepareJSONChunk(uint16_t startindex, DynamicJsonDocument& rootDoc, bool InUpdateMode, String value);
-    bool        SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode);
+    bool        SendControlsToClient(uint16_t startidx, ClientUpdateType_t TransferMode, String FragmentRequest);
 
     bool        SendClientNotification(ClientUpdateType_t value);
 

src/ESPUIclientFsm.cpp

@@ -58,12 +58,20 @@ bool fsm_EspuiClient_state_Idle::NotifyClient()
     return Response;
 }
 
-void fsm_EspuiClient_state_Idle::ProcessAck(uint16_t)
+void fsm_EspuiClient_state_Idle::ProcessAck(uint16_t ControlIndex, String FragmentRequestString)
 {
+    if(!emptyString.equals(FragmentRequestString))
+    {
+        // Serial.println(F("fsm_EspuiClient_state_Idle::ProcessAck:Fragmentation:Got fragment Header"));
+        Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded, FragmentRequestString);
+    }
+    else
+    {
         // This is an unexpected request for control data from the browser
         // treat it as if it was a rebuild operation
         // Serial.println(F("fsm_EspuiClient_state_Idle: ProcessAck"));
         Parent->NotifyClient(ClientUpdateType_t::RebuildNeeded);
+    }
 }
 
 //----------------------------------------------
@@ -75,10 +83,14 @@ bool fsm_EspuiClient_state_SendingUpdate::NotifyClient()
     return true; /* Ignore request */
 }
 
-void fsm_EspuiClient_state_SendingUpdate::ProcessAck(uint16_t ControlIndex)
+void fsm_EspuiClient_state_SendingUpdate::ProcessAck(uint16_t ControlIndex, String FragmentRequest)
 {
+    if(!emptyString.equals(FragmentRequest))
+    {
         // Serial.println(F("fsm_EspuiClient_state_SendingUpdate: ProcessAck"));
-    if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded))
+    }
+    // Serial.println(F("fsm_EspuiClient_state_SendingUpdate: ProcessAck"));
+    if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded, FragmentRequest))
     {
         // No more data to send. Go back to idle or start next request
         Parent->fsm_EspuiClient_state_Idle_imp.Init();
@@ -95,13 +107,29 @@ bool fsm_EspuiClient_state_Rebuilding::NotifyClient()
     return true; /* Ignore request */
 }
 
-void fsm_EspuiClient_state_Rebuilding::ProcessAck(uint16_t ControlIndex)
+void fsm_EspuiClient_state_Rebuilding::ProcessAck(uint16_t ControlIndex, String FragmentRequest)
 {
+    if(!emptyString.equals(FragmentRequest))
+    {
         // Serial.println(F("fsm_EspuiClient_state_Rebuilding: ProcessAck"));
-    if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::RebuildNeeded))
+    }
+    // Serial.println(F("fsm_EspuiClient_state_Rebuilding: ProcessAck"));
+    if(Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::RebuildNeeded, FragmentRequest))
     {
         // No more data to send. Go back to idle or start next request
         Parent->fsm_EspuiClient_state_Idle_imp.Init();
         Parent->fsm_EspuiClient_state_Idle_imp.NotifyClient();
     }
 }
+
+//----------------------------------------------
+//----------------------------------------------
+//----------------------------------------------
+void fsm_EspuiClient_state_Reloading::ProcessAck(uint16_t ControlIndex, String FragmentRequestString)
+{
+    if(!emptyString.equals(FragmentRequestString))
+    {
+        // Serial.println(F("fsm_EspuiClient_state_Reloading::ProcessAck:Fragmentation:Got fragment Header"));
+        Parent->SendControlsToClient(ControlIndex, ClientUpdateType_t::UpdateNeeded, FragmentRequestString);
+    }
+}

src/ESPUIclientFsm.h

@@ -20,7 +20,7 @@ public:
 
             void Init();
     virtual bool NotifyClient() = 0;
-    virtual void ProcessAck(uint16_t id) = 0;
+    virtual void ProcessAck(uint16_t id, String FragmentRequest) = 0;
     virtual String GetStateName () = 0;
             void SetParent(ESPUIclient * value) { Parent = value; }
 
@@ -36,7 +36,7 @@ public:
     virtual     ~fsm_EspuiClient_state_Idle() {}
 
     virtual bool NotifyClient();
-    virtual void ProcessAck(uint16_t id);
+    virtual void ProcessAck(uint16_t id, String FragmentRequest);
             String GetStateName() { return String(F("Idle")); }
 
 }; // fsm_EspuiClient_state_Idle
@@ -48,7 +48,7 @@ public:
     virtual     ~fsm_EspuiClient_state_SendingUpdate() {}
 
     virtual bool NotifyClient();
-    virtual void ProcessAck(uint16_t id);
+    virtual void ProcessAck(uint16_t id, String FragmentRequest);
             String GetStateName() { return String(F("Sending Update")); }
 
 }; // fsm_EspuiClient_state_SendingUpdate
@@ -60,7 +60,7 @@ public:
     virtual     ~fsm_EspuiClient_state_Rebuilding() {}
 
     virtual bool NotifyClient();
-    virtual void ProcessAck(uint16_t id);
+    virtual void ProcessAck(uint16_t id, String FragmentRequest);
             String GetStateName() { return String(F("Sending Rebuild")); }
 
 }; // fsm_EspuiClient_state_Rebuilding
@@ -72,7 +72,7 @@ public:
     virtual     ~fsm_EspuiClient_state_Reloading() {}
 
     virtual bool NotifyClient() { return false; }
-    virtual void ProcessAck(uint16_t) {}
+    virtual void ProcessAck(uint16_t id, String FragmentRequest);
             String GetStateName() { return String(F("Reloading")); }
 
 }; // fsm_EspuiClient_state_Reloading

src/ESPUIcontrol.cpp

@@ -56,8 +56,43 @@ void Control::DeleteControl()
     callback = nullptr;
 }
 
-void Control::MarshalControl(JsonObject & item, bool refresh)
+void Control::MarshalControl(JsonObject & _item, bool refresh, uint32_t StartingOffset)
 {
+    JsonObject & item = _item;
+    uint32_t length = value.length();
+    uint32_t maxLength = uint32_t(double(ESPUI.jsonInitialDocumentSize) * 0.75);
+    if((length > maxLength) || StartingOffset)
+    {
+    	/*
+        Serial.println(String("MarshalControl:Start Fragment Processing"));
+        Serial.println(String("MarshalControl:id:                ") + String(id));
+        Serial.println(String("MarshalControl:length:            ") + String(length));
+        Serial.println(String("MarshalControl:StartingOffset:    ") + String(StartingOffset));
+        Serial.println(String("MarshalControl:maxLength:         ") + String(maxLength));
+		*/
+        // indicate that no additional controls should be sent
+
+        if(0 == StartingOffset)
+        {
+            Serial.println(String("MarshalControl: New control to fragement. ID: ") + String(id));
+        }
+        else
+        {
+            Serial.println(String("MarshalControl: Next fragement. ID: ") + String(id));
+        }
+
+        _item[F("type")] = uint32_t(ControlType::Fragment);
+        _item[F("id")]   = id;
+
+        length = min((length - StartingOffset), maxLength);
+        Serial.println(String("MarshalControl:Final length:      ") + String(length));
+
+        _item[F("offset")] = StartingOffset;
+        _item[F("length")] = length;
+        _item[F("total")] = value.length();
+        item = _item.createNestedObject(F("control"));
+    }
+
     item[F("id")]      = id;
     ControlType TempType = (ControlType::Password == type) ? ControlType::Text : type;
     if(refresh)
@@ -68,8 +103,9 @@ void Control::MarshalControl(JsonObject & item, bool refresh)
     {
         item[F("type")] = uint32_t(TempType);
     }
+
     item[F("label")]   = label;
-    item[F ("value")]   = (ControlType::Password == type) ? F ("--------") : value;
+    item[F ("value")]   = (ControlType::Password == type) ? F ("--------") : value.substring(StartingOffset, length + StartingOffset);
     item[F("visible")] = visible;
     item[F("color")]   = (int)color;
     item[F("enabled")] = enabled;

src/ESPUIcontrol.h

@@ -32,7 +32,6 @@ enum ControlType : uint8_t
     Time,
     Fragment,
 
-    Password = 99,
     UpdateOffset = 100,
 };
 
@@ -85,7 +84,7 @@ public:
 
     void SendCallback(int type);
     bool HasCallback() { return ((nullptr != callback) || (nullptr != extendedCallback)); }
-    void MarshalControl(ArduinoJson::JsonObject& item, bool refresh, ESPUIclient::ClientTransferContext_t & ClientTransferContext);
+    void MarshalControl(ArduinoJson::JsonObject& item, bool refresh, uint32_t StartingOffset);
     void MarshalErrorMessage(ArduinoJson::JsonObject& item);
     bool ToBeDeleted() { return (ControlSyncState_t::deleted == ControlSyncState); }
     void DeleteControl();