[Backend][Frontend] Performance upgrades to data processing and rendering

client/include/client/graph_panel.hpp

@@ -10,8 +10,9 @@ class GraphPanel : public wxPanel {
 public:
     GraphPanel(wxWindow* parent);
 
-    void AddDataPoint(const std::string& sensorName, double value, double timestamp);
-    void SetVisibleSensors(const std::set<std::string>& visisble);
+    void AddDataPoint(const std::string& sensorName, double value, double timestamp, bool refresh = true);
+    void SetVisibleSensors(const std::set<std::string>& visible, bool refresh = true);
+    void UpdateGraph();
 
 private:
     mpWindow* m_plot;
@@ -27,7 +28,5 @@ class GraphPanel : public wxPanel {
     void DrawBackground(wxDC& dc);
     void DrawGrid(wxDC& dc);
     void DrawAxes(wxDC& dc);
-    void UpdateGraph();
-
     wxDECLARE_EVENT_TABLE();
 };

client/include/client/json_reader.hpp

@@ -3,6 +3,7 @@
 #include <rapidjson/document.h>
 #include <rapidjson/filereadstream.h>
 #include <string>
+#include <vector>
 #include "common/panorama_defines.hpp"
 
 class JsonReader {
@@ -16,9 +17,14 @@ class JsonReader {
     // Accessors for JSON data
     const rapidjson::Document& getDocument() const;
 
-    buffer_data_t exportToBuffer(std::string json); 
+    buffer_data_t exportToBuffer(std::string json);
+
+    // Parse all newline-delimited JSON objects from a TCP chunk.
+    // Stores any trailing incomplete data in recvBuffer_ for the next call.
+    std::vector<buffer_data_t> exportAllToBuffer(const std::string& chunk);
 
 private:
     std::string filename_;
     rapidjson::Document document_;
+    std::string recvBuffer_;  // accumulates partial JSON across recv() calls
 };

client/include/client/sensor_data_panel.h

@@ -9,7 +9,8 @@ class SensorDataFrame : public wxPanel {
     SensorDataFrame(wxWindow* parent, const wxArrayString& sensorNames);
 
     void AddSensorRow(const std::string& sensorName);
-    void UpdateReading(const std::string& sensorName, double value, const std::string& unit = "");
+    void UpdateReading(const std::string& sensorName, double value, const std::string& unit = "", bool refresh = true);
+    void RefreshGrid();
     void ClearReadings();
     void SetActiveSensors(const std::vector<std::string>& sensors);
 

client/src/graph_panel.cpp

@@ -129,15 +129,18 @@ void GraphPanel::DrawAxes(wxDC& dc){
 
 }
 
-void GraphPanel::AddDataPoint(const std::string& sensorName, double value, double timestamp){
+void GraphPanel::AddDataPoint(const std::string& sensorName, double value, double timestamp, bool refresh){
 	sensorData_[sensorName].push_back({timestamp, value});
 
 	// Keeps the first 100 data points
 	if (sensorData_[sensorName].size() > 100) {
 		sensorData_[sensorName].erase(sensorData_[sensorName].begin());
 	}
-	UpdateGraph();
-} 
+
+	if (refresh) {
+		UpdateGraph();
+	}
+}
 
 void GraphPanel::UpdateGraph(){
 	for (auto& pair : sensorLayers_){
@@ -188,7 +191,9 @@ void GraphPanel::UpdateGraph(){
 	m_plot->Update();
 }
 
-void GraphPanel::SetVisibleSensors(const std::set<std::string>& visible) {
-	visibleSensors_ = visible; 
-	UpdateGraph();
+void GraphPanel::SetVisibleSensors(const std::set<std::string>& visible, bool refresh) {
+	visibleSensors_ = visible;
+	if (refresh) {
+		UpdateGraph();
+	}
 }

client/src/json_reader.cpp

@@ -43,7 +43,6 @@ buffer_data_t JsonReader::exportToBuffer(std::string json) {
 
     rapidjson::Document doc;
     rapidjson::ParseResult ok = doc.Parse(json.c_str());
-    std::cout << json.c_str() << std::endl;
     if (!ok) {
         std::cerr << "JSON parse error at offset " << ok.Offset()
                   << ": " << rapidjson::GetParseError_En(ok.Code()) << std::endl;
@@ -101,12 +100,35 @@ buffer_data_t JsonReader::exportToBuffer(std::string json) {
         ret.timestamp = (long) doc["timestamp"].GetInt();
     }
 
-
-
-
-
     //ret.timestamp = std::time(nullptr);
 
     return ret;
 
+}
+
+std::vector<buffer_data_t> JsonReader::exportAllToBuffer(const std::string& chunk) {
+    std::vector<buffer_data_t> results;
+
+    // Append new data to any leftover partial data from last recv()
+    recvBuffer_ += chunk;
+
+    // Split on newline delimiters (the JSON the server sends is newline delimited)
+    size_t pos = 0;
+    size_t newlinePos;
+    while ((newlinePos = recvBuffer_.find('\n', pos)) != std::string::npos) {
+        std::string line = recvBuffer_.substr(pos, newlinePos - pos);
+        pos = newlinePos + 1;
+
+        if (line.empty()) continue;
+
+        buffer_data_t parsed = exportToBuffer(line);
+        if (!parsed.datatype.empty()) {
+            results.push_back(std::move(parsed));
+        }
+    }
+
+    // Keep any remaining partial data for next call
+    recvBuffer_ = recvBuffer_.substr(pos);
+
+    return results;
 }

client/src/mainframe.cpp

@@ -190,35 +190,43 @@ void MainFrame::updateDataPanel() {
 
 
     if (dataBuffer_->size() > 0) {
-        for (buffer_data_t latestData : dataBuffer_->consume()) {
 
-            // Skip entries with no data-type (first sensor reading)
+        auto batch = dataBuffer_->consume();
+
+        for (buffer_data_t& latestData : batch) {
+
+            // Skip entries with no data-type (first sensor/ corrupt reading)
             if (latestData.datatype.empty()) continue;
 
-            // Add Sensors to sensor manager and data grid only when they are first seen
+            // Add Sensors to sensor manager and data grid ONLY when they are first seen
             if (registeredSensors_.find(latestData.datatype) == registeredSensors_.end()) {
                 auto sensor = std::make_shared<Sensor>(latestData.datatype, latestData.dataunit);
                 sensorManager_->AddSensor(sensor);
                 sensorDataGrid->AddSensorRow(latestData.datatype);
                 registeredSensors_.insert(latestData.datatype);
             }
 
-            sensorDataGrid->UpdateReading(latestData.datatype, (double)latestData.data, latestData.dataunit);
-
+            sensorDataGrid->UpdateReading(latestData.datatype, (double)latestData.data, latestData.dataunit, false);
 
-            auto enabledNames = sensorManager_->GetEnabledSensorNames();
-            std::set<std::string> visible(enabledNames.begin(), enabledNames.end());
-            graphPanel_->SetVisibleSensors(visible);
-            //std::cout << "Updated " << latestData.datatype << " with value: " << latestData.data << " " << latestData.dataunit << std::endl;
-
-            if(graphPanel_){
+            if (graphPanel_) {
                 graphPanel_->AddDataPoint(
                     latestData.datatype,
                     (double)latestData.data,
-                    (double)latestData.timestamp
+                    (double)latestData.timestamp,
+                    false
                 );
             }
         }
+
+        // Do a SINGLE refresh after the ENTIRE batch is processed
+        sensorDataGrid->RefreshGrid();
+
+        if (graphPanel_) {
+            auto enabledNames = sensorManager_->GetEnabledSensorNames();
+            std::set<std::string> visible(enabledNames.begin(), enabledNames.end());
+            graphPanel_->SetVisibleSensors(visible, false);
+            graphPanel_->UpdateGraph();
+        }
     }
 }
 
@@ -331,10 +339,13 @@ void MainFrame::OnStopStream(wxCommandEvent& event) {
 }
 
 void MainFrame::OnUpdateTimer(wxTimerEvent&) {
-    if (updatePending_.exchange(false)) {
-        updateMessageDisplay();
-        updateDataPanel();
-    }
+    // if (updatePending_.exchange(false)) {
+    // }
+    // updateMessageDisplay(); // NOT NEEDED TO PRINT ENTIRE BUFFER CONTENT
+    // TODO: Create ConsoleMessageBuffer that keeps track of this, have method to append to that buffer instead.
+
+    // Data panel polls the buffer independently of the message model
+    updateDataPanel();
 
     // Makesure the esp is actually detected
     if (esp32BannerPending_.exchange(false)) {

client/src/sensor_data_panel.cpp

@@ -96,29 +96,35 @@ void SensorDataFrame::AddSensorRow(const std::string& sensorName) {
     grid_->ForceRefresh();
 }
 
-void SensorDataFrame::UpdateReading(const std::string& sensorName, double value, const std::string& unit) {
+void SensorDataFrame::UpdateReading(const std::string& sensorName, double value, const std::string& unit, bool refresh) {
     int row = FindSensorRow(sensorName);
     if (row == -1) return;
-    
-    
+
+
     wxString valueStr = wxString::Format("%.2f %s", value, unit);
     grid_->SetCellValue(row, 1, valueStr);
-    
-    
+
+
     wxString timestamp = wxDateTime::Now().FormatISOTime();
     grid_->SetCellValue(row, 2, timestamp);
 
     //std::cout << "Updated " << sensorName << " with value: " << valueStr.ToStdString() << " at " << timestamp.ToStdString() << std::endl;
-    
-    // Color 
+
+    // Color
     if (value > 50.0) {
         grid_->SetCellBackgroundColour(row, 1, PCOLOUR_LIGHT_RED);
     } else if (value > 25.0) {
         grid_->SetCellBackgroundColour(row, 1, PCOLOUR_LIGHT_YELLOW);
     } else {
         grid_->SetCellBackgroundColour(row, 1, PCOLOUR_LIGHT_GREEN);
     }
-
+
+    if (refresh) {
+        grid_->ForceRefresh();
+    }
+}
+
+void SensorDataFrame::RefreshGrid() {
     grid_->ForceRefresh();
 }
 

client/src/tcp_client.cpp

@@ -92,11 +92,11 @@ void TcpClient::run() {
                 logger_->logJsonData(received);
             }
 
-            // Parse JSON and write to DataBuffer
-            buffer_data_t parsedData = reader.exportToBuffer(received);
-            dataBuffer_->writeData(parsedData);
-            model_->addMessage("Data" + std::to_string(dataBuffer_->size()));
-            model_->addMessage("Received: " + received);
+            // Parse all JSON objects in this TCP chunk and write to DataBuffer
+            std::vector<buffer_data_t> parsedItems = reader.exportAllToBuffer(received);
+            for (auto& item : parsedItems) {
+                dataBuffer_->writeData(std::move(item));
+            }
         }
     }
 }

tools/pserver/pserver.py

@@ -5,6 +5,7 @@
 from putils import *
 from pstreamer import PStreamer
 from pstream_json import PStreamJSON
+from pserver_config import *
 
 class PServer:
     def __init__(self):
@@ -86,16 +87,16 @@ def send_stream(self, client_socket, client_address, streamer: PStreamer):
             client_socket.close()
 
 def main():
-    host = '127.0.0.1'
-    port = 3000
+    host = PSERVER_HOST_ADDRESS
+    port = PSERVER_PORT_NUMBER
 
     if len(sys.argv) > 1:
         port = int(sys.argv[1])
 
     server = PServer()
 
     streamer = PStreamer()
-    streamer.build_stream(PStreamJSON()).set_interval(0.4)
+    streamer.build_stream(PStreamJSON()).set_interval(PSTREAMER_REFRESH_PERIOD)
 
     streamer.start()
 

tools/pserver/pserver_config.py

@@ -0,0 +1,5 @@
+# CONFIGURATION PARAMETERS FOR PSERVER
+
+PSTREAMER_REFRESH_PERIOD = 0.01 # Unit: Seconds
+PSERVER_HOST_ADDRESS = '127.0.0.1'
+PSERVER_PORT_NUMBER = 3000