From 46e2c0884da248478576b058413c651909ac9446 Mon Sep 17 00:00:00 2001 From: Alex Zhou Date: Sat, 28 Mar 2026 15:33:57 -0700 Subject: [PATCH 1/2] [Backend][Frontend] Removed constant console logging of data, batcherized sensor data reading for rendering --- client/include/client/graph_panel.hpp | 7 ++-- client/include/client/json_reader.hpp | 8 ++++- client/include/client/sensor_data_panel.h | 3 +- client/src/graph_panel.cpp | 17 +++++---- client/src/json_reader.cpp | 32 ++++++++++++++--- client/src/mainframe.cpp | 43 ++++++++++++++--------- client/src/sensor_data_panel.cpp | 22 +++++++----- client/src/tcp_client.cpp | 10 +++--- 8 files changed, 96 insertions(+), 46 deletions(-) diff --git a/client/include/client/graph_panel.hpp b/client/include/client/graph_panel.hpp index e49f20eb..8f294071 100644 --- a/client/include/client/graph_panel.hpp +++ b/client/include/client/graph_panel.hpp @@ -10,8 +10,9 @@ class GraphPanel : public wxPanel { public: GraphPanel(wxWindow* parent); - void AddDataPoint(const std::string& sensorName, double value, double timestamp); - void SetVisibleSensors(const std::set& visisble); + void AddDataPoint(const std::string& sensorName, double value, double timestamp, bool refresh = true); + void SetVisibleSensors(const std::set& visible, bool refresh = true); + void UpdateGraph(); private: mpWindow* m_plot; @@ -27,7 +28,5 @@ class GraphPanel : public wxPanel { void DrawBackground(wxDC& dc); void DrawGrid(wxDC& dc); void DrawAxes(wxDC& dc); - void UpdateGraph(); - wxDECLARE_EVENT_TABLE(); }; \ No newline at end of file diff --git a/client/include/client/json_reader.hpp b/client/include/client/json_reader.hpp index 6cf8604d..6162a599 100644 --- a/client/include/client/json_reader.hpp +++ b/client/include/client/json_reader.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "common/panorama_defines.hpp" class JsonReader { @@ -16,9 +17,14 @@ class JsonReader { // Accessors for JSON data const rapidjson::Document& getDocument() const; - buffer_data_t exportToBuffer(std::string json); + buffer_data_t exportToBuffer(std::string json); + + // Parse all newline-delimited JSON objects from a TCP chunk. + // Stores any trailing incomplete data in recvBuffer_ for the next call. + std::vector exportAllToBuffer(const std::string& chunk); private: std::string filename_; rapidjson::Document document_; + std::string recvBuffer_; // accumulates partial JSON across recv() calls }; \ No newline at end of file diff --git a/client/include/client/sensor_data_panel.h b/client/include/client/sensor_data_panel.h index 46448bbf..a04aa4e4 100644 --- a/client/include/client/sensor_data_panel.h +++ b/client/include/client/sensor_data_panel.h @@ -9,7 +9,8 @@ class SensorDataFrame : public wxPanel { SensorDataFrame(wxWindow* parent, const wxArrayString& sensorNames); void AddSensorRow(const std::string& sensorName); - void UpdateReading(const std::string& sensorName, double value, const std::string& unit = ""); + void UpdateReading(const std::string& sensorName, double value, const std::string& unit = "", bool refresh = true); + void RefreshGrid(); void ClearReadings(); void SetActiveSensors(const std::vector& sensors); diff --git a/client/src/graph_panel.cpp b/client/src/graph_panel.cpp index 82153b91..31702aff 100644 --- a/client/src/graph_panel.cpp +++ b/client/src/graph_panel.cpp @@ -129,15 +129,18 @@ void GraphPanel::DrawAxes(wxDC& dc){ } -void GraphPanel::AddDataPoint(const std::string& sensorName, double value, double timestamp){ +void GraphPanel::AddDataPoint(const std::string& sensorName, double value, double timestamp, bool refresh){ sensorData_[sensorName].push_back({timestamp, value}); // Keeps the first 100 data points if (sensorData_[sensorName].size() > 100) { sensorData_[sensorName].erase(sensorData_[sensorName].begin()); } - UpdateGraph(); -} + + if (refresh) { + UpdateGraph(); + } +} void GraphPanel::UpdateGraph(){ for (auto& pair : sensorLayers_){ @@ -188,7 +191,9 @@ void GraphPanel::UpdateGraph(){ m_plot->Update(); } -void GraphPanel::SetVisibleSensors(const std::set& visible) { - visibleSensors_ = visible; - UpdateGraph(); +void GraphPanel::SetVisibleSensors(const std::set& visible, bool refresh) { + visibleSensors_ = visible; + if (refresh) { + UpdateGraph(); + } } diff --git a/client/src/json_reader.cpp b/client/src/json_reader.cpp index a1e79834..42c02cb9 100644 --- a/client/src/json_reader.cpp +++ b/client/src/json_reader.cpp @@ -43,7 +43,6 @@ buffer_data_t JsonReader::exportToBuffer(std::string json) { rapidjson::Document doc; rapidjson::ParseResult ok = doc.Parse(json.c_str()); - std::cout << json.c_str() << std::endl; if (!ok) { std::cerr << "JSON parse error at offset " << ok.Offset() << ": " << rapidjson::GetParseError_En(ok.Code()) << std::endl; @@ -101,12 +100,35 @@ buffer_data_t JsonReader::exportToBuffer(std::string json) { ret.timestamp = (long) doc["timestamp"].GetInt(); } - - - - //ret.timestamp = std::time(nullptr); return ret; +} + +std::vector JsonReader::exportAllToBuffer(const std::string& chunk) { + std::vector results; + + // Append new data to any leftover partial data from last recv() + recvBuffer_ += chunk; + + // Split on newline delimiters (the JSON the server sends is newline delimited) + size_t pos = 0; + size_t newlinePos; + while ((newlinePos = recvBuffer_.find('\n', pos)) != std::string::npos) { + std::string line = recvBuffer_.substr(pos, newlinePos - pos); + pos = newlinePos + 1; + + if (line.empty()) continue; + + buffer_data_t parsed = exportToBuffer(line); + if (!parsed.datatype.empty()) { + results.push_back(std::move(parsed)); + } + } + + // Keep any remaining partial data for next call + recvBuffer_ = recvBuffer_.substr(pos); + + return results; } \ No newline at end of file diff --git a/client/src/mainframe.cpp b/client/src/mainframe.cpp index 224c9e6d..fca0aad8 100644 --- a/client/src/mainframe.cpp +++ b/client/src/mainframe.cpp @@ -190,12 +190,15 @@ void MainFrame::updateDataPanel() { if (dataBuffer_->size() > 0) { - for (buffer_data_t latestData : dataBuffer_->consume()) { - // Skip entries with no data-type (first sensor reading) + auto batch = dataBuffer_->consume(); + + for (buffer_data_t& latestData : batch) { + + // Skip entries with no data-type (first sensor/ corrupt reading) if (latestData.datatype.empty()) continue; - // Add Sensors to sensor manager and data grid only when they are first seen + // Add Sensors to sensor manager and data grid ONLY when they are first seen if (registeredSensors_.find(latestData.datatype) == registeredSensors_.end()) { auto sensor = std::make_shared(latestData.datatype, latestData.dataunit); sensorManager_->AddSensor(sensor); @@ -203,22 +206,27 @@ void MainFrame::updateDataPanel() { registeredSensors_.insert(latestData.datatype); } - sensorDataGrid->UpdateReading(latestData.datatype, (double)latestData.data, latestData.dataunit); - + sensorDataGrid->UpdateReading(latestData.datatype, (double)latestData.data, latestData.dataunit, false); - auto enabledNames = sensorManager_->GetEnabledSensorNames(); - std::set visible(enabledNames.begin(), enabledNames.end()); - graphPanel_->SetVisibleSensors(visible); - //std::cout << "Updated " << latestData.datatype << " with value: " << latestData.data << " " << latestData.dataunit << std::endl; - - if(graphPanel_){ + if (graphPanel_) { graphPanel_->AddDataPoint( latestData.datatype, (double)latestData.data, - (double)latestData.timestamp + (double)latestData.timestamp, + false ); } } + + // Do a SINGLE refresh after the ENTIRE batch is processed + sensorDataGrid->RefreshGrid(); + + if (graphPanel_) { + auto enabledNames = sensorManager_->GetEnabledSensorNames(); + std::set visible(enabledNames.begin(), enabledNames.end()); + graphPanel_->SetVisibleSensors(visible, false); + graphPanel_->UpdateGraph(); + } } } @@ -331,10 +339,13 @@ void MainFrame::OnStopStream(wxCommandEvent& event) { } void MainFrame::OnUpdateTimer(wxTimerEvent&) { - if (updatePending_.exchange(false)) { - updateMessageDisplay(); - updateDataPanel(); - } + // if (updatePending_.exchange(false)) { + // } + // updateMessageDisplay(); // NOT NEEDED TO PRINT ENTIRE BUFFER CONTENT + // TODO: Create ConsoleMessageBuffer that keeps track of this, have method to append to that buffer instead. + + // Data panel polls the buffer independently of the message model + updateDataPanel(); // Makesure the esp is actually detected if (esp32BannerPending_.exchange(false)) { diff --git a/client/src/sensor_data_panel.cpp b/client/src/sensor_data_panel.cpp index b78303c7..6bf88692 100644 --- a/client/src/sensor_data_panel.cpp +++ b/client/src/sensor_data_panel.cpp @@ -96,21 +96,21 @@ void SensorDataFrame::AddSensorRow(const std::string& sensorName) { grid_->ForceRefresh(); } -void SensorDataFrame::UpdateReading(const std::string& sensorName, double value, const std::string& unit) { +void SensorDataFrame::UpdateReading(const std::string& sensorName, double value, const std::string& unit, bool refresh) { int row = FindSensorRow(sensorName); if (row == -1) return; - - + + wxString valueStr = wxString::Format("%.2f %s", value, unit); grid_->SetCellValue(row, 1, valueStr); - - + + wxString timestamp = wxDateTime::Now().FormatISOTime(); grid_->SetCellValue(row, 2, timestamp); //std::cout << "Updated " << sensorName << " with value: " << valueStr.ToStdString() << " at " << timestamp.ToStdString() << std::endl; - - // Color + + // Color if (value > 50.0) { grid_->SetCellBackgroundColour(row, 1, PCOLOUR_LIGHT_RED); } else if (value > 25.0) { @@ -118,7 +118,13 @@ void SensorDataFrame::UpdateReading(const std::string& sensorName, double value, } else { grid_->SetCellBackgroundColour(row, 1, PCOLOUR_LIGHT_GREEN); } - + + if (refresh) { + grid_->ForceRefresh(); + } +} + +void SensorDataFrame::RefreshGrid() { grid_->ForceRefresh(); } diff --git a/client/src/tcp_client.cpp b/client/src/tcp_client.cpp index 039c39a6..767bea72 100644 --- a/client/src/tcp_client.cpp +++ b/client/src/tcp_client.cpp @@ -92,11 +92,11 @@ void TcpClient::run() { logger_->logJsonData(received); } - // Parse JSON and write to DataBuffer - buffer_data_t parsedData = reader.exportToBuffer(received); - dataBuffer_->writeData(parsedData); - model_->addMessage("Data" + std::to_string(dataBuffer_->size())); - model_->addMessage("Received: " + received); + // Parse all JSON objects in this TCP chunk and write to DataBuffer + std::vector parsedItems = reader.exportAllToBuffer(received); + for (auto& item : parsedItems) { + dataBuffer_->writeData(std::move(item)); + } } } } From f6359e4dc92dc159ea75659c570cdc0ecca75e35 Mon Sep 17 00:00:00 2001 From: Alex Zhou Date: Sat, 28 Mar 2026 15:34:27 -0700 Subject: [PATCH 2/2] [PServer] Defined magic numbers, created central config file; --- tools/pserver/pserver.py | 7 ++++--- tools/pserver/pserver_config.py | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 tools/pserver/pserver_config.py diff --git a/tools/pserver/pserver.py b/tools/pserver/pserver.py index 6eaafe77..2147cc5f 100644 --- a/tools/pserver/pserver.py +++ b/tools/pserver/pserver.py @@ -5,6 +5,7 @@ from putils import * from pstreamer import PStreamer from pstream_json import PStreamJSON +from pserver_config import * class PServer: def __init__(self): @@ -86,8 +87,8 @@ def send_stream(self, client_socket, client_address, streamer: PStreamer): client_socket.close() def main(): - host = '127.0.0.1' - port = 3000 + host = PSERVER_HOST_ADDRESS + port = PSERVER_PORT_NUMBER if len(sys.argv) > 1: port = int(sys.argv[1]) @@ -95,7 +96,7 @@ def main(): server = PServer() streamer = PStreamer() - streamer.build_stream(PStreamJSON()).set_interval(0.4) + streamer.build_stream(PStreamJSON()).set_interval(PSTREAMER_REFRESH_PERIOD) streamer.start() diff --git a/tools/pserver/pserver_config.py b/tools/pserver/pserver_config.py new file mode 100644 index 00000000..31f2c912 --- /dev/null +++ b/tools/pserver/pserver_config.py @@ -0,0 +1,5 @@ +# CONFIGURATION PARAMETERS FOR PSERVER + +PSTREAMER_REFRESH_PERIOD = 0.01 # Unit: Seconds +PSERVER_HOST_ADDRESS = '127.0.0.1' +PSERVER_PORT_NUMBER = 3000 \ No newline at end of file