-
Notifications
You must be signed in to change notification settings - Fork 0
feat: Create datawriter class #22
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
M1KUS3Q
wants to merge
13
commits into
main
Choose a base branch
from
feat/create-datawriter-class
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from 5 commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
2be929c
feat: add EEGData and Marker struct stubs
M1KUS3Q 291d970
feat: implement DataWriter
M1KUS3Q c89cd8e
chore: add DataWriter to CMakeLists.txt
M1KUS3Q d9d35de
test: add basic datawriter unit tests
M1KUS3Q 7ec3125
Merge branch 'main' into feat/create-datawriter-class
M1KUS3Q fa5daac
refactor: comply with req changes
M1KUS3Q 510683e
fix: properly move includes to forward declarations
M1KUS3Q 08cdd71
fix: add necessary includes
MichalSzandar 61dbc73
fix: bump liblsl
M1KUS3Q 789bbaf
chore: move datawriter stuff to it's own folder
M1KUS3Q a3f7e9e
feat: add format strategy interface, replace atomicbool with stop_token
M1KUS3Q 9ca2a86
fix(datawriter): modularize queue draining
M1KUS3Q 2ebaff7
feat: add csv format strategy
M1KUS3Q File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| #ifndef DATAWRITER_HPP | ||
| #define DATAWRITER_HPP | ||
|
|
||
| #include <blockingconcurrentqueue.h> | ||
|
|
||
| #include <EEGData.hpp> | ||
|
MichalSzandar marked this conversation as resolved.
Outdated
|
||
| #include <Marker.hpp> | ||
| #include <atomic> | ||
| #include <fstream> | ||
| #include <memory> | ||
| #include <string> | ||
| #include <thread> | ||
|
|
||
| class DataWriter { | ||
| public: | ||
| DataWriter() = default; | ||
| ~DataWriter(); | ||
|
|
||
| DataWriter(const DataWriter&) = delete; | ||
| DataWriter& operator=(const DataWriter&) = delete; | ||
| DataWriter(DataWriter&&) = delete; | ||
| DataWriter& operator=(DataWriter&&) = delete; | ||
|
|
||
| void start(const std::string& filePath, | ||
| std::shared_ptr<moodycamel::ConcurrentQueue<EEGData>> eegQueue, | ||
| std::shared_ptr<moodycamel::ConcurrentQueue<Marker>> markerQueue); | ||
| void stop(); | ||
|
|
||
| private: | ||
| void writeLoop(); | ||
| void writeEEGData(const EEGData& data); | ||
| void writeMarker(const Marker& marker); | ||
|
|
||
| std::ofstream outputFile; | ||
| std::shared_ptr<moodycamel::ConcurrentQueue<EEGData>> eegQueue; | ||
| std::shared_ptr<moodycamel::ConcurrentQueue<Marker>> markerQueue; | ||
| std::thread writerThread; | ||
|
MichalSzandar marked this conversation as resolved.
Outdated
|
||
|
|
||
| std::atomic<bool> stopRequested{false}; | ||
| }; | ||
|
|
||
| #endif // DATAWRITER_HPP | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| #ifndef EEGDATA_HPP | ||
| #define EEGDATA_HPP | ||
|
|
||
| #include <utility> | ||
| #include <vector> | ||
|
|
||
| struct EEGData { | ||
| std::vector<float> channels; | ||
| double timestamp = 0.0; | ||
|
|
||
| EEGData() = default; | ||
|
|
||
| EEGData(const std::vector<float>& channelValues, double sampleTimestamp) | ||
| : channels(channelValues), timestamp(sampleTimestamp) {} | ||
|
|
||
| EEGData(std::vector<float>&& channelValues, double sampleTimestamp) | ||
| : channels(std::move(channelValues)), timestamp(sampleTimestamp) {} | ||
| }; | ||
|
|
||
| #endif // EEGDATA_HPP | ||
|
Comment on lines
+1
to
+20
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would consider moving EEGData and Marker classes to seperate directory like include/data_structures or something like that |
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| #ifndef MARKER_HPP | ||
| #define MARKER_HPP | ||
|
|
||
| #include <string> | ||
| #include <utility> | ||
|
|
||
| struct Marker { | ||
| std::string eventName; | ||
| double timestamp = 0.0; | ||
|
|
||
| Marker() = default; | ||
|
|
||
| Marker(const std::string& name, double markerTimestamp) | ||
| : eventName(name), timestamp(markerTimestamp) {} | ||
|
|
||
| Marker(std::string&& name, double markerTimestamp) | ||
| : eventName(std::move(name)), timestamp(markerTimestamp) {} | ||
| }; | ||
|
|
||
| #endif // MARKER_HPP |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,111 @@ | ||
| #include <DataWriter.hpp> | ||
| #include <chrono> | ||
| #include <sstream> | ||
| #include <stdexcept> | ||
| #include <thread> | ||
| #include <utility> | ||
|
|
||
| // anonymous namespace here for private utility-style definitions | ||
| namespace { | ||
| constexpr auto kWriteLoopSleep = std::chrono::milliseconds(10); | ||
|
|
||
| std::string formatChannels(const std::vector<float>& channels) { | ||
|
MichalSzandar marked this conversation as resolved.
Outdated
|
||
| std::ostringstream stream; | ||
| for (std::size_t index = 0; index < channels.size(); ++index) { | ||
| if (index > 0) { | ||
| stream << ','; | ||
| } | ||
| stream << channels[index]; | ||
| } | ||
| return stream.str(); | ||
| } | ||
| } // namespace | ||
|
|
||
| DataWriter::~DataWriter() { stop(); } | ||
|
|
||
| void DataWriter::start(const std::string& filePath, | ||
| std::shared_ptr<moodycamel::ConcurrentQueue<EEGData>> eegQueue, | ||
| std::shared_ptr<moodycamel::ConcurrentQueue<Marker>> markerQueue) { | ||
| stop(); | ||
|
|
||
| outputFile.open(filePath, std::ios::out | std::ios::trunc); | ||
| if (!outputFile.is_open()) { | ||
| throw std::runtime_error("Failed to open data writer output file: " + filePath); | ||
| } | ||
|
|
||
| this->eegQueue = std::move(eegQueue); | ||
| this->markerQueue = std::move(markerQueue); | ||
| stopRequested.store(false); | ||
|
|
||
| outputFile << "type,timestamp,payload\n"; | ||
| writerThread = std::thread(&DataWriter::writeLoop, this); | ||
| } | ||
|
|
||
| void DataWriter::stop() { | ||
| stopRequested.store(true); | ||
|
|
||
| if (writerThread.joinable()) { | ||
| writerThread.join(); | ||
| } | ||
|
|
||
| if (outputFile.is_open()) { | ||
| outputFile.flush(); | ||
| outputFile.close(); | ||
| } | ||
| } | ||
|
|
||
| void DataWriter::writeLoop() { | ||
| while (!stopRequested.load()) { | ||
| bool wroteData = false; | ||
|
|
||
| if (eegQueue) { | ||
| EEGData eegData; | ||
| while (eegQueue->try_dequeue(eegData)) { | ||
| writeEEGData(eegData); | ||
| wroteData = true; | ||
| } | ||
| } | ||
|
|
||
| if (markerQueue) { | ||
| Marker marker; | ||
| while (markerQueue->try_dequeue(marker)) { | ||
| writeMarker(marker); | ||
| wroteData = true; | ||
| } | ||
| } | ||
|
|
||
| if (!wroteData) { | ||
| std::this_thread::sleep_for(kWriteLoopSleep); | ||
| } | ||
| } | ||
|
|
||
| if (eegQueue) { | ||
| EEGData eegData; | ||
| while (eegQueue->try_dequeue(eegData)) { | ||
| writeEEGData(eegData); | ||
| } | ||
| } | ||
|
|
||
| if (markerQueue) { | ||
| Marker marker; | ||
| while (markerQueue->try_dequeue(marker)) { | ||
| writeMarker(marker); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| void DataWriter::writeEEGData(const EEGData& data) { | ||
| if (!outputFile.is_open()) { | ||
|
MichalSzandar marked this conversation as resolved.
Outdated
|
||
| return; | ||
| } | ||
|
|
||
| outputFile << "eeg," << data.timestamp << ",\"" << formatChannels(data.channels) << '"' << '\n'; | ||
| } | ||
|
|
||
| void DataWriter::writeMarker(const Marker& marker) { | ||
| if (!outputFile.is_open()) { | ||
|
MichalSzandar marked this conversation as resolved.
Outdated
|
||
| return; | ||
| } | ||
|
|
||
| outputFile << "marker," << marker.timestamp << ",\"" << marker.eventName << '"' << '\n'; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| #include <gtest/gtest.h> | ||
|
|
||
| #include <DataWriter.hpp> | ||
| #include <chrono> | ||
| #include <filesystem> | ||
| #include <fstream> | ||
| #include <memory> | ||
| #include <string> | ||
| #include <thread> | ||
| #include <vector> | ||
|
|
||
| namespace { | ||
| namespace fs = std::filesystem; | ||
|
|
||
| constexpr float kChannelOne = 1.25F; | ||
| constexpr float kChannelTwo = 2.5F; | ||
| constexpr float kChannelThree = 3.75F; | ||
| constexpr double kEegTimestamp = 12.5; | ||
| constexpr double kMarkerTimestamp = 13.25; | ||
| constexpr auto kWriteWait = std::chrono::milliseconds(50); | ||
|
|
||
| fs::path makeTempFilePath(const std::string& nameStem) { | ||
| const auto uniqueSuffix = | ||
| std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); | ||
| return fs::temp_directory_path() / (nameStem + "_" + uniqueSuffix + ".csv"); | ||
| } | ||
|
|
||
| std::vector<std::string> readAllLines(const fs::path& filePath) { | ||
| std::ifstream input(filePath); | ||
| std::vector<std::string> lines; | ||
| std::string line; | ||
|
|
||
| while (std::getline(input, line)) { | ||
| lines.push_back(line); | ||
| } | ||
|
|
||
| return lines; | ||
| } | ||
| } // namespace | ||
|
|
||
| TEST(DataWriterTest, WritesCsvHeaderOnStart) { | ||
| const auto filePath = makeTempFilePath("datawriter_header"); | ||
|
|
||
| auto eegQueue = std::make_shared<moodycamel::ConcurrentQueue<EEGData>>(); | ||
| auto markerQueue = std::make_shared<moodycamel::ConcurrentQueue<Marker>>(); | ||
|
|
||
| { | ||
| DataWriter writer; | ||
| writer.start(filePath.string(), eegQueue, markerQueue); | ||
| writer.stop(); | ||
| } | ||
|
|
||
| const auto lines = readAllLines(filePath); | ||
| ASSERT_FALSE(lines.empty()); | ||
| EXPECT_EQ(lines.front(), "type,timestamp,payload"); | ||
|
|
||
| fs::remove(filePath); | ||
| } | ||
|
|
||
| TEST(DataWriterTest, FlushesEegAndMarkerRecords) { | ||
| const auto filePath = makeTempFilePath("datawriter_records"); | ||
|
|
||
| auto eegQueue = std::make_shared<moodycamel::ConcurrentQueue<EEGData>>(); | ||
| auto markerQueue = std::make_shared<moodycamel::ConcurrentQueue<Marker>>(); | ||
|
|
||
| eegQueue->enqueue(EEGData{{kChannelOne, kChannelTwo, kChannelThree}, kEegTimestamp}); | ||
| markerQueue->enqueue(Marker{"stimulus_on", kMarkerTimestamp}); | ||
|
|
||
| { | ||
| DataWriter writer; | ||
| writer.start(filePath.string(), eegQueue, markerQueue); | ||
| std::this_thread::sleep_for(kWriteWait); | ||
| writer.stop(); | ||
| } | ||
|
|
||
| const auto lines = readAllLines(filePath); | ||
| ASSERT_GE(lines.size(), 3U); | ||
| EXPECT_EQ(lines[0], "type,timestamp,payload"); | ||
| EXPECT_EQ(lines[1], "eeg,12.5,\"1.25,2.5,3.75\""); | ||
| EXPECT_EQ(lines[2], "marker,13.25,\"stimulus_on\""); | ||
|
|
||
| fs::remove(filePath); | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.