From 51a139bf412d1ea200b1c5c7e2f14ff7c5866d34 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 14 Apr 2026 16:14:36 +0800 Subject: [PATCH 1/2] fix get_metadata datatype --- cpp/src/common/tsfile_common.h | 5 +- .../block/single_device_tsblock_reader.cc | 6 +- cpp/src/reader/tsfile_series_scan_iterator.cc | 2 +- cpp/test/reader/tsfile_reader_test.cc | 74 +++++++++++++++++++ 4 files changed, 82 insertions(+), 5 deletions(-) diff --git a/cpp/src/common/tsfile_common.h b/cpp/src/common/tsfile_common.h index c866e4995..36d2b251e 100644 --- a/cpp/src/common/tsfile_common.h +++ b/cpp/src/common/tsfile_common.h @@ -321,6 +321,7 @@ class ITimeseriesIndex { virtual common::TSDataType get_data_type() const { return common::INVALID_DATATYPE; } + virtual bool is_aligned() const { return false; } virtual Statistic* get_statistic() const { return nullptr; } }; @@ -589,8 +590,10 @@ class AlignedTimeseriesIndex : public ITimeseriesIndex { return value_ts_idx_->get_measurement_name(); } virtual common::TSDataType get_data_type() const { - return time_ts_idx_->get_data_type(); + return value_ts_idx_ == nullptr ? common::INVALID_DATATYPE + : value_ts_idx_->get_data_type(); } + virtual bool is_aligned() const { return true; } virtual Statistic* get_statistic() const { return value_ts_idx_->get_statistic(); } diff --git a/cpp/src/reader/block/single_device_tsblock_reader.cc b/cpp/src/reader/block/single_device_tsblock_reader.cc index 9b21e41b1..93f42efd3 100644 --- a/cpp/src/reader/block/single_device_tsblock_reader.cc +++ b/cpp/src/reader/block/single_device_tsblock_reader.cc @@ -63,7 +63,7 @@ int32_t SingleDeviceTsBlockReader::compute_dense_row_count( int64_t time_count = 0; int64_t value_count = 0; - if (ts_index->get_data_type() == common::VECTOR) { + if (ts_index->is_aligned()) { auto* time_list = ts_index->get_time_chunk_meta_list(); auto* value_list = ts_index->get_value_chunk_meta_list(); if (time_list == nullptr || value_list == nullptr) { @@ -427,9 +427,9 @@ int SingleDeviceTsBlockReader::construct_column_context( int ssi_offset, int ssi_limit) { int ret = common::E_OK; if (time_series_index == nullptr || - (time_series_index->get_data_type() != common::TSDataType::VECTOR && + (!time_series_index->is_aligned() && time_series_index->get_chunk_meta_list()->empty())) { - } else if (time_series_index->get_data_type() == common::VECTOR) { + } else if (time_series_index->is_aligned()) { const AlignedTimeseriesIndex* aligned_time_series_index = dynamic_cast(time_series_index); if (aligned_time_series_index == nullptr) { diff --git a/cpp/src/reader/tsfile_series_scan_iterator.cc b/cpp/src/reader/tsfile_series_scan_iterator.cc index 5add1e9a3..f53e88e7b 100644 --- a/cpp/src/reader/tsfile_series_scan_iterator.cc +++ b/cpp/src/reader/tsfile_series_scan_iterator.cc @@ -171,7 +171,7 @@ void TsFileSeriesScanIterator::revert_tsblock() { int TsFileSeriesScanIterator::init_chunk_reader() { int ret = E_OK; - is_aligned_ = itimeseries_index_->get_data_type() == common::VECTOR; + is_aligned_ = itimeseries_index_->is_aligned(); if (!is_aligned_) { void* buf = common::mem_alloc(sizeof(ChunkReader), common::MOD_CHUNK_READER); diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index 54127e072..c4ec9622d 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -21,7 +21,9 @@ #include #include +#include #include +#include #include #include "common/record.h" @@ -264,6 +266,78 @@ TEST_F(TsFileReaderTest, GetTimeseriesSchema) { reader.close(); } +TEST_F(TsFileReaderTest, GetTimeseriesMetadataTableModelTypeAndDeviceFilter) { + std::vector measurement_schemas = { + new MeasurementSchema("deviceid1", TSDataType::STRING), + new MeasurementSchema("deviceid2", TSDataType::STRING), + new MeasurementSchema("temperature", TSDataType::FLOAT), + new MeasurementSchema("pressure", TSDataType::DOUBLE), + new MeasurementSchema("humidity", TSDataType::INT32)}; + std::vector column_categories = { + ColumnCategory::TAG, ColumnCategory::TAG, ColumnCategory::FIELD, + ColumnCategory::FIELD, ColumnCategory::FIELD}; + auto table_schema = std::make_shared( + "testtable", measurement_schemas, column_categories); + + ASSERT_EQ(tsfile_writer_->register_table(table_schema), E_OK); + + Tablet tablet(table_schema->get_table_name(), + table_schema->get_measurement_names(), + table_schema->get_data_types(), + table_schema->get_column_categories(), 10); + for (int row = 0; row < 5; row++) { + ASSERT_EQ(tablet.add_timestamp(row, row), E_OK); + ASSERT_EQ(tablet.add_value(row, "deviceid1", "device_a"), E_OK); + ASSERT_EQ(tablet.add_value(row, "deviceid2", "device_b"), E_OK); + ASSERT_EQ(tablet.add_value(row, "temperature", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "pressure", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "humidity", static_cast(row)), + E_OK); + } + for (int row = 5; row < 10; row++) { + ASSERT_EQ(tablet.add_timestamp(row, row), E_OK); + ASSERT_EQ(tablet.add_value(row, "deviceid1", "device_b"), E_OK); + ASSERT_EQ(tablet.add_value(row, "deviceid2", "device_a"), E_OK); + ASSERT_EQ(tablet.add_value(row, "temperature", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "pressure", static_cast(row)), + E_OK); + ASSERT_EQ(tablet.add_value(row, "humidity", static_cast(row)), + E_OK); + } + + ASSERT_EQ(tsfile_writer_->write_table(tablet), E_OK); + ASSERT_EQ(tsfile_writer_->flush(), E_OK); + ASSERT_EQ(tsfile_writer_->close(), E_OK); + + storage::TsFileReader reader; + ASSERT_EQ(reader.open(file_name_), common::E_OK); + + auto all_meta = reader.get_timeseries_metadata(); + ASSERT_EQ(all_meta.size(), 2u); + + std::vector selected_device_segments = { + "testtable", "device_a", "device_b"}; + std::vector> selected_devices = { + std::make_shared(selected_device_segments)}; + auto selected_meta = reader.get_timeseries_metadata(selected_devices); + ASSERT_EQ(selected_meta.size(), 1u); + + auto selected_list = selected_meta.begin()->second; + std::unordered_map type_by_measurement; + for (const auto& index : selected_list) { + type_by_measurement[index->get_measurement_name().to_std_string()] = + index->get_data_type(); + } + ASSERT_EQ(type_by_measurement.at("temperature"), TSDataType::FLOAT); + ASSERT_EQ(type_by_measurement.at("pressure"), TSDataType::DOUBLE); + ASSERT_EQ(type_by_measurement.at("humidity"), TSDataType::INT32); + + reader.close(); +} + static const int64_t kLargeFileNumRecords = 300000000; static const int64_t kLargeFileFlushBatch = 100000; From ffc230eaf3b343cdd22a03a37c277a31dcbd37d0 Mon Sep 17 00:00:00 2001 From: 761417898 <761417898@qq.com> Date: Tue, 14 Apr 2026 16:51:48 +0800 Subject: [PATCH 2/2] add it --- cpp/test/reader/tsfile_reader_test.cc | 60 ++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/cpp/test/reader/tsfile_reader_test.cc b/cpp/test/reader/tsfile_reader_test.cc index c4ec9622d..45261cf45 100644 --- a/cpp/test/reader/tsfile_reader_test.cc +++ b/cpp/test/reader/tsfile_reader_test.cc @@ -308,7 +308,40 @@ TEST_F(TsFileReaderTest, GetTimeseriesMetadataTableModelTypeAndDeviceFilter) { E_OK); } + // Append one row whose middle TAG segment is null. + Tablet null_tag_tablet(table_schema->get_table_name(), + table_schema->get_measurement_names(), + table_schema->get_data_types(), + table_schema->get_column_categories(), 1); + int64_t null_tag_ts[1] = {10}; + int32_t null_tag_humidity[1] = {10}; + float null_tag_temperature[1] = {10.0F}; + double null_tag_pressure[1] = {10.0}; + // deviceid1 = null + int32_t id1_offsets[2] = {0, 0}; + uint8_t id1_bitmap[1] = {0x01}; // row0 is null + // deviceid2 = "device_b" + int32_t id2_offsets[2] = {0, 8}; + const char id2_data[] = "device_b"; + ASSERT_EQ(null_tag_tablet.set_timestamps(null_tag_ts, 1), E_OK); + ASSERT_EQ(null_tag_tablet.set_column_string_values(0, id1_offsets, "", + id1_bitmap, 1), + E_OK); + ASSERT_EQ(null_tag_tablet.set_column_string_values(1, id2_offsets, id2_data, + nullptr, 1), + E_OK); + ASSERT_EQ( + null_tag_tablet.set_column_values(2, null_tag_temperature, nullptr, 1), + E_OK); + ASSERT_EQ( + null_tag_tablet.set_column_values(3, null_tag_pressure, nullptr, 1), + E_OK); + ASSERT_EQ( + null_tag_tablet.set_column_values(4, null_tag_humidity, nullptr, 1), + E_OK); + ASSERT_EQ(tsfile_writer_->write_table(tablet), E_OK); + ASSERT_EQ(tsfile_writer_->write_table(null_tag_tablet), E_OK); ASSERT_EQ(tsfile_writer_->flush(), E_OK); ASSERT_EQ(tsfile_writer_->close(), E_OK); @@ -316,7 +349,7 @@ TEST_F(TsFileReaderTest, GetTimeseriesMetadataTableModelTypeAndDeviceFilter) { ASSERT_EQ(reader.open(file_name_), common::E_OK); auto all_meta = reader.get_timeseries_metadata(); - ASSERT_EQ(all_meta.size(), 2u); + ASSERT_EQ(all_meta.size(), 3u); std::vector selected_device_segments = { "testtable", "device_a", "device_b"}; @@ -335,6 +368,31 @@ TEST_F(TsFileReaderTest, GetTimeseriesMetadataTableModelTypeAndDeviceFilter) { ASSERT_EQ(type_by_measurement.at("pressure"), TSDataType::DOUBLE); ASSERT_EQ(type_by_measurement.at("humidity"), TSDataType::INT32); + // Query metadata for the device with null middle TAG segment. + std::vector null_seg_device = { + new std::string("testtable"), nullptr, new std::string("device_b")}; + std::vector> null_seg_devices = { + std::make_shared(null_seg_device)}; + for (auto* seg : null_seg_device) { + if (seg != nullptr) { + delete seg; + } + } + auto null_seg_meta = reader.get_timeseries_metadata(null_seg_devices); + ASSERT_EQ(null_seg_meta.size(), 1u); + auto null_seg_list = null_seg_meta.begin()->second; + ASSERT_EQ(null_seg_list.size(), 3u); + std::unordered_map null_seg_type_by_measurement; + for (const auto& index : null_seg_list) { + null_seg_type_by_measurement[index->get_measurement_name() + .to_std_string()] = + index->get_data_type(); + } + ASSERT_EQ(null_seg_type_by_measurement.at("temperature"), + TSDataType::FLOAT); + ASSERT_EQ(null_seg_type_by_measurement.at("pressure"), TSDataType::DOUBLE); + ASSERT_EQ(null_seg_type_by_measurement.at("humidity"), TSDataType::INT32); + reader.close(); }