Skip to content

Commit 3a688ee

Browse files
jt2594838jt2594838
authored
Add TsFileLastReader for retrieving last points in a TsFile (#498)
* Add TsFileLastReader for retrieving last points in a TsFile * Fix query last of blob series (cherry picked from commit 3ab6c40) * spotless (cherry picked from commit 1f70820) * fix series with empty chunk * fix test name * reduce test scale * fix last empty page (cherry picked from commit 40105d1) * add ignore blob
1 parent dd603b4 commit 3a688ee

17 files changed

Lines changed: 932 additions & 31 deletions

File tree

‎cpp/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<parent>
2323
<groupId>org.apache.tsfile</groupId>
2424
<artifactId>tsfile-parent</artifactId>
25-
<version>2.1.0-250325-SNAPSHOT</version>
25+
<version>2.1.0-SNAPSHOT</version>
2626
</parent>
2727
<artifactId>tsfile-cpp</artifactId>
2828
<packaging>pom</packaging>

‎java/common/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<parent>
2525
<groupId>org.apache.tsfile</groupId>
2626
<artifactId>tsfile-java</artifactId>
27-
<version>2.1.0-250325-SNAPSHOT</version>
27+
<version>2.1.0-SNAPSHOT</version>
2828
</parent>
2929
<artifactId>common</artifactId>
3030
<name>TsFile: Java: Common</name>

‎java/examples/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<parent>
2525
<groupId>org.apache.tsfile</groupId>
2626
<artifactId>tsfile-java</artifactId>
27-
<version>2.1.0-250325-SNAPSHOT</version>
27+
<version>2.1.0-SNAPSHOT</version>
2828
</parent>
2929
<artifactId>examples</artifactId>
3030
<name>TsFile: Java: Examples</name>
@@ -36,7 +36,7 @@
3636
<dependency>
3737
<groupId>org.apache.tsfile</groupId>
3838
<artifactId>tsfile</artifactId>
39-
<version>2.1.0-250325-SNAPSHOT</version>
39+
<version>2.1.0-SNAPSHOT</version>
4040
</dependency>
4141
</dependencies>
4242
<build>

‎java/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@
2424
<parent>
2525
<groupId>org.apache.tsfile</groupId>
2626
<artifactId>tsfile-parent</artifactId>
27-
<version>2.1.0-250325-SNAPSHOT</version>
27+
<version>2.1.0-SNAPSHOT</version>
2828
</parent>
2929
<artifactId>tsfile-java</artifactId>
30-
<version>2.1.0-250325-SNAPSHOT</version>
30+
<version>2.1.0-SNAPSHOT</version>
3131
<packaging>pom</packaging>
3232
<name>TsFile: Java</name>
3333
<modules>

‎java/tools/pom.xml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,15 @@
2424
<parent>
2525
<groupId>org.apache.tsfile</groupId>
2626
<artifactId>tsfile-java</artifactId>
27-
<version>2.1.0-250325-SNAPSHOT</version>
27+
<version>2.1.0-SNAPSHOT</version>
2828
</parent>
2929
<artifactId>tools</artifactId>
3030
<name>TsFile: Java: Tools</name>
3131
<dependencies>
3232
<dependency>
3333
<groupId>org.apache.tsfile</groupId>
3434
<artifactId>common</artifactId>
35-
<version>2.1.0-250325-SNAPSHOT</version>
35+
<version>2.1.0-SNAPSHOT</version>
3636
</dependency>
3737
<dependency>
3838
<groupId>commons-cli</groupId>
@@ -50,7 +50,7 @@
5050
<dependency>
5151
<groupId>org.apache.tsfile</groupId>
5252
<artifactId>tsfile</artifactId>
53-
<version>2.1.0-250325-SNAPSHOT</version>
53+
<version>2.1.0-SNAPSHOT</version>
5454
</dependency>
5555
<dependency>
5656
<groupId>ch.qos.logback</groupId>

‎java/tsfile/pom.xml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
<parent>
2525
<groupId>org.apache.tsfile</groupId>
2626
<artifactId>tsfile-java</artifactId>
27-
<version>2.1.0-250325-SNAPSHOT</version>
27+
<version>2.1.0-SNAPSHOT</version>
2828
</parent>
2929
<artifactId>tsfile</artifactId>
3030
<name>TsFile: Java: TsFile</name>
@@ -38,7 +38,7 @@
3838
<dependency>
3939
<groupId>org.apache.tsfile</groupId>
4040
<artifactId>common</artifactId>
41-
<version>2.1.0-250325-SNAPSHOT</version>
41+
<version>2.1.0-SNAPSHOT</version>
4242
</dependency>
4343
<dependency>
4444
<groupId>com.github.luben</groupId>

‎java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -121,14 +121,20 @@ public TimeseriesMetadata(TimeseriesMetadata timeseriesMetadata) {
121121
}
122122

123123
public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean needChunkMetadata) {
124+
return deserializeFrom(buffer, needChunkMetadata, needChunkMetadata);
125+
}
126+
127+
public static TimeseriesMetadata deserializeFrom(
128+
ByteBuffer buffer, boolean needChunkMetadataForNonBlob, boolean needChunkMetadataForBlob) {
124129
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
125130
timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(buffer));
126131
timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(buffer));
127132
timeseriesMetaData.setTsDataType(ReadWriteIOUtils.readDataType(buffer));
128133
int chunkMetaDataListDataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(buffer);
129134
timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize);
130135
timeseriesMetaData.setStatistics(Statistics.deserialize(buffer, timeseriesMetaData.dataType));
131-
if (needChunkMetadata) {
136+
if ((timeseriesMetaData.getTsDataType() != TSDataType.BLOB && needChunkMetadataForNonBlob)
137+
|| (timeseriesMetaData.getTsDataType() == TSDataType.BLOB && needChunkMetadataForBlob)) {
132138
ByteBuffer byteBuffer = buffer.slice();
133139
byteBuffer.limit(chunkMetaDataListDataSize);
134140
timeseriesMetaData.chunkMetadataList = new ArrayList<>();
@@ -145,6 +151,14 @@ public static TimeseriesMetadata deserializeFrom(ByteBuffer buffer, boolean need
145151

146152
public static TimeseriesMetadata deserializeFrom(
147153
TsFileInput tsFileInput, boolean needChunkMetadata) throws IOException {
154+
return deserializeFrom(tsFileInput, needChunkMetadata, needChunkMetadata);
155+
}
156+
157+
public static TimeseriesMetadata deserializeFrom(
158+
TsFileInput tsFileInput,
159+
boolean needChunkMetadataForNonBlob,
160+
boolean needChunkMetadataForBlob)
161+
throws IOException {
148162
InputStream inputStream = tsFileInput.wrapAsInputStream();
149163
TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata();
150164
timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(inputStream));
@@ -155,7 +169,8 @@ public static TimeseriesMetadata deserializeFrom(
155169
timeseriesMetaData.setStatistics(
156170
Statistics.deserialize(inputStream, timeseriesMetaData.dataType));
157171
long startOffset = tsFileInput.position();
158-
if (needChunkMetadata) {
172+
if ((timeseriesMetaData.getTsDataType() != TSDataType.BLOB && needChunkMetadataForNonBlob)
173+
|| (timeseriesMetaData.getTsDataType() == TSDataType.BLOB && needChunkMetadataForBlob)) {
159174
timeseriesMetaData.chunkMetadataList = new ArrayList<>();
160175
while (tsFileInput.position() < startOffset + chunkMetaDataListDataSize) {
161176
timeseriesMetaData.chunkMetadataList.add(
@@ -175,6 +190,14 @@ public static TimeseriesMetadata deserializeFrom(
175190
*/
176191
public static TimeseriesMetadata deserializeFrom(
177192
ByteBuffer buffer, Set<String> excludedMeasurements, boolean needChunkMetadata) {
193+
return deserializeFrom(buffer, excludedMeasurements, needChunkMetadata, needChunkMetadata);
194+
}
195+
196+
public static TimeseriesMetadata deserializeFrom(
197+
ByteBuffer buffer,
198+
Set<String> excludedMeasurements,
199+
boolean needChunkMetadataForNonBlob,
200+
boolean needChunkMetadataForBlob) {
178201
byte timeseriesType = ReadWriteIOUtils.readByte(buffer);
179202
String measurementID = ReadWriteIOUtils.readVarIntString(buffer);
180203
TSDataType tsDataType = ReadWriteIOUtils.readDataType(buffer);
@@ -188,7 +211,9 @@ public static TimeseriesMetadata deserializeFrom(
188211
timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize);
189212
timeseriesMetaData.setStatistics(statistics);
190213

191-
if (!excludedMeasurements.contains(measurementID) && needChunkMetadata) {
214+
if (!excludedMeasurements.contains(measurementID)
215+
&& ((tsDataType != TSDataType.BLOB && needChunkMetadataForNonBlob)
216+
|| (tsDataType == TSDataType.BLOB && needChunkMetadataForBlob))) {
192217
// measurement is not in the excluded set and need chunk metadata
193218
ByteBuffer byteBuffer = buffer.slice();
194219
byteBuffer.limit(chunkMetaDataListDataSize);

‎java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimeStatistics.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -76,22 +76,22 @@ public void update(long[] time, int batchSize, int arrayOffset) {
7676

7777
@Override
7878
public Long getMinValue() {
79-
throw new StatisticsClassException(String.format(STATS_UNSUPPORTED_MSG, TIME, "min value"));
79+
return getStartTime();
8080
}
8181

8282
@Override
8383
public Long getMaxValue() {
84-
throw new StatisticsClassException(String.format(STATS_UNSUPPORTED_MSG, TIME, "max value"));
84+
return getEndTime();
8585
}
8686

8787
@Override
8888
public Long getFirstValue() {
89-
throw new StatisticsClassException(String.format(STATS_UNSUPPORTED_MSG, TIME, "first value"));
89+
return getStartTime();
9090
}
9191

9292
@Override
9393
public Long getLastValue() {
94-
throw new StatisticsClassException(String.format(STATS_UNSUPPORTED_MSG, TIME, "last value"));
94+
return getEndTime();
9595
}
9696

9797
@Override

0 commit comments

Comments
 (0)