Merge branch 'develop' into tableStatistics

Author: shuwenwei

.github/workflows/unit-test-cpp.yml

@@ -104,12 +104,15 @@ jobs:
               core.setOutput('platform_suffix', ``)
             }
 
-      - name: Install clang-format
+      # Install dependencies
+      - name: Install dependencies
         shell: bash
         run: |
           if [[ "$RUNNER_OS" == "Linux" ]]; then
             sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100
             sudo update-alternatives --set clang-format /usr/bin/clang-format-17
+            sudo apt-get update
+            sudo apt-get install -y uuid-dev
           elif [[ "$RUNNER_OS" == "Windows" ]]; then
             choco install llvm --version 17.0.6 --force
           else

.github/workflows/unit-test-java.yml

@@ -44,7 +44,7 @@ jobs:
       fail-fast: false
       max-parallel: 15
       matrix:
-        java: [ 8, 17, 21 ]
+        java: [ 8, 17, 25 ]
         os: [ ubuntu-latest, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
 

.github/workflows/unit-test-python.yml

@@ -75,6 +75,23 @@ jobs:
               core.setOutput('platform_suffix', ``)
             }
 
+      # Install dependencies
+      - name: Install dependencies
+        shell: bash
+        run: |
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100
+            sudo update-alternatives --set clang-format /usr/bin/clang-format-17
+            sudo apt-get update
+            sudo apt-get install -y uuid-dev
+          elif [[ "$RUNNER_OS" == "Windows" ]]; then
+            choco install llvm --version 17.0.6 --force
+          else
+            brew install llvm@17
+            ln -sf $(brew --prefix llvm@17)/bin/clang-format /opt/homebrew/bin/clang-format
+          fi
+
+
       # Run the actual maven build including all tests.
       - name: Build and test with Maven
         shell: bash

RELEASE_NOTES.md

@@ -18,6 +18,26 @@
     under the License.
 
 -->
+# Apache TsFile 2.2.0
+
+## New Feature
+
+- TsFile-Python fully supports TEXT and STRING data types.
+- The Python interface supports the to_dataframe method.
+- C++ TsFile supports encoding formats such as RLE.
+- Both C++ TsFile and Java TsFile table models support tag filtering.
+- C++ TsFile supports writing data of the TEXT type.
+- Added the CAMEL encoding method.
+- ...
+
+## Improvement/Bugfix
+
+- Fixed the issue where the CppTsFile write and query interfaces had various exceptions when handling null values.
+- Fixed the issue where Cpp TsFile failed to write data when TAG and FIELD columns were all empty (only timestamps existed with no actual data).
+- Fixed the issue where column names were all converted to lowercase when constructing a Tablet in Cpp TsFile, resulting in the failure to add values to Tablets constructed with uppercase column names.
+- Fixed the issue where queries would throw errors when the values of TEXT-type columns in the CPP TsFile table model were partially empty.
+- Fixed security vulnerabilities CVE-2025-12183, CVE-2025-66566 and CVE-2025-11226.
+- ...
 
 # Apache TsFile 2.1.1
 
@@ -103,6 +123,34 @@
 - Fixed the issue that the time of the first data item written to TSFile by measurement cannot be a negative number by @luoluoyuyu in #297
 - Fix float encoder overflow by @HTHou in #342
 
+# Apache TsFile 1.1.3
+
+* perf: Optimize aligned object memory size calculation
+* feat: add markRange / unmarkRange / merge for high-performance bit manipulation
+* Modify the TsFileSequenceReaderTimeseriesMetadataIterator next function to return a LinkedHashMap
+* fix GroupByMonthFilter.getTimeRanges
+* Fix bug in PaginationController
+* change config not found log to debug
+* Init all series writer for AlignedChunkGroupWriter
+* Added memory calculation for tablet
+* Dont print exception log when thread is interrupted
+* Bump lz4-java version to 1.10.1
+
+# Apache TsFile 1.1.2
+
+## Improvement/Bugfix
+
+* Fix the bug in parse date to int when year out of range by @HTHou in #500
+* Add TsFileLastReader for retrieving last points in a TsFile by @jt2594838 in #506
+* Added accountable function to measurementSchema by @Caideyipi in #509
+* Correct the retained size calculation for BinaryColumn and BinaryColumnBuilder by @JackieTien97 in #514
+* add switch to disable native lz4 (#480) by @jt2594838 in #515
+* Correct the memroy calculation of BinaryColumnBuilder by @JackieTien97 in #530
+* Fetch max tsblock line number each time from TSFileConfig by @JackieTien97 in #535
+* Support set default compression by data type & Bump org.apache.commons:commons-lang3 from 3.15.0 to 3.18.0 by @jt2594838 in #547
+* Avoid calculating shallow size of map by @shuwenwei in #566
+* Add methods for RamUsageEstimator by @shuwenwei in #570
+
 # Apache TsFile 1.1.1
 
 ## Improvement/Bugfix

cpp/.clang-format

@@ -65,7 +65,6 @@ ConstructorInitializerIndentWidth: 4
 ContinuationIndentWidth: 4
 Cpp11BracedListStyle: true
 DeriveLineEnding: true
-DerivePointerAlignment: true
 DisableFormat:   false
 EmptyLineAfterAccessModifier: Never
 EmptyLineBeforeAccessModifier: LogicalBlock
@@ -207,7 +206,7 @@ SpacesInParentheses: false
 SpacesInSquareBrackets: false
 SpaceBeforeSquareBrackets: false
 BitFieldColonSpacing: Both
-Standard:        Auto
+Standard:        Cpp11
 StatementAttributeLikeMacros:
   - Q_EMIT
 StatementMacros:

cpp/.gitignore

@@ -5,4 +5,3 @@ build/
 .settings/
 .vscode/
 cmake-build-debug/
-cmake*/

cpp/CMakeLists.txt

@@ -19,9 +19,17 @@ under the License.
 cmake_minimum_required(VERSION 3.11)
 project(TsFile_CPP)
 
+if (DEFINED ToolChain)
+    include(${CMAKE_SOURCE_DIR}/cmake/ToolChain.cmake)
+    message(STATUS "Using ToolChain: ${CMAKE_TOOLCHAIN_FILE}")
+else()
+    message(STATUS "Not using ToolChain")
+endif ()
+
 cmake_policy(SET CMP0079 NEW)
-set(TsFile_CPP_VERSION 2.2.0.dev)
+set(TsFile_CPP_VERSION 2.2.1.dev)
 set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall")
+
 if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wunused -Wuninitialized -D__STDC_FORMAT_MACROS")
 endif ()
@@ -55,6 +63,18 @@ if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
 endif ()
 
+if (NOT DEFINED CMAKE_BUILD_PARALLEL_LEVEL)
+    include(ProcessorCount)
+    ProcessorCount(N)
+    if (N EQUAL 0)
+        set(N 1)
+    endif ()
+    set(CMAKE_BUILD_PARALLEL_LEVEL ${N} CACHE STRING "Number of parallel build jobs")
+    message("CMAKE BUILD PARALLEL LEVEL: ${CMAKE_BUILD_PARALLEL_LEVEL} (auto-detected)")
+else ()
+    message("CMAKE BUILD PARALLEL LEVEL: ${CMAKE_BUILD_PARALLEL_LEVEL} (from environment)")
+endif ()
+
 message("CMAKE BUILD TYPE " ${CMAKE_BUILD_TYPE})
 if (CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
@@ -88,6 +108,40 @@ if (NOT WIN32)
     endif ()
 endif ()
 
+option(BUILD_TEST "Build tests" ON)
+message("cmake using: BUILD_TEST=${BUILD_TEST}")
+
+option(ENABLE_ANTLR4 "Enable ANTLR4 runtime" ON)
+message("cmake using: ENABLE_ANTLR4=${ENABLE_ANTLR4}")
+
+option(ENABLE_SNAPPY "Enable Google Snappy compression" ON)
+message("cmake using: ENABLE_SNAPPY=${ENABLE_SNAPPY}")
+
+if (ENABLE_SNAPPY)
+    add_definitions(-DENABLE_SNAPPY)
+endif()
+
+option(ENABLE_LZ4 "Enable LZ4 compression" ON)
+message("cmake using: ENABLE_LZ4=${ENABLE_LZ4}")
+
+if (ENABLE_LZ4)
+    add_definitions(-DENABLE_LZ4)
+endif()
+
+option(ENABLE_LZOKAY "Enable LZOKAY compression" ON)
+message("cmake using: ENABLE_LZOKAY=${ENABLE_LZOKAY}")
+
+if (ENABLE_LZOKAY)
+    add_definitions(-DENABLE_LZOKAY)
+endif()
+
+option(ENABLE_ZLIB "Enable Zlib compression" ON)
+message("cmake using: ENABLE_ZLIB=${ENABLE_ZLIB}")
+
+if (ENABLE_ZLIB)
+    add_definitions(-DENABLE_ZLIB)
+    add_definitions(-DENABLE_GZIP)
+endif()
 
 # All libs will be stored here, including libtsfile, compress-encoding lib.
 set(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
@@ -106,9 +160,14 @@ add_subdirectory(third_party)
 set(CMAKE_CXX_FLAGS "${SAVED_CXX_FLAGS}")
 
 add_subdirectory(src)
-add_subdirectory(test)
-add_subdirectory(examples)
-if (TESTS_ENABLED)
-    add_dependencies(TsFile_Test tsfile)
+if (BUILD_TEST)
+    add_subdirectory(test)
+    if (TESTS_ENABLED)
+        add_dependencies(TsFile_Test tsfile)
+    endif ()
+else()
+    message("BUILD_TEST is OFF, skipping test directory")
 endif ()
 
+add_subdirectory(examples)
+

cpp/README.md

@@ -42,23 +42,6 @@ The source code can be found in the `./src` directory. C/C++ examples are locate
 
 We use `clang-format` to ensure that our C++ code adheres to a consistent set of rules defined in `./clang-format`. This is similar to the Google style.
 
-**Feature List**:
-
-- [ ] Add unit tests for the reader, writer, compression, etc.
-- [ ] Add unit tests for the C wrapper.
-- [ ] Support multiple data flushes.
-- [ ] Support aligned timeseries.
-- [ ] Support table description in tsfile.
-- [ ] Retrieve all table schemas/names.
-- [ ] Implement automatic flush.
-- [ ] Support out-of-order data writing.
-- [ ] Support TsFile V4. Note: TsFile CPP does not implement support for the table model, therefore there are differences in file output compared to the Java version.
-
-**Bug List**:
-
-- [ ] Flushing without writing after registering a timeseries will cause a core dump.
-- [ ] Misalignment in memory may lead to a bus error.
-
 We welcome any bug reports. You can open an issue with a title starting with [CPP] to describe the bug, like: https://github.com/apache/tsfile/issues/94
 
 ## Build
@@ -67,7 +50,7 @@ We welcome any bug reports. You can open an issue with a title starting with [CP
 
 ```bash
 sudo apt-get update
-sudo apt-get install -y cmake make g++ clang-format
+sudo apt-get install -y cmake make g++ clang-format libuuid-dev
 ```
 
 To build tsfile, you can run: `bash build.sh`. If you have Maven tools, you can run: `mvn package -P with-cpp clean verify`. Then, you can find the shared object at `./build`.
@@ -81,6 +64,21 @@ If you compile using MinGW on windows and encounter an error, you can try replac
 * GCC 12.2.0 + LLVM/Clang/LLD/LLDB 16.0.0 + MinGW-w64 10.0.0 (MSVCRT) - release 5
 * GCC 11.2.0 + MinGW-w64 10.0.0 (MSVCRT) - release 1
 
+### configure the cross-compilation toolchain
+
+Modify the Toolchain File `cmake/ToolChain.cmake`, define the following variables:
+
+- `CMAKE_C_COMPILER`: Specify the path to the C compiler.
+- `CMAKE_CXX_COMPILER`: Specify the path to the C++ compiler.
+- `CMAKE_FIND_ROOT_PATH`: Set the root path for the cross-compilation environment (e.g., the directory of the cross-compilation toolchain).
+
+In the `cpp/` directory, run the following commands to create the build directory and start the compilation:
+```
+mkdir build && cd build
+cmake .. -DToolChain=ON
+make
+```
+
 ## Use TsFile
 
 You can find examples on how to read and write data in `demo_read.cpp` and `demo_write.cpp` located under `./examples/cpp_examples`. There are also examples under `./examples/c_examples`on how to use a C-style API to read and write data in a C environment. You can run `bash build.sh` under `./examples` to generate an executable output under `./examples/build`.

cpp/build.sh

@@ -25,22 +25,38 @@ use_cpp11=1
 enable_cov=0
 debug_se=0
 run_cov_only=0
+enable_antlr4=ON
+
+enable_snappy=ON
+enable_lz4=ON
+enable_lzokay=ON
+enable_zlib=ON
 
 shell_dir=$(cd "$(dirname "$0")";pwd)
 
+# 添加get_key_value函数
+get_key_value() {
+    echo "${1#*=}"
+}
+
 function print_config()
 {
   echo "build_type=$build_type"
   echo "build_test=$build_test"
   echo "use_cpp11=$use_cpp11"
   echo "enable_cov=$enable_cov"
   echo "enable_asan=$enable_asan"
+  echo "enable_antlr4=$enable_antlr4"
+  echo "enable_snappy=$enable_snappy"
+  echo "enable_lz4=$enable_lz4"
+  echo "enable_lzokay=$enable_lzokay"
+  echo "enable_zlib=$enable_zlib"
 }
 
 function run_test_for_cov()
 {
   # sh ${shell_dir}/scripts/regression_unittest.sh
-  sh ${shell_dir}/test/libtsfile_test/run_sdk_tests.sh 
+  sh ${shell_dir}/test/libtsfile_test/run_sdk_tests.sh
 }
 
 parse_options()
@@ -53,20 +69,40 @@ parse_options()
     run_cov)
       run_cov_only=1;;
     -t=*)
-      build_type=`get_key_value "$1"`;;
+      build_type=$(get_key_value "$1");;
     -t)
       shift
-      build_type=`get_key_value "$1"`;;
+      build_type=$(get_key_value "$1");;
     -a=*)
-      enable_asan=`get_key_value "$1"`;;
+      enable_asan=$(get_key_value "$1");;
     -a)
       shift
-      enable_asan=`get_key_value "$1"`;;
+      enable_asan=$(get_key_value "$1");;
     -c=*)
-      enable_cov=`get_key_value "$1"`;;
+      enable_cov=$(get_key_value "$1");;
     -c)
       shift
-      enable_cov=`get_key_value "$1"`;;
+      enable_cov=$(get_key_value "$1");;
+    --enable-antlr4=*)
+      enable_antlr4=$(get_key_value "$1");;
+    --enable-snappy=*)
+      enable_snappy=$(get_key_value "$1");;
+    --enable-lz4=*)
+      enable_lz4=$(get_key_value "$1");;
+    --enable-lzokay=*)
+      enable_lzokay=$(get_key_value "$1");;
+    --enable-zlib=*)
+      enable_zlib=$(get_key_value "$1");;
+    --disable-antlr4)
+      enable_antlr4=OFF;;
+    --disable-snappy)
+      enable_snappy=OFF;;
+    --disable-lz4)
+      enable_lz4=OFF;;
+    --disable-lzokay)
+      enable_lzokay=OFF;;
+    --disable-zlib)
+      enable_zlib=OFF;;
     #-h | --help)
     #  usage
     #  exit 0;;
@@ -127,14 +163,12 @@ cmake ../../                           \
   -DUSE_CPP11=$use_cpp11               \
   -DENABLE_COV=$enable_cov             \
   -DDEBUG_SE=$debug_se                 \
-  -DBUILD_TSFILE_ONLY=$build_tsfile_only
+  -DENABLE_ANTLR4=$enable_antlr4       \
+  -DBUILD_TSFILE_ONLY=$build_tsfile_only \
+  -DENABLE_SNAPPY=$enable_snappy       \
+  -DENABLE_LZ4=$enable_lz4             \
+  -DENABLE_LZOKAY=$enable_lzokay       \
+  -DENABLE_ZLIB=$enable_zlib
 
 VERBOSE=1 make
-VERBOSE=1 make install
-
-
-
-
-
-
-
+VERBOSE=1 make install