add zstd,lzma2 and some ending type.

.github/workflows/compatibility-test.yml

@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Compatibility-Test
+
+on:
+  push:
+    branches:
+      - develop
+      - iotdb
+      - rc/*
+  pull_request:
+    branches:
+      - develop
+      - dev/*
+      - iotdb
+      - rc/*
+  workflow_dispatch:
+    inputs:
+      forceUpdates:
+        description: "Forces a snapshot update"
+        required: false
+        default: 'false'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
+  DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+
+jobs:
+  generate-fixtures:
+    name: Generate ${{ matrix.language }} fixtures
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ java, cpp ]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          distribution: corretto
+          java-version: 17
+
+      - name: Cache Maven packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2-
+
+      - name: Install C++ dependencies
+        if: matrix.language == 'cpp'
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y uuid-dev
+          sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100
+          sudo update-alternatives --set clang-format /usr/bin/clang-format-17
+
+      - name: Generate Java table fixtures
+        if: matrix.language == 'java'
+        shell: bash
+        run: |
+          mkdir -p compat-fixtures/java
+          ./mvnw -P with-java \
+            -Dtest=TableModelEncodingCompressionCompatibilityTest#generateFixtures \
+            -Dsurefire.failIfNoSpecifiedTests=false \
+            -Dtsfile.compat.generate.dir="$PWD/compat-fixtures/java" \
+            test
+
+      - name: Build C++ tests
+        if: matrix.language == 'cpp'
+        shell: bash
+        run: |
+          ./mvnw -P with-cpp -Dbuild.type=Release -DskipTests package
+
+      - name: Generate C++ table fixtures
+        if: matrix.language == 'cpp'
+        shell: bash
+        run: |
+          mkdir -p compat-fixtures/cpp
+          TSFILE_COMPAT_GENERATE_DIR="$PWD/compat-fixtures/cpp" \
+            ./cpp/target/build/test/lib/TsFile_Test \
+            --gtest_filter=TableModelEncodingCompressionCompatibilityTest.GenerateFixtures
+
+      - name: Upload ${{ matrix.language }} fixtures
+        uses: actions/upload-artifact@v7
+        with:
+          name: tsfile-${{ matrix.language }}-compat-fixtures
+          path: compat-fixtures/${{ matrix.language }}/
+          retention-days: 1
+
+  validate-fixtures:
+    name: Validate ${{ matrix.writer }} fixtures with ${{ matrix.reader }}
+    needs: generate-fixtures
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - writer: java
+            reader: cpp
+          - writer: cpp
+            reader: java
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          distribution: corretto
+          java-version: 17
+
+      - name: Cache Maven packages
+        uses: actions/cache@v5
+        with:
+          path: ~/.m2
+          key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
+          restore-keys: ${{ runner.os }}-m2-
+
+      - name: Install C++ dependencies
+        if: matrix.reader == 'cpp'
+        shell: bash
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y uuid-dev
+          sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100
+          sudo update-alternatives --set clang-format /usr/bin/clang-format-17
+
+      - name: Download ${{ matrix.writer }} fixtures
+        uses: actions/download-artifact@v8
+        with:
+          name: tsfile-${{ matrix.writer }}-compat-fixtures
+          path: compat-fixtures/${{ matrix.writer }}
+
+      - name: Validate ${{ matrix.writer }} fixtures with Java
+        if: matrix.reader == 'java'
+        shell: bash
+        run: |
+          ./mvnw -P with-java \
+            -Dtest=TableModelEncodingCompressionCompatibilityTest#validateFixtures \
+            -Dsurefire.failIfNoSpecifiedTests=false \
+            -Dtsfile.compat.validate.dir="$PWD/compat-fixtures/${{ matrix.writer }}" \
+            test
+
+      - name: Build C++ tests
+        if: matrix.reader == 'cpp'
+        shell: bash
+        run: |
+          ./mvnw -P with-cpp -Dbuild.type=Release -DskipTests package
+
+      - name: Validate ${{ matrix.writer }} fixtures with C++
+        if: matrix.reader == 'cpp'
+        shell: bash
+        run: |
+          TSFILE_COMPAT_VALIDATE_DIR="$PWD/compat-fixtures/${{ matrix.writer }}" \
+            ./cpp/target/build/test/lib/TsFile_Test \
+            --gtest_filter=TableModelEncodingCompressionCompatibilityTest.ValidateFixtures

cpp/.gitignore

@@ -1,4 +1,7 @@
 build/
+!third_party/zstd-1.5.7/build/
+!third_party/zstd-1.5.7/build/cmake/
+!third_party/zstd-1.5.7/build/cmake/**
 .idea/
 .cproject
 .project

cpp/CMakeLists.txt

@@ -247,6 +247,23 @@ if (ENABLE_ZLIB)
     add_definitions(-DENABLE_GZIP)
 endif()
 
+option(ENABLE_ZSTD "Enable Zstandard compression" ON)
+message("cmake using: ENABLE_ZSTD=${ENABLE_ZSTD}")
+
+if (ENABLE_ZSTD)
+    add_definitions(-DENABLE_ZSTD)
+endif()
+
+option(ENABLE_LZMA2 "Enable LZMA2 compression through bundled XZ Utils" ON)
+message("cmake using: ENABLE_LZMA2=${ENABLE_LZMA2}")
+
+if (ENABLE_LZMA2)
+    if (CMAKE_VERSION VERSION_LESS 3.20)
+        message(FATAL_ERROR "ENABLE_LZMA2 requires CMake 3.20 or newer because bundled XZ Utils 5.8.3 requires it")
+    endif()
+    add_definitions(-DENABLE_LZMA2)
+endif()
+
 option(ENABLE_THREADS "Enable multi-threaded read/write (requires pthreads)" ON)
 message("cmake using: ENABLE_THREADS=${ENABLE_THREADS}")
 

cpp/build.sh

@@ -32,6 +32,8 @@ enable_snappy=ON
 enable_lz4=ON
 enable_lzokay=ON
 enable_zlib=ON
+enable_zstd=ON
+enable_lzma2=ON
 
 shell_dir=$(cd "$(dirname "$0")";pwd)
 
@@ -62,6 +64,10 @@ Options:
   --disable-lzokay
   --enable-zlib=<ON|OFF>
   --disable-zlib
+  --enable-zstd=<ON|OFF>
+  --disable-zstd
+  --enable-lzma2=<ON|OFF>
+  --disable-lzma2
   -h, --help             Show this help message.
 EOF
 }
@@ -79,6 +85,8 @@ function print_config()
   echo "enable_lz4=$enable_lz4"
   echo "enable_lzokay=$enable_lzokay"
   echo "enable_zlib=$enable_zlib"
+  echo "enable_zstd=$enable_zstd"
+  echo "enable_lzma2=$enable_lzma2"
 }
 
 function run_test_for_cov()
@@ -123,6 +131,10 @@ parse_options()
       enable_lzokay=$(get_key_value "$1");;
     --enable-zlib=*)
       enable_zlib=$(get_key_value "$1");;
+    --enable-zstd=*)
+      enable_zstd=$(get_key_value "$1");;
+    --enable-lzma2=*)
+      enable_lzma2=$(get_key_value "$1");;
     --disable-antlr4)
       enable_antlr4=OFF;;
     --disable-snappy)
@@ -133,6 +145,10 @@ parse_options()
       enable_lzokay=OFF;;
     --disable-zlib)
       enable_zlib=OFF;;
+    --disable-zstd)
+      enable_zstd=OFF;;
+    --disable-lzma2)
+      enable_lzma2=OFF;;
     -h | --help)
       usage
       exit 0;;
@@ -199,7 +215,9 @@ cmake ../../                           \
   -DENABLE_SNAPPY=$enable_snappy       \
   -DENABLE_LZ4=$enable_lz4             \
   -DENABLE_LZOKAY=$enable_lzokay       \
-  -DENABLE_ZLIB=$enable_zlib
+  -DENABLE_ZLIB=$enable_zlib           \
+  -DENABLE_ZSTD=$enable_zstd           \
+  -DENABLE_LZMA2=$enable_lzma2
 
 VERBOSE=1 make
 if [ ${do_install} -eq 1 ]

cpp/pom.xml

@@ -39,6 +39,8 @@
         <enable.lz4>ON</enable.lz4>
         <enable.lzokay>ON</enable.lzokay>
         <enable.zlib>ON</enable.zlib>
+        <enable.zstd>ON</enable.zstd>
+        <enable.lzma2>ON</enable.lzma2>
         <enable.antlr4>ON</enable.antlr4>
         <enable.simde>OFF</enable.simde>
     </properties>
@@ -87,6 +89,8 @@
                                 <option>-DENABLE_LZ4=${enable.lz4}</option>
                                 <option>-DENABLE_LZOKAY=${enable.lzokay}</option>
                                 <option>-DENABLE_ZLIB=${enable.zlib}</option>
+                                <option>-DENABLE_ZSTD=${enable.zstd}</option>
+                                <option>-DENABLE_LZMA2=${enable.lzma2}</option>
                                 <option>-DENABLE_SIMDE=${enable.simde}</option>
                                 <!--
                                     The Visual Studio generator additionally

cpp/src/CMakeLists.txt

@@ -37,6 +37,12 @@ message("cmake using: ENABLE_LZOKAY=${ENABLE_LZOKAY}")
 option(ENABLE_ZLIB "Enable Zlib compression" ON)
 message("cmake using: ENABLE_ZLIB=${ENABLE_ZLIB}")
 
+option(ENABLE_ZSTD "Enable Zstandard compression" ON)
+message("cmake using: ENABLE_ZSTD=${ENABLE_ZSTD}")
+
+option(ENABLE_LZMA2 "Enable LZMA2 compression" ON)
+message("cmake using: ENABLE_LZMA2=${ENABLE_LZMA2}")
+
 # ENABLE_SIMD is defined in the top-level CMakeLists.txt
 message("cmake using: ENABLE_SIMD=${ENABLE_SIMD}")
 
@@ -85,6 +91,18 @@ if (ENABLE_ZLIB)
     )
 endif()
 
+if (ENABLE_ZSTD)
+    list(APPEND PROJECT_INCLUDE_DIR
+            ${CMAKE_SOURCE_DIR}/third_party/zstd-1.5.7/lib
+    )
+endif()
+
+if (ENABLE_LZMA2)
+    list(APPEND PROJECT_INCLUDE_DIR
+            ${CMAKE_SOURCE_DIR}/third_party/xz-5.8.3/src/liblzma/api
+    )
+endif()
+
 if (ENABLE_ANTLR4)
     list(APPEND PROJECT_INCLUDE_DIR
             ${CMAKE_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src
@@ -138,6 +156,14 @@ if (ENABLE_ZLIB)
     list(APPEND COMPRESSION_LIBS zlibstatic)
 endif()
 
+if (ENABLE_ZSTD)
+    list(APPEND COMPRESSION_LIBS libzstd_static)
+endif()
+
+if (ENABLE_LZMA2)
+    list(APPEND COMPRESSION_LIBS liblzma)
+endif()
+
 message("Compression libraries: ${COMPRESSION_LIBS}")
 
 if (ENABLE_ANTLR4)

cpp/src/common/db_common.h

@@ -73,7 +73,10 @@ enum TSEncoding : uint8_t {
     GORILLA = 8,
     ZIGZAG = 9,
     FREQ = 10,
+    CHIMP = 11,
     SPRINTZ = 12,
+    RLBE = 13,
+    CAMEL = 14,
     INVALID_ENCODING = 255
 };
 
@@ -92,12 +95,14 @@ enum CompressionType : uint8_t {
     PAA = 5,
     PLA = 6,
     LZ4 = 7,
+    ZSTD = 8,
+    LZMA2 = 9,
     INVALID_COMPRESSION = 255
 };
 
-extern TSFILE_API const char* s_data_type_names[8];
-extern TSFILE_API const char* s_encoding_names[12];
-extern TSFILE_API const char* s_compression_names[8];
+extern TSFILE_API const char* s_data_type_names[12];
+extern TSFILE_API const char* s_encoding_names[15];
+extern TSFILE_API const char* s_compression_names[10];
 }  // namespace common
 
 #if defined(__GLIBCXX__) && (__GNUC__ < 7)
@@ -154,11 +159,12 @@ FORCE_INLINE bool parse_data_type_name(const std::string& s, TSDataType& out) {
 }
 
 FORCE_INLINE const char* get_encoding_name(TSEncoding encoding) {
-    ASSERT(encoding >= PLAIN && encoding <= FREQ);
+    ASSERT(encoding >= PLAIN && encoding <= CAMEL);
     return s_encoding_names[encoding];
 }
 
 FORCE_INLINE const char* get_compression_name(CompressionType type) {
+    ASSERT(type >= UNCOMPRESSED && type <= LZMA2);
     return s_compression_names[type];
 }