From be074da1f51928c873f796d86dd6226ed76d4732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Tue, 28 Jul 2026 12:29:08 +0100 Subject: [PATCH 01/17] MINOR: Build test artifacts once in CI --- .github/workflows/test.yml | 230 +++++++++++++++++++++++++++++-------- ci/scripts/build.sh | 6 +- ci/scripts/test.sh | 68 +++++++++-- compose.yaml | 85 ++++++++++++++ 4 files changed, 331 insertions(+), 58 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63384b5d32..276209b987 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,25 +37,11 @@ env: DOCKER_VOLUME_PREFIX: ".docker/" jobs: - ubuntu: - name: AMD64 ${{ matrix.name }} JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + build-java: + name: Build Java artifacts runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [17, 21, 25] - maven: [3.9.16] - image: [ubuntu, conda-jni-cdata] - include: - - image: ubuntu - name: "Ubuntu" - - image: conda-jni-cdata - name: "Conda JNI" - env: - JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -65,34 +51,75 @@ jobs: - name: Cache Docker Volumes uses: actions/cache@v6 with: - path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - - name: Execute Docker Build + path: .docker/maven-cache + key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-3.9.9- + - name: Build without tests + shell: bash env: - # Enables build caching, but not strictly required DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + JDK: 17 + MAVEN: 3.9.9 run: | docker compose run \ + --rm \ -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ - ${{ matrix.image }} + ubuntu-artifacts + if [ -d .docker ]; then + sudo chown -R "$(id -u):$(id -g)" .docker + fi + - name: Pack reusable artifacts + shell: bash + run: | + tar -czf java-build.tgz \ + .docker/java-build \ + .docker/jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + - name: Upload reusable artifacts + uses: actions/upload-artifact@v5 + with: + name: java-build + path: java-build.tgz + retention-days: 1 - macos: - name: ${{ matrix.arch }} macOS ${{ matrix.macos }} Java JDK ${{ matrix.jdk }} - runs-on: macos-${{ matrix.macos }} + test-java: + name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven {1}', matrix.jdk, matrix.maven) }} + needs: build-java + runs-on: ${{ matrix.os }} if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 strategy: fail-fast: false matrix: + target: [ubuntu] + jdk: [17, 21, 23] + maven: [3.9.9] include: - - arch: AMD64 + - target: ubuntu + os: ubuntu-latest + compose_service: ubuntu-test-prebuilt + - target: macos-intel + name: AMD64 macOS 15-intel Java JDK 17 + os: macos-15-intel + jdk: 17 + maven: 3.9.9 + compose_service: '' + - target: macos-arm + name: AArch64 macOS latest Java JDK 17 + os: macos-latest jdk: 17 - macos: 15-intel - - arch: AArch64 + maven: 3.9.9 + compose_service: '' + - target: windows + name: AMD64 Windows Server 2022 Java JDK 17 + os: windows-latest jdk: 17 - macos: latest + maven: 3.9.9 + compose_service: '' + env: + JDK: ${{ matrix.jdk }} + MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -100,50 +127,153 @@ jobs: fetch-depth: 0 submodules: recursive - name: Set up Java - uses: actions/setup-java@v6 + if: ${{ matrix.compose_service == '' }} + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} cache: 'maven' - - name: Build + - name: Cache Docker Volumes + if: ${{ matrix.compose_service != '' }} + uses: actions/cache@v6 + with: + path: .docker/maven-cache + key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-3.9.9- + - name: Download reusable artifacts + uses: actions/download-artifact@v6 + with: + name: java-build + - name: Restore reusable artifacts + shell: bash + run: | + mkdir -p .docker + rm -rf \ + .docker/java-build \ + .docker/jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + tar -xzf java-build.tgz + - name: Restore reusable artifacts for hosted runner + if: ${{ matrix.compose_service == '' }} + shell: bash + run: | + rm -rf build jni + cp -a .docker/java-build/build build + cp -a .docker/jni-dist jni + mkdir -p "${HOME}/.m2/repository/org/apache" + cp -a .docker/maven-cache/repository/org/apache/arrow "${HOME}/.m2/repository/org/apache/" + - name: Test prebuilt artifacts on macOS/Windows + if: ${{ matrix.compose_service == '' }} shell: bash env: + ARROW_JAVA_TEST_PREBUILT: "ON" DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/build.sh . build jni - - name: Test + run: ci/scripts/test.sh . build jni + - name: Test prebuilt artifacts on Ubuntu Docker + if: ${{ matrix.compose_service != '' }} shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/test.sh . build jni + run: | + docker compose run \ + --rm \ + -e CI=true \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ + ${{ matrix.compose_service }} - windows: - name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} - runs-on: windows-latest + build-cdata: + name: Build C Data artifacts + runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - jdk: [17] + env: + JDK: 17 + MAVEN: 3.9.9 steps: - name: Checkout Arrow uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive - - name: Set up Java - uses: actions/setup-java@v6 + - name: Cache Docker Volumes + uses: actions/cache@v6 with: - java-version: ${{ matrix.jdk }} - distribution: 'temurin' - cache: 'maven' - - name: Build + path: .docker/maven-cache + key: cdata-${{ env.JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-${{ env.JDK }}-${{ env.MAVEN }}- + - name: Build C Data without tests shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/build.sh . build jni - - name: Test + run: | + docker compose run \ + --rm \ + -e CI=true \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ + cdata-artifacts + if [ -d .docker ]; then + sudo chown -R "$(id -u):$(id -g)" .docker + fi + - name: Pack reusable C Data artifacts + shell: bash + run: | + tar -czf cdata-build.tgz \ + .docker/cdata-build \ + .docker/cdata-jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + - name: Upload reusable C Data artifacts + uses: actions/upload-artifact@v5 + with: + name: cdata-build + path: cdata-build.tgz + retention-days: 1 + + test-cdata: + name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + needs: build-cdata + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + jdk: [17, 21, 23] + maven: [3.9.9] + env: + JDK: ${{ matrix.jdk }} + MAVEN: ${{ matrix.maven }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker/maven-cache + key: cdata-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-17-3.9.9- + - name: Download reusable C Data artifacts + uses: actions/download-artifact@v6 + with: + name: cdata-build + - name: Restore reusable C Data artifacts + shell: bash + run: | + mkdir -p .docker + rm -rf \ + .docker/cdata-build \ + .docker/cdata-jni-dist \ + .docker/maven-cache/repository/org/apache/arrow + tar -xzf cdata-build.tgz + - name: Test C Data without compiler lifecycle shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - run: ci/scripts/test.sh . build jni + run: | + docker compose run \ + --rm \ + -e CI=true \ + -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ + cdata-test-prebuilt diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 146a40cf7a..3fcb10e242 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -46,7 +46,11 @@ cp -r "${source_dir}/dev" "${build_dir}" # crawl back up to the top. GNU realpath has --relative-to but this does not # work on macOS -poms=$(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml) +poms=$( + find "${source_dir}" \ + \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune \ + -o -type f -name pom.xml -print +) if [[ "$OSTYPE" == "darwin"* ]]; then poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") else diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 8061ee455d..55b2808f76 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -31,14 +31,61 @@ if [ -d "${java_jni_dist_dir}" ]; then java_jni_dist_dir="$(cd "${java_jni_dist_dir}" && pwd)" fi -mvn="mvn -B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn" -# Use `2 * ncores` threads -mvn="${mvn} -T 2C" -mvn="${mvn} -Denforcer.skip=true" +mvn=( + mvn + -B + -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + -T + 2C + -Denforcer.skip=true +) + +run_tests() { + local log_name=$1 + shift + + if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + run_prebuilt_tests "${log_name}" "${@}" -DfailIfNoTests=false surefire:test + else + "${@}" test + fi +} + +run_prebuilt_tests() { + local log_name=$1 + shift + + set -o pipefail + "${@}" | tee "${source_dir}/${log_name}" + + if grep -E "Compiling [0-9]+ source files?" "${source_dir}/${log_name}"; then + echo "Unexpected compilation occurred while running prebuilt tests." + exit 1 + fi + + if ! grep -q "Tests run:" "${source_dir}/${log_name}"; then + echo "No surefire test summary found; tests may have been skipped." + exit 1 + fi +} pushd "${build_dir}" -${mvn} -Darrow.test.dataRoot="${source_dir}/testing/data" test +if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then + run_tests \ + surefire.log \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" + + if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + run_prebuilt_tests \ + opens-surefire.log \ + "${mvn[@]}" \ + -DfailIfNoTests=false \ + -pl memory/memory-core \ + org.apache.maven.plugins:maven-surefire-plugin:test@opens-tests + fi +fi projects=() if [ "${ARROW_JAVA_JNI}" = "ON" ]; then @@ -47,7 +94,9 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then projects+=(gandiva) fi if [ "${#projects[@]}" -gt 0 ]; then - ${mvn} test \ + run_tests \ + jni-surefire.log \ + "${mvn[@]}" \ -Parrow-jni \ -pl "$( IFS=, @@ -57,7 +106,12 @@ if [ "${#projects[@]}" -gt 0 ]; then fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then - ${mvn} test -Parrow-c-data -pl c -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" + run_tests \ + cdata-surefire.log \ + "${mvn[@]}" \ + -Parrow-c-data \ + -pl c \ + -Darrow.c.jni.dist.dir="${java_jni_dist_dir}" fi popd diff --git a/compose.yaml b/compose.yaml index 4fd825e5a5..43435fd1af 100644 --- a/compose.yaml +++ b/compose.yaml @@ -30,6 +30,14 @@ volumes: name: ccache-cache maven-cache: name: maven-cache + java-build: + name: java-build + jni-dist: + name: jni-dist + cdata-build: + name: cdata-build + cdata-jni-dist: + name: cdata-jni-dist services: ubuntu: @@ -50,6 +58,33 @@ services: /arrow-java/ci/scripts/build.sh /arrow-java /build /jni && /arrow-java/ci/scripts/test.sh /arrow-java /build /jni" + ubuntu-artifacts: + # Builds reusable pure Java artifacts for CI test jobs. + image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated + command: + /bin/bash -c " + rm -rf /build-output/build && + find /jni -mindepth 1 -maxdepth 1 -exec rm -rf {} + && + /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /jni" + + ubuntu-test-prebuilt: + # Runs pure Java tests against prebuilt CI artifacts on Ubuntu. + image: ${ARCH}/maven:${MAVEN}-eclipse-temurin-${JDK} + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated + environment: + ARROW_JAVA_TEST_PREBUILT: "ON" + command: + /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /jni" + conda-jni-cdata: # Builds and tests just the C Data Interface JNI library and JARs. # (No dependencies on arrow-cpp.) @@ -84,6 +119,56 @@ services: /arrow-java/ci/scripts/build.sh /arrow-java /build /jni && /arrow-java/ci/scripts/test.sh /arrow-java /build /jni" + cdata-artifacts: + # Builds reusable C Data Interface JNI artifacts for CI test jobs. + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + environment: + ARROW_JAVA_CDATA: "ON" + command: + /bin/bash -c " + rm -rf /build-output/build && + find /jni -mindepth 1 -maxdepth 1 -exec rm -rf {} + && + /arrow-java/ci/scripts/jni_build.sh /arrow-java /tmp/cdata-jni /tmp/cdata-native /jni && + /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /jni" + + cdata-test-prebuilt: + # Runs C Data Interface tests against prebuilt CI artifacts. + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + environment: + ARROW_JAVA_CDATA: "ON" + ARROW_JAVA_TEST_BASE: "OFF" + ARROW_JAVA_TEST_PREBUILT: "ON" + command: + /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /jni" + vcpkg-jni: # Builds all the JNI libraries, but not the JARs. # (Requires arrow-cpp.) From ee37e92a78c15fe9fd6b441224ba389d94440615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Wed, 29 Jul 2026 15:23:51 +0100 Subject: [PATCH 02/17] Simplify build-once test workflow --- .github/workflows/test.yml | 73 +++++++++++--------------------------- ci/scripts/build.sh | 6 +--- ci/scripts/test.sh | 24 +++++-------- compose.yaml | 58 +++++++++++------------------- 4 files changed, 52 insertions(+), 109 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 276209b987..ad51e1295e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,6 +35,7 @@ permissions: env: DOCKER_VOLUME_PREFIX: ".docker/" + MAVEN: 3.9.9 jobs: build-java: @@ -52,29 +53,23 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-build-17-3.9.9- + key: java-build-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-${{ env.MAVEN }}- - name: Build without tests - shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} JDK: 17 - MAVEN: 3.9.9 run: | docker compose run \ --rm \ -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ ubuntu-artifacts - if [ -d .docker ]; then - sudo chown -R "$(id -u):$(id -g)" .docker - fi + sudo chown -R "$(id -u):$(id -g)" .docker - name: Pack reusable artifacts - shell: bash run: | tar -czf java-build.tgz \ .docker/java-build \ - .docker/jni-dist \ .docker/maven-cache/repository/org/apache/arrow - name: Upload reusable artifacts uses: actions/upload-artifact@v5 @@ -84,7 +79,7 @@ jobs: retention-days: 1 test-java: - name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven {1}', matrix.jdk, matrix.maven) }} + name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven 3.9.9', matrix.jdk) }} needs: build-java runs-on: ${{ matrix.os }} if: ${{ !contains(github.event.pull_request.title, 'WIP') }} @@ -92,34 +87,22 @@ jobs: strategy: fail-fast: false matrix: - target: [ubuntu] jdk: [17, 21, 23] - maven: [3.9.9] + os: [ubuntu-latest] include: - - target: ubuntu - os: ubuntu-latest + - os: ubuntu-latest compose_service: ubuntu-test-prebuilt - - target: macos-intel - name: AMD64 macOS 15-intel Java JDK 17 + - name: AMD64 macOS 15-intel Java JDK 17 os: macos-15-intel jdk: 17 - maven: 3.9.9 - compose_service: '' - - target: macos-arm - name: AArch64 macOS latest Java JDK 17 + - name: AArch64 macOS latest Java JDK 17 os: macos-latest jdk: 17 - maven: 3.9.9 - compose_service: '' - - target: windows - name: AMD64 Windows Server 2022 Java JDK 17 + - name: AMD64 Windows Server 2022 Java JDK 17 os: windows-latest jdk: 17 - maven: 3.9.9 - compose_service: '' env: JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -127,19 +110,19 @@ jobs: fetch-depth: 0 submodules: recursive - name: Set up Java - if: ${{ matrix.compose_service == '' }} + if: ${{ !matrix.compose_service }} uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} cache: 'maven' - name: Cache Docker Volumes - if: ${{ matrix.compose_service != '' }} + if: ${{ matrix.compose_service }} uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-build-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-build-17-3.9.9- + key: java-build-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-17-${{ env.MAVEN }}- - name: Download reusable artifacts uses: actions/download-artifact@v6 with: @@ -147,30 +130,26 @@ jobs: - name: Restore reusable artifacts shell: bash run: | - mkdir -p .docker rm -rf \ .docker/java-build \ - .docker/jni-dist \ .docker/maven-cache/repository/org/apache/arrow tar -xzf java-build.tgz - name: Restore reusable artifacts for hosted runner - if: ${{ matrix.compose_service == '' }} + if: ${{ !matrix.compose_service }} shell: bash run: | - rm -rf build jni cp -a .docker/java-build/build build - cp -a .docker/jni-dist jni mkdir -p "${HOME}/.m2/repository/org/apache" cp -a .docker/maven-cache/repository/org/apache/arrow "${HOME}/.m2/repository/org/apache/" - name: Test prebuilt artifacts on macOS/Windows - if: ${{ matrix.compose_service == '' }} + if: ${{ !matrix.compose_service }} shell: bash env: ARROW_JAVA_TEST_PREBUILT: "ON" DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: ci/scripts/test.sh . build jni - name: Test prebuilt artifacts on Ubuntu Docker - if: ${{ matrix.compose_service != '' }} + if: ${{ matrix.compose_service }} shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} @@ -188,7 +167,6 @@ jobs: timeout-minutes: 30 env: JDK: 17 - MAVEN: 3.9.9 steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -202,7 +180,6 @@ jobs: key: cdata-${{ env.JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} restore-keys: cdata-${{ env.JDK }}-${{ env.MAVEN }}- - name: Build C Data without tests - shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: | @@ -211,11 +188,8 @@ jobs: -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ cdata-artifacts - if [ -d .docker ]; then - sudo chown -R "$(id -u):$(id -g)" .docker - fi + sudo chown -R "$(id -u):$(id -g)" .docker - name: Pack reusable C Data artifacts - shell: bash run: | tar -czf cdata-build.tgz \ .docker/cdata-build \ @@ -229,7 +203,7 @@ jobs: retention-days: 1 test-cdata: - name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven 3.9.9 needs: build-cdata runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} @@ -238,10 +212,8 @@ jobs: fail-fast: false matrix: jdk: [17, 21, 23] - maven: [3.9.9] env: JDK: ${{ matrix.jdk }} - MAVEN: ${{ matrix.maven }} steps: - name: Checkout Arrow uses: actions/checkout@v6 @@ -252,23 +224,20 @@ jobs: uses: actions/cache@v5 with: path: .docker/maven-cache - key: cdata-17-3.9.9-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: cdata-17-3.9.9- + key: cdata-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-17-${{ env.MAVEN }}- - name: Download reusable C Data artifacts uses: actions/download-artifact@v6 with: name: cdata-build - name: Restore reusable C Data artifacts - shell: bash run: | - mkdir -p .docker rm -rf \ .docker/cdata-build \ .docker/cdata-jni-dist \ .docker/maven-cache/repository/org/apache/arrow tar -xzf cdata-build.tgz - name: Test C Data without compiler lifecycle - shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} run: | diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 3fcb10e242..4e3f2ad517 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -46,11 +46,7 @@ cp -r "${source_dir}/dev" "${build_dir}" # crawl back up to the top. GNU realpath has --relative-to but this does not # work on macOS -poms=$( - find "${source_dir}" \ - \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune \ - -o -type f -name pom.xml -print -) +poms=$(find "${source_dir}" \( -path "${source_dir}"/build -o -path "${source_dir}"/.docker \) -prune -o -type f -name pom.xml -print) if [[ "$OSTYPE" == "darwin"* ]]; then poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") else diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 55b2808f76..f6c1a68d8e 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -set -ex +set -exo pipefail if [[ "${ARROW_JAVA_TEST:-ON}" != "ON" ]]; then exit @@ -41,45 +41,41 @@ mvn=( ) run_tests() { - local log_name=$1 - shift - if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then - run_prebuilt_tests "${log_name}" "${@}" -DfailIfNoTests=false surefire:test + run_prebuilt_tests "${@}" -DfailIfNoTests=false surefire:test else "${@}" test fi } run_prebuilt_tests() { - local log_name=$1 - shift + local log + log=$(mktemp) - set -o pipefail - "${@}" | tee "${source_dir}/${log_name}" + "${@}" | tee "${log}" - if grep -E "Compiling [0-9]+ source files?" "${source_dir}/${log_name}"; then + if grep -E "Compiling [0-9]+ source files?" "${log}"; then echo "Unexpected compilation occurred while running prebuilt tests." exit 1 fi - if ! grep -q "Tests run:" "${source_dir}/${log_name}"; then + if ! grep -q "Tests run:" "${log}"; then echo "No surefire test summary found; tests may have been skipped." exit 1 fi + + rm -f "${log}" } pushd "${build_dir}" if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then run_tests \ - surefire.log \ "${mvn[@]}" \ -Darrow.test.dataRoot="${source_dir}/testing/data" if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then run_prebuilt_tests \ - opens-surefire.log \ "${mvn[@]}" \ -DfailIfNoTests=false \ -pl memory/memory-core \ @@ -95,7 +91,6 @@ if [ "${ARROW_JAVA_JNI}" = "ON" ]; then fi if [ "${#projects[@]}" -gt 0 ]; then run_tests \ - jni-surefire.log \ "${mvn[@]}" \ -Parrow-jni \ -pl "$( @@ -107,7 +102,6 @@ fi if [ "${ARROW_JAVA_CDATA}" = "ON" ]; then run_tests \ - cdata-surefire.log \ "${mvn[@]}" \ -Parrow-c-data \ -pl c \ diff --git a/compose.yaml b/compose.yaml index 43435fd1af..cd682b99c2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -32,13 +32,28 @@ volumes: name: maven-cache java-build: name: java-build - jni-dist: - name: jni-dist cdata-build: name: cdata-build cdata-jni-dist: name: cdata-jni-dist +x-cdata-prebuilt: &cdata-prebuilt + image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + build: + context: . + dockerfile: ci/docker/conda-jni.dockerfile + cache_from: + - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration + args: + jdk: ${JDK} + maven: ${MAVEN} + user: root + volumes: + - .:/arrow-java:delegated + - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated + - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + services: ubuntu: # Build and test arrow-java on Ubuntu. @@ -65,12 +80,10 @@ services: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated command: /bin/bash -c " rm -rf /build-output/build && - find /jni -mindepth 1 -maxdepth 1 -exec rm -rf {} + && - /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /jni" + /arrow-java/ci/scripts/build.sh /arrow-java /build-output/build /tmp/jni" ubuntu-test-prebuilt: # Runs pure Java tests against prebuilt CI artifacts on Ubuntu. @@ -79,11 +92,10 @@ services: - .:/arrow-java:delegated - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - ${DOCKER_VOLUME_PREFIX}java-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}jni-dist:/jni:delegated environment: ARROW_JAVA_TEST_PREBUILT: "ON" command: - /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /jni" + /bin/bash -c "/arrow-java/ci/scripts/test.sh /arrow-java /build-output/build /tmp/jni" conda-jni-cdata: # Builds and tests just the C Data Interface JNI library and JARs. @@ -121,21 +133,7 @@ services: cdata-artifacts: # Builds reusable C Data Interface JNI artifacts for CI test jobs. - image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - build: - context: . - dockerfile: ci/docker/conda-jni.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - args: - jdk: ${JDK} - maven: ${MAVEN} - user: root - volumes: - - .:/arrow-java:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + <<: *cdata-prebuilt environment: ARROW_JAVA_CDATA: "ON" command: @@ -147,21 +145,7 @@ services: cdata-test-prebuilt: # Runs C Data Interface tests against prebuilt CI artifacts. - image: ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - build: - context: . - dockerfile: ci/docker/conda-jni.dockerfile - cache_from: - - ${REPO}:${ARCH}-conda-java-${JDK}-maven-${MAVEN}-jni-integration - args: - jdk: ${JDK} - maven: ${MAVEN} - user: root - volumes: - - .:/arrow-java:delegated - - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-build:/build-output:delegated - - ${DOCKER_VOLUME_PREFIX}cdata-jni-dist:/jni:delegated + <<: *cdata-prebuilt environment: ARROW_JAVA_CDATA: "ON" ARROW_JAVA_TEST_BASE: "OFF" From 569333e6997fbdf5455adf7b08250fbeb03f2cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Fri, 31 Jul 2026 16:21:25 +0100 Subject: [PATCH 03/17] Restore custom Surefire test coverage --- .github/workflows/test.yml | 4 ++-- ci/scripts/test.sh | 22 ++++++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ad51e1295e..f04827ce29 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -216,12 +216,12 @@ jobs: JDK: ${{ matrix.jdk }} steps: - name: Checkout Arrow - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: fetch-depth: 0 submodules: recursive - name: Cache Docker Volumes - uses: actions/cache@v5 + uses: actions/cache@v6 with: path: .docker/maven-cache key: cdata-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index f6c1a68d8e..93abe6f0e7 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -70,16 +70,30 @@ run_prebuilt_tests() { pushd "${build_dir}" if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then - run_tests \ - "${mvn[@]}" \ - -Darrow.test.dataRoot="${source_dir}/testing/data" - if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl "!vector" \ + surefire:test + for execution in default-test run-unsafe; do + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl vector \ + "org.apache.maven.plugins:maven-surefire-plugin:test@${execution}" + done run_prebuilt_tests \ "${mvn[@]}" \ -DfailIfNoTests=false \ -pl memory/memory-core \ org.apache.maven.plugins:maven-surefire-plugin:test@opens-tests + else + run_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" fi fi From 727c23f11666908bc0a64b1ef7a83e6a1dd96c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Fri, 31 Jul 2026 18:18:57 +0100 Subject: [PATCH 04/17] Make Vector test passes explicit --- ci/scripts/test.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 93abe6f0e7..491f3cf861 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -77,14 +77,19 @@ if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then -DfailIfNoTests=false \ -pl "!vector" \ surefire:test - for execution in default-test run-unsafe; do - run_prebuilt_tests \ - "${mvn[@]}" \ - -Darrow.test.dataRoot="${source_dir}/testing/data" \ - -DfailIfNoTests=false \ - -pl vector \ - "org.apache.maven.plugins:maven-surefire-plugin:test@${execution}" - done + # Direct Surefire skips Vector's lifecycle-bound allocator test passes. + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl vector \ + org.apache.maven.plugins:maven-surefire-plugin:test@default-test + run_prebuilt_tests \ + "${mvn[@]}" \ + -Darrow.test.dataRoot="${source_dir}/testing/data" \ + -DfailIfNoTests=false \ + -pl vector \ + org.apache.maven.plugins:maven-surefire-plugin:test@run-unsafe run_prebuilt_tests \ "${mvn[@]}" \ -DfailIfNoTests=false \ From 123a2d54b051cc66e4de6bef87773873efa0d011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Wed, 2 Sep 2026 20:53:40 +0100 Subject: [PATCH 05/17] MINOR: Align build-once workflow with current CI matrix --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f04827ce29..b045f98042 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ permissions: env: DOCKER_VOLUME_PREFIX: ".docker/" - MAVEN: 3.9.9 + MAVEN: 3.9.16 jobs: build-java: @@ -79,7 +79,7 @@ jobs: retention-days: 1 test-java: - name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven 3.9.9', matrix.jdk) }} + name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven 3.9.16', matrix.jdk) }} needs: build-java runs-on: ${{ matrix.os }} if: ${{ !contains(github.event.pull_request.title, 'WIP') }} @@ -87,7 +87,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [17, 21, 23] + jdk: [17, 21, 25] os: [ubuntu-latest] include: - os: ubuntu-latest @@ -203,7 +203,7 @@ jobs: retention-days: 1 test-cdata: - name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven 3.9.9 + name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven 3.9.16 needs: build-cdata runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} @@ -211,7 +211,7 @@ jobs: strategy: fail-fast: false matrix: - jdk: [17, 21, 23] + jdk: [17, 21, 25] env: JDK: ${{ matrix.jdk }} steps: From 4683d0fc6311fc3091bf6bec04fd7617f4ba696e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Wed, 2 Sep 2026 22:52:52 +0100 Subject: [PATCH 06/17] Fix stale Arrow artifacts in hosted tests --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b045f98042..4dda5c5dfa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,6 +139,7 @@ jobs: shell: bash run: | cp -a .docker/java-build/build build + rm -rf "${HOME}/.m2/repository/org/apache/arrow" mkdir -p "${HOME}/.m2/repository/org/apache" cp -a .docker/maven-cache/repository/org/apache/arrow "${HOME}/.m2/repository/org/apache/" - name: Test prebuilt artifacts on macOS/Windows From a06fa44a580d722723d68f90d67b763f48c2127c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Wed, 2 Sep 2026 22:53:48 +0100 Subject: [PATCH 07/17] Verify prebuilt test artifacts --- ci/scripts/build.sh | 3 +++ ci/scripts/test.sh | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 4e3f2ad517..8333732fa8 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -79,6 +79,9 @@ fi # Use `2 * ncores` threads ${mvn} -T 2C clean install +# Prebuilt test jobs use this to verify that test classes survived artifact restore. +find . -type d -path "*/target/test-classes" -print | LC_ALL=C sort > .arrow-java-prebuilt-test-classes + if [ "${ARROW_JAVA_BUILD_DOCS:-OFF}" == "ON" ]; then # HTTP pooling is turned off to avoid download issues: # https://github.com/apache/arrow/issues/27496 diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 491f3cf861..1a28ecc9e1 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -52,7 +52,7 @@ run_prebuilt_tests() { local log log=$(mktemp) - "${@}" | tee "${log}" + "${@}" -Ddevelocity.cache.local.enabled=false | tee "${log}" if grep -E "Compiling [0-9]+ source files?" "${log}"; then echo "Unexpected compilation occurred while running prebuilt tests." @@ -67,8 +67,29 @@ run_prebuilt_tests() { rm -f "${log}" } +verify_prebuilt_test_classes() { + local manifest=.arrow-java-prebuilt-test-classes + local test_classes + + if [[ ! -s "${manifest}" ]]; then + echo "No prebuilt test-class manifest found." + exit 1 + fi + + while IFS= read -r test_classes; do + if [[ ! -d "${test_classes}" ]]; then + echo "Missing prebuilt test classes: ${test_classes}" + exit 1 + fi + done < "${manifest}" +} + pushd "${build_dir}" +if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then + verify_prebuilt_test_classes +fi + if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then run_prebuilt_tests \ From 639a888a7607e980aa050009759e674e37c7a434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Wed, 2 Sep 2026 23:35:15 +0100 Subject: [PATCH 08/17] Simplify prebuilt test validation --- ci/scripts/build.sh | 3 --- ci/scripts/test.sh | 27 +-------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 8333732fa8..4e3f2ad517 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -79,9 +79,6 @@ fi # Use `2 * ncores` threads ${mvn} -T 2C clean install -# Prebuilt test jobs use this to verify that test classes survived artifact restore. -find . -type d -path "*/target/test-classes" -print | LC_ALL=C sort > .arrow-java-prebuilt-test-classes - if [ "${ARROW_JAVA_BUILD_DOCS:-OFF}" == "ON" ]; then # HTTP pooling is turned off to avoid download issues: # https://github.com/apache/arrow/issues/27496 diff --git a/ci/scripts/test.sh b/ci/scripts/test.sh index 1a28ecc9e1..c1bfe65752 100755 --- a/ci/scripts/test.sh +++ b/ci/scripts/test.sh @@ -42,7 +42,7 @@ mvn=( run_tests() { if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then - run_prebuilt_tests "${@}" -DfailIfNoTests=false surefire:test + run_prebuilt_tests "${@}" surefire:test else "${@}" test fi @@ -67,53 +67,28 @@ run_prebuilt_tests() { rm -f "${log}" } -verify_prebuilt_test_classes() { - local manifest=.arrow-java-prebuilt-test-classes - local test_classes - - if [[ ! -s "${manifest}" ]]; then - echo "No prebuilt test-class manifest found." - exit 1 - fi - - while IFS= read -r test_classes; do - if [[ ! -d "${test_classes}" ]]; then - echo "Missing prebuilt test classes: ${test_classes}" - exit 1 - fi - done < "${manifest}" -} - pushd "${build_dir}" -if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then - verify_prebuilt_test_classes -fi - if [[ "${ARROW_JAVA_TEST_BASE:-ON}" = "ON" ]]; then if [[ "${ARROW_JAVA_TEST_PREBUILT:-OFF}" = "ON" ]]; then run_prebuilt_tests \ "${mvn[@]}" \ -Darrow.test.dataRoot="${source_dir}/testing/data" \ - -DfailIfNoTests=false \ -pl "!vector" \ surefire:test # Direct Surefire skips Vector's lifecycle-bound allocator test passes. run_prebuilt_tests \ "${mvn[@]}" \ -Darrow.test.dataRoot="${source_dir}/testing/data" \ - -DfailIfNoTests=false \ -pl vector \ org.apache.maven.plugins:maven-surefire-plugin:test@default-test run_prebuilt_tests \ "${mvn[@]}" \ -Darrow.test.dataRoot="${source_dir}/testing/data" \ - -DfailIfNoTests=false \ -pl vector \ org.apache.maven.plugins:maven-surefire-plugin:test@run-unsafe run_prebuilt_tests \ "${mvn[@]}" \ - -DfailIfNoTests=false \ -pl memory/memory-core \ org.apache.maven.plugins:maven-surefire-plugin:test@opens-tests else From f52284be274a928293b0c6ee259780d9398af895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 3 Sep 2026 22:11:17 +0100 Subject: [PATCH 09/17] Optimize build-once workflow caches --- .github/workflows/test.yml | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4dda5c5dfa..9ea7cbbdbf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,7 @@ permissions: contents: read env: + BUILD_JDK: 17 DOCKER_VOLUME_PREFIX: ".docker/" MAVEN: 3.9.16 @@ -53,12 +54,12 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-build-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-build-17-${{ env.MAVEN }}- + key: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Build without tests env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} - JDK: 17 + JDK: ${{ env.BUILD_JDK }} run: | docker compose run \ --rm \ @@ -77,6 +78,8 @@ jobs: name: java-build path: java-build.tgz retention-days: 1 + - name: Exclude reactor artifacts from Maven dependency cache + run: rm -rf .docker/maven-cache/repository/org/apache/arrow test-java: name: ${{ matrix.name || format('AMD64 Ubuntu JDK {0} Maven 3.9.16', matrix.jdk) }} @@ -107,7 +110,7 @@ jobs: - name: Checkout Arrow uses: actions/checkout@v7 with: - fetch-depth: 0 + fetch-depth: 1 submodules: recursive - name: Set up Java if: ${{ !matrix.compose_service }} @@ -121,8 +124,8 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-build-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-build-17-${{ env.MAVEN }}- + key: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Download reusable artifacts uses: actions/download-artifact@v6 with: @@ -160,6 +163,10 @@ jobs: -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ ${{ matrix.compose_service }} + - name: Exclude reactor artifacts from hosted Maven dependency cache + if: ${{ !matrix.compose_service && always() }} + shell: bash + run: rm -rf "${HOME}/.m2/repository/org/apache/arrow" build-cdata: name: Build C Data artifacts @@ -167,7 +174,7 @@ jobs: if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 env: - JDK: 17 + JDK: ${{ env.BUILD_JDK }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -178,8 +185,8 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: cdata-${{ env.JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: cdata-${{ env.JDK }}-${{ env.MAVEN }}- + key: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Build C Data without tests env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} @@ -202,6 +209,8 @@ jobs: name: cdata-build path: cdata-build.tgz retention-days: 1 + - name: Exclude reactor artifacts from Maven dependency cache + run: rm -rf .docker/maven-cache/repository/org/apache/arrow test-cdata: name: AMD64 Conda JNI JDK ${{ matrix.jdk }} Maven 3.9.16 @@ -219,14 +228,14 @@ jobs: - name: Checkout Arrow uses: actions/checkout@v7 with: - fetch-depth: 0 + fetch-depth: 1 submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v6 with: path: .docker/maven-cache - key: cdata-17-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: cdata-17-${{ env.MAVEN }}- + key: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Download reusable C Data artifacts uses: actions/download-artifact@v6 with: From 9ad2010452ab5b7994c732d1f34649db7323a78f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 3 Sep 2026 22:12:20 +0100 Subject: [PATCH 10/17] Quote workflow build JDK --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ea7cbbdbf..8e924eb348 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,7 +34,7 @@ permissions: contents: read env: - BUILD_JDK: 17 + BUILD_JDK: "17" DOCKER_VOLUME_PREFIX: ".docker/" MAVEN: 3.9.16 From 082c2ecf29bcd93ccc30b6423b487934d41382ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 3 Sep 2026 22:13:28 +0100 Subject: [PATCH 11/17] Fix build JDK workflow context --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8e924eb348..3c0caabd82 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -173,8 +173,6 @@ jobs: runs-on: ubuntu-latest if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 - env: - JDK: ${{ env.BUILD_JDK }} steps: - name: Checkout Arrow uses: actions/checkout@v7 @@ -190,6 +188,7 @@ jobs: - name: Build C Data without tests env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + JDK: ${{ env.BUILD_JDK }} run: | docker compose run \ --rm \ From 24fbc90f447685e0b9a878e7f9b9a0f83c876e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 3 Sep 2026 22:15:05 +0100 Subject: [PATCH 12/17] Use established Maven cache keys --- .github/workflows/test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c0caabd82..d736f5b152 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,8 +54,8 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- + key: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Build without tests env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} @@ -124,8 +124,8 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: java-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- + key: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: java-build-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Download reusable artifacts uses: actions/download-artifact@v6 with: @@ -183,8 +183,8 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- + key: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Build C Data without tests env: DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} @@ -233,8 +233,8 @@ jobs: uses: actions/cache@v6 with: path: .docker/maven-cache - key: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} - restore-keys: cdata-deps-v2-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- + key: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}-${{ hashFiles('compose.yaml', '**/pom.xml') }} + restore-keys: cdata-${{ env.BUILD_JDK }}-${{ env.MAVEN }}- - name: Download reusable C Data artifacts uses: actions/download-artifact@v6 with: From 280e3c3f9e312a9206162f9f7d1605a25c81de4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Thu, 3 Sep 2026 22:16:53 +0100 Subject: [PATCH 13/17] Use shallow test checkouts --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d736f5b152..ff99296352 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,7 +110,6 @@ jobs: - name: Checkout Arrow uses: actions/checkout@v7 with: - fetch-depth: 1 submodules: recursive - name: Set up Java if: ${{ !matrix.compose_service }} @@ -227,7 +226,6 @@ jobs: - name: Checkout Arrow uses: actions/checkout@v7 with: - fetch-depth: 1 submodules: recursive - name: Cache Docker Volumes uses: actions/cache@v6 From 318e38a1634ad89cce2e95c3e3ef49bc386fd44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lder=20Greg=C3=B3rio?= Date: Fri, 4 Sep 2026 19:44:21 +0100 Subject: [PATCH 14/17] Sync setup-java version with upstream --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff99296352..c709743950 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -113,7 +113,7 @@ jobs: submodules: recursive - name: Set up Java if: ${{ !matrix.compose_service }} - uses: actions/setup-java@v5 + uses: actions/setup-java@v6 with: distribution: 'temurin' java-version: ${{ matrix.jdk }} From d035fc8c5b4c414fa2e5eec44dc3c711b5e074f0 Mon Sep 17 00:00:00 2001 From: Abhishek Pathania Date: Wed, 16 Sep 2026 12:13:41 +0530 Subject: [PATCH 15/17] GH-1293: Use floor division when splitting epoch millis into day and time (#1294) ## What's Changed `DateTimeUtils.getTimestampValue(long)` used `/` and `%` to split epoch milliseconds into an epoch day and a time within that day. These operators round toward zero. For negative values that were not exactly midnight, the existing code fixed the remainder but not the epoch day. The two parts then referred to different days, so the timestamp came back one day late. For example, `-618102000000` ms is 1950-06-01 01:00:00 UTC. The old division produced epoch day `-7153`, which is 1950-06-02, while the remainder was 01:00. The method returned 1950-06-02 01:00:00. This affects DATE values before 1970 when `ArrowFlightJdbcDateVectorAccessor.getDate(Calendar)` applies a non-zero calendar offset. The offset moves the value away from midnight and exposes the division bug. Closes #1293. --- .../arrow/driver/jdbc/utils/DateTimeUtils.java | 13 +++++-------- .../arrow/driver/jdbc/utils/DateTimeUtilsTest.java | 11 +++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtils.java b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtils.java index 9363e3486c..c4e7fda59b 100644 --- a/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtils.java +++ b/flight/flight-sql-jdbc-core/src/main/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtils.java @@ -55,15 +55,12 @@ public static long applyCalendarOffset(long milliseconds, Calendar calendar) { * @return a {@link Timestamp} object representing the given Epoch millis */ public static Timestamp getTimestampValue(long millisWithCalendar) { - long milliseconds = millisWithCalendar; - if (milliseconds < 0) { - // LocalTime#ofNanoDay only accepts positive values - milliseconds -= ((milliseconds / MILLIS_PER_DAY) - 1) * MILLIS_PER_DAY; - } - + // Millis are negative before 1970, where only floor semantics keep the epoch day + // and the time-of-day remainder on the same day (and the remainder non-negative). return Timestamp.valueOf( LocalDateTime.of( - LocalDate.ofEpochDay(millisWithCalendar / MILLIS_PER_DAY), - LocalTime.ofNanoOfDay(TimeUnit.MILLISECONDS.toNanos(milliseconds % MILLIS_PER_DAY)))); + LocalDate.ofEpochDay(Math.floorDiv(millisWithCalendar, MILLIS_PER_DAY)), + LocalTime.ofNanoOfDay( + TimeUnit.MILLISECONDS.toNanos(Math.floorMod(millisWithCalendar, MILLIS_PER_DAY))))); } } diff --git a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtilsTest.java b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtilsTest.java index 9c66352023..70bcb9b4c2 100644 --- a/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtilsTest.java +++ b/flight/flight-sql-jdbc-core/src/test/java/org/apache/arrow/driver/jdbc/utils/DateTimeUtilsTest.java @@ -95,4 +95,15 @@ public void testShouldGetTimestampNegative() { assertThat(expected, is(actual)); } + + @Test + public void testShouldGetTimestampNegativeNotAlignedToDay() { + final long epochMilli = negativeEpochMilli + 3600000L; // 1950-06-01 01:00:00 UTC + final Instant instant = Instant.ofEpochMilli(epochMilli); + + final Timestamp expected = Timestamp.from(instant); + final Timestamp actual = DateTimeUtils.getTimestampValue(epochMilli); + + assertThat(expected, is(actual)); + } } From 55868214cd4b3c1f1e7294b5d74155ba8cceeaaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 21 Sep 2026 13:05:22 +0200 Subject: [PATCH 16/17] GH-1305: [CI] Drop x86_64 macOS from JNI builds (#1306) ### Rationale for this change Homebrew treats x86_64 macOS as a [tier 3 configuration](https://docs.brew.sh/Support-Tiers#tier-3) and rarely builds bottles for it. `brew bundle` doesn't build from source, so the `JNI macos-15-intel x86_64` job in the RC workflow aborts in `Install dependencies` with `no bottle available!` before anything is compiled. We've been patching this formula by formula: * #1269 skipped `aws-sdk-cpp` and gRPC. * #1270 skipped Node.js. Any new dependency added to `apache/arrow`'s `cpp/Brewfile` can break the job again the same way, and nightly RC runs go red on `main` when it does. We can't keep this job green on a configuration Homebrew doesn't support. #1269 noted that dropping the `macos-15-intel` entry needed its own discussion because it's user-facing. #1305 is that discussion. ### What changes are included in this PR? `rc.yml`: * Removed the `macos-15-intel` / `x86_64` entry from the `jni-macos` matrix. * `binaries` no longer extracts `jni-macos-x86_64.tar.gz` nor asserts the four `jni/*/x86_64/*.dylib` files. * Dropped the `/usr/local` `python@XXX --overwrite` conflict loop. Its comment already said it could go once we dropped `macos-15-intel`, since `macos-14` and later on arm64 use `/opt/homebrew`. `brew update` is kept. * Unwrapped the `pkg-config` uninstall from its `if [ "$(uname -m)" = "arm64" ]` guard, which is now always true. * Trimmed the tier-3 justification from the `HOMEBREW_BUNDLE_BREW_SKIP` comment. The skip list itself is unchanged: `aws-sdk-cpp` and gRPC are still skipped so the bundled static ones are used, and Node.js is still skipped because the JNI build doesn't need it. `test.yml`: * Removed the `AMD64` / `macos-15-intel` entry from the `macos` test matrix, leaving `AArch64` on `macos-latest`. `ci/scripts/jni_macos_build.sh` is untouched, so building the JNI libraries locally on an Intel Mac still works. ### Are these changes tested? Yes, by CI. The remaining `JNI macos-14 aarch_64` job exercises the changed `Install dependencies` step, and `binaries` exercises the changed artifact extraction and verification. ### Are there any user-facing changes? Yes. Release JARs no longer bundle x86_64 macOS `.dylib` files for `arrow_cdata_jni`, `arrow_dataset_jni`, `arrow_orc_jni` and `gandiva_jni`. Intel Mac users of those modules will need to build the JNI libraries themselves. aarch64 macOS, x86_64 and aarch64 Linux, and x86_64 Windows are unaffected. Closes #1305. --- .github/workflows/rc.yml | 46 +++++--------------------------------- .github/workflows/test.yml | 3 --- 2 files changed, 6 insertions(+), 43 deletions(-) diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml index 34623b4018..15d4177255 100644 --- a/.github/workflows/rc.yml +++ b/.github/workflows/rc.yml @@ -162,7 +162,6 @@ jobs: fail-fast: false matrix: platform: - - { runs_on: macos-15-intel, arch: "x86_64"} - { runs_on: macos-14, arch: "aarch_64" } env: MACOSX_DEPLOYMENT_TARGET: "14.0" @@ -211,46 +210,19 @@ jobs: # llvm@14 because llvm is newer than llvm@14. brew uninstall llvm || : - # We can remove this when we drop support for - # macos-15-intel. because macos-14 or later with arm64 uses /opt/homebrew/ - # not /usr/local/. - # - # Ensure updating python@XXX with the "--overwrite" option. - # If python@XXX is updated without "--overwrite", it causes - # a conflict error. Because Python 3 installed not by - # Homebrew exists in /usr/local on GitHub Actions. If - # Homebrew's python@XXX is updated without "--overwrite", it - # tries to replace /usr/local/bin/2to3 and so on and causes - # a conflict error. brew update - for python_package in $(brew list | grep python@ | sort -r); do - brew install --overwrite ${python_package} - done - brew install --overwrite python3 - if [ "$(uname -m)" = "arm64" ]; then - # pkg-config formula is deprecated but it's still installed - # in GitHub Actions runner now. We can remove this once - # pkg-config formula is removed from GitHub Actions runner. - brew uninstall pkg-config || : - brew uninstall pkg-config@0.29.2 || : - fi + # pkg-config formula is deprecated but it's still installed + # in GitHub Actions runner now. We can remove this once + # pkg-config formula is removed from GitHub Actions runner. + brew uninstall pkg-config || : + brew uninstall pkg-config@0.29.2 || : # We don't use Homebrew's aws-sdk-cpp and gRPC. See the # "brew uninstall" calls below for details. So we don't install # them instead of installing and uninstalling them. # - # This is also needed because Homebrew doesn't provide bottles - # for them on x86_64 macOS. Homebrew treats x86_64 macOS as a - # tier 3 configuration and rarely builds bottles for it: - # - # https://docs.brew.sh/Support-Tiers#tier-3 - # - # "brew bundle" fails with "no bottle available!" without this - # because it doesn't build them from source. - # - # We don't use Homebrew's Node.js too. Homebrew doesn't provide - # a bottle for it on x86_64 macOS for the same reason. + # We don't need Homebrew's Node.js for the JNI build either. HOMEBREW_BUNDLE_BREW_SKIP="aws-sdk-cpp grpc node" \ brew bundle --file=arrow/cpp/Brewfile # We want to link aws-sdk-cpp statically but Homebrew's @@ -388,7 +360,6 @@ jobs: tar -xf apache-arrow-java-*.tar.gz --strip-components=1 tar -xvzf jni-linux-x86_64.tar.gz tar -xvzf jni-linux-aarch_64.tar.gz - tar -xvzf jni-macos-x86_64.tar.gz tar -xvzf jni-macos-aarch_64.tar.gz tar -xvzf jni-windows-x86_64.tar.gz - name: Test that shared libraries exist @@ -405,11 +376,6 @@ jobs: test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.so test -f jni/gandiva_jni/aarch_64/libgandiva_jni.so - test -f jni/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib - test -f jni/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib - test -f jni/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib - test -f jni/gandiva_jni/x86_64/libgandiva_jni.dylib - test -f jni/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib test -f jni/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63384b5d32..b9cd01a770 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -87,9 +87,6 @@ jobs: fail-fast: false matrix: include: - - arch: AMD64 - jdk: 17 - macos: 15-intel - arch: AArch64 jdk: 17 macos: latest From 53a9ccd8f272bd576ec0524b704c1a1fdab20948 Mon Sep 17 00:00:00 2001 From: Sotaro Hikita <70102274+lawofcycles@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:40:40 +0900 Subject: [PATCH 17/17] GH-1300: Use Locale.ROOT when building C Data Interface format strings (#1301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Format.asString` built the `FixedSizeList` (`+w:%d`), `FixedSizeBinary` (`w:%d`) and `Decimal` (`d:%d,%d[,%d]`) format strings with `String.format` and no explicit `Locale`. `java.util.Formatter` localises the digits of `%d` with the default locale, so a JVM running under `ar-EG`, `bn-BD`, `mr-IN` and similar locales exported `+w:٨` instead of `+w:8`, which arrow-rs and other implementations reject. This passes `Locale.ROOT` to the four calls and adds `FormatTest.testAsStringIgnoresDefaultLocale`, which sets the default locale to `ar-EG` for the duration of the test and asserts the ASCII output. Closes #1300. --- c/src/main/java/org/apache/arrow/c/Format.java | 8 ++++---- .../java/org/apache/arrow/c/FormatTest.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/c/src/main/java/org/apache/arrow/c/Format.java b/c/src/main/java/org/apache/arrow/c/Format.java index 7ce99614d2..7f546fca2c 100644 --- a/c/src/main/java/org/apache/arrow/c/Format.java +++ b/c/src/main/java/org/apache/arrow/c/Format.java @@ -63,10 +63,10 @@ static String asString(ArrowType arrowType) { { ArrowType.Decimal type = (ArrowType.Decimal) arrowType; if (type.getBitWidth() == 128) { - return String.format("d:%d,%d", type.getPrecision(), type.getScale()); + return String.format(Locale.ROOT, "d:%d,%d", type.getPrecision(), type.getScale()); } return String.format( - "d:%d,%d,%d", type.getPrecision(), type.getScale(), type.getBitWidth()); + Locale.ROOT, "d:%d,%d,%d", type.getPrecision(), type.getScale(), type.getBitWidth()); } case Duration: { @@ -88,12 +88,12 @@ static String asString(ArrowType arrowType) { case FixedSizeBinary: { ArrowType.FixedSizeBinary type = (ArrowType.FixedSizeBinary) arrowType; - return String.format("w:%d", type.getByteWidth()); + return String.format(Locale.ROOT, "w:%d", type.getByteWidth()); } case FixedSizeList: { ArrowType.FixedSizeList type = (ArrowType.FixedSizeList) arrowType; - return String.format("+w:%d", type.getListSize()); + return String.format(Locale.ROOT, "+w:%d", type.getListSize()); } case FloatingPoint: { diff --git a/c/src/test/java/org/apache/arrow/c/FormatTest.java b/c/src/test/java/org/apache/arrow/c/FormatTest.java index c773324330..3691ca095d 100644 --- a/c/src/test/java/org/apache/arrow/c/FormatTest.java +++ b/c/src/test/java/org/apache/arrow/c/FormatTest.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.Locale; import org.apache.arrow.vector.types.DateUnit; import org.apache.arrow.vector.types.FloatingPointPrecision; import org.apache.arrow.vector.types.IntervalUnit; @@ -156,4 +157,20 @@ public void testAsType() assertThrows(UnsupportedOperationException.class, () -> Format.asType(":", 0L)); assertThrows(NumberFormatException.class, () -> Format.asType("w:1,2,3", 0L)); } + + @Test + public void testAsStringIgnoresDefaultLocale() { + // Locales that use digits other than 0-9 (Arabic-Indic, Bengali, Devanagari, ...) + // must not leak into the format string, which other implementations parse as ASCII. + Locale saved = Locale.getDefault(); + try { + Locale.setDefault(Locale.forLanguageTag("ar-EG")); + assertEquals("d:10,2", Format.asString(new ArrowType.Decimal(10, 2, 128))); + assertEquals("d:10,2,256", Format.asString(new ArrowType.Decimal(10, 2, 256))); + assertEquals("w:16", Format.asString(new ArrowType.FixedSizeBinary(16))); + assertEquals("+w:8", Format.asString(new ArrowType.FixedSizeList(8))); + } finally { + Locale.setDefault(saved); + } + } }