Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
316 changes: 254 additions & 62 deletions .github/workflows/04-android-build.yml

Large diffs are not rendered by default.

186 changes: 171 additions & 15 deletions .github/workflows/06-ios-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ permissions:
jobs:
build-ios:
runs-on: macos-15
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -50,8 +51,12 @@ jobs:
-DCMAKE_OSX_ARCHITECTURES="${{ matrix.arch }}" \
-DCMAKE_OSX_SYSROOT="$SDK_PATH" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_ZVEC_SHARED=OFF \
-DBUILD_ZVEC_AILEGO_SHARED=OFF \
-DBUILD_ZVEC_CORE_SHARED=OFF \
-DBUILD_PYTHON_BINDINGS=OFF \
-DBUILD_TOOLS=OFF \
-DBUILD_CPP_EXAMPLES=ON \
-DENABLE_WERROR=ON \
-DCMAKE_INSTALL_PREFIX="./install" \
-DIOS=ON \
Expand All @@ -61,6 +66,36 @@ jobs:

cmake --build build_ios_${{ matrix.platform }} --parallel $NPROC

- name: Build public static C++ examples
run: |
NPROC=$(sysctl -n hw.ncpu)
BUILD_DIR=build_ios_${{ matrix.platform }}
cmake --build "$BUILD_DIR" --target zvec_cpp_examples --parallel "$NPROC"

for example in ailego-example core-example external-vector-example db-example; do
example_binary="$BUILD_DIR/bin/$example.app/$example"
if [ ! -f "$example_binary" ]; then
echo "Missing iOS example binary: $example_binary"
exit 1
fi
done

- name: Build required DiskAnn test targets
run: |
NPROC=$(sysctl -n hw.ncpu)
BUILD_DIR=build_ios_${{ matrix.platform }}
cmake --build "$BUILD_DIR" \
--target diskann_mobile_compat_test diskann_mobile_collection_test \
--parallel "$NPROC"

for test_name in diskann_mobile_compat_test diskann_mobile_collection_test; do
test_app="$BUILD_DIR/bin/${test_name}.app"
if [ ! -d "$test_app" ]; then
echo "::error::Missing required iOS DiskAnn test app: $test_app"
exit 1
fi
done

- name: Build test targets
if: matrix.test_on_simulator
run: |
Expand All @@ -70,6 +105,11 @@ jobs:
- name: Boot iOS Simulator
if: matrix.test_on_simulator
run: |
if [ "$(uname -m)" != "arm64" ]; then
echo "::error::SIMULATORARM64 tests require an arm64 macOS runner"
exit 1
fi

DEVICE_ID=$(xcrun simctl list devices available -j \
| python3 -c "
import json, sys
Expand All @@ -84,6 +124,7 @@ jobs:
")
echo "DEVICE_ID=$DEVICE_ID" >> $GITHUB_ENV
xcrun simctl boot "$DEVICE_ID"
xcrun simctl bootstatus "$DEVICE_ID" -b
echo "Booted simulator: $DEVICE_ID"

- name: Run all tests on simulator
Expand All @@ -92,52 +133,167 @@ jobs:
FAILED_TESTS=""
PASSED=0
TOTAL=0
DISKANN_COMPAT_PASSED=0
DISKANN_COLLECTION_PASSED=0
BUILD_DIR=build_ios_${{ matrix.platform }}

for APP in build_ios_${{ matrix.platform }}/bin/*_test.app; do
for test_name in diskann_mobile_compat_test diskann_mobile_collection_test; do
test_app="$BUILD_DIR/bin/${test_name}.app"
if [ ! -d "$test_app" ]; then
echo "::error::Missing required iOS DiskAnn test app: $test_app"
exit 1
fi
done

for APP in "$BUILD_DIR"/bin/*_test.app; do
[ -d "$APP" ] || continue
TEST_NAME=$(basename "$APP" .app)

# All test cases in this legacy target are disabled with #if 0.
# Do not count an empty GTest binary as an executed test suite.
if [ "$TEST_NAME" = "cosine_distance_matrix_int8_test" ]; then
echo "::notice::Skipping ${TEST_NAME}: it contains no enabled GTest cases"
continue
fi

BUNDLE_ID="com.zvec.${TEST_NAME}"
LOG_FILE="$RUNNER_TEMP/${TEST_NAME}.log"
TOTAL=$((TOTAL + 1))

echo "::group::Running ${TEST_NAME}"
xcrun simctl install "$DEVICE_ID" "$APP"
set +eo pipefail
for attempt in 1 2 3; do
xcrun simctl launch --console "$DEVICE_ID" "$BUNDLE_ID" 2>&1 | tee /tmp/${TEST_NAME}.log
LAUNCH_EXIT=${PIPESTATUS[0]}
if ! grep -q "unknown to FrontBoard" /tmp/${TEST_NAME}.log; then
set +e
python3 - "$DEVICE_ID" "$BUNDLE_ID" "$LOG_FILE" <<'PY'
import os
import pathlib
import subprocess
import sys

device_id, bundle_id, log_file = sys.argv[1:]
env = os.environ.copy()
env["SIMCTL_CHILD_GTEST_COLOR"] = "no"
try:
result = subprocess.run(
["xcrun", "simctl", "launch", "--console", device_id, bundle_id],
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
timeout=600,
)
output = result.stdout or ""
return_code = result.returncode
except subprocess.TimeoutExpired as error:
output = error.stdout or ""
if isinstance(output, bytes):
output = output.decode("utf-8", errors="replace")
output += "\nTest timed out after 600 seconds.\n"
return_code = 124

pathlib.Path(log_file).write_text(output, encoding="utf-8")
print(output, end="")
sys.exit(return_code)
PY
LAUNCH_EXIT=$?
set -e

SUMMARY_FOUND=0
if grep -Eq '^\[==========\][[:space:]]+[0-9]+ tests?[[:space:]]+from[[:space:]]+[0-9]+ test (suites?|cases?)[[:space:]]+ran\.' "$LOG_FILE"; then
SUMMARY_FOUND=1
elif [ "$TEST_NAME" = "c_api_test" ] && grep -Eq '^Failed: [0-9]+[[:space:]]*$' "$LOG_FILE"; then
SUMMARY_FOUND=1
fi

RETRY_REASON=""
if grep -q "unknown to FrontBoard" "$LOG_FILE"; then
RETRY_REASON="FrontBoard has not registered ${TEST_NAME} yet"
elif [ "$LAUNCH_EXIT" -eq 0 ] && [ "$SUMMARY_FOUND" -eq 0 ] && \
! grep -Eq '^\[ FAILED \]' "$LOG_FILE"; then
RETRY_REASON="${TEST_NAME} exited without a complete test summary"
else
break
fi
echo "::warning::Attempt ${attempt}/3: FrontBoard has not registered ${TEST_NAME} yet, retrying in 3s..."

if [ "$attempt" -eq 3 ]; then
break
fi

echo "::warning::Attempt ${attempt}/3: ${RETRY_REASON}; reinstalling and retrying in 3s..."
xcrun simctl terminate "$DEVICE_ID" "$BUNDLE_ID" 2>/dev/null || true
xcrun simctl uninstall "$DEVICE_ID" "$BUNDLE_ID" 2>/dev/null || true
sleep 3
xcrun simctl install "$DEVICE_ID" "$APP"
done
set -eo pipefail

if grep -q '\[ FAILED \]' /tmp/${TEST_NAME}.log; then
EXPECTED_COUNT=""
case "$TEST_NAME" in
diskann_mobile_compat_test)
EXPECTED_COUNT=7
;;
diskann_mobile_collection_test)
EXPECTED_COUNT=4
;;
esac

if [ "$LAUNCH_EXIT" -eq 124 ]; then
echo "::error::${TEST_NAME} timed out after 600 seconds"
xcrun simctl terminate "$DEVICE_ID" "$BUNDLE_ID" 2>/dev/null || true
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
elif [ "$LAUNCH_EXIT" -ne 0 ]; then
echo "::error::${TEST_NAME} launch exited ${LAUNCH_EXIT}"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
elif grep -Eq '^\[ FAILED \]' "$LOG_FILE"; then
echo "::error::${TEST_NAME} has failing tests"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
elif grep -q '\[ PASSED \]' /tmp/${TEST_NAME}.log; then
elif [ -n "$EXPECTED_COUNT" ] && \
! grep -Eq "^\\[ PASSED \\][[:space:]]+${EXPECTED_COUNT} tests?\\.[[:space:]]*$" "$LOG_FILE"; then
echo "::error::${TEST_NAME} did not report the required ${EXPECTED_COUNT}/${EXPECTED_COUNT} passing tests"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
elif [ -n "$EXPECTED_COUNT" ] && \
! grep -Eq "^\\[==========\\][[:space:]]+${EXPECTED_COUNT} tests?[[:space:]]+from[[:space:]]+[1-9][0-9]* test (suites?|cases?)[[:space:]]+ran\\." "$LOG_FILE"; then
echo "::error::${TEST_NAME} did not run exactly ${EXPECTED_COUNT} tests"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
elif grep -Eq '^\[==========\][[:space:]]+[1-9][0-9]* tests?[[:space:]]+from[[:space:]]+[1-9][0-9]* test (suites?|cases?)[[:space:]]+ran\.' "$LOG_FILE"; then
PASSED=$((PASSED + 1))
elif grep -qE 'Failed: 0$' /tmp/${TEST_NAME}.log; then
if [ "$TEST_NAME" = "diskann_mobile_compat_test" ]; then
DISKANN_COMPAT_PASSED=1
elif [ "$TEST_NAME" = "diskann_mobile_collection_test" ]; then
DISKANN_COLLECTION_PASSED=1
fi
elif [ "$TEST_NAME" = "c_api_test" ] && \
grep -Eq '^Passed: [1-9][0-9]*[[:space:]]*$' "$LOG_FILE" && \
grep -Eq '^Failed: 0[[:space:]]*$' "$LOG_FILE"; then
# c_api_test uses a custom test framework (not GTest)
PASSED=$((PASSED + 1))
elif [ "$LAUNCH_EXIT" -eq 0 ]; then
echo "::warning::${TEST_NAME} exited 0 but produced no recognisable test summary"
PASSED=$((PASSED + 1))
else
echo "::error::${TEST_NAME} exited ${LAUNCH_EXIT} with no test summary"
echo "::error::${TEST_NAME} produced no recognisable passing test summary"
FAILED_TESTS="${FAILED_TESTS} ${TEST_NAME}"
fi
echo "::endgroup::"
done

echo "Test summary: ${PASSED}/${TOTAL} passed"
if [ "$TOTAL" -eq 0 ]; then
echo "::error::No iOS test apps were discovered"
exit 1
fi
if [ -n "$FAILED_TESTS" ]; then
echo "::error::Failed tests:${FAILED_TESTS}"
exit 1
fi
if [ "$DISKANN_COMPAT_PASSED" -ne 1 ] || [ "$DISKANN_COLLECTION_PASSED" -ne 1 ]; then
echo "::error::Required DiskAnn tests did not both complete successfully"
exit 1
fi
if [ "$PASSED" -ne "$TOTAL" ]; then
echo "::error::Only ${PASSED}/${TOTAL} iOS tests completed successfully"
exit 1
fi

- name: Shutdown Simulator
if: matrix.test_on_simulator && always()
run: |
xcrun simctl shutdown "$DEVICE_ID" || true
if [ -n "${DEVICE_ID:-}" ]; then
xcrun simctl shutdown "$DEVICE_ID" || true
fi
22 changes: 19 additions & 3 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -114,20 +114,30 @@ endif()
include_directories(${PROJECT_ROOT_DIR}/src/include)
include_directories(${PROJECT_ROOT_DIR}/src)

option(BUILD_ZVEC_SHARED "Build all-in-one C++ shared library libzvec" ON)
option(BUILD_ZVEC_AILEGO_SHARED "Build all-in-one zvec-ailego shared library libzvec_ailego" ON)
option(BUILD_ZVEC_CORE_SHARED "Build all-in-one zvec-core shared library libzvec_core" ON)
set(ZVEC_CPP_SHARED_DEFAULT ON)
if(ANDROID OR IOS)
# A C++ shared library built with a static libc++ cannot safely exchange STL
# objects with a mobile application. Mobile C++ consumers use the static SDK
# targets below; the shared C API remains available through BUILD_C_BINDINGS.
set(ZVEC_CPP_SHARED_DEFAULT OFF)
endif()

option(BUILD_ZVEC_SHARED "Build all-in-one C++ shared library libzvec" ${ZVEC_CPP_SHARED_DEFAULT})
option(BUILD_ZVEC_AILEGO_SHARED "Build all-in-one zvec-ailego shared library libzvec_ailego" ${ZVEC_CPP_SHARED_DEFAULT})
option(BUILD_ZVEC_CORE_SHARED "Build all-in-one zvec-core shared library libzvec_core" ${ZVEC_CPP_SHARED_DEFAULT})

option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF)
option(BUILD_C_BINDINGS "Build C bindings" ON)
option(BUILD_TOOLS "Build tools" ON)
option(BUILD_CPP_EXAMPLES "Build C++ examples" OFF)

message(STATUS "BUILD_ZVEC_SHARED:${BUILD_ZVEC_SHARED}")
message(STATUS "BUILD_ZVEC_AILEGO_SHARED:${BUILD_ZVEC_AILEGO_SHARED}")
message(STATUS "BUILD_ZVEC_CORE_SHARED:${BUILD_ZVEC_CORE_SHARED}")
message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}")
message(STATUS "BUILD_C_BINDINGS:${BUILD_C_BINDINGS}")
message(STATUS "BUILD_TOOLS:${BUILD_TOOLS}")
message(STATUS "BUILD_CPP_EXAMPLES:${BUILD_CPP_EXAMPLES}")

if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64" AND NOT ANDROID AND NOT IOS)
include(CheckCXXCompilerFlag)
Expand Down Expand Up @@ -163,10 +173,12 @@ message(STATUS "RABITQ_SUPPORTED: ${RABITQ_SUPPORTED}")

# DiskAnn support:
# - Linux x86_64 and ARM64 with io_uring, libaio, or pread
# - 64-bit Android and iOS with the portable synchronous pread backend
# - macOS ARM64 (Apple Silicon) with synchronous pread
# - Windows x86_64 with overlapped I/O
if((CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|AMD64|aarch64|arm64)$" AND NOT ANDROID AND NOT IOS)
OR (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64)$" AND NOT IOS)
OR ((ANDROID OR IOS) AND CMAKE_SIZEOF_VOID_P EQUAL 8)
OR (WIN32 AND CMAKE_SIZEOF_VOID_P EQUAL 8 AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64|amd64)$"))
set(DISKANN_SUPPORTED ON)
add_definitions(-DDISKANN_SUPPORTED=1)
Expand All @@ -186,6 +198,10 @@ message(STATUS "USE_OSS_MIRROR:${USE_OSS_MIRROR}")
cc_directory(thirdparty)
cc_directories(src)

if(BUILD_CPP_EXAMPLES)
add_subdirectory(examples/c++ EXCLUDE_FROM_ALL)
endif()

cc_directories(tests)

add_custom_target(clang_tidy_deps DEPENDS ARROW.BUILD glog gflags Lz4.BUILD)
Expand Down
21 changes: 17 additions & 4 deletions cmake/bazel.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -619,8 +619,9 @@ endfunction()
## Add both shared and static library
macro(_add_library _NAME _OPTION)
add_library(${_NAME}_objects OBJECT ${_OPTION} ${ARGN})
if(IOS)
# iOS has no shared libraries, so the main target is static as well.
if(IOS OR (ANDROID AND ANDROID_STL STREQUAL "c++_static"))
# Mobile c++_static builds expose one archive under both target names so
# the C++ runtime is linked into an executable exactly once.
# Building a second, identical archive under the ${_NAME}_static name is
# not just wasteful, it breaks the build: giving both the same OUTPUT_NAME
# makes Ninja fail ("multiple rules generate ..."), while distinct names
Expand All @@ -645,7 +646,7 @@ macro(_add_library _NAME _OPTION)
endmacro()

## Check whether <name>_static is a target of its own rather than an alias of
## <name> (see _add_library: on iOS the two names share a single archive).
## <name> (see _add_library: mobile c++_static builds share one archive).
function(_has_own_static_variant _RESULT _NAME)
set(${_RESULT} FALSE PARENT_SCOPE)
if(NOT TARGET ${_NAME}_static)
Expand Down Expand Up @@ -785,7 +786,16 @@ function(_target_link_libraries _NAME)
endif()

if(NOT MSVC)
if(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "iOS")
if(ANDROID AND ANDROID_STL STREQUAL "c++_static")
# Keep the target in the normal link graph so CMake can resolve its
# transitive dependencies, but force-load the archive exactly once via
# a link option. Wrapping the target directly in --whole-archive here
# lets CMake emit another transitive occurrence later; lld then loads
# the same objects twice and reports duplicate symbols.
list(APPEND LINK_LIBS ${LIB})
list(APPEND ANDROID_WHOLEARCHIVE_OPTS
-Wl,--whole-archive,$<TARGET_FILE:${LIB}>,--no-whole-archive)
elseif(NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "iOS")
list(APPEND LINK_LIBS -Wl,--whole-archive ${LIB} -Wl,--no-whole-archive)
else()
list(APPEND LINK_LIBS -Wl,-force_load ${LIB})
Expand All @@ -811,6 +821,9 @@ function(_target_link_libraries _NAME)
endforeach()

target_link_libraries(${_NAME} ${LINK_LIBS})
if(ANDROID_WHOLEARCHIVE_OPTS)
target_link_options(${_NAME} PRIVATE ${ANDROID_WHOLEARCHIVE_OPTS})
endif()
if(MSVC_WHOLEARCHIVE_OPTS)
target_link_options(${_NAME} PRIVATE ${MSVC_WHOLEARCHIVE_OPTS})
endif()
Expand Down
Loading