diff --git a/recipes/pion/build.sh b/recipes/pion/build.sh new file mode 100755 index 00000000..350c0aba --- /dev/null +++ b/recipes/pion/build.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Builds pion-server from source only: the open reference vector kernels, no +# vendor/ archive, as `pixi run build-open` does. The channel carries no closed +# binaries; the release tarballs and Homebrew link the tuned closed kernels, and +# vector search at 1536 dims is the only thing that differs. +set -euo pipefail + +case "${target_platform}" in + osx-arm64) cpu=apple-m1 ;; + linux-64) cpu=x86-64-v2 ;; + linux-aarch64) cpu=neoverse-n1 ;; + *) echo "unsupported platform: ${target_platform}" >&2; exit 1 ;; +esac + +shims=(crash_wrap build_pool_wrap worker_spawn_wrap fcntl_wrap geo_math uring_wrap xdp_wrap ssm_state_wrap) +[[ "${target_platform}" == osx-* ]] && shims+=(dev_macos_stubs) +link=() +for f in "${shims[@]}"; do + gcc -O3 -c "src/ffi/$f.c" -o "src/ffi/$f.o" + link+=(-Xlinker "src/ffi/$f.o") +done + +if [[ "${target_platform}" == osx-* ]]; then + for f in metal_wrap nle_wrap; do + clang -O3 -c -fobjc-arc "src/ffi/$f.m" -o "src/ffi/$f.o" + link+=(-Xlinker "src/ffi/$f.o") + done +fi + +bash scripts/build_lua.sh +link+=(-Xlinker src/ffi/lua_wrap.o -Xlinker src/ffi/lua/liblua.a) + +if [[ "${target_platform}" == osx-* ]]; then + link+=(-Xlinker -export_dynamic + -Xlinker -u -Xlinker _pion_gh199_build_lane + -Xlinker -u -Xlinker _pion_vector_auto_build + -Xlinker -u -Xlinker _pion_worker_entry + -Xlinker -u -Xlinker _pion_script_dispatch + -Xlinker -framework -Xlinker Metal + -Xlinker -framework -Xlinker Foundation + -Xlinker -framework -Xlinker NaturalLanguage) +else + link+=(-Xlinker --export-dynamic + -Xlinker -u -Xlinker pion_gh199_build_lane + -Xlinker -u -Xlinker pion_vector_auto_build + -Xlinker -u -Xlinker pion_worker_entry + -Xlinker -u -Xlinker pion_script_dispatch + -Xlinker -lm -Xlinker -lpthread) +fi + +# Stamp src/common/version.mojo from VERSION, as `pixi run build` does. A tag +# archive has no .git, so the stamp reads the commit from .export_sha. GIT_DIR +# keeps git from finding a repository the build directory happens to sit in. +echo "${PION_COMMIT}" > .export_sha +GIT_DIR=/nonexistent bash scripts/stamp_version.sh + +mkdir -p "${PREFIX}/bin" +mojo build -I . -O3 --target-cpu "${cpu}" src/main.mojo "${link[@]}" -o "${PREFIX}/bin/pion-server" + +# The --metal-attention shader. pion-server looks for it next to its own +# executable, including /../share/pion. Compiling it needs full Xcode; +# without it the flag reports NOT ACTIVE and nothing else changes. +if [[ "${target_platform}" == osx-* ]]; then + if xcrun -sdk macosx metal -c src/ffi/metal_compute.metal -o metal_compute.air && + xcrun -sdk macosx metallib metal_compute.air -o metal_compute.metallib; then + mkdir -p "${PREFIX}/share/pion" + cp metal_compute.metallib "${PREFIX}/share/pion/" + else + echo "[Metal] no shader compiler (needs full Xcode): --metal-attention will report NOT ACTIVE" >&2 + fi +fi + +# install_name_tool tries and fails to rewrite paths inside these. +find "${PREFIX}" -type d -name .mojo_cache -prune -exec rm -rf {} + +# The compiler writes these into the prefix it runs from; they are not Pion's. +rm -rf "${PREFIX}/share/max/crashdb" "${PREFIX}/share/max/firstActivation" diff --git a/recipes/pion/recipe.yaml b/recipes/pion/recipe.yaml new file mode 100644 index 00000000..27ae0e95 --- /dev/null +++ b/recipes/pion/recipe.yaml @@ -0,0 +1,77 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json +# Built from source only, with Pion's open reference vector kernels: this channel +# carries no closed binaries. + +context: + version: "1.1.0" + # The commit the v${{ version }} tag points at, stamped into the binary as the + # release binaries stamp it. + commit: "8221878" + mojo_version: "1.2.0" + +package: + name: pion + version: ${{ version }} + +source: + url: https://github.com/pavelhorak/pion/archive/refs/tags/v${{ version }}.tar.gz + sha256: e71259ccb38f5370b6cdaae70c8cab017a4a3a6876e56fe63facb99d7bf83116 + +build: + number: 0 + skip: + - win + script: + file: build.sh + env: + PION_COMMIT: ${{ commit }} + +requirements: + host: + - mojo-compiler ==${{ mojo_version }} + # pion-server loads libKGENCompilerRTShared (and the two libraries it pulls + # in) from the compiler package at run time. + run: + - mojo-compiler ==${{ mojo_version }} + +tests: + - script: + - bash test_server.sh ${{ version }} + requirements: + run: + - python >=3.10 + files: + recipe: + - test_server.sh + source: + - tests/test_raw.py + - tests/resp_strict.py + - tests/test_vector_recall_smoke.py + +about: + homepage: https://pion.pavelhorak.com + repository: https://github.com/pavelhorak/pion + documentation: https://pion.pavelhorak.com + license: Apache-2.0 + license_file: + - LICENSE + - NOTICE + summary: Redis-compatible key-value and vector database written in Mojo, built entirely from source + description: | + Pion speaks RESP2/RESP3, so existing Redis clients work against it. It is a + durable key-value store with a write-ahead log, a cache with Redis's eviction + policies, vector search over the FT.* commands, and a persistent prompt cache + for local models. + + This package is the Apache-2.0 build, compiled entirely from source with the + open reference vector kernels. The project's release tarballs and Homebrew + formula link a closed library of tuned 1536-dim kernels instead, which return + identical results, faster. Nothing else differs. + + Run `pion-server`; it listens on port 1974 and keeps its state in the working + directory. + +extra: + maintainers: + - pavelhorak + project_name: Pion diff --git a/recipes/pion/test_server.sh b/recipes/pion/test_server.sh new file mode 100755 index 00000000..8394280a --- /dev/null +++ b/recipes/pion/test_server.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Starts the packaged server and runs the two standard-library checks Pion's +# release workflow runs before packaging: the wire basics (test_raw.py) and a +# 1536-dim recall check that only a correct vector build passes. +set -euo pipefail + +expected_version="$1" + +pion-server --version | tee version.txt +grep -q "^pion-server ${expected_version}+" version.txt +# This package is the open build: the closed kernels must not be linked. +if grep -q "(closed" version.txt; then + echo "the closed vector library is linked" >&2 + exit 1 +fi + +port=$((20000 + RANDOM % 20000)) +pion-server -p "${port}" -w 1 --no-auto-detect --no-auto-embed > server.log 2>&1 & +pid=$! +trap 'kill "${pid}" 2>/dev/null || true; wait "${pid}" 2>/dev/null || true' EXIT + +for _ in $(seq 1 60); do + if python -c "import socket, sys; socket.create_connection(('127.0.0.1', int(sys.argv[1])), timeout=1)" "${port}" 2>/dev/null; then + break + fi + if ! kill -0 "${pid}" 2>/dev/null; then + cat server.log + exit 1 + fi + sleep 1 +done + +python tests/test_raw.py --port "${port}" || { cat server.log; exit 1; } +python tests/test_vector_recall_smoke.py --port "${port}" || { cat server.log; exit 1; }