From 58f6ea907a6e2f0c8284c8eb11107524858f01cd Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Wed, 26 Aug 2026 09:34:05 +0200 Subject: [PATCH] Build the in-process hub in extension CI instead of compiling a stub. wirebone.test requires linked=true; Docker test_release was silently disabling Wirebone when pkg-config could not see vcpkg OpenSSL/nghttp2/zstd. --- .../workflows/MainDistributionPipeline.yml | 1 + .gitmodules | 3 + CMakeLists.txt | 2 +- cmake/Wirebone.cmake | 84 +++++++++++++++---- docs/DEVELOPMENT.md | 11 +-- third_party/wirebone | 1 + vcpkg.json | 7 +- 7 files changed, 88 insertions(+), 21 deletions(-) create mode 160000 third_party/wirebone diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index 1769040..1615d02 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -3,6 +3,7 @@ # name: Main Extension Distribution Pipeline on: + pull_request: release: workflow_dispatch: diff --git a/.gitmodules b/.gitmodules index 937325b..11724c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,3 +9,6 @@ [submodule "third_party/libtailscale"] path = third_party/libtailscale url = https://github.com/tailscale/libtailscale.git +[submodule "third_party/wirebone"] + path = third_party/wirebone + url = https://github.com/lmangani/wirebone.cpp.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 84c95ff..47809fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.5) set(TARGET_NAME quackscale) option(QUACKSCALE_WITH_TAILSCALE "Embed libtailscale (requires Go)" ON) -option(QUACKSCALE_WITH_WIREBONE "Embed Wirebone coordinator when sources + deps are available" ON) +option(QUACKSCALE_WITH_WIREBONE "Embed Wirebone coordinator (required on POSIX unless OFF)" ON) set(EXTENSION_NAME ${TARGET_NAME}_extension) set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) diff --git a/cmake/Wirebone.cmake b/cmake/Wirebone.cmake index 3224734..aab5857 100644 --- a/cmake/Wirebone.cmake +++ b/cmake/Wirebone.cmake @@ -4,6 +4,33 @@ # 1. QUACKSCALE_WIREBONE_DIR (cache/path) # 2. ${CMAKE_CURRENT_SOURCE_DIR}/third_party/wirebone # 3. ${CMAKE_CURRENT_SOURCE_DIR}/../wirebone.cpp (sibling checkout) +# 4. FetchContent from github.com/lmangani/wirebone.cpp +# +# DuckDB extension CI installs those libraries via vcpkg and sets +# OPENSSL_ROOT_DIR. Prepend that prefix to PKG_CONFIG_PATH so this file +# does not silently disable the hub (which made wirebone.test fail). + +function(_quackscale_prepend_pkgconfig prefix) + if(prefix AND IS_DIRECTORY "${prefix}/lib/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${prefix}/lib/pkgconfig:$ENV{PKG_CONFIG_PATH}") + endif() + if(prefix AND IS_DIRECTORY "${prefix}/lib64/pkgconfig") + set(ENV{PKG_CONFIG_PATH} "${prefix}/lib64/pkgconfig:$ENV{PKG_CONFIG_PATH}") + endif() + if(prefix AND EXISTS "${prefix}") + list(APPEND CMAKE_PREFIX_PATH "${prefix}") + set(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH}" PARENT_SCOPE) + endif() +endfunction() + +_quackscale_prepend_pkgconfig("$ENV{OPENSSL_ROOT_DIR}") +_quackscale_prepend_pkgconfig("$ENV{OPENSSL_DIR}") +if(DEFINED ENV{VCPKG_TARGET_TRIPLET} AND NOT "$ENV{VCPKG_TARGET_TRIPLET}" STREQUAL "") + _quackscale_prepend_pkgconfig("${CMAKE_BINARY_DIR}/vcpkg_installed/$ENV{VCPKG_TARGET_TRIPLET}") +endif() +if(VCPKG_TARGET_TRIPLET) + _quackscale_prepend_pkgconfig("${CMAKE_BINARY_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}") +endif() set(_qs_wirebone_candidates "") if(QUACKSCALE_WIREBONE_DIR) @@ -21,11 +48,35 @@ foreach(_qs_wb_dir IN LISTS _qs_wirebone_candidates) endif() endforeach() +if(NOT QUACKSCALE_WIREBONE_SOURCE AND NOT WIN32) + include(FetchContent) + FetchContent_Declare( + wirebone_src + GIT_REPOSITORY https://github.com/lmangani/wirebone.cpp.git + GIT_TAG 6cf6aaabe704f51f8a4a62fc9f87d4265a8d8446 + ) + message(STATUS "QuackScale: fetching Wirebone from GitHub") + FetchContent_GetProperties(wirebone_src) + if(NOT wirebone_src_POPULATED) + FetchContent_Populate(wirebone_src) + endif() + if(EXISTS "${wirebone_src_SOURCE_DIR}/CMakeLists.txt" + AND EXISTS "${wirebone_src_SOURCE_DIR}/include/wirebone/wirebone.h") + set(QUACKSCALE_WIREBONE_SOURCE "${wirebone_src_SOURCE_DIR}") + endif() +endif() + if(NOT QUACKSCALE_WIREBONE_SOURCE) - message(STATUS "QuackScale: Wirebone sources not found; coordinator SQL disabled " - "(set QUACKSCALE_WIREBONE_DIR or checkout wirebone.cpp next to this repo)") - set(QUACKSCALE_WITH_WIREBONE OFF CACHE BOOL "Embed Wirebone coordinator" FORCE) - return() + if(WIN32) + message(STATUS "QuackScale: Wirebone is POSIX-only; coordinator SQL disabled on Windows") + set(QUACKSCALE_WITH_WIREBONE OFF CACHE BOOL "Embed Wirebone coordinator" FORCE) + return() + endif() + message(FATAL_ERROR + "QuackScale: Wirebone sources are required (in-process hub). " + "Init the submodule (git submodule update --init third_party/wirebone), " + "check out wirebone.cpp next to this repo, set QUACKSCALE_WIREBONE_DIR, " + "or pass -DQUACKSCALE_WITH_WIREBONE=OFF for a stub build.") endif() if(WIN32) @@ -34,20 +85,25 @@ if(WIN32) return() endif() -find_package(PkgConfig QUIET) +find_package(nlohmann_json CONFIG QUIET) + +find_package(PkgConfig) if(NOT PkgConfig_FOUND) - message(STATUS "QuackScale: pkg-config missing; Wirebone coordinator SQL disabled") - set(QUACKSCALE_WITH_WIREBONE OFF CACHE BOOL "Embed Wirebone coordinator" FORCE) - return() + message(FATAL_ERROR + "QuackScale: pkg-config is required to link Wirebone (OpenSSL, nghttp2, zstd). " + "Install pkg-config or pass -DQUACKSCALE_WITH_WIREBONE=OFF.") endif() -pkg_check_modules(_QS_WB_CRYPTO QUIET libcrypto) -pkg_check_modules(_QS_WB_NGHTTP2 QUIET libnghttp2) -pkg_check_modules(_QS_WB_ZSTD QUIET libzstd) +pkg_check_modules(_QS_WB_CRYPTO libcrypto) +pkg_check_modules(_QS_WB_NGHTTP2 libnghttp2) +pkg_check_modules(_QS_WB_ZSTD libzstd) if(NOT _QS_WB_CRYPTO_FOUND OR NOT _QS_WB_NGHTTP2_FOUND OR NOT _QS_WB_ZSTD_FOUND) - message(STATUS "QuackScale: OpenSSL/nghttp2/zstd not found; Wirebone coordinator SQL disabled") - set(QUACKSCALE_WITH_WIREBONE OFF CACHE BOOL "Embed Wirebone coordinator" FORCE) - return() + message(FATAL_ERROR + "QuackScale: Wirebone needs OpenSSL (libcrypto), nghttp2, and libzstd. " + "Install them (or let vcpkg.json provide openssl/nghttp2/zstd), " + "or pass -DQUACKSCALE_WITH_WIREBONE=OFF.\n" + " libcrypto=${_QS_WB_CRYPTO_FOUND} nghttp2=${_QS_WB_NGHTTP2_FOUND} zstd=${_QS_WB_ZSTD_FOUND}\n" + " PKG_CONFIG_PATH=$ENV{PKG_CONFIG_PATH}") endif() set(QUACKSCALE_WITH_WIREBONE ON CACHE BOOL "Embed Wirebone coordinator") diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index f0afe57..cf2d8a2 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -11,7 +11,7 @@ QuackScale does **not** reimplement Quack. It provides tailnet lifecycle SQL, a ```text DuckDB + quackscale + libtailscale → tailscale_up, tailscale_quack_forward, quack_uri, attach_ducklake -DuckDB + quackscale + wirebone (optional hub library) +DuckDB + quackscale + wirebone (in-process hub) → quackscale_hub (control plane + client in one process) DuckDB + quack (core) → quack_serve, ATTACH, quack_query @@ -38,7 +38,7 @@ Disable libtailscale (stub build): make CMAKE_VARS="-DQUACKSCALE_WITH_TAILSCALE=OFF" ``` -The in-process hub is enabled automatically on POSIX when `../wirebone.cpp` or `third_party/wirebone` exists and pkg-config can see OpenSSL (`libcrypto`), nghttp2, and libzstd. Hub state is stored in DuckDB tables (`quackscale.meta`, `quackscale.preauth_keys`, `quackscale.nodes`) or in an attached DuckLake catalog. Override the source path with `-DQUACKSCALE_WIREBONE_DIR=…`, or force it off: +The in-process hub is **on by default** on POSIX. Sources come from `third_party/wirebone` (submodule), a sibling `../wirebone.cpp`, `QUACKSCALE_WIREBONE_DIR`, or a CMake FetchContent of [wirebone.cpp](https://github.com/lmangani/wirebone.cpp). OpenSSL (`libcrypto`), nghttp2, and libzstd are required (`vcpkg.json` supplies them in DuckDB extension CI). Hub state is stored in DuckDB tables (`quackscale.meta`, `quackscale.preauth_keys`, `quackscale.nodes`) or in an attached DuckLake catalog. Force the stub off: ```sh make CMAKE_VARS="-DQUACKSCALE_WITH_WIREBONE=OFF" @@ -50,8 +50,9 @@ Docker Compose images build from source by default — see [examples/Dockerfile] ```text cmake/Libtailscale.cmake Go c-archive build + Go 1.25.5 bootstrap -cmake/Wirebone.cmake Optional in-process coordinator (sibling or QUACKSCALE_WIREBONE_DIR) +cmake/Wirebone.cmake In-process coordinator (submodule, sibling, or FetchContent) third_party/libtailscale/ git submodule +third_party/wirebone/ git submodule (lmangani/wirebone.cpp) src/ C++ extension (bridge, forwarder, attach_ducklake, wirebone catalog) scripts/e2e/ Compose entrypoint, bootstrap, verify-image examples/ Headscale Compose demo; examples/wirebone is local coordinator+peer @@ -113,7 +114,7 @@ Each Pages deploy replaces the whole site (one DuckDB version hosted). To host m | Headscale + Compose e2e | Done | | `ATTACH … TYPE quacktail_lake` (Tier 3 native catalog) | Planned | | `ducklake_discover()` enriched discovery | Planned | -| `quackscale_hub()` in-process control plane + client | Done (optional Wirebone build) | +| `quackscale_hub()` in-process control plane + client | Done (Wirebone; `-DQUACKSCALE_WITH_WIREBONE=OFF` stub) | | Community extension descriptor publish | Done (GitHub Pages on release) | ## Risks @@ -130,7 +131,7 @@ Each Pages deploy replaces the whole site (one DuckDB version hosted). To host m make test ``` -SQL unit tests do not require a live tailnet. `test/sql/wirebone.test` needs a hub-linked build (`../wirebone.cpp` or `third_party/wirebone`). Two-process hub smoke (CI): +SQL unit tests do not require a live tailnet. `test/sql/wirebone.test` expects a hub-linked build (`SELECT linked FROM quackscale_status()` is `true`). Two-process hub smoke (CI): ```sh ./scripts/ci_hub_smoke.sh diff --git a/third_party/wirebone b/third_party/wirebone new file mode 160000 index 0000000..6cf6aaa --- /dev/null +++ b/third_party/wirebone @@ -0,0 +1 @@ +Subproject commit 6cf6aaabe704f51f8a4a62fc9f87d4265a8d8446 diff --git a/vcpkg.json b/vcpkg.json index 1e41e0b..fce73d8 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -1,5 +1,10 @@ { - "dependencies": [], + "dependencies": [ + "openssl", + "nghttp2", + "zstd", + "nlohmann-json" + ], "vcpkg-configuration": { "overlay-ports": [ "./extension-ci-tools/vcpkg_ports"