diff --git a/docs/descriptor-examples.md b/docs/descriptor-examples.md index 53f1fe0a..4370905b 100644 --- a/docs/descriptor-examples.md +++ b/docs/descriptor-examples.md @@ -14,7 +14,7 @@ For descriptor templates see [package-types.md](package-types.md). | Native module library (Form A) | [`mcpplibs.xpkg`](../pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](../pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](../pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](../pkgs/f/ffmpeg.lua) (module layer; sources compiled directly through `compat.ffmpeg`) · [`opencv`](../pkgs/o/opencv.opencv.lua) (single repository: the module layer and the full OpenCV 5 source build both live in the package, and only this descriptor stays on the index side) · [`mcpplibs.grpc`](../pkgs/g/grpc.lua) (gRPC 1.83.0 — the one library here that CANNOT be a compat descriptor: upstream publishes no self-contained source artifact, its tag archive carrying abseil/protobuf/re2/boringssl/zlib as empty submodule placeholders, so [grpc-m](https://github.com/mcpplibs/grpc-m)'s release tarball IS that artifact. It vendors only gRPC's own source and takes the five dependencies from this index, so a consumer that also uses protobuf links one copy rather than two) | | Form A whose consumer deps must be written by hand | [`huxerui.huxerui`](../pkgs/h/huxerui.huxerui.lua) (HuxerUI declares its GTK4 stack on the TARGET axis, which is the form mcpp recommends and which a descriptor structurally cannot carry — three platform blocks, and a cfg selector is not a platform. `mcpp emit xpkg` says so and emits empty `deps`, so the 36-entry closure is transcribed into `xpm.linux.deps` at PLATFORM level (a per-version `deps` is inert). Its `licenses`/`repo` also deliberately disagree with what emit produces) | | Native multi-module library with feature-scoped sources | [`gzj-creator.galay`](../pkgs/g/gzj-creator.galay.lua) (Galay 5.0.2 — the upstream Form-A manifest exposes `galay.utils` and `galay.kernel` by default, while SSL, HTTP, database, RPC, MCP, and tracing modules stay behind named features and their corresponding dependencies. The index keeps the upstream manifest intact and tests the default module surface on Unix.) | -| C-source compat (with `features`) | [`compat.cjson`](../pkgs/c/compat.cjson.lua) · [`compat.zlib`](../pkgs/c/compat.zlib.lua) · [`compat.hiredis`](../pkgs/c/compat.hiredis.lua) (the classic 1.2.0 — a 7-TU C build whose flat tarball headers get `hiredis/`-prefixed wrapper headers via `generated_files`, so consumers write `#include ` exactly like upstream's install layout) · [`compat.sqlite3`](../pkgs/c/compat.sqlite3.lua) (plain C-source, no features: the single `sqlite3.c` amalgamation; 3.45.3, the final maintenance release of the most widely deployed 3.45.x line) · [`compat.libuv`](../pkgs/c/compat.libuv.lua) (libuv 1.48.0 — the per-OS source sets transcribed from upstream's CMakeLists, because a `src/unix/*.c` glob would compile every OS's backend at once; linux/macos get explicit unix subsets, windows globs `src/win/*.c`) · [`compat.xxhash`](../pkgs/c/compat.xxhash.lua) (one TU, one header, no features at all — the interesting decision is what is NOT compiled: `xxh_x86dispatch.c` selects an AVX2/AVX512 path at RUNTIME and needs per-file `-mavx2` plus `XXH_X86DISPATCH` at every call site, so the package ships the flagless SSE2 baseline instead. Nor is the header-only `XXH_INLINE_ALL` mode chosen: it re-emits the implementation in every TU that hashes anything, which is the right trade only when there is exactly one such TU — something a package cannot know) · [`compat.quickjs-ng`](../pkgs/c/compat.quickjs-ng.lua) (QuickJS-ng 0.17.0 — upstream's four engine TUs and nothing else. `quickjs-libc.c`, the optional standard library that gives scripts files, the environment, timers and child processes, is NOT compiled, so an embedder gets an engine with no host access unless it adds its own. The tarball keeps `quickjs.h` beside private headers with generic names (`list.h`, `cutils.h`), so a `generated_files` forwarder is the only include directory, as in compat.libaio) | +| C-source compat (with `features`) | [`compat.cjson`](../pkgs/c/compat.cjson.lua) · [`compat.zlib`](../pkgs/c/compat.zlib.lua) · [`compat.zlib-ng`](../pkgs/c/compat.zlib-ng.lua) (zlib-ng 2.3.3 in its zlib-compatible form — CMake's `WITH_RUNTIME_CPU_DETECTION=OFF` build, so each function binds at compile time to the best path the architecture's baseline guarantees: NEON on aarch64, SSE2 on x86_64, which an ok image needs because openkal names no processor feature. The architecture sources gate themselves and a generated header sets the gate from the compiler's own macro; the one source without a gate is compiled through a generated forwarder. install() derives `zlib.h`, `zconf.h`, `zlib_name_mangling.h` and `gzread_mangle.h` from the tarball's templates, writing zconf.h's `HAVE_UNISTD_H` line as `#if !defined(_WIN32)` so the library and every consumer read one text. The test reads a stream madler's zlib wrote) · [`compat.hiredis`](../pkgs/c/compat.hiredis.lua) (the classic 1.2.0 — a 7-TU C build whose flat tarball headers get `hiredis/`-prefixed wrapper headers via `generated_files`, so consumers write `#include ` exactly like upstream's install layout) · [`compat.sqlite3`](../pkgs/c/compat.sqlite3.lua) (plain C-source, no features: the single `sqlite3.c` amalgamation; 3.45.3, the final maintenance release of the most widely deployed 3.45.x line) · [`compat.libuv`](../pkgs/c/compat.libuv.lua) (libuv 1.48.0 — the per-OS source sets transcribed from upstream's CMakeLists, because a `src/unix/*.c` glob would compile every OS's backend at once; linux/macos get explicit unix subsets, windows globs `src/win/*.c`) · [`compat.xxhash`](../pkgs/c/compat.xxhash.lua) (one TU, one header, no features at all — the interesting decision is what is NOT compiled: `xxh_x86dispatch.c` selects an AVX2/AVX512 path at RUNTIME and needs per-file `-mavx2` plus `XXH_X86DISPATCH` at every call site, so the package ships the flagless SSE2 baseline instead. Nor is the header-only `XXH_INLINE_ALL` mode chosen: it re-emits the implementation in every TU that hashes anything, which is the right trade only when there is exactly one such TU — something a package cannot know) · [`compat.quickjs-ng`](../pkgs/c/compat.quickjs-ng.lua) (QuickJS-ng 0.17.0 — upstream's four engine TUs and nothing else. `quickjs-libc.c`, the optional standard library that gives scripts files, the environment, timers and child processes, is NOT compiled, so an embedder gets an engine with no host access unless it adds its own. The tarball keeps `quickjs.h` beside private headers with generic names (`list.h`, `cutils.h`), so a `generated_files` forwarder is the only include directory, as in compat.libaio) | | C-source compat whose sources are chosen by ARCHITECTURE | [`compat.wamr`](../pkgs/c/compat.wamr.lua) (WAMR 2.4.5 — the descriptor schema varies `sources` and `cflags` per OS but has no per-architecture hook, and `archs` is package metadata rather than a selector. WAMR needs both: one of `BUILD_TARGET_X86_64`/`BUILD_TARGET_AARCH64` must be defined or `wasm_runtime_common.c` compiles its whole invoke-native section away and the link fails on `invokeNative`, and the implementation of that symbol is a hand-written assembly file per architecture. Both halves are handed to the preprocessor instead: a `generated_files` config header maps `__x86_64__`/`__aarch64__` to the matching `BUILD_TARGET_*` and reaches every TU through `-include`, and a generated `.S` — `.S`, because upstream's `arch/invokeNative_*.s` are lowercase and clang assembles those WITHOUT the preprocessor, so unlike libffi's `.S` files they cannot guard themselves — `#include`s the chosen one as text. Two further notes worth copying: `-std=gnu11` arrives through `cflags` because `c_standard = "c11"` defines `__STRICT_ANSI__`, under which the bare `asm` in `platform_internal.h` is not a keyword; and the POSIX files upstream drops when WASI is off are carried unconditionally rather than `!`-excluded and re-added by the `libc-wasi` feature, because an exclusion glob is global and wins over the feature's own entry for the same file) | | C-source compat where the library IS a kernel ABI | [`compat.libaio`](../pkgs/c/compat.libaio.lua) (libaio 0.3.113 — twelve syscall-wrapper TUs, and the only `xpm` section is `linux`, because there is no port to declare: `struct iocb` is the kernel's and every TU is `syscall(__NR_io_*, …)`. Consumers gate it with `[target.'cfg(linux)'.dependencies]`, the mirror image of compat.wil. Three things it teaches. **One public header out of a source dir**: upstream installs exactly one, `libaio.h`, but the tarball keeps it in `src/` beside the private headers — one of which is named `syscall.h` and would SHADOW glibc's for every consumer TU — so `include_dirs` names a `generated_files` forwarder and nothing else; the package's own sources reach the real header through it while their quote-form `#include "syscall.h"` still resolves next to the including `.c`, so no `-I` into `src/` is needed at all. **A `c_standard` that is a trap**: `-std=c11` sets `__STRICT_ANSI__`, which hides `syscall()` and `sigset_t`, and the public header then fails to parse at `io_pgetevents`; declaring `c_standard = "gnu11"` LOOKS like the fix but mcpp 2026.8.27.2 accepts the string and still emits `-std=c11` (visible in the emitted `compile_commands.json`), so `-D_GNU_SOURCE` in `cflags` is the spelling that takes effect. **Symbol versioning in a static package**: `io_getevents` and `io_cancel` have no ordinary definitions upstream — the functions are `io_getevents_0_4` etc. publishing short names through `.symver … @@LIBAIO_0.4` — which resolves for an executable under both ld.bfd and lld, but not when a consumer builds a `.so` straight out of these objects; that needs upstream's `src/libaio.map`, exactly as upstream's own `libaio.a` does) | | C++-source compat, Linux-only POSIX API + consumer-visible header fixes | [`compat.libserial`](../pkgs/c/compat.libserial.lua) (LibSerial 1.0.0 — `SerialPort`/`SerialStream`, three TUs from upstream's `LIBSERIAL_SOURCES`, and the only `xpm` section is `linux`: the headers open with `` and every TU reaches ``, so it is gated like compat.libaio. **A header that only compiles by accident gets a shim**: `src/libserial/SerialPortConstants.h` uses `uint8_t` but includes neither `` nor ``, so a consumer TU that opens a libserial header first fails with an undeclared-type error; a `generated_files` shim placed ahead of `*/src` includes `` and `#include_next`s upstream's header (the compat.yaml-cpp / compat.cpptrace pattern). The test proves it by including the libserial header first of all and asserts the rest over a real openpty() pair. **A header a shim cannot fix is patched in `install()`**: `SerialPort.h`'s public template `call_with_retry` is spelled with `std::result_of`, removed in C++20 — libstdc++ still ships it, libc++ does not, so on an llvm toolchain the library and every consumer TU fail to compile. A shim cannot reach a template inside the header, and libc++'s `_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS` only works when defined before every TU's first libc++ header, so the hook rewrites both spellings to the equivalent `std::invoke_result_t` and asserts the match counts) | diff --git a/docs/zh/descriptor-examples.md b/docs/zh/descriptor-examples.md index 89590854..35aabd75 100644 --- a/docs/zh/descriptor-examples.md +++ b/docs/zh/descriptor-examples.md @@ -13,7 +13,7 @@ | 原生模块库(Form A) | [`mcpplibs.xpkg`](../../pkgs/x/xpkg.lua) · [`mcpplibs.tinyhttps`](../../pkgs/t/tinyhttps.lua) · [`tensorvia-cpu`](../../pkgs/t/tensorvia-cpu.lua) · [`ffmpeg`](../../pkgs/f/ffmpeg.lua)(模块层,源码经 `compat.ffmpeg` 直编) · [`opencv`](../../pkgs/o/opencv.opencv.lua)(单仓库:模块层与 OpenCV 5 全源码构建同在包内,索引侧只留本描述符) · [`mcpplibs.grpc`](../../pkgs/g/grpc.lua)(gRPC 1.83.0 —— 本索引里唯一**无法**做成 compat 描述符的库:上游不发布任何自包含源码产物,其 tag 归档里 abseil/protobuf/re2/boringssl/zlib 全是空 submodule 占位,因此 [grpc-m](https://github.com/mcpplibs/grpc-m) 的 release tarball 才是那个产物。它只 vendor gRPC 自己的源码,五个依赖全取自本索引,故同时直接使用 protobuf 的消费者链进去的是同一份而非两份)| | 消费者依赖须手写的 Form A | [`huxerui.huxerui`](../../pkgs/h/huxerui.huxerui.lua)(HuxerUI 把 GTK4 栈声明在 TARGET 轴上 —— 这是 mcpp 推荐的写法,而描述符在结构上承载不了:只有三个平台块,cfg 选择器不是平台。`mcpp emit xpkg` 会如实说明并输出空 `deps`,因此 36 项闭包被手工抄进 PLATFORM 级的 `xpm.linux.deps`(按版本写的 `deps` 不生效)。它的 `licenses`/`repo` 也刻意与 emit 的产出不同) | | 原生多模块库(按 feature 管理源码) | [`gzj-creator.galay`](../../pkgs/g/gzj-creator.galay.lua)(Galay 5.0.2 —— 上游 Form-A manifest 默认提供 `galay.utils` 与 `galay.kernel`,SSL、HTTP、数据库、RPC、MCP、tracing 等模块及其依赖按具名 feature 开启。索引保持上游 manifest 原样,Unix 成员测试默认模块表面)| -| C 源码 compat(含 `features`) | [`compat.cjson`](../../pkgs/c/compat.cjson.lua) · [`compat.zlib`](../../pkgs/c/compat.zlib.lua) · [`compat.hiredis`](../../pkgs/c/compat.hiredis.lua)(经典 1.2.0 —— 7 个 C TU;tarball 平铺头经 `generated_files` 补 `hiredis/` 前缀薄包装头,消费者可写 `#include `,与上游安装布局一致) · [`compat.sqlite3`](../../pkgs/c/compat.sqlite3.lua)(纯 C 源码、无 feature:单一 `sqlite3.c` amalgamation;3.45.3,部署最广的 3.45.x 线) · [`compat.libuv`](../../pkgs/c/compat.libuv.lua)(libuv 1.48.0 —— 逐 OS 源清单转录自上游 CMakeLists,因为 `src/unix/*.c` 通配会一次编进所有 OS 的后端;linux/macos 显式列 unix 子集,windows 用 `src/win/*.c` glob) · [`compat.xxhash`](../../pkgs/c/compat.xxhash.lua)(单 TU、单头、无 feature —— 值得说的是**没有**编译什么：`xxh_x86dispatch.c` 在运行期选择 AVX2/AVX512 路径，需要 per-file `-mavx2` 并要求每个调用点定义 `XXH_X86DISPATCH`，故本包只出无需任何 flag 的 SSE2 基线。也没有选 header-only 的 `XXH_INLINE_ALL` 模式：它会在每个做哈希的 TU 里重新展开整份实现，那只有在「恰好只有一个这样的 TU」时才划算 —— 而这件事包本身无从知道) · [`compat.quickjs-ng`](../../pkgs/c/compat.quickjs-ng.lua)(QuickJS-ng 0.17.0 —— 只编上游的四个引擎 TU。可选标准库 `quickjs-libc.c`(给脚本文件、环境变量、定时器与子进程能力)不编，嵌入方拿到的引擎碰不到宿主，除非自己提供。tarball 把 `quickjs.h` 和 `list.h`、`cutils.h` 这类名字很通用的私有头放在一起，所以 include 路径上只放一个 `generated_files` 转发头，与 compat.libaio 同法) | +| C 源码 compat(含 `features`) | [`compat.cjson`](../../pkgs/c/compat.cjson.lua) · [`compat.zlib`](../../pkgs/c/compat.zlib.lua) · [`compat.zlib-ng`](../../pkgs/c/compat.zlib-ng.lua)(zlib-ng 2.3.3 的 zlib 兼容形态 —— 即 CMake 的 `WITH_RUNTIME_CPU_DETECTION=OFF` 构建：每个函数在编译期绑定到该架构基线保证的最优实现，aarch64 为 NEON，x86_64 为 SSE2；ok 映像需要这种形态，因为 openkal 不提供处理器特性。架构源文件自带门控，由生成的配置头按编译器自身的宏设置；唯一没有门控的源文件经生成的转发文件编译。install() 由发布包中的模板生成 `zlib.h`、`zconf.h`、`zlib_name_mangling.h` 与 `gzread_mangle.h`，并把 zconf.h 的 `HAVE_UNISTD_H` 一行写为 `#if !defined(_WIN32)`，使库与每个使用者读同一份文本。测试读取 madler zlib 写出的流）· [`compat.hiredis`](../../pkgs/c/compat.hiredis.lua)(经典 1.2.0 —— 7 个 C TU;tarball 平铺头经 `generated_files` 补 `hiredis/` 前缀薄包装头,消费者可写 `#include `,与上游安装布局一致) · [`compat.sqlite3`](../../pkgs/c/compat.sqlite3.lua)(纯 C 源码、无 feature:单一 `sqlite3.c` amalgamation;3.45.3,部署最广的 3.45.x 线) · [`compat.libuv`](../../pkgs/c/compat.libuv.lua)(libuv 1.48.0 —— 逐 OS 源清单转录自上游 CMakeLists,因为 `src/unix/*.c` 通配会一次编进所有 OS 的后端;linux/macos 显式列 unix 子集,windows 用 `src/win/*.c` glob) · [`compat.xxhash`](../../pkgs/c/compat.xxhash.lua)(单 TU、单头、无 feature —— 值得说的是**没有**编译什么：`xxh_x86dispatch.c` 在运行期选择 AVX2/AVX512 路径，需要 per-file `-mavx2` 并要求每个调用点定义 `XXH_X86DISPATCH`，故本包只出无需任何 flag 的 SSE2 基线。也没有选 header-only 的 `XXH_INLINE_ALL` 模式：它会在每个做哈希的 TU 里重新展开整份实现，那只有在「恰好只有一个这样的 TU」时才划算 —— 而这件事包本身无从知道) · [`compat.quickjs-ng`](../../pkgs/c/compat.quickjs-ng.lua)(QuickJS-ng 0.17.0 —— 只编上游的四个引擎 TU。可选标准库 `quickjs-libc.c`(给脚本文件、环境变量、定时器与子进程能力)不编，嵌入方拿到的引擎碰不到宿主，除非自己提供。tarball 把 `quickjs.h` 和 `list.h`、`cutils.h` 这类名字很通用的私有头放在一起，所以 include 路径上只放一个 `generated_files` 转发头，与 compat.libaio 同法) | | C 源码 compat(源码按**架构**选择) | [`compat.wamr`](../../pkgs/c/compat.wamr.lua)(WAMR 2.4.5 —— 描述符能按 OS 分 `sources`/`cflags`,但**没有按架构分**的钩子,`archs` 是包级元数据而非选择器。而 WAMR 两处都要按架构走:不定义 `BUILD_TARGET_X86_64`/`BUILD_TARGET_AARCH64` 之一,`wasm_runtime_common.c` 会把整段 invoke-native 编没,链接期报 `invokeNative` 未定义;而该符号的实现本身就是每架构一份手写汇编。两处都改交给预处理器:`generated_files` 生成的配置头把 `__x86_64__`/`__aarch64__` 映射到对应 `BUILD_TARGET_*`,经 `-include` 到达每个 TU;另一份生成的 `.S` 用 `#include` 把选中的汇编原样拉进来 —— 之所以必须是 `.S`,是因为上游的 `arch/invokeNative_*.s` 是小写后缀,clang 汇编它们**不过预处理器**,因此不像 libffi 的 `.S` 那样能自己加架构守卫。另有两点值得抄:`-std=gnu11` 经 `cflags` 追加,因为 `c_standard = "c11"` 会定义 `__STRICT_ANSI__`,此时 `platform_internal.h` 里裸写的 `asm` 不是关键字;以及上游在关掉 WASI 时会剔除的那几个 POSIX 文件,这里**无条件常含**,而不是 base 用 `!` 排除、再由 `libc-wasi` feature 加回来 —— 排除 glob 是全局的,会盖过 feature 里同名文件的条目) | | C 源码 compat(库本身就是内核 ABI) | [`compat.libaio`](../../pkgs/c/compat.libaio.lua)(libaio 0.3.113 —— 12 个系统调用封装 TU,`xpm` 只有 `linux` 一段,因为根本不存在「移植」可声明:`struct iocb` 就是内核的结构体,每个 TU 都是 `syscall(__NR_io_*, …)`。消费者用 `[target.'cfg(linux)'.dependencies]` 门控,与 compat.wil 互为镜像。它给出三条经验。**把唯一的公开头从源码目录里择出来**:上游只安装 `libaio.h` 一个头,但 tarball 把它放在 `src/` 里、与私有头并列 —— 其中一个恰好叫 `syscall.h`,一旦上了 include 路径就会**遮蔽** glibc 的同名头。故 `include_dirs` 只指向一个 `generated_files` 转发头;包自身的源码经它拿到真头文件,而它们引号形式的 `#include "syscall.h"` 仍按「包含者所在目录优先」解析,于是整包**不需要任何指向 `src/` 的 `-I`**。**一个会骗人的 `c_standard`**:`-std=c11` 会定义 `__STRICT_ANSI__`,从而藏掉 `syscall()` 与 `sigset_t`,连公开头都会在 `io_pgetevents` 处解析失败;写 `c_standard = "gnu11"` **看起来**是解法,但 mcpp 2026.8.27.2 接受这个字符串却依然发 `-std=c11`(在产出的 `compile_commands.json` 里可见),真正生效的写法是 `cflags` 里的 `-D_GNU_SOURCE`。**静态包里的符号版本**:`io_getevents` / `io_cancel` 在上游并没有普通定义 —— 函数名是 `io_getevents_0_4` 之类,短名经 `.symver … @@LIBAIO_0.4` 发布 —— 链接**可执行文件**时 ld.bfd 与 lld 都能解析,但消费者若直接拿这些对象去构建 `.so` 就不行,那需要上游的 `src/libaio.map`,与上游自己的 `libaio.a` 完全同理) | | C++ 源码 compat,仅 Linux 的 POSIX 接口 + 消费者可见的头文件修复 | [`compat.libserial`](../../pkgs/c/compat.libserial.lua)(LibSerial 1.0.0 —— `SerialPort`/`SerialStream`,上游 `LIBSERIAL_SOURCES` 的三个 TU,`xpm` 只有 `linux` 一段：各头文件以 `` 开头,每个 TU 都触及 ``,故与 compat.libaio 同样门控。**一个「只是碰巧能编」的头被 shim 修复**:`src/libserial/SerialPortConstants.h` 用了 `uint8_t`,却既没含 `` 也没含 ``,消费者 TU 若先打开 libserial 头就会报未声明类型;`generated_files` 的 shim 排在 `*/src` 之前,先含 `` 再 `#include_next` 上游真头(compat.yaml-cpp / compat.cpptrace 的做法)。测试把 libserial 头放在**第一行**来证明 shim 生效,其余断言跑在真实的 openpty() 对上。**shim 修不了的头在 `install()` 里改**:`SerialPort.h` 的公开模板 `call_with_retry` 用了 C++20 已移除的 `std::result_of` —— libstdc++ 仍保留,libc++ 没有,于是 llvm 工具链下库本身和每个消费者 TU 都编不过。shim 碰不到头文件内部的模板,libc++ 的 `_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS` 又必须在每个 TU 的第一个 libc++ 头之前定义,所以由安装钩子把两处改写为等价的 `std::invoke_result_t` 并断言匹配次数) | diff --git a/mcpp.toml b/mcpp.toml index 88fcd3d3..5650e1f2 100644 --- a/mcpp.toml +++ b/mcpp.toml @@ -228,6 +228,7 @@ members = [ "tests/examples/yaml-cpp-v080", "tests/examples/yyjson", "tests/examples/zlib", + "tests/examples/zlib-ng", "tests/examples/magic_enum", "tests/examples/mimalloc", "tests/examples/harfbuzz", diff --git a/pkgs/c/compat.zlib-ng.lua b/pkgs/c/compat.zlib-ng.lua new file mode 100644 index 00000000..4fe3a250 --- /dev/null +++ b/pkgs/c/compat.zlib-ng.lua @@ -0,0 +1,267 @@ +-- compat.zlib-ng — zlib-ng in its zlib-compatible form, built from source. +-- +-- The zlib API (`zlib.h`, `compress2`, `uncompress`, `inflate`, the gz files) +-- over zlib-ng's implementation: the same streams and the same names as +-- compat.zlib, with the inner loops of inflate, adler32 and the deflate match +-- search written for the processor's vector unit. A consumer that wants zlib's +-- interface and not madler's code chooses this package instead of compat.zlib; +-- the two define the same symbols and do not share a link. +-- +-- ───────────────────────────────────────────────────────────────────────── +-- WHY IT EXISTS (mcpp-community/mcpp design 2026-11-08, W5) +-- +-- mcppls-clangd's ok image reads its index shards through zlib. Against a +-- native clangd linked to macOS's own libz, whose inflate and adler32 use +-- NEON, the image's warm start took 1.214–1.332 times the processor time with +-- compat.zlib and 0.973–1.107 with no zlib at all, on the same runner (runs +-- 42369457883 and 42386660489, four pairs each). The difference is the +-- decompression of the shards, and zlib-ng's NEON inflate is what closes it. +-- +-- ───────────────────────────────────────────────────────────────────────── +-- THE PROCESSOR'S PATHS ARE CHOSEN AT COMPILE TIME +-- +-- Upstream chooses among its implementations at run time (functable.c over +-- cpu_features.c, which reads the auxiliary vector, cpuid or sysctl). An ok +-- image has none of these: openkal names no processor feature. This package +-- is therefore CMake's `WITH_RUNTIME_CPU_DETECTION=OFF` build, in which +-- arch_functions.h binds each function to the best implementation the +-- target's BASELINE guarantees: +-- +-- aarch64 NEON (Armv8-A has it): inflate_fast, adler32, chunkmemset, +-- slide_hash, compare256 and longest_match +-- x86_64 SSE2 (x86-64 has it): chunkmemset, inflate_fast, slide_hash, +-- compare256, longest_match and the CRC's chorba fold +-- +-- Every architecture source but one opens with its own guard (`#ifdef +-- ARM_NEON`, `#ifdef X86_SSE2`), so all of them are listed and the +-- configuration header below sets the guard from the compiler's architecture +-- macro, the way compat.wamr chooses its target. arch/x86/slide_hash_sse2.c +-- has no guard, because upstream's CMake lists it for x86 only; it is +-- compiled through a generated source that includes it under `X86_SSE2`. A path beyond the baseline (the Armv8 CRC32 +-- instructions, SSSE3, AVX2, PCLMULQDQ) needs either a flag the whole program +-- would then require or detection at run time, and is not built. The CRC32 +-- instructions matter to gzip streams only: a zlib stream, which is what LLVM +-- writes, is checked with adler32. +-- +-- ───────────────────────────────────────────────────────────────────────── +-- THE GENERATED HEADERS +-- +-- CMake writes three public headers and one private one. install() derives +-- them from the tarball's templates rather than inlining 2000 lines, so a +-- version bump cannot ship an old header: +-- +-- zlib.h zlib.h.in with @ZLIB_SYMBOL_PREFIX@ empty (no +-- prefix: the names are zlib's own) +-- zconf.h zconf.h.in, with configure's `#ifdef +-- HAVE_UNISTD_H` written as `#if !defined(_WIN32)` +-- zlib_name_mangling.h zlib_name_mangling.h.empty, as CMake copies it +-- gzread_mangle.h gzread_mangle.h.in with the prefix empty (gzread.c +-- undefines zlib.h's gzgetc macro through it) +-- +-- CMake writes `#if 1` into zconf.h on a POSIX system and leaves the line on +-- Windows. The condition written here is the same decision taken by the +-- header itself, so the library and every consumer compute z_off_t from one +-- text and cannot disagree (compat.zlib needs a package-private `-include` +-- for the same line and documents the split; this package has none). On +-- Windows with musl as the C library (an openkal graph) `_WIN32` is absent, +-- because openkal-musl presents POSIX, and the header takes the POSIX branch +-- there as well. +package = { + spec = "1", + namespace = "compat", + name = "zlib-ng", + description = "zlib-ng in its zlib-compatible form: zlib's API with vectorised inflate, adler32 and deflate", + licenses = {"Zlib"}, + repo = "https://github.com/zlib-ng/zlib-ng", + type = "package", + + xpm = { + linux = { + ["2.3.3"] = { + url = { + GLOBAL = "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.3.3.tar.gz", + CN = "https://gitcode.com/mcpp-res/zlib-ng/releases/download/2.3.3/zlib-ng-2.3.3.tar.gz", + }, + sha256 = "f9c65aa9c852eb8255b636fd9f07ce1c406f061ec19a2e7d508b318ca0c907d1", + }, + }, + macosx = { + ["2.3.3"] = { + url = { + GLOBAL = "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.3.3.tar.gz", + CN = "https://gitcode.com/mcpp-res/zlib-ng/releases/download/2.3.3/zlib-ng-2.3.3.tar.gz", + }, + sha256 = "f9c65aa9c852eb8255b636fd9f07ce1c406f061ec19a2e7d508b318ca0c907d1", + }, + }, + windows = { + ["2.3.3"] = { + url = { + GLOBAL = "https://github.com/zlib-ng/zlib-ng/archive/refs/tags/2.3.3.tar.gz", + CN = "https://gitcode.com/mcpp-res/zlib-ng/releases/download/2.3.3/zlib-ng-2.3.3.tar.gz", + }, + sha256 = "f9c65aa9c852eb8255b636fd9f07ce1c406f061ec19a2e7d508b318ca0c907d1", + }, + }, + }, + + mcpp = { + language = "c++23", + import_std = false, + c_standard = "c11", + -- The tree's root holds the public headers install() writes and the + -- private ones every architecture source includes by name + -- (`#include "zbuild.h"` from arch/arm/), as compat.zlib's root does. + include_dirs = {"*", "mcpp_generated/include"}, + -- Package-private: the configuration of zlib-ng's own translation + -- units. No line of it changes a public declaration; zlib.h and + -- zconf.h read none of these names. + cflags = { "-include", "mcpp_zlibng_config.h" }, + linux = { + cflags = { "-D_GNU_SOURCE" }, + }, + generated_files = { + ["mcpp_generated/include/mcpp_zlibng_config.h"] = +[==[ +#ifndef MCPP_ZLIBNG_CONFIG_H +#define MCPP_ZLIBNG_CONFIG_H +/* CMake's definitions for zlib-ng 2.3.3 with ZLIB_COMPAT=ON and + WITH_RUNTIME_CPU_DETECTION=OFF, taken from its compile_commands.json, with + the architecture taken from the compiler instead of from CMake's probe. */ +#define ZLIB_COMPAT 1 +#define WITH_GZFILEOP 1 +#define WITH_OPTIM 1 +#define DISABLE_RUNTIME_CPU_DETECTION 1 +#if defined(__GNUC__) || defined(__clang__) +# define HAVE_ATTRIBUTE_ALIGNED 1 +# define HAVE_BUILTIN_ASSUME_ALIGNED 1 +# define HAVE_BUILTIN_CTZ 1 +# define HAVE_BUILTIN_CTZLL 1 +# define HAVE_VISIBILITY_HIDDEN 1 +# define HAVE_VISIBILITY_INTERNAL 1 +#endif +#if defined(__aarch64__) || defined(_M_ARM64) +# define ARM_FEATURES 1 +# define ARM_NEON 1 +# define ARM_NEON_HASLD4 1 +#elif defined(__x86_64__) || defined(_M_X64) +# define X86_FEATURES 1 +# define X86_SSE2 1 +#endif +#endif +]==], + -- The one architecture source without a guard of its own. + ["mcpp_generated/slide_hash_sse2.c"] = +[==[ +#ifdef X86_SSE2 +#include "arch/x86/slide_hash_sse2.c" +#endif +]==], + }, + sources = { + "*/adler32.c", + "*/compress.c", + "*/crc32.c", + "*/crc32_braid_comb.c", + "*/deflate.c", + "*/deflate_fast.c", + "*/deflate_huff.c", + "*/deflate_medium.c", + "*/deflate_quick.c", + "*/deflate_rle.c", + "*/deflate_slow.c", + "*/deflate_stored.c", + "*/functable.c", + "*/gzlib.c", + "*/gzread.c", + "*/gzwrite.c", + "*/infback.c", + "*/inflate.c", + "*/inftrees.c", + "*/insert_string.c", + "*/insert_string_roll.c", + "*/trees.c", + "*/uncompr.c", + "*/zutil.c", + -- The portable implementations: the baseline every architecture + -- falls back to, and the only one of the others. + "*/arch/generic/adler32_c.c", + "*/arch/generic/adler32_fold_c.c", + "*/arch/generic/chunkset_c.c", + "*/arch/generic/compare256_c.c", + "*/arch/generic/crc32_braid_c.c", + "*/arch/generic/crc32_chorba_c.c", + "*/arch/generic/crc32_fold_c.c", + "*/arch/generic/slide_hash_c.c", + -- Each guarded by its own `#ifdef`; see the header comment. + "*/arch/arm/adler32_neon.c", + "*/arch/arm/chunkset_neon.c", + "*/arch/arm/compare256_neon.c", + "*/arch/arm/slide_hash_neon.c", + "*/arch/x86/chorba_sse2.c", + "*/arch/x86/chunkset_sse2.c", + "*/arch/x86/compare256_sse2.c", + "mcpp_generated/slide_hash_sse2.c", + }, + targets = { ["zlib"] = { kind = "lib" } }, + deps = {}, + }, +} + +import("xim.libxpkg.pkginfo") +import("xim.libxpkg.log") + +-- What CMake's configure_file does to the four headers. +function install() + -- Once a descriptor defines install(), the archive is extracted beside the + -- install dir; the tree is moved into it under its own name, so the layout + -- is the one an extraction without install() has (`*/zlib.h`). + local name = "zlib-ng-" .. pkginfo.version() + local srcroot = pkginfo.install_file():replace(".tar.gz", "") + if not os.isdir(srcroot) then srcroot = name end + if not os.isdir(srcroot) then + log.error("[zlib-ng] extracted tree not found (looked for %s)", srcroot) + return false + end + os.tryrm(pkginfo.install_dir()) + os.mkdir(pkginfo.install_dir()) + local root = path.join(pkginfo.install_dir(), name) + os.mv(srcroot, root) + + local function derive(template, header, edit) + local text = io.readfile(path.join(root, template)) + if not text then + log.error("[zlib-ng] %s missing from the release tarball", template) + return false + end + text = edit(text) + if text:find("@[A-Za-z_]+@") then + log.error("[zlib-ng] %s has a placeholder this descriptor does not substitute", template) + return false + end + io.writefile(path.join(root, header), text) + return true + end + + if not derive("zlib.h.in", "zlib.h", function(t) + return (t:gsub("@ZLIB_SYMBOL_PREFIX@", "")) + end) then return false end + + local unistd = "#ifdef HAVE_UNISTD_H /* may be set to #if 1 by configure/cmake/etc */" + if not derive("zconf.h.in", "zconf.h", function(t) + local s, e = t:find(unistd, 1, true) + if not s then return t .. "\n@UNISTD_LINE_NOT_FOUND@\n" end + return t:sub(1, s - 1) .. "#if !defined(_WIN32) /* CMake writes #if 1 here on a POSIX system */" .. t:sub(e + 1) + end) then return false end + + if not derive("gzread_mangle.h.in", "gzread_mangle.h", function(t) + return (t:gsub("@ZLIB_SYMBOL_PREFIX@", "")) + end) then return false end + + if not os.isfile(path.join(root, "zlib_name_mangling.h.empty")) then + log.error("[zlib-ng] zlib_name_mangling.h.empty missing from the release tarball") + return false + end + os.cp(path.join(root, "zlib_name_mangling.h.empty"), path.join(root, "zlib_name_mangling.h")) + return true +end diff --git a/tests/examples/zlib-ng/mcpp.toml b/tests/examples/zlib-ng/mcpp.toml new file mode 100644 index 00000000..cf39f114 --- /dev/null +++ b/tests/examples/zlib-ng/mcpp.toml @@ -0,0 +1,13 @@ +# compat.zlib-ng test project. It asserts that zlib-ng in its zlib-compatible +# form reads what madler's zlib writes (a stream Python's zlib 1.3.1 produced, +# as a native engine linked to its system's libz writes a shard), that its own +# streams and gz files round-trip through the vectorised inflate on data long +# enough to reach it, that the checksums give their published values, and that +# the library and this consumer agree on the width of z_off_t. +# `compat` redirect inherited from the workspace root. +[package] +name = "zlib-ng-tests" +version = "0.1.0" + +[dependencies] +compat.zlib-ng = "2.3.3" diff --git a/tests/examples/zlib-ng/tests/zlib-ng.cpp b/tests/examples/zlib-ng/tests/zlib-ng.cpp new file mode 100644 index 00000000..42c8a99a --- /dev/null +++ b/tests/examples/zlib-ng/tests/zlib-ng.cpp @@ -0,0 +1,145 @@ +// compat.zlib-ng: zlib's interface, madler's streams, and agreement between the +// library and its consumer. +#include + +#include +#include +#include + +namespace { + +int failures = 0; + +void check(bool held, const char* what) { + if (!held) { std::printf("FAIL: %s\n", what); ++failures; } +} + +// zlibCompileFlags() bits 6 and 7 state the size of z_off_t the library was +// compiled with: 0 is 2 bytes, 1 is 4, 2 is 8, 3 is another size. +unsigned long library_off_t_size() { + switch ((zlibCompileFlags() >> 6) & 3u) { + case 0: return 2; + case 1: return 4; + case 2: return 8; + default: return 0; + } +} + +// The text Python's zlib 1.3.1 compressed at level 6 into `known' below. +std::string shard_text() { + std::string text; + char line[96]; + for (int i = 0; i < 64; ++i) { + std::snprintf(line, sizeof line, "shard %05d: the engine's index of src/main.cpp, symbol table and refs\n", i); + text += line; + } + return text; +} + +const unsigned char known[] = { + 0x78, 0x9c, 0xad, 0xd2, 0xb9, 0x8d, 0x02, 0x41, 0x14, 0x00, 0x51, 0x9f, 0x28, 0xbe, 0x87, 0x83, + 0x80, 0x7f, 0x35, 0x47, 0x36, 0x03, 0xd3, 0xc0, 0x48, 0xd0, 0xa0, 0x69, 0x0c, 0x36, 0xfb, 0xd5, + 0x3a, 0x9b, 0x00, 0x55, 0x01, 0x94, 0xf5, 0xfa, 0x6d, 0x98, 0x47, 0xd9, 0xfe, 0x75, 0x94, 0xf7, + 0xad, 0x4a, 0x6d, 0xd7, 0xa9, 0xd5, 0x65, 0x97, 0xa9, 0x8d, 0xf5, 0x23, 0xcf, 0x8b, 0xf4, 0xf9, + 0xbc, 0x79, 0x0c, 0x53, 0x5b, 0x9f, 0x5f, 0xaf, 0x95, 0xf4, 0x9f, 0xc7, 0xe9, 0x79, 0x97, 0xf7, + 0x70, 0xba, 0x57, 0x19, 0xda, 0x28, 0x73, 0xbd, 0xf4, 0x45, 0xff, 0xdf, 0x28, 0xb3, 0x31, 0x66, + 0xe3, 0xcc, 0x26, 0x98, 0x4d, 0x32, 0x9b, 0xc2, 0x6c, 0x76, 0xcc, 0x66, 0xcf, 0x6c, 0x0e, 0xc8, + 0x46, 0x19, 0xc5, 0xca, 0x28, 0x56, 0x46, 0xb1, 0x32, 0x8a, 0x95, 0x51, 0xac, 0x8c, 0x62, 0x65, + 0x14, 0x2b, 0xa3, 0x58, 0x19, 0xc5, 0xca, 0x28, 0x36, 0x46, 0xb1, 0x31, 0x8a, 0x8d, 0x51, 0x6c, + 0x8c, 0x62, 0x63, 0x14, 0x1b, 0xa3, 0xd8, 0x18, 0xc5, 0xc6, 0x28, 0x36, 0x46, 0xb1, 0x31, 0x8a, + 0x9d, 0x51, 0xec, 0x8c, 0x62, 0x67, 0x14, 0x3b, 0xa3, 0xd8, 0x19, 0xc5, 0xce, 0x28, 0x76, 0x46, + 0xb1, 0x33, 0x8a, 0x9d, 0x51, 0xec, 0x8c, 0xe2, 0x60, 0x14, 0x07, 0xa3, 0x38, 0x18, 0xc5, 0xc1, + 0x28, 0x0e, 0x46, 0x71, 0x30, 0x8a, 0x83, 0x51, 0x1c, 0x8c, 0xe2, 0x60, 0x14, 0x07, 0xa3, 0x38, + 0x19, 0xc5, 0xc9, 0x28, 0x4e, 0x46, 0x71, 0x32, 0x8a, 0x93, 0x51, 0x9c, 0x8c, 0xe2, 0x64, 0x14, + 0x27, 0xa3, 0x38, 0x19, 0xc5, 0xc9, 0x28, 0x2e, 0x8c, 0xe2, 0xc2, 0x28, 0x2e, 0x8c, 0xe2, 0xf2, + 0x95, 0xe2, 0x5f, 0x40, 0xad, 0x01, 0x9d +}; + +// Bytes that compress neither to nothing nor not at all: runs, repeats at +// varying distances and noise, so inflate copies matches of every length the +// vector paths handle (chunkmemset) and literals between them. +std::string mixed(std::size_t size) { + std::string out; + out.reserve(size); + unsigned state = 2463534242u; + while (out.size() < size) { + state ^= state << 13; state ^= state >> 17; state ^= state << 5; + const unsigned kind = state % 4; + if (kind == 0) out.append(1 + state % 300, static_cast('a' + state % 26)); + else if (kind == 1 && out.size() > 64) { + const std::size_t distance = 1 + (state >> 8) % (out.size() < 32768 ? out.size() : 32768); + const std::size_t length = 3 + (state >> 4) % 258; + for (std::size_t i = 0; i < length; ++i) out.push_back(out[out.size() - distance]); + } else out.push_back(static_cast(state >> 24)); + } + out.resize(size); + return out; +} + +bool round_trip(const std::string& text, int level) { + uLongf bound = compressBound(static_cast(text.size())); + std::string packed(bound, '\0'); + if (compress2(reinterpret_cast(packed.data()), &bound, + reinterpret_cast(text.data()), static_cast(text.size()), level) != Z_OK) + return false; + std::string unpacked(text.size(), '\0'); + uLongf size = static_cast(unpacked.size()); + return uncompress(reinterpret_cast(unpacked.data()), &size, + reinterpret_cast(packed.data()), bound) == Z_OK + && size == text.size() && unpacked == text; +} + +} // namespace + +int main() { + check(library_off_t_size() == sizeof(z_off_t), + "the library and this consumer agree on the width of z_off_t"); + check(std::strstr(zlibVersion(), "zlib-ng") != nullptr && std::strcmp(zlibVersion(), ZLIB_VERSION) == 0, + "the library is zlib-ng, of the version its header names"); + + const std::string text = shard_text(); + std::string unpacked(text.size(), '\0'); + uLongf size = static_cast(unpacked.size()); + check(uncompress(reinterpret_cast(unpacked.data()), &size, known, sizeof known) == Z_OK + && size == text.size() && unpacked == text, + "a stream madler's zlib wrote decompresses to its text"); + + check(adler32(1, reinterpret_cast("Wikipedia"), 9) == 0x11e60398ul, + "adler32 of \"Wikipedia\" is 0x11e60398"); + check(crc32(0, reinterpret_cast("123456789"), 9) == 0xcbf43926ul, + "crc32 of \"123456789\" is 0xcbf43926"); + check(adler32(1, reinterpret_cast(text.data()), static_cast(text.size())) == 0x40ad019dul + && crc32(0, reinterpret_cast(text.data()), static_cast(text.size())) == 0xbea69a54ul, + "and of the shard text, the values madler's zlib gives"); + + const std::string data = mixed(1 << 20); + for (int level : {1, 6, 9}) + check(round_trip(data, level), "a mebibyte of mixed bytes round-trips at levels 1, 6 and 9"); + check(round_trip(std::string(1 << 16, 'z'), 6), "and a run of one byte"); + check(round_trip(std::string(), 6), "and nothing"); + + const char* name = "zlib-ng-test.gz"; + gzFile out = gzopen(name, "wb"); + check(out != nullptr, "a gz file is opened for writing"); + if (out) { + check(gzwrite(out, data.data(), static_cast(data.size())) == static_cast(data.size()), + "the gz file is written"); + gzclose(out); + } + gzFile in = gzopen(name, "rb"); + check(in != nullptr, "the gz file is opened for reading"); + if (in) { + std::string back(data.size(), '\0'); + check(gzread(in, back.data(), static_cast(back.size())) == static_cast(data.size()) + && back == data, + "and reads back what was written"); + check(gzseek(in, 10, SEEK_SET) == 10 && gztell(in) == 10, + "a gz file seeks and reports its offset through z_off_t"); + check(gzgetc(in) == static_cast(data[10]), "and gzgetc reads the byte there"); + gzclose(in); + } + std::remove(name); + + if (failures == 0) std::printf("compat.zlib-ng: every observation held\n"); + return failures == 0 ? 0 : 1; +} diff --git a/tests/openkal/members.toml b/tests/openkal/members.toml index 1e073a64..f8046f6e 100644 --- a/tests/openkal/members.toml +++ b/tests/openkal/members.toml @@ -17,6 +17,7 @@ md4c = "Markdown parsing, C" xxhash = "hashing, C" brotli = "compression, C" zlib = "compression, C; its configuration differs on Windows with musl" +zlib-ng = "compression, C; zlib's API, its vector paths chosen at compile time" gzip-hpp = "compression, C++ over compat.zlib" archive = "archive formats, C (libarchive)" "fmtlib.fmt" = "formatting, C++"