diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8a6c241 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Preserve upstream document bytes and original whitespace on every platform. +upstream/** -text -whitespace diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c4f7da5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,92 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + mcpp: + name: mcpp (${{ matrix.os }}, ${{ matrix.toolchain }}) + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-24.04, toolchain: 'gcc@16.1.0' } + - { os: ubuntu-24.04, toolchain: 'llvm@23.1.3' } + - { os: ubuntu-24.04-arm, toolchain: 'gcc@16.1.0' } + - { os: macos-15, toolchain: 'llvm@23.1.3' } + - { os: windows-2025, toolchain: 'llvm@23.1.3' } + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + defaults: + run: + shell: bash + env: + XLINGS_NON_INTERACTIVE: 1 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Mwrap identity and bilingual README contract + run: python3 tools/check_mwrap_release.py + + - name: Install Xlings (Linux/macOS) + if: runner.os != 'Windows' + run: | + curl -fsSL https://raw.githubusercontent.com/openxlings/xlings/refs/heads/main/tools/other/quick_install.sh | bash + echo "$HOME/.xlings/subos/current/bin" >> "$GITHUB_PATH" + + - name: Install Xlings (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + irm https://raw.githubusercontent.com/openxlings/xlings/main/tools/other/quick_install.ps1 | iex + "$env:USERPROFILE\.xlings\subos\current\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Install project tools (.xlings.json) + run: | + for attempt in 1 2 3; do + xlings update > /dev/null 2>&1 || true + if xlings install; then break; fi + [ "$attempt" = 3 ] && exit 1 + sleep 30 + done + mcpp --version + + - name: Select the toolchain + run: | + name="${{ matrix.toolchain }}" + mcpp toolchain install "${name%@*}" "${name#*@}" + mcpp toolchain default "$name" + + # Until mcpp-index publishes compat.sheenbidi (mcpplibs/mcpp-index#607), + # the compat namespace of this checkout's manifests resolves from that + # PR's branch, as an unchanged namespace view of its descriptors. Once + # the published index has the descriptor, this step does nothing. + - name: Candidate compat index (only while compat.sheenbidi is unpublished) + run: python3 tools/candidate_index.py --ci + + - name: Build the modules + run: mcpp build + + - name: Tests (the API, every macro and enumerator mapping against the header, header mixing) + run: mcpp test + + - name: Export audit (the generated sources are fresh; every declaration and macro of the public headers) + run: python3 tools/check_exports.py + + - name: Unicode's conformance tests (BidiTest.txt, BidiCharacterTest.txt) on programs written against the module + run: python3 tools/run_conformance.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f0c6447 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,49 @@ +name: Release +on: + push: + tags: ["mwrap-*"] +permissions: + contents: read +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: write + actions: read + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Check tag, upstream version and bilingual entry pages + run: python3 tools/check_mwrap_release.py --tag "$GITHUB_REF_NAME" + - name: Require successful source CI at this exact commit + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + result=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow ci.yml --commit "$GITHUB_SHA" --limit 1 --json status,conclusion --jq '.[0] | .status == "completed" and .conclusion == "success"') + test "$result" = true || { echo 'No successful source CI at the tag commit'; exit 1; } + - name: Build immutable source asset and publish + env: + GH_TOKEN: ${{ github.token }} + shell: bash + run: | + set -euo pipefail + version="${GITHUB_REF_NAME#mwrap-}" + repo="${GITHUB_REPOSITORY#*/}" + asset="${repo}-${version}.tar.gz" + git archive --format=tar.gz --prefix="${repo}-${version}/" "$GITHUB_SHA" > "$RUNNER_TEMP/$asset" + (cd "$RUNNER_TEMP" && sha256sum "$asset" > "$asset.sha256") + if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo 'Release already exists; refusing to change immutable assets'; exit 1 + fi + extra=() + if [[ "$version" == *-rc* ]]; then extra+=(--prerelease); fi + gh release create "$GITHUB_REF_NAME" "$RUNNER_TEMP/$asset" "$RUNNER_TEMP/$asset.sha256" \ + --repo "$GITHUB_REPOSITORY" --verify-tag --title "$repo $version" \ + --notes "Module wrapper for upstream $version. New mwrap package identities become installable after mcpp-index publication; see the bilingual README for the tested support scope." "${extra[@]}" diff --git a/.github/workflows/templates.yml b/.github/workflows/templates.yml new file mode 100644 index 0000000..abcc3d9 --- /dev/null +++ b/.github/workflows/templates.yml @@ -0,0 +1,33 @@ +name: Actual project templates +on: + push: + branches: [main] + pull_request: + workflow_dispatch: +permissions: + contents: read +concurrency: + group: templates-${{ github.ref }} + cancel-in-progress: true +env: + MCPP_VERSION: "2026.10.26.1" +jobs: + templates: + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - name: Install the pinned mcpp + shell: bash + run: | + archive="mcpp-${MCPP_VERSION}-linux-x86_64.tar.gz" + curl -L -fsS -o "$archive" "https://github.com/mcpp-community/mcpp/releases/download/v${MCPP_VERSION}/$archive" + tar -xzf "$archive" + echo "$PWD/mcpp-${MCPP_VERSION}-linux-x86_64/bin" >> "$GITHUB_PATH" + # The candidate index (tools/candidate_index.py) only while mcpp-index + # does not publish compat.sheenbidi. + - name: Actual scaffold, strict build and run of the default template + shell: bash + run: | + index=$(python3 tools/candidate_index.py --checkout-only) + python3 tools/test_template.py --toolchain gcc@16.1.0 ${index:+--index "$index"} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b6dfbae --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +target +compile_commands.json +mcpp.lock +.mcpp/ +.xlings/ +doctor.log +__pycache__/ diff --git a/.xlings.json b/.xlings.json new file mode 100644 index 0000000..730a8ff --- /dev/null +++ b/.xlings.json @@ -0,0 +1,5 @@ +{ + "workspace": { + "xim:mcpp": "2026.10.26.1" + } +} diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..6a4b8ba --- /dev/null +++ b/NOTICE @@ -0,0 +1,22 @@ +sheenbidi-m: C++ module wrapper +Copyright (c) 2026 mcpp-community contributors + +The wrapper contributions are licensed under the Apache License, Version 2.0. +Upstream code, copied interfaces and dependencies retain their original +licenses and copyright notices. SheenBidi is distributed under the Apache +License, Version 2.0: + + Copyright (C) 2014-2026 Muhammad Tayyab Akram + +Its original README and license are retained in upstream/sheenbidi/. Its +character tables (Source/Data) are generated from the Unicode Character +Database 17.0.0, (c) Unicode, Inc., distributed under the Unicode License v3, +retained in upstream/unicode/license.txt. The library itself is built by +compat.sheenbidi from the same release; this wrapper links it, it does not +modify it. + +The conformance runners in examples/sheenbidi read Unicode's BidiTest.txt and +BidiCharacterTest.txt (17.0.0) from the pinned release; the files are not +redistributed here. + +Provenance and checksums are recorded in upstream/documents.json. diff --git a/README.md b/README.md index 71a7e9a..426e6d1 100644 --- a/README.md +++ b/README.md @@ -1 +1,105 @@ -# sheenbidi-m +# sheenbidi + +> SheenBidi, the Unicode Bidirectional Algorithm, through C++23 modules. + +**English** · [简体中文](README.zh.md) · [Upstream](https://github.com/Tehreer/SheenBidi) · [Releases](https://github.com/mcpplibs/sheenbidi-m/releases) · [mcpp](https://github.com/mcpp-community/mcpp) · [Index](https://github.com/mcpplibs/mcpp-index) + +[![C++23](https://img.shields.io/badge/C%2B%2B-23-blue.svg)](https://en.cppreference.com/w/cpp/23) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) +[![SheenBidi](https://img.shields.io/badge/SheenBidi-Apache--2.0-blue.svg)](upstream/sheenbidi/LICENSE) +[![mcpp-community](https://img.shields.io/badge/mcpp--community-mwrap-blue)](https://github.com/mcpp-community) +[![CI](https://github.com/mcpplibs/sheenbidi-m/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mcpplibs/sheenbidi-m/actions/workflows/ci.yml) + +SheenBidi turns text stored in logical order into the visual order a display +needs for Arabic, Hebrew and mixed-direction text (UAX #9, Unicode 17.0.0). It +is Apache-2.0 C with no dependencies, so it links statically without copyleft +terms. The upstream C API is unchanged: `SBAlgorithmCreate`, +`SBParagraphGetLevelsPtr`, `SBLineGetRunsPtr` and `SBMirrorLocatorMoveNext` +are the upstream globals of ``. The library is +`compat.sheenbidi` 3.0.0. This package maps SheenBidi's macros and the +enumerators of its unnamed enums to constants and constexpr functions, each +checked against the header. Unicode's conformance tests pass on every CI run: +all 770,241 cases of BidiTest.txt and all 91,707 of BidiCharacterTest.txt, on +programs written against the module. + +## Modules + +- `import mcpplibs.mwrap.sheenbidi;`: the whole API of `` (SBAlgorithm, SBParagraph, SBLine, SBRun, SBMirrorLocator, SBScriptLocator, SBCodepoint, SBCodepointSequence, SBAllocator, SBVersion), re-exporting the module below +- `import mcpplibs.mwrap.sheenbidi.macros;`: `SBBidiTypeR`, `SBScriptARAB`, `SBStringEncodingUTF8`, `SBLevelDefaultLTR`, `SBBidiTypeIsStrong` … in `mcpplibs::mwrap::sheenbidi::macros` + +> More Modules API -> [Usage and API](docs/usage.md) + +## Getting started + +```sh +mcpp new bidi --template mcpplibs.mwrap.sheenbidi@3.0.0 +cd bidi +mcpp run +``` + +The default template lays out UTF-8 lines as a renderer does: it resolves a +paragraph's embedding levels, then draws the line's runs in visual order with +mirrored brackets. Run `mcpp run -- "your text"` for your own lines. + +```toml +[dependencies] +"mcpplibs.mwrap.sheenbidi" = "3.0.0" +``` + +```cpp +import std; +import mcpplibs.mwrap.sheenbidi; +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBStringEncodingUTF8, SBLevelDefaultLTR … + +int main() { + std::string_view text = "abc אבג (123)"; + SBCodepointSequence seq{SBStringEncodingUTF8, text.data(), text.size()}; + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, 0, text.size(), SBLevelDefaultLTR); + SBLineRef line = SBParagraphCreateLine(paragraph, 0, SBParagraphGetLength(paragraph)); + const SBRun* runs = SBLineGetRunsPtr(line); // in visual order, offsets in UTF-8 bytes + for (SBUInteger i = 0; i < SBLineGetRunCount(line); ++i) + std::println("[{}, {}) level {}", runs[i].offset, runs[i].offset + runs[i].length, runs[i].level); + SBLineRelease(line); + SBParagraphRelease(paragraph); + SBAlgorithmRelease(algorithm); +} +``` + +## Macros and enumerators + +Constants such as `SBLevelMax` keep the type their macro has (`int`). +Computations such as `SBBidiTypeIsStrong` and `SBCodepointIsValid` are +constexpr functions whose body is the macro itself, matching its value and +type for every argument type. SheenBidi's enumerators (`SBBidiTypeL`, +`SBScriptARAB` …) belong to unnamed enums, which a module cannot export, so +they are constants of the typedef the header gives them (`SBBidiType`, +`SBScript` …). `SB_PUBLIC`, `SB_DEPRECATED` and the `extern "C"` brackets stay +macros in ``, included before the import. Details: +[docs/usage.md](docs/usage.md). + +## Examples + +- [examples/sheenbidi](examples/sheenbidi): a layout tool (logical to visual order, levels, runs and + script runs of each paragraph) and Unicode's two conformance runners, written against the module. + [tools/run_conformance.py](tools/run_conformance.py) runs BidiTest.txt and BidiCharacterTest.txt on them. + +## Build and testing + +Linux (x86_64 GCC and LLVM, arm64 GCC), macOS and Windows CI build the +modules, then run the tests and the export audit +([tools/check_exports.py](tools/check_exports.py)). The audit checks that the +generated sources are fresh, that every declaration is exported and that +every macro is mapped or listed with its reason. CI also runs Unicode 17.0.0's +conformance tests from the pinned release, each file checked against the one +Unicode publishes: BidiTest.txt (770,241 cases) and BidiCharacterTest.txt +(91,707 cases, with mirroring). A separate job scaffolds the default template +as an installed package and runs it. + +## License + +[Apache-2.0](LICENSE) for the wrapper, by mcpp-community ([NOTICE](NOTICE)). +SheenBidi is Apache-2.0 too ([upstream/sheenbidi/LICENSE](upstream/sheenbidi/LICENSE)). +Its tables come from the Unicode Character Database +([Unicode License v3](upstream/unicode/license.txt)). Both notices are +deployed beside every program that depends on this package. diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..5035777 --- /dev/null +++ b/README.zh.md @@ -0,0 +1,97 @@ +# sheenbidi + +> 以 C++23 模块使用 SheenBidi——Unicode 双向文本算法。 + +[English](README.md) · **简体中文** · [Upstream](https://github.com/Tehreer/SheenBidi) · [Releases](https://github.com/mcpplibs/sheenbidi-m/releases) · [mcpp](https://github.com/mcpp-community/mcpp) · [Index](https://github.com/mcpplibs/mcpp-index) + +[![C++23](https://img.shields.io/badge/C%2B%2B-23-blue.svg)](https://en.cppreference.com/w/cpp/23) +[![License](https://img.shields.io/badge/license-Apache--2.0-blue.svg)](LICENSE) +[![SheenBidi](https://img.shields.io/badge/SheenBidi-Apache--2.0-blue.svg)](upstream/sheenbidi/LICENSE) +[![mcpp-community](https://img.shields.io/badge/mcpp--community-mwrap-blue)](https://github.com/mcpp-community) +[![CI](https://github.com/mcpplibs/sheenbidi-m/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mcpplibs/sheenbidi-m/actions/workflows/ci.yml) + +SheenBidi 把按逻辑顺序存储的文本转换成显示所需的视觉顺序，用于阿拉伯文、 +希伯来文和混合方向文本（UAX #9，Unicode 17.0.0）。它是 Apache-2.0 许可、 +没有依赖的 C 库，静态链接不带来著佐权（copyleft）条款。上游 C API 保持不变： +`SBAlgorithmCreate`、`SBParagraphGetLevelsPtr`、`SBLineGetRunsPtr`、 +`SBMirrorLocatorMoveNext` 都是 `` 的上游全局名字。 +库本身是 `compat.sheenbidi` 3.0.0。本包把 SheenBidi 的宏和无名枚举的枚举值 +映射为常量和 constexpr 函数，并逐一与头文件对照检查。Unicode 的一致性测试在 +每次 CI 中全部通过：BidiTest.txt 的全部 770,241 个用例和 BidiCharacterTest.txt +的全部 91,707 个用例，运行在基于模块编写的程序上。 + +## 模块 + +- `import mcpplibs.mwrap.sheenbidi;`：`` 的完整 API（SBAlgorithm、SBParagraph、SBLine、SBRun、SBMirrorLocator、SBScriptLocator、SBCodepoint、SBCodepointSequence、SBAllocator、SBVersion），并重新导出下面的模块 +- `import mcpplibs.mwrap.sheenbidi.macros;`：`SBBidiTypeR`、`SBScriptARAB`、`SBStringEncodingUTF8`、`SBLevelDefaultLTR`、`SBBidiTypeIsStrong` 等，位于 `mcpplibs::mwrap::sheenbidi::macros` + +> 更多模块 API -> [用法与 API](docs/usage.md) + +## 快速开始 + +```sh +mcpp new bidi --template mcpplibs.mwrap.sheenbidi@3.0.0 +cd bidi +mcpp run +``` + +默认模板像渲染器那样排布 UTF-8 文本行：先求出段落的嵌入层级，再按视觉顺序 +绘制该行的各个 run，并镜像括号。运行 `mcpp run -- "你的文本"` 处理自己的文本。 + +```toml +[dependencies] +"mcpplibs.mwrap.sheenbidi" = "3.0.0" +``` + +```cpp +import std; +import mcpplibs.mwrap.sheenbidi; +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBStringEncodingUTF8、SBLevelDefaultLTR … + +int main() { + std::string_view text = "abc אבג (123)"; + SBCodepointSequence seq{SBStringEncodingUTF8, text.data(), text.size()}; + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, 0, text.size(), SBLevelDefaultLTR); + SBLineRef line = SBParagraphCreateLine(paragraph, 0, SBParagraphGetLength(paragraph)); + const SBRun* runs = SBLineGetRunsPtr(line); // 视觉顺序，偏移以 UTF-8 字节计 + for (SBUInteger i = 0; i < SBLineGetRunCount(line); ++i) + std::println("[{}, {}) level {}", runs[i].offset, runs[i].offset + runs[i].length, runs[i].level); + SBLineRelease(line); + SBParagraphRelease(paragraph); + SBAlgorithmRelease(algorithm); +} +``` + +## 宏与枚举值 + +`SBLevelMax` 这样的常量保持其宏的类型（`int`）。`SBBidiTypeIsStrong`、 +`SBCodepointIsValid` 这样的计算是函数体即宏本身的 constexpr 函数，对每种参数 +类型都与宏的值和类型一致。SheenBidi 的枚举值（`SBBidiTypeL`、`SBScriptARAB` +等）属于无名枚举，模块无法导出，因此它们是头文件为其给出的 typedef +（`SBBidiType`、`SBScript` 等）类型的常量。`SB_PUBLIC`、`SB_DEPRECATED` 和 +`extern "C"` 括号仍是宏，位于 ``，在 import 之前 +包含。详见 [docs/usage.md](docs/usage.md)。 + +## 示例 + +- [examples/sheenbidi](examples/sheenbidi)：一个排版工具（每个段落的逻辑到视觉顺序、层级、run 和文字 + run）和 Unicode 的两个一致性测试运行器，都基于模块编写。 + [tools/run_conformance.py](tools/run_conformance.py) 在它们上运行 BidiTest.txt 和 BidiCharacterTest.txt。 + +## 构建与测试 + +Linux（x86_64 的 GCC 和 LLVM、arm64 的 GCC）、macOS 和 Windows 的 CI 构建模块， +然后运行测试和导出审计（[tools/check_exports.py](tools/check_exports.py)）。 +审计检查生成的源文件是最新的、每个声明都已导出、每个宏都已映射或注明保留原因。 +CI 还从固定的上游版本运行 Unicode 17.0.0 的一致性测试，并核对每个文件与 +Unicode 官方发布的一致：BidiTest.txt（770,241 个用例）和 +BidiCharacterTest.txt（91,707 个用例，含镜像）。另有一个作业把默认模板作为 +已安装的包生成并运行。 + +## 许可 + +封装部分为 [Apache-2.0](LICENSE)，由 mcpp-community 提供（[NOTICE](NOTICE)）。 +SheenBidi 同样是 Apache-2.0（[upstream/sheenbidi/LICENSE](upstream/sheenbidi/LICENSE)）。 +其数据表来自 Unicode 字符数据库（[Unicode License v3](upstream/unicode/license.txt)）。 +两份声明都会部署到每个依赖本包的程序旁边。 diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..4139d23 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,65 @@ +# Using mcpplibs.mwrap.sheenbidi + +`import mcpplibs.mwrap.sheenbidi;` declares SheenBidi 3.0.0's API as `` does. Types, struct tags and functions are the upstream globals, the very same entities as the header's, so a translation unit may include the header too. The library is `compat.sheenbidi` 3.0.0, built from the release's unity source with its Unicode 17.0.0 tables. + +## Modules + +| Module | What it holds | +|---|---| +| `mcpplibs.mwrap.sheenbidi` | every type (`SBCodepoint`, `SBLevel`, `SBBidiType`, `SBGeneralCategory`, `SBScript`, `SBStringEncoding`, `SBBoolean`, `SBUInteger` …, the structs `SBCodepointSequence`, `SBRun`, `SBMirrorAgent`, `SBScriptAgent`, `SBAllocatorProtocol`, the object references `SBAlgorithmRef`, `SBParagraphRef`, `SBLineRef`, `SBMirrorLocatorRef`, `SBScriptLocatorRef`, `SBAllocatorRef` and the allocator callback types), the 11 struct tags (`_SBRun` …; the opaque ones declared, not defined) and all 51 `SB_PUBLIC` functions; re-exports the module below | +| `mcpplibs.mwrap.sheenbidi.macros` | the macros and the enumerators, in `mcpplibs::mwrap::sheenbidi::macros` | + +The public headers are the ones upstream installs (`SHEENBIDI_HEADERS` in CMakeLists.txt). `SBConfig.h` is not among them: no public header includes it, and its switches are the build's. + +### The API in one paragraph + +`SBAlgorithmCreate` takes an `SBCodepointSequence` (UTF-8, UTF-16 or UTF-32 code units; offsets everywhere are in those code units) and finds the bidi types and the paragraph boundaries (P1, `SBAlgorithmGetParagraphBoundary`). `SBAlgorithmCreateParagraph` resolves one paragraph at a base level, explicit (0, 1) or from its first strong character (`SBLevelDefaultLTR`, `SBLevelDefaultRTL`), and gives the level of every code unit (`SBParagraphGetLevelsPtr`). `SBParagraphCreateLine` applies L1 to a line and gives its runs in visual order (`SBLineGetRunsPtr`: draw a run with an odd level right to left). `SBMirrorLocator` walks the characters of a line that are drawn mirrored (L4); `SBScriptLocator` walks the script runs of a sequence (for shaping). Every object is reference counted (`…Retain`, `…Release`); a paragraph keeps its algorithm, a locator its line. `SBAllocatorCreate` / `SBAllocatorSetDefault` route the library's allocations. + +### Not exported: the experimental text API + +`compat.sheenbidi` does not define `SB_CONFIG_EXPERIMENTAL_TEXT_API` (upstream's default builds do not), so `SB_TEXT_API_SUPPORTED` is 0 and `SBText`, `SBTextConfig`, the attribute registry and the run iterators declare nothing: neither the header nor the module has them. + +### Upstream behavior worth knowing + +- **Release a mirror locator only after it has loaded a line.** In 3.0.0 its finalizer releases the loaded line unconditionally, and `SBLineRelease(NULL)` dereferences the null pointer. `SBMirrorLocatorLoadLine` also leaves no line when `stringBuffer` is not the buffer the line was created over. +- `SBScriptGetOpenTypeTag` is `SB_DEPRECATED` upstream (its replacement is `SBScriptGetUnicodeTag`); the module exports it as it is, deprecation included. + +## Macros and enumerators + +A named module cannot export a macro, nor an enumerator of an unnamed enum (the enumeration is TU-local). `mcpplibs::mwrap::sheenbidi::macros` holds both under their original names; `using namespace mcpplibs::mwrap::sheenbidi::macros;` makes code read as it does with the header. + +- **Constants** as `inline constexpr`, each of the type its macro expands to: `SBLevelInvalid`, `SBLevelMax`, `SBLevelDefaultLTR`, `SBLevelDefaultRTL`, `SBCodepointFaulty`, `SBCodepointMax`, `SHEENBIDI_VERSION_MAJOR` / `_MINOR` / `_PATCH` and `SB_TEXT_API_SUPPORTED` are `int`; `SBCodepointInvalid` is `uint32_t` (`UINT32_MAX`). `SHEENBIDI_VERSION_STRING` is a reference to a constexpr char array. +- **Enumerators** as constants of the typedef the header declares right after their enum: the 24 `SBBidiType*` are `SBBidiType`, the 31 `SBGeneralCategory*` `SBGeneralCategory`, the 176 `SBScript*` `SBScript`, the 3 `SBStringEncoding*` `SBStringEncoding` and `SBTrue` / `SBFalse` `SBBoolean`. They work in constant expressions and case labels as the enumerators do. +- **Computations** as `constexpr` function templates: `SBUInt8InRange`, `SBUInt16InRange`, `SBUInt32InRange`, `SBBidiTypeIsStrong` / `IsWeak` / `IsNeutral` / `IsFormat`, `SBCodepointIsSurrogate`, `SBCodepointIsValid`, `SBGeneralCategoryIsLetter` / `Mark` / `Number` / `Punctuation` / `Symbol` / `Separator` / `Other`. Each body is the macro itself, expanded on the parameters: the result has the macro's value and type (`bool`) for every argument type. Each argument is evaluated once, before the call; `SBCodepointIsValid(c)` and the `…InRange` macros evaluate an argument more than once, so for an argument without side effects the two agree. + +Each mapping is checked against the raw header for every sample argument type (every 8-bit value, 16- and 32-bit boundaries, code points around the surrogates and past U+10FFFF, ints), and each enumerator constant against the header's enumerator, in [tests/macros_mix_test.cpp](../tests/macros_mix_test.cpp). + +### What stays a macro + +These stay macros in ``, which is ``: + +| Macros | Why | +|---|---| +| `SB_PUBLIC`, `SB_DEPRECATED` | attributes of declarations (`SB_PUBLIC` is empty here: the library is static and built without `SB_CONFIG_DLL_*`) | +| `SB_EXTERN_C_BEGIN`, `SB_EXTERN_C_END` | open and close an `extern "C"` block | + +`SB_TEXT_API_SUPPORTED` is both a constant and, through the side header, a macro for `#if`. + +Include the side header before the import. While a header macro is defined, it expands even inside a qualified name: an object-like macro therefore hides `macros::SBLevelMax`; use the plain macro, or set it aside with `#pragma push_macro`. A function-like macro does not expand when its name is followed by `)`, so `(macros::SBBidiTypeIsStrong)(t)` calls the function beside the header. With the header included, its enumerators are its own (`::SBBidiTypeL`); do not also bring `macros` into scope with a using-directive, or the unqualified names are ambiguous ([tests/header_mix_test.cpp](../tests/header_mix_test.cpp)). [tools/check_exports.py](../tools/check_exports.py) accounts for all 54 macros and 236 enumerators. + +## Licensing + +The wrapper's code is Apache-2.0. SheenBidi is Apache-2.0 ([upstream/sheenbidi/LICENSE](../upstream/sheenbidi/LICENSE)); a program built with it, statically linked, carries the Apache-2.0 notice requirements only. Its tables are generated from the Unicode Character Database 17.0.0 ([Unicode License v3](../upstream/unicode/license.txt)). + +`runtime.deploy` copies the wrapper's LICENSE and NOTICE, SheenBidi's README and LICENSE and the Unicode license into `licenses/sheenbidi/` beside every program that depends on this package. + +## Checks + +- `mcpp test` runs four tests: + - [tests/api_test.cpp](../tests/api_test.cpp): the API through the module only — bidi types (Unicode 17.0's Sidetic letters included), general categories, mirrors, scripts and their ISO 15924 tags, UTF-8 / UTF-16 decoding and code point sequences, paragraph boundaries (CR LF) and base levels, levels and visual order, UAX #9's N0 bracket-pair example with the mirror locator, isolates, the script locator, retain/release, a custom default allocator, the version; + - [tests/macros_mix_test.cpp](../tests/macros_mix_test.cpp) (generated): every constant, enumerator and function against the header; + - [tests/header_mix_test.cpp](../tests/header_mix_test.cpp): the header beside the import — the same entities, `SB_EXTERN_C_BEGIN` and `SB_PUBLIC` on a declaration of one's own, `SB_DEPRECATED`; + - [tests/macros_test.cpp](../tests/macros_test.cpp): the macros module alone, at compile time. +- `python3 tools/check_exports.py` checks that the sources are what [tools/gen_exports.py](../tools/gen_exports.py) writes from the pinned headers, and that every declaration is exported, every enumerator mapped, and every macro mapped, kept or an include guard. +- `python3 tools/run_conformance.py` runs Unicode's conformance tests on [examples/sheenbidi](../examples/sheenbidi), written against the module: all 770,241 cases of BidiTest.txt (each line under each paragraph direction it lists: levels after L1 and visual order) and all 91,707 of BidiCharacterTest.txt (paragraph level, levels, visual order, and the mirrored characters against BidiMirroring.txt). The three files are the release's, each checked to be byte for byte the one Unicode publishes for 17.0.0. Nothing is excluded. +- `python3 tools/test_template.py` scaffolds the default template from an installed copy of the package, then builds and runs it. diff --git a/examples/sheenbidi/conformance/BidiCharacterTest.cpp b/examples/sheenbidi/conformance/BidiCharacterTest.cpp new file mode 100644 index 0000000..c8ee7cc --- /dev/null +++ b/examples/sheenbidi/conformance/BidiCharacterTest.cpp @@ -0,0 +1,164 @@ +// BidiCharacterTest — Unicode's BidiCharacterTest.txt through import +// mcpplibs.mwrap.sheenbidi, every case checked. +// +// BidiCharacterTest +// +// Each line is one paragraph: code points; the paragraph direction (0 LTR, +// 1 RTL, 2 auto, the first strong character deciding and LTR without one); +// the resolved paragraph level; the resolved level of each character after +// rule L1 (x for the characters X9 removes); and the visual order of the +// characters that are not removed. A case passes when SheenBidi gives +// +// - the paragraph: one paragraph covering the whole line (P1) at the +// expected level (P2, P3); +// - the levels: the level of each run of the line (SBLineGetRunsPtr), +// which is the level after L1, on every character that is not x; +// - the order: the runs in visual order, an odd run reversed (L2), the x +// characters skipped; +// - the mirrors: SBMirrorLocator reports exactly the characters at an odd +// level that BidiMirroring.txt maps, each with that mapping (L4). +// +// Output: one line per failing case (its line number and what differed), then +// "BidiCharacterTest: tests, errors". Exit 0 when nothing +// failed. +import std; +import mcpplibs.mwrap.sheenbidi; + +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBLevelDefaultLTR + +namespace { + +std::vector split(std::string_view s, char sep) { + std::vector out; + for (std::size_t start = 0;;) { + const std::size_t end = s.find(sep, start); + out.push_back(s.substr(start, end - start)); + if (end == std::string_view::npos) return out; + start = end + 1; + } +} + +std::vector words(std::string_view s) { + std::vector out; + for (std::size_t i = 0; i < s.size();) { + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; + const std::size_t start = i; + while (i < s.size() && s[i] != ' ' && s[i] != '\t') ++i; + if (i > start) out.push_back(s.substr(start, i - start)); + } + return out; +} + +std::uint32_t number(std::string_view s, int base) { + std::uint32_t value = 0; + const auto [end, error] = std::from_chars(s.data(), s.data() + s.size(), value, base); + if (error != std::errc{} || end != s.data() + s.size()) throw std::runtime_error("not a number: " + std::string(s)); + return value; +} + +constexpr int removed = -1; // an "x" level + +std::map mirroring(const char* path) { + std::ifstream in(path); + if (!in) throw std::runtime_error(std::string("cannot read ") + path); + std::map out; + for (std::string line; std::getline(in, line);) { + std::string_view data = line; + data = data.substr(0, data.find('#')); + const auto fields = split(data, ';'); + if (fields.size() != 2) continue; + const auto from = words(fields[0]), to = words(fields[1]); + if (from.size() == 1 && to.size() == 1) out[number(from[0], 16)] = number(to[0], 16); + } + return out; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 3) { + std::println(std::cerr, "usage: BidiCharacterTest "); + return 2; + } + const auto mirrors = mirroring(argv[2]); + std::ifstream in(argv[1]); + if (!in) { + std::println(std::cerr, "cannot read {}", argv[1]); + return 2; + } + std::size_t cases = 0, failures = 0, number_of_line = 0; + SBMirrorLocatorRef locator = SBMirrorLocatorCreate(); + bool loaded = false; + for (std::string line; std::getline(in, line);) { + ++number_of_line; + if (!line.empty() && line.back() == '\r') line.pop_back(); + if (line.empty() || line[0] == '#') continue; + const auto fields = split(line, ';'); + if (fields.size() != 5) throw std::runtime_error("line " + std::to_string(number_of_line) + ": not 5 fields"); + ++cases; + + std::vector text; + for (auto w : words(fields[0])) text.push_back(number(w, 16)); + const std::uint32_t direction = number(words(fields[1]).at(0), 10); + const SBLevel base = direction == 0 ? 0 : direction == 1 ? 1 : SBLevelDefaultLTR; + const SBLevel expected_paragraph = static_cast(number(words(fields[2]).at(0), 10)); + std::vector expected_levels; + for (auto w : words(fields[3])) expected_levels.push_back(w == "x" ? removed : static_cast(number(w, 10))); + std::vector expected_order; + for (auto w : words(fields[4])) expected_order.push_back(number(w, 10)); + + std::string problem; + const SBCodepointSequence seq{SBStringEncodingUTF32, text.data(), text.size()}; + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, 0, text.size(), base); + if (SBParagraphGetLength(paragraph) != text.size()) { + problem = std::format("paragraph length {} of {}", SBParagraphGetLength(paragraph), text.size()); + } else if (SBParagraphGetBaseLevel(paragraph) != expected_paragraph) { + problem = std::format("paragraph level {} != {}", SBParagraphGetBaseLevel(paragraph), expected_paragraph); + } else { + SBLineRef sbline = SBParagraphCreateLine(paragraph, 0, text.size()); + std::vector levels(text.size(), removed); + std::vector order; + const SBRun* runs = SBLineGetRunsPtr(sbline); + for (SBUInteger r = 0; r < SBLineGetRunCount(sbline); ++r) { + for (SBUInteger k = 0; k < runs[r].length; ++k) { + const std::size_t i = runs[r].level & 1 ? runs[r].offset + runs[r].length - 1 - k : runs[r].offset + k; + levels.at(i) = runs[r].level; + if (expected_levels.at(i) != removed) order.push_back(i); + } + } + for (std::size_t i = 0; i < text.size() && problem.empty(); ++i) { + if (expected_levels[i] != removed && levels[i] != expected_levels[i]) + problem = std::format("level of {} is {}, expected {}", i, levels[i], expected_levels[i]); + } + if (problem.empty() && order != expected_order) problem = "visual order differs"; + if (problem.empty()) { + std::map expected_mirrors, found_mirrors; + for (std::size_t i = 0; i < text.size(); ++i) { + const auto m = mirrors.find(text[i]); + if (expected_levels[i] != removed && (expected_levels[i] & 1) && m != mirrors.end()) + expected_mirrors[i] = m->second; + } + SBMirrorLocatorLoadLine(locator, sbline, text.data()); + loaded = true; + while (SBMirrorLocatorMoveNext(locator)) { + const SBMirrorAgent* agent = SBMirrorLocatorGetAgent(locator); + found_mirrors[agent->index] = agent->mirror; + } + if (found_mirrors != expected_mirrors) problem = "mirrors differ"; + } + SBLineRelease(sbline); + } + SBParagraphRelease(paragraph); + SBAlgorithmRelease(algorithm); + if (!problem.empty()) { + ++failures; + if (failures <= 50) std::println("line {}: {}", number_of_line, problem); + } + } + // A mirror locator that never loaded a line must not be released in + // SheenBidi 3.0.0 (its finalizer releases a NULL line). + if (loaded) SBMirrorLocatorRelease(locator); + std::println("BidiCharacterTest: {} tests, {} errors", cases, failures); + return failures == 0 && cases > 0 ? 0 : 1; +} diff --git a/examples/sheenbidi/conformance/BidiTest.cpp b/examples/sheenbidi/conformance/BidiTest.cpp new file mode 100644 index 0000000..e6f4368 --- /dev/null +++ b/examples/sheenbidi/conformance/BidiTest.cpp @@ -0,0 +1,153 @@ +// BidiTest — Unicode's BidiTest.txt through import mcpplibs.mwrap.sheenbidi, +// every case checked. +// +// BidiTest +// +// Each data line is a sequence of bidi classes and a bitset of paragraph +// directions (1 auto, 2 LTR, 4 RTL); the @Levels and @Reorder lines before it +// give the expected level of each class after rule L1 (x for what X9 removes) +// and the visual order of the classes not removed. Every class is given as a +// code point of that class (the table below, each checked against +// SBCodepointGetBidiType first), and each listed direction is one case. A +// case passes when SheenBidi gives one paragraph covering the sequence (B is +// only ever last), the levels of the line's runs on every class that is not +// x, and the runs' visual order (an odd run reversed, x skipped). +// +// Output: one line per failing case, then "BidiTest: tests, +// errors". Exit 0 when nothing failed. +import std; +import mcpplibs.mwrap.sheenbidi; + +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBBidiTypeL …, SBLevelDefaultLTR + +namespace { + +struct Class { + std::string_view name; + std::uint32_t codepoint; + SBBidiType type; +}; + +// One code point of each of the 23 classes. +constexpr Class classes[] = { + {"L", 0x0041, SBBidiTypeL}, {"R", 0x05D0, SBBidiTypeR}, {"AL", 0x0627, SBBidiTypeAL}, + {"EN", 0x0030, SBBidiTypeEN}, {"ES", 0x002B, SBBidiTypeES}, {"ET", 0x0023, SBBidiTypeET}, + {"AN", 0x0660, SBBidiTypeAN}, {"CS", 0x002C, SBBidiTypeCS}, {"NSM", 0x0300, SBBidiTypeNSM}, + {"BN", 0x00AD, SBBidiTypeBN}, {"B", 0x2029, SBBidiTypeB}, {"S", 0x0009, SBBidiTypeS}, + {"WS", 0x0020, SBBidiTypeWS}, {"ON", 0x0021, SBBidiTypeON}, {"LRE", 0x202A, SBBidiTypeLRE}, + {"LRO", 0x202D, SBBidiTypeLRO}, {"RLE", 0x202B, SBBidiTypeRLE}, {"RLO", 0x202E, SBBidiTypeRLO}, + {"PDF", 0x202C, SBBidiTypePDF}, {"LRI", 0x2066, SBBidiTypeLRI}, {"RLI", 0x2067, SBBidiTypeRLI}, + {"FSI", 0x2068, SBBidiTypeFSI}, {"PDI", 0x2069, SBBidiTypePDI}, +}; + +std::vector words(std::string_view s) { + std::vector out; + for (std::size_t i = 0; i < s.size();) { + while (i < s.size() && (s[i] == ' ' || s[i] == '\t')) ++i; + const std::size_t start = i; + while (i < s.size() && s[i] != ' ' && s[i] != '\t') ++i; + if (i > start) out.push_back(s.substr(start, i - start)); + } + return out; +} + +std::uint32_t number(std::string_view s, int base) { + std::uint32_t value = 0; + const auto [end, error] = std::from_chars(s.data(), s.data() + s.size(), value, base); + if (error != std::errc{} || end != s.data() + s.size()) throw std::runtime_error("not a number: " + std::string(s)); + return value; +} + +constexpr int removed = -1; // an "x" level + +std::uint32_t codepoint_of(std::string_view name) { + for (const Class& c : classes) + if (c.name == name) return c.codepoint; + throw std::runtime_error("unknown bidi class " + std::string(name)); +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 2) { + std::println(std::cerr, "usage: BidiTest "); + return 2; + } + for (const Class& c : classes) { + if (SBCodepointGetBidiType(c.codepoint) != c.type) { + std::println("U+{:04X} is not of class {}", c.codepoint, c.name); + return 1; + } + } + std::ifstream in(argv[1]); + if (!in) { + std::println(std::cerr, "cannot read {}", argv[1]); + return 2; + } + std::vector expected_levels; + std::vector expected_order; + std::size_t cases = 0, failures = 0, number_of_line = 0; + for (std::string line; std::getline(in, line);) { + ++number_of_line; + if (!line.empty() && line.back() == '\r') line.pop_back(); + std::string_view data = line; + data = data.substr(0, data.find('#')); + if (words(data).empty()) continue; + if (data.starts_with("@Levels:")) { + expected_levels.clear(); + for (auto w : words(data.substr(8))) expected_levels.push_back(w == "x" ? removed : static_cast(number(w, 10))); + continue; + } + if (data.starts_with("@Reorder:")) { + expected_order.clear(); + for (auto w : words(data.substr(9))) expected_order.push_back(number(w, 10)); + continue; + } + if (data.starts_with('@')) continue; // forward compatibility, as the file asks + const std::size_t semicolon = data.find(';'); + if (semicolon == std::string_view::npos) throw std::runtime_error("line " + std::to_string(number_of_line) + ": no bitset"); + std::vector text; + for (auto w : words(data.substr(0, semicolon))) text.push_back(codepoint_of(w)); + const std::uint32_t bitset = number(words(data.substr(semicolon + 1)).at(0), 16); + if (text.size() != expected_levels.size()) throw std::runtime_error("line " + std::to_string(number_of_line) + ": levels do not match"); + + const SBCodepointSequence seq{SBStringEncodingUTF32, text.data(), text.size()}; + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + for (const auto [bit, base, name] : {std::tuple{1u, SBLevel{SBLevelDefaultLTR}, "auto"}, + std::tuple{2u, SBLevel{0}, "LTR"}, std::tuple{4u, SBLevel{1}, "RTL"}}) { + if (!(bitset & bit)) continue; + ++cases; + std::string problem; + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, 0, text.size(), base); + if (SBParagraphGetLength(paragraph) != text.size()) { + problem = std::format("paragraph length {} of {}", SBParagraphGetLength(paragraph), text.size()); + } else { + SBLineRef sbline = SBParagraphCreateLine(paragraph, 0, text.size()); + std::vector levels(text.size(), removed); + std::vector order; + const SBRun* runs = SBLineGetRunsPtr(sbline); + for (SBUInteger r = 0; r < SBLineGetRunCount(sbline); ++r) { + for (SBUInteger k = 0; k < runs[r].length; ++k) { + const std::size_t i = runs[r].level & 1 ? runs[r].offset + runs[r].length - 1 - k : runs[r].offset + k; + levels.at(i) = runs[r].level; + if (expected_levels.at(i) != removed) order.push_back(i); + } + } + for (std::size_t i = 0; i < text.size() && problem.empty(); ++i) { + if (expected_levels[i] != removed && levels[i] != expected_levels[i]) + problem = std::format("level of {} is {}, expected {}", i, levels[i], expected_levels[i]); + } + if (problem.empty() && order != expected_order) problem = "visual order differs"; + SBLineRelease(sbline); + } + SBParagraphRelease(paragraph); + if (!problem.empty()) { + ++failures; + if (failures <= 50) std::println("line {} ({}): {}", number_of_line, name, problem); + } + } + SBAlgorithmRelease(algorithm); + } + std::println("BidiTest: {} tests, {} errors", cases, failures); + return failures == 0 && cases > 0 ? 0 : 1; +} diff --git a/examples/sheenbidi/mcpp.toml b/examples/sheenbidi/mcpp.toml new file mode 100644 index 0000000..bc5e68a --- /dev/null +++ b/examples/sheenbidi/mcpp.toml @@ -0,0 +1,21 @@ +[package] +name = "sheenbidi" +version = "0.1.0" +description = "A bidirectional text layout tool (logical to visual order, levels, runs, script runs) and Unicode's two conformance tests, written against import mcpplibs.mwrap.sheenbidi; the hosts tools/run_conformance.py runs them on" +license = "Apache-2.0" + +[targets.sheenbidi] +kind = "bin" +main = "src/main.cpp" + +# Unicode's BidiTest.txt and BidiCharacterTest.txt, every case. +[targets.BidiTest] +kind = "bin" +main = "conformance/BidiTest.cpp" + +[targets.BidiCharacterTest] +kind = "bin" +main = "conformance/BidiCharacterTest.cpp" + +[dependencies] +"mcpplibs.mwrap.sheenbidi" = { path = "../.." } diff --git a/examples/sheenbidi/src/main.cpp b/examples/sheenbidi/src/main.cpp new file mode 100644 index 0000000..3582f7d --- /dev/null +++ b/examples/sheenbidi/src/main.cpp @@ -0,0 +1,143 @@ +// sheenbidi — lay out bidirectional text as a renderer does, through import +// mcpplibs.mwrap.sheenbidi: each UTF-8 line in logical (storage) order is +// split into paragraphs, its embedding levels resolved (UAX #9), and printed +// in visual order with mirrored brackets, with its levels, its runs and its +// script runs. A terminal without bidi support shows the visual line as it +// should be read. +// +// sheenbidi [--ltr | --rtl] [--runs] text ... each argument +// sheenbidi [--ltr | --rtl] [--runs] - lines from stdin +// +// The base direction is the first strong character's (P2, P3) unless --ltr or +// --rtl says otherwise. Offsets are in UTF-8 code units, as SheenBidi reports +// them for UTF-8 input. +import std; +import mcpplibs.mwrap.sheenbidi; + +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBStringEncodingUTF8, SBLevelDefaultLTR … + +namespace { + +void append_utf8(std::string& out, SBCodepoint c) { + if (c < 0x80) { + out += static_cast(c); + } else if (c < 0x800) { + out += static_cast(0xC0 | (c >> 6)); + out += static_cast(0x80 | (c & 0x3F)); + } else if (c < 0x10000) { + out += static_cast(0xE0 | (c >> 12)); + out += static_cast(0x80 | ((c >> 6) & 0x3F)); + out += static_cast(0x80 | (c & 0x3F)); + } else { + out += static_cast(0xF0 | (c >> 18)); + out += static_cast(0x80 | ((c >> 12) & 0x3F)); + out += static_cast(0x80 | ((c >> 6) & 0x3F)); + out += static_cast(0x80 | (c & 0x3F)); + } +} + +std::string tag(SBUInt32 t) { + return {static_cast(t >> 24), static_cast(t >> 16), static_cast(t >> 8), static_cast(t)}; +} + +struct Options { + SBLevel base = SBLevelDefaultLTR; + bool runs = false; +}; + +void lay_out(std::string_view text, const Options& options) { + const auto* bytes = reinterpret_cast(text.data()); + const SBCodepointSequence seq{SBStringEncodingUTF8, text.data(), text.size()}; + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + for (SBUInteger offset = 0; offset < text.size();) { + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, offset, text.size() - offset, options.base); + const SBUInteger length = SBParagraphGetLength(paragraph); + SBLineRef line = SBParagraphCreateLine(paragraph, offset, length); + + // Mirrored glyphs: the characters at odd levels that have a mirror (L4). + std::map mirrored; + SBMirrorLocatorRef locator = SBMirrorLocatorCreate(); + SBMirrorLocatorLoadLine(locator, line, text.data()); + while (SBMirrorLocatorMoveNext(locator)) { + const SBMirrorAgent* agent = SBMirrorLocatorGetAgent(locator); + mirrored[agent->index] = agent->mirror; + } + SBMirrorLocatorRelease(locator); + + // The runs in visual order, an odd one read backwards (L2). + std::string visual, runs; + const SBRun* run = SBLineGetRunsPtr(line); + for (SBUInteger r = 0; r < SBLineGetRunCount(line); ++r) { + runs += std::format(" [{},{}) L{}", run[r].offset, run[r].offset + run[r].length, run[r].level); + if (run[r].level & 1) { + SBUInteger i = run[r].offset + run[r].length; + while (i > run[r].offset) { + const SBCodepoint c = SBCodepointDecodePreviousFromUTF8(bytes, run[r].offset + run[r].length, &i); + const auto m = mirrored.find(i); + append_utf8(visual, m != mirrored.end() ? m->second : c); + } + } else { + visual.append(text.substr(run[r].offset, run[r].length)); + } + } + + std::string levels; + const SBLevel* level = SBParagraphGetLevelsPtr(paragraph); + for (SBUInteger i = 0; i < length;) { // one digit per code point + const SBUInteger at = i; + SBCodepointDecodeNextFromUTF8(bytes + offset, length, &i); + levels += static_cast('0' + level[at]); + } + + std::println("logical {}", text.substr(offset, length)); + std::println("visual {}", visual); + std::println("levels {} (base {}; odd levels run right to left)", levels, SBParagraphGetBaseLevel(paragraph)); + if (options.runs) { + std::println("runs {}", runs); + const SBCodepointSequence part{SBStringEncodingUTF8, text.data() + offset, length}; + SBScriptLocatorRef scripts = SBScriptLocatorCreate(); + SBScriptLocatorLoadCodepoints(scripts, &part); + std::string script_runs; + while (SBScriptLocatorMoveNext(scripts)) { + const SBScriptAgent* agent = SBScriptLocatorGetAgent(scripts); + script_runs += std::format(" [{},{}) {}", offset + agent->offset, offset + agent->offset + agent->length, + tag(SBScriptGetUnicodeTag(agent->script))); + } + SBScriptLocatorRelease(scripts); + std::println("scripts {}", script_runs); + } + std::println(); + SBLineRelease(line); + SBParagraphRelease(paragraph); + offset += length; + } + SBAlgorithmRelease(algorithm); +} + +} // namespace + +int main(int argc, char** argv) { + Options options; + std::vector texts; + bool from_stdin = false; + for (int i = 1; i < argc; ++i) { + const std::string_view arg = argv[i]; + if (arg == "--ltr") options.base = 0; + else if (arg == "--rtl") options.base = 1; + else if (arg == "--runs") options.runs = true; + else if (arg == "-") from_stdin = true; + else if (arg == "-h" || arg == "--help") { + std::println("usage: sheenbidi [--ltr | --rtl] [--runs] (text ... | -)"); + return 0; + } else texts.emplace_back(arg); + } + std::println("SheenBidi {} (Unicode 17.0.0) through import mcpplibs.mwrap.sheenbidi\n", SBVersionGetString()); + if (from_stdin) { + for (std::string line; std::getline(std::cin, line);) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + texts.push_back(line); + } + } + if (texts.empty()) texts = {"The title is מזל טוב in Hebrew.", "הספר (book) עולה 120 ש״ח.", "مرحبا [world] 2026!"}; + for (const auto& text : texts) lay_out(text, options); +} diff --git a/include/mwrap/sheenbidi/macros.hpp b/include/mwrap/sheenbidi/macros.hpp new file mode 100644 index 0000000..97201c1 --- /dev/null +++ b/include/mwrap/sheenbidi/macros.hpp @@ -0,0 +1,15 @@ +// — SheenBidi's macros as macros, for the ones a +// C++ entity cannot be: the declaration attributes (SB_PUBLIC, SB_DEPRECATED) +// and the extern "C" brackets (SB_EXTERN_C_BEGIN / SB_EXTERN_C_END). +// +// Include it before `import mcpplibs.mwrap.sheenbidi;`: it is +// , as the module sees it (compat.sheenbidi is a static +// library built without SB_CONFIG_DLL_*, so SB_PUBLIC is empty everywhere, and +// without SB_CONFIG_EXPERIMENTAL_TEXT_API, so SB_TEXT_API_SUPPORTED is 0). Every +// other macro is defined too, and while a macro is defined it expands even +// inside `mcpplibs::mwrap::sheenbidi::macros::NAME`; a function-like one does +// not when its name is followed by `)`: `(macros::SBBidiTypeIsStrong)(t)` is +// the function. The header's enumerators (SBBidiTypeL …) are its own here, and +// `macros::SBBidiTypeL` the module's constant of the same value. +#pragma once +#include diff --git a/mcpp.toml b/mcpp.toml new file mode 100644 index 0000000..7ce3f04 --- /dev/null +++ b/mcpp.toml @@ -0,0 +1,47 @@ +# Module wrapper; upstream API preserved. Policy: mcpp-index/docs/mwrap.md. +[package] +namespace = "mcpplibs.mwrap" +name = "sheenbidi" +version = "3.0.0" +description = "SheenBidi 3.0.0 C++ modules for mcpp (import mcpplibs.mwrap.sheenbidi;): the Unicode Bidirectional Algorithm (UAX #9, Unicode 17.0.0) of SheenBidi.h over compat.sheenbidi, its macros and enumerators as constants and constexpr functions" +license = "Apache-2.0" +authors = ["mcpp-community"] +repo = "https://github.com/mcpplibs/sheenbidi-m" +platforms = ["linux", "macos", "windows"] + +[targets.sheenbidi] +kind = "lib" + +[build] +sources = ["src/*.cppm"] +# : the macros a C++ entity cannot be. +include_dirs = ["include"] + +[dependencies] +compat.sheenbidi = "3.0.0" + +[dev-dependencies] +compat.gtest = "1.15.2" + +[package.metadata.mwrap] +upstream-version = "3.0.0" + +[[runtime.deploy]] +from = "LICENSE" +to = "licenses/sheenbidi" + +[[runtime.deploy]] +from = "NOTICE" +to = "licenses/sheenbidi" + +[[runtime.deploy]] +from = "upstream/sheenbidi/README.md" +to = "licenses/sheenbidi/upstream/sheenbidi" + +[[runtime.deploy]] +from = "upstream/sheenbidi/LICENSE" +to = "licenses/sheenbidi/upstream/sheenbidi" + +[[runtime.deploy]] +from = "upstream/unicode/license.txt" +to = "licenses/sheenbidi/upstream/unicode" diff --git a/src/macros.cppm b/src/macros.cppm new file mode 100644 index 0000000..7242aae --- /dev/null +++ b/src/macros.cppm @@ -0,0 +1,385 @@ +// Generated by tools/gen_exports.py from the SheenBidi 3.0.0 headers — do not +// edit by hand. +// +// mcpplibs.mwrap.sheenbidi.macros — what declares that +// a named module cannot export as it is, under the original names in +// mcpplibs::mwrap::sheenbidi::macros: +// +// - object-like macros (SBLevelInvalid, SBLevelMax, SBLevelDefaultLTR, +// SBLevelDefaultRTL, SBCodepointInvalid, SBCodepointFaulty, +// SBCodepointMax, SB_TEXT_API_SUPPORTED, SHEENBIDI_VERSION_*) as `inline +// constexpr`, each of the type its macro expands to (SBLevelMax is an +// `int`, as 125 is; SBCodepointInvalid a `uint32_t`, as UINT32_MAX is), the +// version string as a reference to a constexpr char array; +// - function-like macros (SBUInt8InRange, SBBidiTypeIsStrong, +// SBCodepointIsValid, SBGeneralCategoryIsLetter …) as constexpr function +// templates. Each body is the macro itself, expanded on the parameters +// while the header's macros are defined (the declarator is parenthesised +// so the name is not expanded): the result has the macro's value and +// type for every argument type. Each argument is evaluated once, before +// the call; +// - the enumerators of the headers' unnamed enums (SBBidiType*, +// SBGeneralCategory*, SBScript*, SBStringEncoding*, SBTrue, SBFalse). An +// unnamed enumeration is TU-local, so they cannot be exported as they are: +// each is a constant of the typedef the header gives its values (the enum +// of SBBidiTypeL is followed by `typedef SBUInt8 SBBidiType;`). +// +// Attributes and the extern "C" brackets stay macros, in +// , included before the import. +// tools/check_exports.py accounts for every macro. +module; +#include + +export module mcpplibs.mwrap.sheenbidi.macros; + +// The header's values, taken while its macros are defined (module linkage). +namespace mwrap_sheenbidi_capture { +inline constexpr auto value_SB_TEXT_API_SUPPORTED = SB_TEXT_API_SUPPORTED; +inline constexpr auto value_SBLevelInvalid = SBLevelInvalid; +inline constexpr auto value_SBLevelMax = SBLevelMax; +inline constexpr auto value_SBLevelDefaultLTR = SBLevelDefaultLTR; +inline constexpr auto value_SBLevelDefaultRTL = SBLevelDefaultRTL; +inline constexpr auto value_SBCodepointInvalid = SBCodepointInvalid; +inline constexpr auto value_SBCodepointFaulty = SBCodepointFaulty; +inline constexpr auto value_SBCodepointMax = SBCodepointMax; +inline constexpr auto value_SHEENBIDI_VERSION_MAJOR = SHEENBIDI_VERSION_MAJOR; +inline constexpr auto value_SHEENBIDI_VERSION_MINOR = SHEENBIDI_VERSION_MINOR; +inline constexpr auto value_SHEENBIDI_VERSION_PATCH = SHEENBIDI_VERSION_PATCH; +inline constexpr char value_SHEENBIDI_VERSION_STRING[] = SHEENBIDI_VERSION_STRING; +} // namespace mwrap_sheenbidi_capture + +// ── computation macros, defined while the header's macros are ────────────── +export namespace mcpplibs::mwrap::sheenbidi::macros { + +// #define SBUInt8InRange(v, s, e) ( (SBUInt8)((v) - (s)) <= (SBUInt8)((e) - (s)) ) +template +constexpr auto (SBUInt8InRange)(T0 v, T1 s, T2 e) noexcept { return SBUInt8InRange(v, s, e); } +// #define SBUInt16InRange(v, s, e) ( (SBUInt16)((v) - (s)) <= (SBUInt16)((e) - (s)) ) +template +constexpr auto (SBUInt16InRange)(T0 v, T1 s, T2 e) noexcept { return SBUInt16InRange(v, s, e); } +// #define SBUInt32InRange(v, s, e) ( (SBUInt32)((v) - (s)) <= (SBUInt32)((e) - (s)) ) +template +constexpr auto (SBUInt32InRange)(T0 v, T1 s, T2 e) noexcept { return SBUInt32InRange(v, s, e); } +// #define SBBidiTypeIsStrong(t) SBUInt8InRange(t, SBBidiTypeL, SBBidiTypeAL) +template +constexpr auto (SBBidiTypeIsStrong)(T0 t) noexcept { return SBBidiTypeIsStrong(t); } +// #define SBBidiTypeIsWeak(t) SBUInt8InRange(t, SBBidiTypeBN, SBBidiTypeCS) +template +constexpr auto (SBBidiTypeIsWeak)(T0 t) noexcept { return SBBidiTypeIsWeak(t); } +// #define SBBidiTypeIsNeutral(t) SBUInt8InRange(t, SBBidiTypeWS, SBBidiTypeON) +template +constexpr auto (SBBidiTypeIsNeutral)(T0 t) noexcept { return SBBidiTypeIsNeutral(t); } +// #define SBBidiTypeIsFormat(t) SBUInt8InRange(t, SBBidiTypeLRI, SBBidiTypePDF) +template +constexpr auto (SBBidiTypeIsFormat)(T0 t) noexcept { return SBBidiTypeIsFormat(t); } +// #define SBCodepointIsSurrogate(c) SBUInt32InRange(c, 0xD800, 0xDFFF) +template +constexpr auto (SBCodepointIsSurrogate)(T0 c) noexcept { return SBCodepointIsSurrogate(c); } +// #define SBCodepointIsValid(c) (!SBCodepointIsSurrogate(c) && (c) <= SBCodepointMax) +template +constexpr auto (SBCodepointIsValid)(T0 c) noexcept { return SBCodepointIsValid(c); } +// #define SBGeneralCategoryIsLetter(gc) SBUInt8InRange(gc, SBGeneralCategoryLU, SBGeneralCategoryLO) +template +constexpr auto (SBGeneralCategoryIsLetter)(T0 gc) noexcept { return SBGeneralCategoryIsLetter(gc); } +// #define SBGeneralCategoryIsMark(gc) SBUInt8InRange(gc, SBGeneralCategoryMN, SBGeneralCategoryME) +template +constexpr auto (SBGeneralCategoryIsMark)(T0 gc) noexcept { return SBGeneralCategoryIsMark(gc); } +// #define SBGeneralCategoryIsNumber(gc) SBUInt8InRange(gc, SBGeneralCategoryND, SBGeneralCategoryNO) +template +constexpr auto (SBGeneralCategoryIsNumber)(T0 gc) noexcept { return SBGeneralCategoryIsNumber(gc); } +// #define SBGeneralCategoryIsPunctuation(gc) SBUInt8InRange(gc, SBGeneralCategoryPC, SBGeneralCategoryPO) +template +constexpr auto (SBGeneralCategoryIsPunctuation)(T0 gc) noexcept { return SBGeneralCategoryIsPunctuation(gc); } +// #define SBGeneralCategoryIsSymbol(gc) SBUInt8InRange(gc, SBGeneralCategorySM, SBGeneralCategorySO) +template +constexpr auto (SBGeneralCategoryIsSymbol)(T0 gc) noexcept { return SBGeneralCategoryIsSymbol(gc); } +// #define SBGeneralCategoryIsSeparator(gc) SBUInt8InRange(gc, SBGeneralCategoryZS, SBGeneralCategoryZP) +template +constexpr auto (SBGeneralCategoryIsSeparator)(T0 gc) noexcept { return SBGeneralCategoryIsSeparator(gc); } +// #define SBGeneralCategoryIsOther(gc) SBUInt8InRange(gc, SBGeneralCategoryCC, SBGeneralCategoryCN) +template +constexpr auto (SBGeneralCategoryIsOther)(T0 gc) noexcept { return SBGeneralCategoryIsOther(gc); } + +} // namespace mcpplibs::mwrap::sheenbidi::macros + +// Every mapped object-like name, so the definitions below are not rewritten. +#undef SB_TEXT_API_SUPPORTED +#undef SBLevelInvalid +#undef SBLevelMax +#undef SBLevelDefaultLTR +#undef SBLevelDefaultRTL +#undef SBCodepointInvalid +#undef SBCodepointFaulty +#undef SBCodepointMax +#undef SHEENBIDI_VERSION_MAJOR +#undef SHEENBIDI_VERSION_MINOR +#undef SHEENBIDI_VERSION_PATCH +#undef SHEENBIDI_VERSION_STRING + +namespace c = mwrap_sheenbidi_capture; + +export namespace mcpplibs::mwrap::sheenbidi::macros { + +// ── constants ─────────────────────────────────────────────────────────────── +inline constexpr auto SB_TEXT_API_SUPPORTED = c::value_SB_TEXT_API_SUPPORTED; +inline constexpr auto SBLevelInvalid = c::value_SBLevelInvalid; +inline constexpr auto SBLevelMax = c::value_SBLevelMax; +inline constexpr auto SBLevelDefaultLTR = c::value_SBLevelDefaultLTR; +inline constexpr auto SBLevelDefaultRTL = c::value_SBLevelDefaultRTL; +inline constexpr auto SBCodepointInvalid = c::value_SBCodepointInvalid; +inline constexpr auto SBCodepointFaulty = c::value_SBCodepointFaulty; +inline constexpr auto SBCodepointMax = c::value_SBCodepointMax; +inline constexpr auto SHEENBIDI_VERSION_MAJOR = c::value_SHEENBIDI_VERSION_MAJOR; +inline constexpr auto SHEENBIDI_VERSION_MINOR = c::value_SHEENBIDI_VERSION_MINOR; +inline constexpr auto SHEENBIDI_VERSION_PATCH = c::value_SHEENBIDI_VERSION_PATCH; + +// ── strings ───────────────────────────────────────────────────────────────── +inline constexpr auto& SHEENBIDI_VERSION_STRING = c::value_SHEENBIDI_VERSION_STRING; + +// ── enumerators of the unnamed enum whose values are SBBoolean (2) ─────────── +inline constexpr ::SBBoolean SBFalse = ::SBFalse; +inline constexpr ::SBBoolean SBTrue = ::SBTrue; + +// ── enumerators of the unnamed enum whose values are SBBidiType (24) ────────── +inline constexpr ::SBBidiType SBBidiTypeNil = ::SBBidiTypeNil; +inline constexpr ::SBBidiType SBBidiTypeL = ::SBBidiTypeL; +inline constexpr ::SBBidiType SBBidiTypeR = ::SBBidiTypeR; +inline constexpr ::SBBidiType SBBidiTypeAL = ::SBBidiTypeAL; +inline constexpr ::SBBidiType SBBidiTypeBN = ::SBBidiTypeBN; +inline constexpr ::SBBidiType SBBidiTypeNSM = ::SBBidiTypeNSM; +inline constexpr ::SBBidiType SBBidiTypeAN = ::SBBidiTypeAN; +inline constexpr ::SBBidiType SBBidiTypeEN = ::SBBidiTypeEN; +inline constexpr ::SBBidiType SBBidiTypeET = ::SBBidiTypeET; +inline constexpr ::SBBidiType SBBidiTypeES = ::SBBidiTypeES; +inline constexpr ::SBBidiType SBBidiTypeCS = ::SBBidiTypeCS; +inline constexpr ::SBBidiType SBBidiTypeWS = ::SBBidiTypeWS; +inline constexpr ::SBBidiType SBBidiTypeS = ::SBBidiTypeS; +inline constexpr ::SBBidiType SBBidiTypeB = ::SBBidiTypeB; +inline constexpr ::SBBidiType SBBidiTypeON = ::SBBidiTypeON; +inline constexpr ::SBBidiType SBBidiTypeLRI = ::SBBidiTypeLRI; +inline constexpr ::SBBidiType SBBidiTypeRLI = ::SBBidiTypeRLI; +inline constexpr ::SBBidiType SBBidiTypeFSI = ::SBBidiTypeFSI; +inline constexpr ::SBBidiType SBBidiTypePDI = ::SBBidiTypePDI; +inline constexpr ::SBBidiType SBBidiTypeLRE = ::SBBidiTypeLRE; +inline constexpr ::SBBidiType SBBidiTypeRLE = ::SBBidiTypeRLE; +inline constexpr ::SBBidiType SBBidiTypeLRO = ::SBBidiTypeLRO; +inline constexpr ::SBBidiType SBBidiTypeRLO = ::SBBidiTypeRLO; +inline constexpr ::SBBidiType SBBidiTypePDF = ::SBBidiTypePDF; + +// ── enumerators of the unnamed enum whose values are SBStringEncoding (3) ──── +inline constexpr ::SBStringEncoding SBStringEncodingUTF8 = ::SBStringEncodingUTF8; +inline constexpr ::SBStringEncoding SBStringEncodingUTF16 = ::SBStringEncodingUTF16; +inline constexpr ::SBStringEncoding SBStringEncodingUTF32 = ::SBStringEncodingUTF32; + +// ── enumerators of the unnamed enum whose values are SBGeneralCategory (31) ─── +inline constexpr ::SBGeneralCategory SBGeneralCategoryNil = ::SBGeneralCategoryNil; +inline constexpr ::SBGeneralCategory SBGeneralCategoryLU = ::SBGeneralCategoryLU; +inline constexpr ::SBGeneralCategory SBGeneralCategoryLL = ::SBGeneralCategoryLL; +inline constexpr ::SBGeneralCategory SBGeneralCategoryLT = ::SBGeneralCategoryLT; +inline constexpr ::SBGeneralCategory SBGeneralCategoryLM = ::SBGeneralCategoryLM; +inline constexpr ::SBGeneralCategory SBGeneralCategoryLO = ::SBGeneralCategoryLO; +inline constexpr ::SBGeneralCategory SBGeneralCategoryMN = ::SBGeneralCategoryMN; +inline constexpr ::SBGeneralCategory SBGeneralCategoryMC = ::SBGeneralCategoryMC; +inline constexpr ::SBGeneralCategory SBGeneralCategoryME = ::SBGeneralCategoryME; +inline constexpr ::SBGeneralCategory SBGeneralCategoryND = ::SBGeneralCategoryND; +inline constexpr ::SBGeneralCategory SBGeneralCategoryNL = ::SBGeneralCategoryNL; +inline constexpr ::SBGeneralCategory SBGeneralCategoryNO = ::SBGeneralCategoryNO; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPC = ::SBGeneralCategoryPC; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPD = ::SBGeneralCategoryPD; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPS = ::SBGeneralCategoryPS; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPE = ::SBGeneralCategoryPE; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPI = ::SBGeneralCategoryPI; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPF = ::SBGeneralCategoryPF; +inline constexpr ::SBGeneralCategory SBGeneralCategoryPO = ::SBGeneralCategoryPO; +inline constexpr ::SBGeneralCategory SBGeneralCategorySM = ::SBGeneralCategorySM; +inline constexpr ::SBGeneralCategory SBGeneralCategorySC = ::SBGeneralCategorySC; +inline constexpr ::SBGeneralCategory SBGeneralCategorySK = ::SBGeneralCategorySK; +inline constexpr ::SBGeneralCategory SBGeneralCategorySO = ::SBGeneralCategorySO; +inline constexpr ::SBGeneralCategory SBGeneralCategoryZS = ::SBGeneralCategoryZS; +inline constexpr ::SBGeneralCategory SBGeneralCategoryZL = ::SBGeneralCategoryZL; +inline constexpr ::SBGeneralCategory SBGeneralCategoryZP = ::SBGeneralCategoryZP; +inline constexpr ::SBGeneralCategory SBGeneralCategoryCC = ::SBGeneralCategoryCC; +inline constexpr ::SBGeneralCategory SBGeneralCategoryCF = ::SBGeneralCategoryCF; +inline constexpr ::SBGeneralCategory SBGeneralCategoryCS = ::SBGeneralCategoryCS; +inline constexpr ::SBGeneralCategory SBGeneralCategoryCO = ::SBGeneralCategoryCO; +inline constexpr ::SBGeneralCategory SBGeneralCategoryCN = ::SBGeneralCategoryCN; + +// ── enumerators of the unnamed enum whose values are SBScript (176) ──────────── +inline constexpr ::SBScript SBScriptNil = ::SBScriptNil; +inline constexpr ::SBScript SBScriptZINH = ::SBScriptZINH; +inline constexpr ::SBScript SBScriptZYYY = ::SBScriptZYYY; +inline constexpr ::SBScript SBScriptZZZZ = ::SBScriptZZZZ; +inline constexpr ::SBScript SBScriptARAB = ::SBScriptARAB; +inline constexpr ::SBScript SBScriptARMN = ::SBScriptARMN; +inline constexpr ::SBScript SBScriptBENG = ::SBScriptBENG; +inline constexpr ::SBScript SBScriptBOPO = ::SBScriptBOPO; +inline constexpr ::SBScript SBScriptCYRL = ::SBScriptCYRL; +inline constexpr ::SBScript SBScriptDEVA = ::SBScriptDEVA; +inline constexpr ::SBScript SBScriptGEOR = ::SBScriptGEOR; +inline constexpr ::SBScript SBScriptGREK = ::SBScriptGREK; +inline constexpr ::SBScript SBScriptGUJR = ::SBScriptGUJR; +inline constexpr ::SBScript SBScriptGURU = ::SBScriptGURU; +inline constexpr ::SBScript SBScriptHANG = ::SBScriptHANG; +inline constexpr ::SBScript SBScriptHANI = ::SBScriptHANI; +inline constexpr ::SBScript SBScriptHEBR = ::SBScriptHEBR; +inline constexpr ::SBScript SBScriptHIRA = ::SBScriptHIRA; +inline constexpr ::SBScript SBScriptKANA = ::SBScriptKANA; +inline constexpr ::SBScript SBScriptKNDA = ::SBScriptKNDA; +inline constexpr ::SBScript SBScriptLAOO = ::SBScriptLAOO; +inline constexpr ::SBScript SBScriptLATN = ::SBScriptLATN; +inline constexpr ::SBScript SBScriptMLYM = ::SBScriptMLYM; +inline constexpr ::SBScript SBScriptORYA = ::SBScriptORYA; +inline constexpr ::SBScript SBScriptTAML = ::SBScriptTAML; +inline constexpr ::SBScript SBScriptTELU = ::SBScriptTELU; +inline constexpr ::SBScript SBScriptTHAI = ::SBScriptTHAI; +inline constexpr ::SBScript SBScriptTIBT = ::SBScriptTIBT; +inline constexpr ::SBScript SBScriptBRAI = ::SBScriptBRAI; +inline constexpr ::SBScript SBScriptCANS = ::SBScriptCANS; +inline constexpr ::SBScript SBScriptCHER = ::SBScriptCHER; +inline constexpr ::SBScript SBScriptETHI = ::SBScriptETHI; +inline constexpr ::SBScript SBScriptKHMR = ::SBScriptKHMR; +inline constexpr ::SBScript SBScriptMONG = ::SBScriptMONG; +inline constexpr ::SBScript SBScriptMYMR = ::SBScriptMYMR; +inline constexpr ::SBScript SBScriptOGAM = ::SBScriptOGAM; +inline constexpr ::SBScript SBScriptRUNR = ::SBScriptRUNR; +inline constexpr ::SBScript SBScriptSINH = ::SBScriptSINH; +inline constexpr ::SBScript SBScriptSYRC = ::SBScriptSYRC; +inline constexpr ::SBScript SBScriptTHAA = ::SBScriptTHAA; +inline constexpr ::SBScript SBScriptYIII = ::SBScriptYIII; +inline constexpr ::SBScript SBScriptDSRT = ::SBScriptDSRT; +inline constexpr ::SBScript SBScriptGOTH = ::SBScriptGOTH; +inline constexpr ::SBScript SBScriptITAL = ::SBScriptITAL; +inline constexpr ::SBScript SBScriptBUHD = ::SBScriptBUHD; +inline constexpr ::SBScript SBScriptHANO = ::SBScriptHANO; +inline constexpr ::SBScript SBScriptTAGB = ::SBScriptTAGB; +inline constexpr ::SBScript SBScriptTGLG = ::SBScriptTGLG; +inline constexpr ::SBScript SBScriptCPRT = ::SBScriptCPRT; +inline constexpr ::SBScript SBScriptLIMB = ::SBScriptLIMB; +inline constexpr ::SBScript SBScriptLINB = ::SBScriptLINB; +inline constexpr ::SBScript SBScriptOSMA = ::SBScriptOSMA; +inline constexpr ::SBScript SBScriptSHAW = ::SBScriptSHAW; +inline constexpr ::SBScript SBScriptTALE = ::SBScriptTALE; +inline constexpr ::SBScript SBScriptUGAR = ::SBScriptUGAR; +inline constexpr ::SBScript SBScriptBUGI = ::SBScriptBUGI; +inline constexpr ::SBScript SBScriptCOPT = ::SBScriptCOPT; +inline constexpr ::SBScript SBScriptGLAG = ::SBScriptGLAG; +inline constexpr ::SBScript SBScriptKHAR = ::SBScriptKHAR; +inline constexpr ::SBScript SBScriptSYLO = ::SBScriptSYLO; +inline constexpr ::SBScript SBScriptTALU = ::SBScriptTALU; +inline constexpr ::SBScript SBScriptTFNG = ::SBScriptTFNG; +inline constexpr ::SBScript SBScriptXPEO = ::SBScriptXPEO; +inline constexpr ::SBScript SBScriptBALI = ::SBScriptBALI; +inline constexpr ::SBScript SBScriptNKOO = ::SBScriptNKOO; +inline constexpr ::SBScript SBScriptPHAG = ::SBScriptPHAG; +inline constexpr ::SBScript SBScriptPHNX = ::SBScriptPHNX; +inline constexpr ::SBScript SBScriptXSUX = ::SBScriptXSUX; +inline constexpr ::SBScript SBScriptCARI = ::SBScriptCARI; +inline constexpr ::SBScript SBScriptCHAM = ::SBScriptCHAM; +inline constexpr ::SBScript SBScriptKALI = ::SBScriptKALI; +inline constexpr ::SBScript SBScriptLEPC = ::SBScriptLEPC; +inline constexpr ::SBScript SBScriptLYCI = ::SBScriptLYCI; +inline constexpr ::SBScript SBScriptLYDI = ::SBScriptLYDI; +inline constexpr ::SBScript SBScriptOLCK = ::SBScriptOLCK; +inline constexpr ::SBScript SBScriptRJNG = ::SBScriptRJNG; +inline constexpr ::SBScript SBScriptSAUR = ::SBScriptSAUR; +inline constexpr ::SBScript SBScriptSUND = ::SBScriptSUND; +inline constexpr ::SBScript SBScriptVAII = ::SBScriptVAII; +inline constexpr ::SBScript SBScriptARMI = ::SBScriptARMI; +inline constexpr ::SBScript SBScriptAVST = ::SBScriptAVST; +inline constexpr ::SBScript SBScriptBAMU = ::SBScriptBAMU; +inline constexpr ::SBScript SBScriptEGYP = ::SBScriptEGYP; +inline constexpr ::SBScript SBScriptJAVA = ::SBScriptJAVA; +inline constexpr ::SBScript SBScriptKTHI = ::SBScriptKTHI; +inline constexpr ::SBScript SBScriptLANA = ::SBScriptLANA; +inline constexpr ::SBScript SBScriptLISU = ::SBScriptLISU; +inline constexpr ::SBScript SBScriptMTEI = ::SBScriptMTEI; +inline constexpr ::SBScript SBScriptORKH = ::SBScriptORKH; +inline constexpr ::SBScript SBScriptPHLI = ::SBScriptPHLI; +inline constexpr ::SBScript SBScriptPRTI = ::SBScriptPRTI; +inline constexpr ::SBScript SBScriptSAMR = ::SBScriptSAMR; +inline constexpr ::SBScript SBScriptSARB = ::SBScriptSARB; +inline constexpr ::SBScript SBScriptTAVT = ::SBScriptTAVT; +inline constexpr ::SBScript SBScriptBATK = ::SBScriptBATK; +inline constexpr ::SBScript SBScriptBRAH = ::SBScriptBRAH; +inline constexpr ::SBScript SBScriptMAND = ::SBScriptMAND; +inline constexpr ::SBScript SBScriptCAKM = ::SBScriptCAKM; +inline constexpr ::SBScript SBScriptMERC = ::SBScriptMERC; +inline constexpr ::SBScript SBScriptMERO = ::SBScriptMERO; +inline constexpr ::SBScript SBScriptPLRD = ::SBScriptPLRD; +inline constexpr ::SBScript SBScriptSHRD = ::SBScriptSHRD; +inline constexpr ::SBScript SBScriptSORA = ::SBScriptSORA; +inline constexpr ::SBScript SBScriptTAKR = ::SBScriptTAKR; +inline constexpr ::SBScript SBScriptAGHB = ::SBScriptAGHB; +inline constexpr ::SBScript SBScriptBASS = ::SBScriptBASS; +inline constexpr ::SBScript SBScriptDUPL = ::SBScriptDUPL; +inline constexpr ::SBScript SBScriptELBA = ::SBScriptELBA; +inline constexpr ::SBScript SBScriptGRAN = ::SBScriptGRAN; +inline constexpr ::SBScript SBScriptHMNG = ::SBScriptHMNG; +inline constexpr ::SBScript SBScriptKHOJ = ::SBScriptKHOJ; +inline constexpr ::SBScript SBScriptLINA = ::SBScriptLINA; +inline constexpr ::SBScript SBScriptMAHJ = ::SBScriptMAHJ; +inline constexpr ::SBScript SBScriptMANI = ::SBScriptMANI; +inline constexpr ::SBScript SBScriptMEND = ::SBScriptMEND; +inline constexpr ::SBScript SBScriptMODI = ::SBScriptMODI; +inline constexpr ::SBScript SBScriptMROO = ::SBScriptMROO; +inline constexpr ::SBScript SBScriptNARB = ::SBScriptNARB; +inline constexpr ::SBScript SBScriptNBAT = ::SBScriptNBAT; +inline constexpr ::SBScript SBScriptPALM = ::SBScriptPALM; +inline constexpr ::SBScript SBScriptPAUC = ::SBScriptPAUC; +inline constexpr ::SBScript SBScriptPERM = ::SBScriptPERM; +inline constexpr ::SBScript SBScriptPHLP = ::SBScriptPHLP; +inline constexpr ::SBScript SBScriptSIDD = ::SBScriptSIDD; +inline constexpr ::SBScript SBScriptSIND = ::SBScriptSIND; +inline constexpr ::SBScript SBScriptTIRH = ::SBScriptTIRH; +inline constexpr ::SBScript SBScriptWARA = ::SBScriptWARA; +inline constexpr ::SBScript SBScriptAHOM = ::SBScriptAHOM; +inline constexpr ::SBScript SBScriptHATR = ::SBScriptHATR; +inline constexpr ::SBScript SBScriptHLUW = ::SBScriptHLUW; +inline constexpr ::SBScript SBScriptHUNG = ::SBScriptHUNG; +inline constexpr ::SBScript SBScriptMULT = ::SBScriptMULT; +inline constexpr ::SBScript SBScriptSGNW = ::SBScriptSGNW; +inline constexpr ::SBScript SBScriptADLM = ::SBScriptADLM; +inline constexpr ::SBScript SBScriptBHKS = ::SBScriptBHKS; +inline constexpr ::SBScript SBScriptMARC = ::SBScriptMARC; +inline constexpr ::SBScript SBScriptNEWA = ::SBScriptNEWA; +inline constexpr ::SBScript SBScriptOSGE = ::SBScriptOSGE; +inline constexpr ::SBScript SBScriptTANG = ::SBScriptTANG; +inline constexpr ::SBScript SBScriptGONM = ::SBScriptGONM; +inline constexpr ::SBScript SBScriptNSHU = ::SBScriptNSHU; +inline constexpr ::SBScript SBScriptSOYO = ::SBScriptSOYO; +inline constexpr ::SBScript SBScriptZANB = ::SBScriptZANB; +inline constexpr ::SBScript SBScriptDOGR = ::SBScriptDOGR; +inline constexpr ::SBScript SBScriptGONG = ::SBScriptGONG; +inline constexpr ::SBScript SBScriptMAKA = ::SBScriptMAKA; +inline constexpr ::SBScript SBScriptMEDF = ::SBScriptMEDF; +inline constexpr ::SBScript SBScriptROHG = ::SBScriptROHG; +inline constexpr ::SBScript SBScriptSOGD = ::SBScriptSOGD; +inline constexpr ::SBScript SBScriptSOGO = ::SBScriptSOGO; +inline constexpr ::SBScript SBScriptELYM = ::SBScriptELYM; +inline constexpr ::SBScript SBScriptHMNP = ::SBScriptHMNP; +inline constexpr ::SBScript SBScriptNAND = ::SBScriptNAND; +inline constexpr ::SBScript SBScriptWCHO = ::SBScriptWCHO; +inline constexpr ::SBScript SBScriptCHRS = ::SBScriptCHRS; +inline constexpr ::SBScript SBScriptDIAK = ::SBScriptDIAK; +inline constexpr ::SBScript SBScriptKITS = ::SBScriptKITS; +inline constexpr ::SBScript SBScriptYEZI = ::SBScriptYEZI; +inline constexpr ::SBScript SBScriptCPMN = ::SBScriptCPMN; +inline constexpr ::SBScript SBScriptOUGR = ::SBScriptOUGR; +inline constexpr ::SBScript SBScriptTNSA = ::SBScriptTNSA; +inline constexpr ::SBScript SBScriptTOTO = ::SBScriptTOTO; +inline constexpr ::SBScript SBScriptVITH = ::SBScriptVITH; +inline constexpr ::SBScript SBScriptKAWI = ::SBScriptKAWI; +inline constexpr ::SBScript SBScriptNAGM = ::SBScriptNAGM; +inline constexpr ::SBScript SBScriptGARA = ::SBScriptGARA; +inline constexpr ::SBScript SBScriptGUKH = ::SBScriptGUKH; +inline constexpr ::SBScript SBScriptKRAI = ::SBScriptKRAI; +inline constexpr ::SBScript SBScriptONAO = ::SBScriptONAO; +inline constexpr ::SBScript SBScriptSUNU = ::SBScriptSUNU; +inline constexpr ::SBScript SBScriptTODR = ::SBScriptTODR; +inline constexpr ::SBScript SBScriptTUTG = ::SBScriptTUTG; +inline constexpr ::SBScript SBScriptBERF = ::SBScriptBERF; +inline constexpr ::SBScript SBScriptSIDT = ::SBScriptSIDT; +inline constexpr ::SBScript SBScriptTAYO = ::SBScriptTAYO; +inline constexpr ::SBScript SBScriptTOLS = ::SBScriptTOLS; + +} // namespace mcpplibs::mwrap::sheenbidi::macros diff --git a/src/sheenbidi.cppm b/src/sheenbidi.cppm new file mode 100644 index 0000000..a224f8f --- /dev/null +++ b/src/sheenbidi.cppm @@ -0,0 +1,116 @@ +// Generated by tools/gen_exports.py from the SheenBidi 3.0.0 headers — do not +// edit by hand. +// +// mcpplibs.mwrap.sheenbidi — SheenBidi, the Unicode Bidirectional Algorithm +// (UAX #9, Unicode 17.0.0), as a C++ module. Every exported name is the +// upstream global of , unchanged: the types and struct +// tags, and every SB_PUBLIC function (SBAlgorithm, SBParagraph, SBLine, +// SBMirrorLocator, SBScriptLocator, SBCodepoint, SBAllocator, SBVersion). The +// root also re-exports the macro and enumerator mappings. The library is +// compat.sheenbidi, built from the same release. +module; +#include + +export module mcpplibs.mwrap.sheenbidi; + +export import mcpplibs.mwrap.sheenbidi.macros; + +// types (32) +export using ::SBAlgorithmRef; +export using ::SBAllocatorAllocateBlockFunc; +export using ::SBAllocatorAllocateScratchFunc; +export using ::SBAllocatorDeallocateBlockFunc; +export using ::SBAllocatorFinalizeFunc; +export using ::SBAllocatorProtocol; +export using ::SBAllocatorReallocateBlockFunc; +export using ::SBAllocatorRef; +export using ::SBAllocatorResetScratchFunc; +export using ::SBBidiType; +export using ::SBBoolean; +export using ::SBCodepoint; +export using ::SBCodepointSequence; +export using ::SBGeneralCategory; +export using ::SBInt16; +export using ::SBInt32; +export using ::SBInt8; +export using ::SBInteger; +export using ::SBLevel; +export using ::SBLineRef; +export using ::SBMirrorAgent; +export using ::SBMirrorLocatorRef; +export using ::SBParagraphRef; +export using ::SBRun; +export using ::SBScript; +export using ::SBScriptAgent; +export using ::SBScriptLocatorRef; +export using ::SBStringEncoding; +export using ::SBUInt16; +export using ::SBUInt32; +export using ::SBUInt8; +export using ::SBUInteger; + +// struct tags (11; the opaque ones are declared, not defined) +export using ::_SBAlgorithm; +export using ::_SBAllocator; +export using ::_SBAllocatorProtocol; +export using ::_SBCodepointSequence; +export using ::_SBLine; +export using ::_SBMirrorAgent; +export using ::_SBMirrorLocator; +export using ::_SBParagraph; +export using ::_SBRun; +export using ::_SBScriptAgent; +export using ::_SBScriptLocator; + +// functions (51; SBScriptGetOpenTypeTag is SB_DEPRECATED upstream) +export using ::SBAlgorithmCreate; +export using ::SBAlgorithmCreateParagraph; +export using ::SBAlgorithmGetBidiTypesPtr; +export using ::SBAlgorithmGetParagraphBoundary; +export using ::SBAlgorithmRelease; +export using ::SBAlgorithmRetain; +export using ::SBAllocatorCreate; +export using ::SBAllocatorGetDefault; +export using ::SBAllocatorRelease; +export using ::SBAllocatorRetain; +export using ::SBAllocatorSetDefault; +export using ::SBCodepointDecodeNextFromUTF16; +export using ::SBCodepointDecodeNextFromUTF8; +export using ::SBCodepointDecodePreviousFromUTF16; +export using ::SBCodepointDecodePreviousFromUTF8; +export using ::SBCodepointGetBidiType; +export using ::SBCodepointGetGeneralCategory; +export using ::SBCodepointGetMirror; +export using ::SBCodepointGetScript; +export using ::SBCodepointSequenceGetCodepointAt; +export using ::SBCodepointSequenceGetCodepointBefore; +export using ::SBLineGetLength; +export using ::SBLineGetOffset; +export using ::SBLineGetRunCount; +export using ::SBLineGetRunsPtr; +export using ::SBLineRelease; +export using ::SBLineRetain; +export using ::SBMirrorLocatorCreate; +export using ::SBMirrorLocatorGetAgent; +export using ::SBMirrorLocatorLoadLine; +export using ::SBMirrorLocatorMoveNext; +export using ::SBMirrorLocatorRelease; +export using ::SBMirrorLocatorReset; +export using ::SBMirrorLocatorRetain; +export using ::SBParagraphCreateLine; +export using ::SBParagraphGetBaseLevel; +export using ::SBParagraphGetLength; +export using ::SBParagraphGetLevelsPtr; +export using ::SBParagraphGetOffset; +export using ::SBParagraphRelease; +export using ::SBParagraphRetain; +export using ::SBScriptGetOpenTypeTag; +export using ::SBScriptGetUnicodeTag; +export using ::SBScriptLocatorCreate; +export using ::SBScriptLocatorGetAgent; +export using ::SBScriptLocatorLoadCodepoints; +export using ::SBScriptLocatorMoveNext; +export using ::SBScriptLocatorRelease; +export using ::SBScriptLocatorReset; +export using ::SBScriptLocatorRetain; +export using ::SBVersionGetString; diff --git a/templates/bidi/mcpp.toml.in b/templates/bidi/mcpp.toml.in new file mode 100644 index 0000000..a637560 --- /dev/null +++ b/templates/bidi/mcpp.toml.in @@ -0,0 +1,10 @@ +[package] +name = "{{project.name}}" +version = "0.1.0" + +[targets.{{project.name}}] +kind = "bin" +main = "src/main.cpp" + +[dependencies] +"{{self.name}}" = "{{self.version}}" diff --git a/templates/bidi/src/main.cpp.in b/templates/bidi/src/main.cpp.in new file mode 100644 index 0000000..c1c5a42 --- /dev/null +++ b/templates/bidi/src/main.cpp.in @@ -0,0 +1,90 @@ +// {{project.name}} — bidirectional text through import mcpplibs.mwrap.sheenbidi: +// each UTF-8 line in logical (storage) order is laid out as a renderer does +// it — a paragraph and its embedding levels (UAX #9), then the line's runs in +// visual order with mirrored brackets — and printed with its levels. A +// terminal without bidi support shows the visual line as it should be read. +// +// {{project.name}} the samples below +// {{project.name}} "text" ... each argument +// {{project.name}} - lines from stdin +import std; +import mcpplibs.mwrap.sheenbidi; + +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBStringEncodingUTF8, SBLevelDefaultLTR … + +namespace { + +constexpr std::string_view samples[] = { + "The title is מזל טוב in Hebrew.", + "הספר (book) עולה 120 ש״ח.", + "مرحبا [world] 2026!", +}; + +void append_utf8(std::string& out, SBCodepoint c) { + if (c < 0x80) { + out += static_cast(c); + } else if (c < 0x800) { + out += {static_cast(0xC0 | (c >> 6)), static_cast(0x80 | (c & 0x3F))}; + } else if (c < 0x10000) { + out += {static_cast(0xE0 | (c >> 12)), static_cast(0x80 | ((c >> 6) & 0x3F)), + static_cast(0x80 | (c & 0x3F))}; + } else { + out += {static_cast(0xF0 | (c >> 18)), static_cast(0x80 | ((c >> 12) & 0x3F)), + static_cast(0x80 | ((c >> 6) & 0x3F)), static_cast(0x80 | (c & 0x3F))}; + } +} + +void show(std::string_view text) { + // SheenBidi reads the UTF-8 code units directly; offsets are in bytes. + const SBCodepointSequence seq{SBStringEncodingUTF8, text.data(), text.size()}; + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, 0, text.size(), SBLevelDefaultLTR); + const SBUInteger length = SBParagraphGetLength(paragraph); // up to the first paragraph separator + SBLineRef line = SBParagraphCreateLine(paragraph, 0, length); + + // The runs come in visual order; an odd (right-to-left) one is read + // backwards, and a character with a mirror is drawn as it. + const auto* bytes = reinterpret_cast(text.data()); + std::string visual; + const SBRun* runs = SBLineGetRunsPtr(line); + for (SBUInteger r = 0; r < SBLineGetRunCount(line); ++r) { + const SBRun& run = runs[r]; + if (run.level & 1) { + for (SBUInteger i = run.offset + run.length; i > run.offset;) { + const SBCodepoint c = SBCodepointDecodePreviousFromUTF8(bytes, run.offset + run.length, &i); + const SBCodepoint mirror = SBCodepointGetMirror(c); + append_utf8(visual, mirror != 0 ? mirror : c); + } + } else { + visual += text.substr(run.offset, run.length); + } + } + + std::string levels; // one digit per character + const SBLevel* level = SBParagraphGetLevelsPtr(paragraph); + for (SBUInteger i = 0; i < length;) { + levels += static_cast('0' + level[i]); + SBCodepointDecodeNextFromUTF8(bytes, length, &i); + } + std::println("logical {}", text.substr(0, length)); + std::println("visual {}", visual); + std::println("levels {} ({} paragraph; odd levels run right to left)\n", levels, + SBParagraphGetBaseLevel(paragraph) & 1 ? "RTL" : "LTR"); + + SBLineRelease(line); + SBParagraphRelease(paragraph); + SBAlgorithmRelease(algorithm); +} + +} // namespace + +int main(int argc, char** argv) { + std::println("SheenBidi {} (Unicode 17.0.0) through import mcpplibs.mwrap.sheenbidi\n", SBVersionGetString()); + if (argc > 1 && std::string_view(argv[1]) == "-") { + for (std::string line; std::getline(std::cin, line);) show(line); + } else if (argc > 1) { + for (int i = 1; i < argc; ++i) show(argv[i]); + } else { + for (auto sample : samples) show(sample); + } +} diff --git a/templates/bidi/template.toml b/templates/bidi/template.toml new file mode 100644 index 0000000..84f4f59 --- /dev/null +++ b/templates/bidi/template.toml @@ -0,0 +1,4 @@ +[template] +description = "Bidirectional text with import mcpplibs.mwrap.sheenbidi: UTF-8 lines in logical order to visual order, with mirrored brackets and their embedding levels, as a renderer lays them out" +default = true +post_message = "Run `mcpp run`, or `mcpp run -- \"your text\"` (or pipe lines in: `mcpp run -- -`)." diff --git a/tests/api_test.cpp b/tests/api_test.cpp new file mode 100644 index 0000000..7b06500 --- /dev/null +++ b/tests/api_test.cpp @@ -0,0 +1,289 @@ +// The SheenBidi API through the module, spelled as +// documents it: code point properties (bidi types, general categories, +// mirrors, scripts and their tags), UTF-8 / UTF-16 decoding and code point +// sequences, paragraph boundaries and base levels, embedding levels (with the +// UAX #9 N0 bracket pair example and explicit isolates), lines and their runs +// in visual order, the mirror locator, the script locator, retain/release, a +// custom allocator, and the version. Only the module is imported; no +// SheenBidi header is included. +#include + +import std; +import mcpplibs.mwrap.sheenbidi; + +using namespace mcpplibs::mwrap::sheenbidi::macros; // SBBidiTypeL, SBLevelDefaultLTR, SBBidiTypeIsStrong … + +namespace { + +SBCodepointSequence utf32(const std::u32string& s) { + return {SBStringEncodingUTF32, s.data(), s.size()}; +} + +// One paragraph and its first line, released together. +struct Laid { + SBAlgorithmRef algorithm; + SBParagraphRef paragraph; + SBLineRef line; + + Laid(const SBCodepointSequence& seq, SBLevel base) + : algorithm(SBAlgorithmCreate(&seq)), + paragraph(SBAlgorithmCreateParagraph(algorithm, 0, seq.stringLength, base)), + line(SBParagraphCreateLine(paragraph, 0, SBParagraphGetLength(paragraph))) {} + ~Laid() { + SBLineRelease(line); + SBParagraphRelease(paragraph); + SBAlgorithmRelease(algorithm); + } + Laid(const Laid&) = delete; + Laid& operator=(const Laid&) = delete; + + std::vector levels() const { + const SBLevel* p = SBParagraphGetLevelsPtr(paragraph); + return {p, p + SBParagraphGetLength(paragraph)}; + } + std::vector runs() const { + const SBRun* p = SBLineGetRunsPtr(line); + return {p, p + SBLineGetRunCount(line)}; + } + // The logical indices in visual order, as a renderer draws them. + std::vector visual() const { + std::vector out; + for (const SBRun& r : runs()) { + for (SBUInteger i = 0; i < r.length; ++i) + out.push_back(r.level & 1 ? r.offset + r.length - 1 - i : r.offset + i); + } + return out; + } +}; + +std::string digits(const std::vector& levels) { + std::string out; + for (SBLevel l : levels) out += static_cast('0' + l); + return out; +} + +} // namespace + +TEST(Api, Version) { + EXPECT_STREQ(SBVersionGetString(), "3.0.0"); + EXPECT_STREQ(SBVersionGetString(), SHEENBIDI_VERSION_STRING); + EXPECT_EQ(SHEENBIDI_VERSION_MAJOR, 3); +} + +TEST(Api, BidiTypes) { + EXPECT_EQ(SBCodepointGetBidiType(U'a'), SBBidiTypeL); + EXPECT_EQ(SBCodepointGetBidiType(U'א'), SBBidiTypeR); + EXPECT_EQ(SBCodepointGetBidiType(U'ا'), SBBidiTypeAL); + EXPECT_EQ(SBCodepointGetBidiType(U'1'), SBBidiTypeEN); + EXPECT_EQ(SBCodepointGetBidiType(U'١'), SBBidiTypeAN); + EXPECT_EQ(SBCodepointGetBidiType(U' '), SBBidiTypeWS); + EXPECT_EQ(SBCodepointGetBidiType(U'⁧'), SBBidiTypeRLI); + EXPECT_EQ(SBCodepointGetBidiType(U'̀'), SBBidiTypeNSM); + EXPECT_TRUE(SBBidiTypeIsStrong(SBCodepointGetBidiType(U'א'))); + EXPECT_TRUE(SBBidiTypeIsWeak(SBBidiTypeEN)); + EXPECT_TRUE(SBBidiTypeIsNeutral(SBBidiTypeON)); + EXPECT_TRUE(SBBidiTypeIsFormat(SBBidiTypePDI)); + // Unicode 17.0: U+10940 SIDETIC LETTER N01 is right to left. + EXPECT_EQ(SBCodepointGetBidiType(0x10940), SBBidiTypeR); +} + +TEST(Api, CategoriesMirrorsScripts) { + EXPECT_EQ(SBCodepointGetGeneralCategory(U'A'), SBGeneralCategoryLU); + EXPECT_EQ(SBCodepointGetGeneralCategory(U'٣'), SBGeneralCategoryND); + EXPECT_TRUE(SBGeneralCategoryIsLetter(SBCodepointGetGeneralCategory(U'ب'))); + EXPECT_TRUE(SBGeneralCategoryIsPunctuation(SBCodepointGetGeneralCategory(U'('))); + EXPECT_EQ(SBCodepointGetMirror(U'('), U')'); + EXPECT_EQ(SBCodepointGetMirror(U'«'), U'»'); + EXPECT_EQ(SBCodepointGetMirror(U'a'), 0u); + EXPECT_EQ(SBCodepointGetScript(U'ب'), SBScriptARAB); + EXPECT_EQ(SBCodepointGetScript(U'א'), SBScriptHEBR); + EXPECT_EQ(SBCodepointGetScript(U' '), SBScriptZYYY); + EXPECT_EQ(SBCodepointGetScript(0x10940), SBScriptSIDT); + EXPECT_EQ(SBScriptGetUnicodeTag(SBScriptARAB), 0x41726162u); // 'Arab' + EXPECT_EQ(SBScriptGetUnicodeTag(SBScriptSIDT), 0x53696474u); // 'Sidt' +} + +TEST(Api, Decoding) { + const std::string utf8 = "a\xD7\x90\xF0\x90\xA5\x80"; // a, U+05D0, U+10940 + const auto* bytes = reinterpret_cast(utf8.data()); + SBUInteger i = 0; + EXPECT_EQ(SBCodepointDecodeNextFromUTF8(bytes, utf8.size(), &i), U'a'); + EXPECT_EQ(SBCodepointDecodeNextFromUTF8(bytes, utf8.size(), &i), U'א'); + EXPECT_EQ(SBCodepointDecodeNextFromUTF8(bytes, utf8.size(), &i), 0x10940u); + EXPECT_EQ(i, utf8.size()); + EXPECT_EQ(SBCodepointDecodeNextFromUTF8(bytes, utf8.size(), &i), SBCodepointInvalid); + EXPECT_EQ(SBCodepointDecodePreviousFromUTF8(bytes, utf8.size(), &i), 0x10940u); + + const SBUInt16 utf16[] = {0xD802, 0xDD40, 0x05D0, 0xDC00}; // U+10940, U+05D0, a lone surrogate + i = 0; + EXPECT_EQ(SBCodepointDecodeNextFromUTF16(utf16, 4, &i), 0x10940u); + EXPECT_EQ(SBCodepointDecodeNextFromUTF16(utf16, 4, &i), U'א'); + EXPECT_EQ(SBCodepointDecodeNextFromUTF16(utf16, 4, &i), static_cast(SBCodepointFaulty)); + i = 4; + EXPECT_EQ(SBCodepointDecodePreviousFromUTF16(utf16, 4, &i), static_cast(SBCodepointFaulty)); + EXPECT_FALSE(SBCodepointIsValid(0xD800u)); + EXPECT_TRUE(SBCodepointIsSurrogate(0xDFFFu)); + + const SBCodepointSequence seq{SBStringEncodingUTF8, utf8.data(), utf8.size()}; + i = 0; + EXPECT_EQ(SBCodepointSequenceGetCodepointAt(&seq, &i), U'a'); + i = utf8.size(); + EXPECT_EQ(SBCodepointSequenceGetCodepointBefore(&seq, &i), 0x10940u); + EXPECT_EQ(i, 3u); +} + +TEST(Api, ParagraphBoundaries) { + const std::u32string text = U"abc\r\nאבג
x"; + const auto seq = utf32(text); + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + const SBBidiType* types = SBAlgorithmGetBidiTypesPtr(algorithm); + EXPECT_EQ(types[3], SBBidiTypeB); + EXPECT_EQ(types[5], SBBidiTypeR); + SBUInteger length = 0, separator = 0; + SBAlgorithmGetParagraphBoundary(algorithm, 0, text.size(), &length, &separator); + EXPECT_EQ(length, 5u); // "abc\r\n": CR LF is one separator + EXPECT_EQ(separator, 2u); + SBAlgorithmGetParagraphBoundary(algorithm, 5, text.size() - 5, &length, &separator); + EXPECT_EQ(length, 4u); + EXPECT_EQ(separator, 1u); + + SBParagraphRef second = SBAlgorithmCreateParagraph(algorithm, 5, text.size() - 5, SBLevelDefaultLTR); + EXPECT_EQ(SBParagraphGetOffset(second), 5u); + EXPECT_EQ(SBParagraphGetLength(second), 4u); + EXPECT_EQ(SBParagraphGetBaseLevel(second), 1); // first strong is Hebrew + SBParagraphRelease(second); + SBAlgorithmRelease(algorithm); +} + +TEST(Api, LevelsAndVisualOrder) { + // "abc אבג" in an RTL paragraph: the Hebrew run first and reversed. + const std::u32string text = U"abc אבג"; + Laid rtl(utf32(text), 1); + EXPECT_EQ(digits(rtl.levels()), "2221111"); + EXPECT_EQ(rtl.visual(), (std::vector{6, 5, 4, 3, 0, 1, 2})); + Laid ltr(utf32(text), SBLevelDefaultLTR); + EXPECT_EQ(SBParagraphGetBaseLevel(ltr.paragraph), 0); + EXPECT_EQ(digits(ltr.levels()), "0000111"); + EXPECT_EQ(ltr.visual(), (std::vector{0, 1, 2, 3, 6, 5, 4})); + // No strong character: the default decides. + Laid neutral_rtl(utf32(U"123 !"), SBLevelDefaultRTL); + EXPECT_EQ(SBParagraphGetBaseLevel(neutral_rtl.paragraph), 1); +} + +TEST(Api, BracketPairs) { + // UAX #9's N0 example "AB(CD[&ef]!)gh" (uppercase right to left; here + // Hebrew letters for A–D) in an RTL paragraph: both bracket pairs follow + // the R before them, so they resolve to level 1 with "&" and "!", and only + // "ef" and "gh" are at level 2 — the levels the example lists. + const std::u32string text = U"אב(גד[&ef]!)gh"; + Laid laid(utf32(text), 1); + EXPECT_EQ(digits(laid.levels()), "11111112211122"); + // Every bracket is at an odd level, so all four are drawn mirrored. The + // locator reports them in the line's visual order. + SBMirrorLocatorRef locator = SBMirrorLocatorCreate(); + SBMirrorLocatorLoadLine(locator, laid.line, text.data()); + std::vector> mirrored; + while (SBMirrorLocatorMoveNext(locator)) { + const SBMirrorAgent* agent = SBMirrorLocatorGetAgent(locator); + EXPECT_EQ(agent->codepoint, static_cast(text[agent->index])); + mirrored.emplace_back(agent->index, agent->mirror); + } + EXPECT_EQ(mirrored, (std::vector>{{9, U'['}, {11, U'('}, {2, U')'}, {5, U']'}})); + SBMirrorLocatorReset(locator); + EXPECT_TRUE(SBMirrorLocatorMoveNext(locator)); + EXPECT_EQ(SBMirrorLocatorGetAgent(locator)->index, mirrored.front().first); + SBMirrorLocatorRelease(locator); +} + +TEST(Api, Isolates) { + // An RLI … PDI isolate inside an LTR paragraph keeps "abc" outside it at 0. + const std::u32string text = U"abc ⁧אב 12⁩ def"; + Laid laid(utf32(text), 0); + const auto levels = laid.levels(); + EXPECT_EQ(levels[0], 0); + EXPECT_EQ(levels[5], 1); // Hebrew inside the isolate + EXPECT_EQ(levels[8], 2); // the digits after it, EN in an RTL isolate + EXPECT_EQ(levels[text.size() - 1], 0); +} + +TEST(Api, ScriptLocator) { + const std::string utf8 = "abc \xD8\xB9\xD8\xB1\xD8\xA8\xD9\x8A \xD7\x90"; // "abc عربي א" + const SBCodepointSequence seq{SBStringEncodingUTF8, utf8.data(), utf8.size()}; + SBScriptLocatorRef locator = SBScriptLocatorCreate(); + SBScriptLocatorLoadCodepoints(locator, &seq); + std::vector runs; + while (SBScriptLocatorMoveNext(locator)) runs.push_back(*SBScriptLocatorGetAgent(locator)); + ASSERT_EQ(runs.size(), 3u); + EXPECT_EQ(runs[0].script, SBScriptLATN); + EXPECT_EQ(runs[1].script, SBScriptARAB); + EXPECT_EQ(runs[2].script, SBScriptHEBR); + EXPECT_EQ(runs[2].offset + runs[2].length, utf8.size()); + SBScriptLocatorReset(locator); + EXPECT_TRUE(SBScriptLocatorMoveNext(locator)); + EXPECT_EQ(SBScriptLocatorGetAgent(locator)->offset, 0u); + SBScriptLocatorRelease(SBScriptLocatorRetain(locator)); + SBScriptLocatorRelease(locator); +} + +TEST(Api, RetainRelease) { + const std::u32string text = U"abc"; + const auto seq = utf32(text); + SBAlgorithmRef algorithm = SBAlgorithmCreate(&seq); + EXPECT_EQ(SBAlgorithmRetain(algorithm), algorithm); + SBParagraphRef paragraph = SBAlgorithmCreateParagraph(algorithm, 0, 3, 0); + SBAlgorithmRelease(algorithm); + SBAlgorithmRelease(algorithm); // the paragraph keeps what it needs + EXPECT_EQ(SBParagraphRetain(paragraph), paragraph); + SBParagraphRelease(paragraph); + SBLineRef line = SBParagraphCreateLine(paragraph, 0, 3); + SBParagraphRelease(paragraph); + EXPECT_EQ(SBLineGetOffset(line), 0u); + EXPECT_EQ(SBLineGetLength(line), 3u); + EXPECT_EQ(SBLineRetain(line), line); + SBLineRelease(line); + // The locator retains the line it loads, so it outlives this release. + // (Release a mirror locator only after it has loaded a line: in 3.0.0 its + // finalizer releases the line unconditionally, and releasing NULL crashes.) + SBMirrorLocatorRef mirrors = SBMirrorLocatorCreate(); + SBMirrorLocatorLoadLine(mirrors, line, text.data()); + SBLineRelease(line); + EXPECT_EQ(SBMirrorLocatorRetain(mirrors), mirrors); + SBMirrorLocatorRelease(mirrors); + EXPECT_FALSE(SBMirrorLocatorMoveNext(mirrors)); // "abc": nothing mirrored + SBMirrorLocatorRelease(mirrors); +} + +namespace { + +struct Counting { + int blocks = 0; + int finalized = 0; +}; + +} // namespace + +TEST(Api, CustomAllocator) { + static Counting counting; + SBAllocatorProtocol protocol{}; + protocol.allocateBlock = [](SBUInteger size, void* info) -> void* { + ++static_cast(info)->blocks; + return std::malloc(size); + }; + protocol.reallocateBlock = [](void* p, SBUInteger size, void*) -> void* { return std::realloc(p, size); }; + protocol.deallocateBlock = [](void* p, void*) { std::free(p); }; + protocol.finalize = [](void* info) { ++static_cast(info)->finalized; }; + SBAllocatorRef allocator = SBAllocatorCreate(&protocol, &counting); + ASSERT_NE(allocator, nullptr); + SBAllocatorRef previous = SBAllocatorGetDefault(); + SBAllocatorSetDefault(allocator); + { + Laid laid(utf32(U"abc אבג"), 1); + EXPECT_EQ(digits(laid.levels()), "2221111"); + } + EXPECT_GT(counting.blocks, 0); + SBAllocatorSetDefault(previous); + EXPECT_EQ(SBAllocatorRetain(allocator), allocator); + SBAllocatorRelease(allocator); + SBAllocatorRelease(allocator); + EXPECT_EQ(counting.finalized, 1); +} diff --git a/tests/header_mix_test.cpp b/tests/header_mix_test.cpp new file mode 100644 index 0000000..6da94bd --- /dev/null +++ b/tests/header_mix_test.cpp @@ -0,0 +1,70 @@ +// The raw header and the import in one translation unit (the side header +// first, the canonical order): the module's names are the very same entities +// as the header's; the macros a C++ entity cannot be work beside them — the +// extern "C" brackets and SB_PUBLIC on a declaration of one's own, and +// SB_DEPRECATED. +#include +#include + +import std; +import mcpplibs.mwrap.sheenbidi; + +// A declaration of one's own in SheenBidi's style, inside its extern "C" brackets. +SB_EXTERN_C_BEGIN +SB_PUBLIC int mwrap_sheenbidi_test_entry(int value); +SB_EXTERN_C_END +int mwrap_sheenbidi_test_entry(int value) { return value + 1; } + +SB_DEPRECATED int mwrap_sheenbidi_old(int value); +int mwrap_sheenbidi_old(int value) { return value; } + +TEST(HeaderMix, SameEntities) { + // One declaration each, whichever way it was introduced. + static_assert(std::is_same_v); + static_assert(std::is_same_v<::SBLevel, std::uint8_t> && std::is_same_v<::SBUInteger, std::uintptr_t>); + static_assert(std::is_same_v<::SBRun, ::_SBRun> && sizeof(SBRun) == sizeof(::SBRun)); + EXPECT_EQ(&::SBCodepointGetBidiType, &SBCodepointGetBidiType); + EXPECT_EQ(::SBCodepointGetBidiType(0x05D0), SBBidiTypeR); // the header's enumerator + EXPECT_EQ(mwrap_sheenbidi_test_entry(41), 42); +} + +TEST(HeaderMix, EnumeratorsBesideTheConstants) { + namespace m = mcpplibs::mwrap::sheenbidi::macros; + // The header's enumerators (of an unnamed enum) and the module's constants + // have the same values; the constants have the header's typedef. + static_assert(m::SBBidiTypeAL == SBBidiTypeAL && m::SBScriptHEBR == SBScriptHEBR); + static_assert(std::is_same_v); + for (SBCodepoint c : {U'a', U'א', U'ا', U'1', U'١'}) { + const SBBidiType t = SBCodepointGetBidiType(c); + EXPECT_EQ(SBBidiTypeIsStrong(t), (m::SBBidiTypeIsStrong)(t)); + EXPECT_EQ(SBBidiTypeIsWeak(t), (m::SBBidiTypeIsWeak)(t)); + } +} + +TEST(HeaderMix, MacrosBesideTheMappings) { + namespace m = mcpplibs::mwrap::sheenbidi::macros; + // An object-like macro expands even in a qualified name: set it aside to + // name the constant. + constexpr auto raw = SBLevelDefaultLTR; +#pragma push_macro("SBLevelDefaultLTR") +#undef SBLevelDefaultLTR + static_assert(m::SBLevelDefaultLTR == raw); +#pragma pop_macro("SBLevelDefaultLTR") + // A function-like macro is not expanded when its name is followed by `)`. + EXPECT_EQ(SBCodepointIsValid(0x10FFFFu), (m::SBCodepointIsValid)(0x10FFFFu)); + // The experimental text API is compiled out, in the header as in the module. +#if SB_TEXT_API_SUPPORTED + FAIL() << "compat.sheenbidi is built without SB_CONFIG_EXPERIMENTAL_TEXT_API"; +#endif +} + +TEST(HeaderMix, DeprecatedFunction) { + // SBScriptGetOpenTypeTag is SB_DEPRECATED upstream; the module exports it + // as it is. Its replacement is SBScriptGetUnicodeTag. +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + EXPECT_EQ(SBScriptGetOpenTypeTag(SBScriptARAB), 0x61726162u); // 'arab' + EXPECT_EQ(mwrap_sheenbidi_old(7), 7); +#pragma GCC diagnostic pop + EXPECT_EQ(SBScriptGetUnicodeTag(SBScriptARAB), 0x41726162u); // 'Arab' +} diff --git a/tests/macros_mix_test.cpp b/tests/macros_mix_test.cpp new file mode 100644 index 0000000..fe1547e --- /dev/null +++ b/tests/macros_mix_test.cpp @@ -0,0 +1,461 @@ +// Generated by tools/gen_exports.py from the SheenBidi 3.0.0 headers — do not +// edit by hand. +// +// Every mapping against the raw header, in one translation unit that includes +// (that is, ) beside the +// import: +// +// - each constant has its macro's type and value (the macro is set aside +// with push_macro around the comparison: while it is defined it expands +// even inside `m::NAME`); +// - each enumerator constant has the value of the header's enumerator and +// the typedef the header gives it; +// - each function gives the macro's value and type for every argument of +// the sample sets (every 8-bit value, 16- and 32-bit boundaries, code +// points around the surrogates and the end of Unicode, ints); a +// function-like macro is not expanded when its name is followed by `)`, +// so `(m::NAME)(args)` calls the function beside the macro `NAME(args)`. +#include +#include + +import std; +import mcpplibs.mwrap.sheenbidi; + +namespace m = mcpplibs::mwrap::sheenbidi::macros; + +// ── constants ─────────────────────────────────────────────────────────────── +constexpr auto raw_SB_TEXT_API_SUPPORTED = SB_TEXT_API_SUPPORTED; +constexpr auto raw_SBLevelInvalid = SBLevelInvalid; +constexpr auto raw_SBLevelMax = SBLevelMax; +constexpr auto raw_SBLevelDefaultLTR = SBLevelDefaultLTR; +constexpr auto raw_SBLevelDefaultRTL = SBLevelDefaultRTL; +constexpr auto raw_SBCodepointInvalid = SBCodepointInvalid; +constexpr auto raw_SBCodepointFaulty = SBCodepointFaulty; +constexpr auto raw_SBCodepointMax = SBCodepointMax; +constexpr auto raw_SHEENBIDI_VERSION_MAJOR = SHEENBIDI_VERSION_MAJOR; +constexpr auto raw_SHEENBIDI_VERSION_MINOR = SHEENBIDI_VERSION_MINOR; +constexpr auto raw_SHEENBIDI_VERSION_PATCH = SHEENBIDI_VERSION_PATCH; +using raw_type_SHEENBIDI_VERSION_STRING = decltype(SHEENBIDI_VERSION_STRING); +constexpr std::string_view raw_SHEENBIDI_VERSION_STRING = SHEENBIDI_VERSION_STRING; +#pragma push_macro("SB_TEXT_API_SUPPORTED") +#undef SB_TEXT_API_SUPPORTED +static_assert(std::is_same_v && m::SB_TEXT_API_SUPPORTED == raw_SB_TEXT_API_SUPPORTED); +#pragma pop_macro("SB_TEXT_API_SUPPORTED") +#pragma push_macro("SBLevelInvalid") +#undef SBLevelInvalid +static_assert(std::is_same_v && m::SBLevelInvalid == raw_SBLevelInvalid); +#pragma pop_macro("SBLevelInvalid") +#pragma push_macro("SBLevelMax") +#undef SBLevelMax +static_assert(std::is_same_v && m::SBLevelMax == raw_SBLevelMax); +#pragma pop_macro("SBLevelMax") +#pragma push_macro("SBLevelDefaultLTR") +#undef SBLevelDefaultLTR +static_assert(std::is_same_v && m::SBLevelDefaultLTR == raw_SBLevelDefaultLTR); +#pragma pop_macro("SBLevelDefaultLTR") +#pragma push_macro("SBLevelDefaultRTL") +#undef SBLevelDefaultRTL +static_assert(std::is_same_v && m::SBLevelDefaultRTL == raw_SBLevelDefaultRTL); +#pragma pop_macro("SBLevelDefaultRTL") +#pragma push_macro("SBCodepointInvalid") +#undef SBCodepointInvalid +static_assert(std::is_same_v && m::SBCodepointInvalid == raw_SBCodepointInvalid); +#pragma pop_macro("SBCodepointInvalid") +#pragma push_macro("SBCodepointFaulty") +#undef SBCodepointFaulty +static_assert(std::is_same_v && m::SBCodepointFaulty == raw_SBCodepointFaulty); +#pragma pop_macro("SBCodepointFaulty") +#pragma push_macro("SBCodepointMax") +#undef SBCodepointMax +static_assert(std::is_same_v && m::SBCodepointMax == raw_SBCodepointMax); +#pragma pop_macro("SBCodepointMax") +#pragma push_macro("SHEENBIDI_VERSION_MAJOR") +#undef SHEENBIDI_VERSION_MAJOR +static_assert(std::is_same_v && m::SHEENBIDI_VERSION_MAJOR == raw_SHEENBIDI_VERSION_MAJOR); +#pragma pop_macro("SHEENBIDI_VERSION_MAJOR") +#pragma push_macro("SHEENBIDI_VERSION_MINOR") +#undef SHEENBIDI_VERSION_MINOR +static_assert(std::is_same_v && m::SHEENBIDI_VERSION_MINOR == raw_SHEENBIDI_VERSION_MINOR); +#pragma pop_macro("SHEENBIDI_VERSION_MINOR") +#pragma push_macro("SHEENBIDI_VERSION_PATCH") +#undef SHEENBIDI_VERSION_PATCH +static_assert(std::is_same_v && m::SHEENBIDI_VERSION_PATCH == raw_SHEENBIDI_VERSION_PATCH); +#pragma pop_macro("SHEENBIDI_VERSION_PATCH") +#pragma push_macro("SHEENBIDI_VERSION_STRING") +#undef SHEENBIDI_VERSION_STRING +static_assert(std::is_same_v, std::remove_cvref_t>); +static_assert(std::string_view(m::SHEENBIDI_VERSION_STRING) == raw_SHEENBIDI_VERSION_STRING); +#pragma pop_macro("SHEENBIDI_VERSION_STRING") + +// ── enumerators ───────────────────────────────────────────────────────────── +static_assert(std::is_same_v && m::SBFalse == ::SBFalse); +static_assert(std::is_same_v && m::SBTrue == ::SBTrue); +static_assert(std::is_same_v && m::SBBidiTypeNil == ::SBBidiTypeNil); +static_assert(std::is_same_v && m::SBBidiTypeL == ::SBBidiTypeL); +static_assert(std::is_same_v && m::SBBidiTypeR == ::SBBidiTypeR); +static_assert(std::is_same_v && m::SBBidiTypeAL == ::SBBidiTypeAL); +static_assert(std::is_same_v && m::SBBidiTypeBN == ::SBBidiTypeBN); +static_assert(std::is_same_v && m::SBBidiTypeNSM == ::SBBidiTypeNSM); +static_assert(std::is_same_v && m::SBBidiTypeAN == ::SBBidiTypeAN); +static_assert(std::is_same_v && m::SBBidiTypeEN == ::SBBidiTypeEN); +static_assert(std::is_same_v && m::SBBidiTypeET == ::SBBidiTypeET); +static_assert(std::is_same_v && m::SBBidiTypeES == ::SBBidiTypeES); +static_assert(std::is_same_v && m::SBBidiTypeCS == ::SBBidiTypeCS); +static_assert(std::is_same_v && m::SBBidiTypeWS == ::SBBidiTypeWS); +static_assert(std::is_same_v && m::SBBidiTypeS == ::SBBidiTypeS); +static_assert(std::is_same_v && m::SBBidiTypeB == ::SBBidiTypeB); +static_assert(std::is_same_v && m::SBBidiTypeON == ::SBBidiTypeON); +static_assert(std::is_same_v && m::SBBidiTypeLRI == ::SBBidiTypeLRI); +static_assert(std::is_same_v && m::SBBidiTypeRLI == ::SBBidiTypeRLI); +static_assert(std::is_same_v && m::SBBidiTypeFSI == ::SBBidiTypeFSI); +static_assert(std::is_same_v && m::SBBidiTypePDI == ::SBBidiTypePDI); +static_assert(std::is_same_v && m::SBBidiTypeLRE == ::SBBidiTypeLRE); +static_assert(std::is_same_v && m::SBBidiTypeRLE == ::SBBidiTypeRLE); +static_assert(std::is_same_v && m::SBBidiTypeLRO == ::SBBidiTypeLRO); +static_assert(std::is_same_v && m::SBBidiTypeRLO == ::SBBidiTypeRLO); +static_assert(std::is_same_v && m::SBBidiTypePDF == ::SBBidiTypePDF); +static_assert(std::is_same_v && m::SBStringEncodingUTF8 == ::SBStringEncodingUTF8); +static_assert(std::is_same_v && m::SBStringEncodingUTF16 == ::SBStringEncodingUTF16); +static_assert(std::is_same_v && m::SBStringEncodingUTF32 == ::SBStringEncodingUTF32); +static_assert(std::is_same_v && m::SBGeneralCategoryNil == ::SBGeneralCategoryNil); +static_assert(std::is_same_v && m::SBGeneralCategoryLU == ::SBGeneralCategoryLU); +static_assert(std::is_same_v && m::SBGeneralCategoryLL == ::SBGeneralCategoryLL); +static_assert(std::is_same_v && m::SBGeneralCategoryLT == ::SBGeneralCategoryLT); +static_assert(std::is_same_v && m::SBGeneralCategoryLM == ::SBGeneralCategoryLM); +static_assert(std::is_same_v && m::SBGeneralCategoryLO == ::SBGeneralCategoryLO); +static_assert(std::is_same_v && m::SBGeneralCategoryMN == ::SBGeneralCategoryMN); +static_assert(std::is_same_v && m::SBGeneralCategoryMC == ::SBGeneralCategoryMC); +static_assert(std::is_same_v && m::SBGeneralCategoryME == ::SBGeneralCategoryME); +static_assert(std::is_same_v && m::SBGeneralCategoryND == ::SBGeneralCategoryND); +static_assert(std::is_same_v && m::SBGeneralCategoryNL == ::SBGeneralCategoryNL); +static_assert(std::is_same_v && m::SBGeneralCategoryNO == ::SBGeneralCategoryNO); +static_assert(std::is_same_v && m::SBGeneralCategoryPC == ::SBGeneralCategoryPC); +static_assert(std::is_same_v && m::SBGeneralCategoryPD == ::SBGeneralCategoryPD); +static_assert(std::is_same_v && m::SBGeneralCategoryPS == ::SBGeneralCategoryPS); +static_assert(std::is_same_v && m::SBGeneralCategoryPE == ::SBGeneralCategoryPE); +static_assert(std::is_same_v && m::SBGeneralCategoryPI == ::SBGeneralCategoryPI); +static_assert(std::is_same_v && m::SBGeneralCategoryPF == ::SBGeneralCategoryPF); +static_assert(std::is_same_v && m::SBGeneralCategoryPO == ::SBGeneralCategoryPO); +static_assert(std::is_same_v && m::SBGeneralCategorySM == ::SBGeneralCategorySM); +static_assert(std::is_same_v && m::SBGeneralCategorySC == ::SBGeneralCategorySC); +static_assert(std::is_same_v && m::SBGeneralCategorySK == ::SBGeneralCategorySK); +static_assert(std::is_same_v && m::SBGeneralCategorySO == ::SBGeneralCategorySO); +static_assert(std::is_same_v && m::SBGeneralCategoryZS == ::SBGeneralCategoryZS); +static_assert(std::is_same_v && m::SBGeneralCategoryZL == ::SBGeneralCategoryZL); +static_assert(std::is_same_v && m::SBGeneralCategoryZP == ::SBGeneralCategoryZP); +static_assert(std::is_same_v && m::SBGeneralCategoryCC == ::SBGeneralCategoryCC); +static_assert(std::is_same_v && m::SBGeneralCategoryCF == ::SBGeneralCategoryCF); +static_assert(std::is_same_v && m::SBGeneralCategoryCS == ::SBGeneralCategoryCS); +static_assert(std::is_same_v && m::SBGeneralCategoryCO == ::SBGeneralCategoryCO); +static_assert(std::is_same_v && m::SBGeneralCategoryCN == ::SBGeneralCategoryCN); +static_assert(std::is_same_v && m::SBScriptNil == ::SBScriptNil); +static_assert(std::is_same_v && m::SBScriptZINH == ::SBScriptZINH); +static_assert(std::is_same_v && m::SBScriptZYYY == ::SBScriptZYYY); +static_assert(std::is_same_v && m::SBScriptZZZZ == ::SBScriptZZZZ); +static_assert(std::is_same_v && m::SBScriptARAB == ::SBScriptARAB); +static_assert(std::is_same_v && m::SBScriptARMN == ::SBScriptARMN); +static_assert(std::is_same_v && m::SBScriptBENG == ::SBScriptBENG); +static_assert(std::is_same_v && m::SBScriptBOPO == ::SBScriptBOPO); +static_assert(std::is_same_v && m::SBScriptCYRL == ::SBScriptCYRL); +static_assert(std::is_same_v && m::SBScriptDEVA == ::SBScriptDEVA); +static_assert(std::is_same_v && m::SBScriptGEOR == ::SBScriptGEOR); +static_assert(std::is_same_v && m::SBScriptGREK == ::SBScriptGREK); +static_assert(std::is_same_v && m::SBScriptGUJR == ::SBScriptGUJR); +static_assert(std::is_same_v && m::SBScriptGURU == ::SBScriptGURU); +static_assert(std::is_same_v && m::SBScriptHANG == ::SBScriptHANG); +static_assert(std::is_same_v && m::SBScriptHANI == ::SBScriptHANI); +static_assert(std::is_same_v && m::SBScriptHEBR == ::SBScriptHEBR); +static_assert(std::is_same_v && m::SBScriptHIRA == ::SBScriptHIRA); +static_assert(std::is_same_v && m::SBScriptKANA == ::SBScriptKANA); +static_assert(std::is_same_v && m::SBScriptKNDA == ::SBScriptKNDA); +static_assert(std::is_same_v && m::SBScriptLAOO == ::SBScriptLAOO); +static_assert(std::is_same_v && m::SBScriptLATN == ::SBScriptLATN); +static_assert(std::is_same_v && m::SBScriptMLYM == ::SBScriptMLYM); +static_assert(std::is_same_v && m::SBScriptORYA == ::SBScriptORYA); +static_assert(std::is_same_v && m::SBScriptTAML == ::SBScriptTAML); +static_assert(std::is_same_v && m::SBScriptTELU == ::SBScriptTELU); +static_assert(std::is_same_v && m::SBScriptTHAI == ::SBScriptTHAI); +static_assert(std::is_same_v && m::SBScriptTIBT == ::SBScriptTIBT); +static_assert(std::is_same_v && m::SBScriptBRAI == ::SBScriptBRAI); +static_assert(std::is_same_v && m::SBScriptCANS == ::SBScriptCANS); +static_assert(std::is_same_v && m::SBScriptCHER == ::SBScriptCHER); +static_assert(std::is_same_v && m::SBScriptETHI == ::SBScriptETHI); +static_assert(std::is_same_v && m::SBScriptKHMR == ::SBScriptKHMR); +static_assert(std::is_same_v && m::SBScriptMONG == ::SBScriptMONG); +static_assert(std::is_same_v && m::SBScriptMYMR == ::SBScriptMYMR); +static_assert(std::is_same_v && m::SBScriptOGAM == ::SBScriptOGAM); +static_assert(std::is_same_v && m::SBScriptRUNR == ::SBScriptRUNR); +static_assert(std::is_same_v && m::SBScriptSINH == ::SBScriptSINH); +static_assert(std::is_same_v && m::SBScriptSYRC == ::SBScriptSYRC); +static_assert(std::is_same_v && m::SBScriptTHAA == ::SBScriptTHAA); +static_assert(std::is_same_v && m::SBScriptYIII == ::SBScriptYIII); +static_assert(std::is_same_v && m::SBScriptDSRT == ::SBScriptDSRT); +static_assert(std::is_same_v && m::SBScriptGOTH == ::SBScriptGOTH); +static_assert(std::is_same_v && m::SBScriptITAL == ::SBScriptITAL); +static_assert(std::is_same_v && m::SBScriptBUHD == ::SBScriptBUHD); +static_assert(std::is_same_v && m::SBScriptHANO == ::SBScriptHANO); +static_assert(std::is_same_v && m::SBScriptTAGB == ::SBScriptTAGB); +static_assert(std::is_same_v && m::SBScriptTGLG == ::SBScriptTGLG); +static_assert(std::is_same_v && m::SBScriptCPRT == ::SBScriptCPRT); +static_assert(std::is_same_v && m::SBScriptLIMB == ::SBScriptLIMB); +static_assert(std::is_same_v && m::SBScriptLINB == ::SBScriptLINB); +static_assert(std::is_same_v && m::SBScriptOSMA == ::SBScriptOSMA); +static_assert(std::is_same_v && m::SBScriptSHAW == ::SBScriptSHAW); +static_assert(std::is_same_v && m::SBScriptTALE == ::SBScriptTALE); +static_assert(std::is_same_v && m::SBScriptUGAR == ::SBScriptUGAR); +static_assert(std::is_same_v && m::SBScriptBUGI == ::SBScriptBUGI); +static_assert(std::is_same_v && m::SBScriptCOPT == ::SBScriptCOPT); +static_assert(std::is_same_v && m::SBScriptGLAG == ::SBScriptGLAG); +static_assert(std::is_same_v && m::SBScriptKHAR == ::SBScriptKHAR); +static_assert(std::is_same_v && m::SBScriptSYLO == ::SBScriptSYLO); +static_assert(std::is_same_v && m::SBScriptTALU == ::SBScriptTALU); +static_assert(std::is_same_v && m::SBScriptTFNG == ::SBScriptTFNG); +static_assert(std::is_same_v && m::SBScriptXPEO == ::SBScriptXPEO); +static_assert(std::is_same_v && m::SBScriptBALI == ::SBScriptBALI); +static_assert(std::is_same_v && m::SBScriptNKOO == ::SBScriptNKOO); +static_assert(std::is_same_v && m::SBScriptPHAG == ::SBScriptPHAG); +static_assert(std::is_same_v && m::SBScriptPHNX == ::SBScriptPHNX); +static_assert(std::is_same_v && m::SBScriptXSUX == ::SBScriptXSUX); +static_assert(std::is_same_v && m::SBScriptCARI == ::SBScriptCARI); +static_assert(std::is_same_v && m::SBScriptCHAM == ::SBScriptCHAM); +static_assert(std::is_same_v && m::SBScriptKALI == ::SBScriptKALI); +static_assert(std::is_same_v && m::SBScriptLEPC == ::SBScriptLEPC); +static_assert(std::is_same_v && m::SBScriptLYCI == ::SBScriptLYCI); +static_assert(std::is_same_v && m::SBScriptLYDI == ::SBScriptLYDI); +static_assert(std::is_same_v && m::SBScriptOLCK == ::SBScriptOLCK); +static_assert(std::is_same_v && m::SBScriptRJNG == ::SBScriptRJNG); +static_assert(std::is_same_v && m::SBScriptSAUR == ::SBScriptSAUR); +static_assert(std::is_same_v && m::SBScriptSUND == ::SBScriptSUND); +static_assert(std::is_same_v && m::SBScriptVAII == ::SBScriptVAII); +static_assert(std::is_same_v && m::SBScriptARMI == ::SBScriptARMI); +static_assert(std::is_same_v && m::SBScriptAVST == ::SBScriptAVST); +static_assert(std::is_same_v && m::SBScriptBAMU == ::SBScriptBAMU); +static_assert(std::is_same_v && m::SBScriptEGYP == ::SBScriptEGYP); +static_assert(std::is_same_v && m::SBScriptJAVA == ::SBScriptJAVA); +static_assert(std::is_same_v && m::SBScriptKTHI == ::SBScriptKTHI); +static_assert(std::is_same_v && m::SBScriptLANA == ::SBScriptLANA); +static_assert(std::is_same_v && m::SBScriptLISU == ::SBScriptLISU); +static_assert(std::is_same_v && m::SBScriptMTEI == ::SBScriptMTEI); +static_assert(std::is_same_v && m::SBScriptORKH == ::SBScriptORKH); +static_assert(std::is_same_v && m::SBScriptPHLI == ::SBScriptPHLI); +static_assert(std::is_same_v && m::SBScriptPRTI == ::SBScriptPRTI); +static_assert(std::is_same_v && m::SBScriptSAMR == ::SBScriptSAMR); +static_assert(std::is_same_v && m::SBScriptSARB == ::SBScriptSARB); +static_assert(std::is_same_v && m::SBScriptTAVT == ::SBScriptTAVT); +static_assert(std::is_same_v && m::SBScriptBATK == ::SBScriptBATK); +static_assert(std::is_same_v && m::SBScriptBRAH == ::SBScriptBRAH); +static_assert(std::is_same_v && m::SBScriptMAND == ::SBScriptMAND); +static_assert(std::is_same_v && m::SBScriptCAKM == ::SBScriptCAKM); +static_assert(std::is_same_v && m::SBScriptMERC == ::SBScriptMERC); +static_assert(std::is_same_v && m::SBScriptMERO == ::SBScriptMERO); +static_assert(std::is_same_v && m::SBScriptPLRD == ::SBScriptPLRD); +static_assert(std::is_same_v && m::SBScriptSHRD == ::SBScriptSHRD); +static_assert(std::is_same_v && m::SBScriptSORA == ::SBScriptSORA); +static_assert(std::is_same_v && m::SBScriptTAKR == ::SBScriptTAKR); +static_assert(std::is_same_v && m::SBScriptAGHB == ::SBScriptAGHB); +static_assert(std::is_same_v && m::SBScriptBASS == ::SBScriptBASS); +static_assert(std::is_same_v && m::SBScriptDUPL == ::SBScriptDUPL); +static_assert(std::is_same_v && m::SBScriptELBA == ::SBScriptELBA); +static_assert(std::is_same_v && m::SBScriptGRAN == ::SBScriptGRAN); +static_assert(std::is_same_v && m::SBScriptHMNG == ::SBScriptHMNG); +static_assert(std::is_same_v && m::SBScriptKHOJ == ::SBScriptKHOJ); +static_assert(std::is_same_v && m::SBScriptLINA == ::SBScriptLINA); +static_assert(std::is_same_v && m::SBScriptMAHJ == ::SBScriptMAHJ); +static_assert(std::is_same_v && m::SBScriptMANI == ::SBScriptMANI); +static_assert(std::is_same_v && m::SBScriptMEND == ::SBScriptMEND); +static_assert(std::is_same_v && m::SBScriptMODI == ::SBScriptMODI); +static_assert(std::is_same_v && m::SBScriptMROO == ::SBScriptMROO); +static_assert(std::is_same_v && m::SBScriptNARB == ::SBScriptNARB); +static_assert(std::is_same_v && m::SBScriptNBAT == ::SBScriptNBAT); +static_assert(std::is_same_v && m::SBScriptPALM == ::SBScriptPALM); +static_assert(std::is_same_v && m::SBScriptPAUC == ::SBScriptPAUC); +static_assert(std::is_same_v && m::SBScriptPERM == ::SBScriptPERM); +static_assert(std::is_same_v && m::SBScriptPHLP == ::SBScriptPHLP); +static_assert(std::is_same_v && m::SBScriptSIDD == ::SBScriptSIDD); +static_assert(std::is_same_v && m::SBScriptSIND == ::SBScriptSIND); +static_assert(std::is_same_v && m::SBScriptTIRH == ::SBScriptTIRH); +static_assert(std::is_same_v && m::SBScriptWARA == ::SBScriptWARA); +static_assert(std::is_same_v && m::SBScriptAHOM == ::SBScriptAHOM); +static_assert(std::is_same_v && m::SBScriptHATR == ::SBScriptHATR); +static_assert(std::is_same_v && m::SBScriptHLUW == ::SBScriptHLUW); +static_assert(std::is_same_v && m::SBScriptHUNG == ::SBScriptHUNG); +static_assert(std::is_same_v && m::SBScriptMULT == ::SBScriptMULT); +static_assert(std::is_same_v && m::SBScriptSGNW == ::SBScriptSGNW); +static_assert(std::is_same_v && m::SBScriptADLM == ::SBScriptADLM); +static_assert(std::is_same_v && m::SBScriptBHKS == ::SBScriptBHKS); +static_assert(std::is_same_v && m::SBScriptMARC == ::SBScriptMARC); +static_assert(std::is_same_v && m::SBScriptNEWA == ::SBScriptNEWA); +static_assert(std::is_same_v && m::SBScriptOSGE == ::SBScriptOSGE); +static_assert(std::is_same_v && m::SBScriptTANG == ::SBScriptTANG); +static_assert(std::is_same_v && m::SBScriptGONM == ::SBScriptGONM); +static_assert(std::is_same_v && m::SBScriptNSHU == ::SBScriptNSHU); +static_assert(std::is_same_v && m::SBScriptSOYO == ::SBScriptSOYO); +static_assert(std::is_same_v && m::SBScriptZANB == ::SBScriptZANB); +static_assert(std::is_same_v && m::SBScriptDOGR == ::SBScriptDOGR); +static_assert(std::is_same_v && m::SBScriptGONG == ::SBScriptGONG); +static_assert(std::is_same_v && m::SBScriptMAKA == ::SBScriptMAKA); +static_assert(std::is_same_v && m::SBScriptMEDF == ::SBScriptMEDF); +static_assert(std::is_same_v && m::SBScriptROHG == ::SBScriptROHG); +static_assert(std::is_same_v && m::SBScriptSOGD == ::SBScriptSOGD); +static_assert(std::is_same_v && m::SBScriptSOGO == ::SBScriptSOGO); +static_assert(std::is_same_v && m::SBScriptELYM == ::SBScriptELYM); +static_assert(std::is_same_v && m::SBScriptHMNP == ::SBScriptHMNP); +static_assert(std::is_same_v && m::SBScriptNAND == ::SBScriptNAND); +static_assert(std::is_same_v && m::SBScriptWCHO == ::SBScriptWCHO); +static_assert(std::is_same_v && m::SBScriptCHRS == ::SBScriptCHRS); +static_assert(std::is_same_v && m::SBScriptDIAK == ::SBScriptDIAK); +static_assert(std::is_same_v && m::SBScriptKITS == ::SBScriptKITS); +static_assert(std::is_same_v && m::SBScriptYEZI == ::SBScriptYEZI); +static_assert(std::is_same_v && m::SBScriptCPMN == ::SBScriptCPMN); +static_assert(std::is_same_v && m::SBScriptOUGR == ::SBScriptOUGR); +static_assert(std::is_same_v && m::SBScriptTNSA == ::SBScriptTNSA); +static_assert(std::is_same_v && m::SBScriptTOTO == ::SBScriptTOTO); +static_assert(std::is_same_v && m::SBScriptVITH == ::SBScriptVITH); +static_assert(std::is_same_v && m::SBScriptKAWI == ::SBScriptKAWI); +static_assert(std::is_same_v && m::SBScriptNAGM == ::SBScriptNAGM); +static_assert(std::is_same_v && m::SBScriptGARA == ::SBScriptGARA); +static_assert(std::is_same_v && m::SBScriptGUKH == ::SBScriptGUKH); +static_assert(std::is_same_v && m::SBScriptKRAI == ::SBScriptKRAI); +static_assert(std::is_same_v && m::SBScriptONAO == ::SBScriptONAO); +static_assert(std::is_same_v && m::SBScriptSUNU == ::SBScriptSUNU); +static_assert(std::is_same_v && m::SBScriptTODR == ::SBScriptTODR); +static_assert(std::is_same_v && m::SBScriptTUTG == ::SBScriptTUTG); +static_assert(std::is_same_v && m::SBScriptBERF == ::SBScriptBERF); +static_assert(std::is_same_v && m::SBScriptSIDT == ::SBScriptSIDT); +static_assert(std::is_same_v && m::SBScriptTAYO == ::SBScriptTAYO); +static_assert(std::is_same_v && m::SBScriptTOLS == ::SBScriptTOLS); + +namespace { + +std::vector u8s() { // every 8-bit value: every bidi type, category, script + std::vector out; + for (int i = 0; i < 256; ++i) out.push_back(static_cast(i)); + return out; +} +std::vector u16s() { return {0, 1, 0x7f, 0xff, 0x100, 0xd7ff, 0xd800, 0xdbff, 0xdc00, 0xdfff, 0xe000, 0xfffd, 0xffff}; } +std::vector u32s() { + return {0u, 0x41u, 0x5d0u, 0xd7ffu, 0xd800u, 0xdbffu, 0xdc00u, 0xdfffu, 0xe000u, 0xfffdu, 0xffffu, + 0x10000u, 0x10940u, 0x10ffffu, 0x110000u, 0x7fffffffu, 0x80000000u, 0xffffffffu}; +} +std::vector ints() { return {0, 1, 2, 3, 4, 10, 14, 15, 23, 24, 125, 253, 254, 255, 256, -1, -2, 0xd800, 0x10ffff, 0x110000}; } + +// Calls `check` with every sample of every argument type. +template +void unary(F check) { + for (auto v : u8s()) check(v); + for (auto v : u16s()) check(v); + for (auto v : u32s()) check(v); + for (auto v : ints()) check(v); +} +template +void ternary(F check) { + auto over = [&](const auto& values) { + for (auto v : values) + for (auto s : values) + for (auto e : values) check(v, s, e); + }; + over(u16s()); + over(u32s()); + over(ints()); + over(std::vector{0, 1, 3, 4, 10, 11, 14, 15, 23, 24, 127, 128, 254, 255}); +} + +} // namespace + +// ── computation macros ────────────────────────────────────────────────────── +TEST(MacroFunctions, SBUInt8InRange) { + ternary([](auto v, auto s, auto e) { + static_assert(std::is_same_v); + EXPECT_EQ(SBUInt8InRange(v, s, e), (m::SBUInt8InRange)(v, s, e)); + }); +} +TEST(MacroFunctions, SBUInt16InRange) { + ternary([](auto v, auto s, auto e) { + static_assert(std::is_same_v); + EXPECT_EQ(SBUInt16InRange(v, s, e), (m::SBUInt16InRange)(v, s, e)); + }); +} +TEST(MacroFunctions, SBUInt32InRange) { + ternary([](auto v, auto s, auto e) { + static_assert(std::is_same_v); + EXPECT_EQ(SBUInt32InRange(v, s, e), (m::SBUInt32InRange)(v, s, e)); + }); +} +TEST(MacroFunctions, SBBidiTypeIsStrong) { + unary([](auto t) { + static_assert(std::is_same_v); + EXPECT_EQ(SBBidiTypeIsStrong(t), (m::SBBidiTypeIsStrong)(t)); + }); +} +TEST(MacroFunctions, SBBidiTypeIsWeak) { + unary([](auto t) { + static_assert(std::is_same_v); + EXPECT_EQ(SBBidiTypeIsWeak(t), (m::SBBidiTypeIsWeak)(t)); + }); +} +TEST(MacroFunctions, SBBidiTypeIsNeutral) { + unary([](auto t) { + static_assert(std::is_same_v); + EXPECT_EQ(SBBidiTypeIsNeutral(t), (m::SBBidiTypeIsNeutral)(t)); + }); +} +TEST(MacroFunctions, SBBidiTypeIsFormat) { + unary([](auto t) { + static_assert(std::is_same_v); + EXPECT_EQ(SBBidiTypeIsFormat(t), (m::SBBidiTypeIsFormat)(t)); + }); +} +TEST(MacroFunctions, SBCodepointIsSurrogate) { + unary([](auto c) { + static_assert(std::is_same_v); + EXPECT_EQ(SBCodepointIsSurrogate(c), (m::SBCodepointIsSurrogate)(c)); + }); +} +TEST(MacroFunctions, SBCodepointIsValid) { + unary([](auto c) { + static_assert(std::is_same_v); + EXPECT_EQ(SBCodepointIsValid(c), (m::SBCodepointIsValid)(c)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsLetter) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsLetter(gc), (m::SBGeneralCategoryIsLetter)(gc)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsMark) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsMark(gc), (m::SBGeneralCategoryIsMark)(gc)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsNumber) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsNumber(gc), (m::SBGeneralCategoryIsNumber)(gc)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsPunctuation) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsPunctuation(gc), (m::SBGeneralCategoryIsPunctuation)(gc)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsSymbol) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsSymbol(gc), (m::SBGeneralCategoryIsSymbol)(gc)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsSeparator) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsSeparator(gc), (m::SBGeneralCategoryIsSeparator)(gc)); + }); +} +TEST(MacroFunctions, SBGeneralCategoryIsOther) { + unary([](auto gc) { + static_assert(std::is_same_v); + EXPECT_EQ(SBGeneralCategoryIsOther(gc), (m::SBGeneralCategoryIsOther)(gc)); + }); +} diff --git a/tests/macros_test.cpp b/tests/macros_test.cpp new file mode 100644 index 0000000..b87ce1e --- /dev/null +++ b/tests/macros_test.cpp @@ -0,0 +1,63 @@ +// The macros module on its own (no header, no root module): the constants, +// enumerators and functions are constant expressions, as the macros and +// enumerators are, so they work in constant-initialised tables, case labels +// and static_asserts. +import std; +import mcpplibs.mwrap.sheenbidi.macros; + +namespace macros = mcpplibs::mwrap::sheenbidi::macros; +using namespace mcpplibs::mwrap::sheenbidi::macros; + +// Every constant has the type its macro expands to. +static_assert(std::is_same_v); // 125 +static_assert(std::is_same_v); // 0xFF +static_assert(std::is_same_v); // UINT32_MAX +static_assert(std::is_same_v); +static_assert(SBLevelMax == 125 && SBLevelInvalid == 0xFF); +static_assert(SBLevelDefaultLTR == 0xFE && SBLevelDefaultRTL == 0xFD); +static_assert(SBCodepointMax == 0x10FFFF && SBCodepointFaulty == 0xFFFD); +static_assert(std::string_view(SHEENBIDI_VERSION_STRING) == "3.0.0" && SHEENBIDI_VERSION_MINOR == 0); +static_assert(SB_TEXT_API_SUPPORTED == 0); // compat.sheenbidi: the experimental text API is off + +// Every enumerator is a constant of the typedef the header gives its values. +static_assert(std::is_same_v); // SBBidiType +static_assert(std::is_same_v); // SBStringEncoding +static_assert(std::is_same_v); // SBBoolean +static_assert(SBBidiTypeNil == 0 && SBBidiTypeL == 1 && SBBidiTypeAL == 3 && SBBidiTypePDF == 0x17); +static_assert(SBGeneralCategoryLU == 1 && SBScriptARAB == 4 && SBScriptTOLS == 0xAF); // Tolong Siki: Unicode 17.0 +static_assert(SBStringEncodingUTF8 == 0 && SBStringEncodingUTF32 == 2 && SBTrue == 1 && SBFalse == 0); + +// The functions, at compile time. +static_assert(SBBidiTypeIsStrong(SBBidiTypeR) && !SBBidiTypeIsStrong(SBBidiTypeEN)); +static_assert(SBBidiTypeIsWeak(SBBidiTypeCS) && SBBidiTypeIsNeutral(SBBidiTypeWS) && SBBidiTypeIsFormat(SBBidiTypeLRI)); +static_assert(SBCodepointIsSurrogate(0xD800u) && !SBCodepointIsValid(0xDFFFu) && !SBCodepointIsValid(0x110000u)); +static_assert(SBCodepointIsValid(0x10FFFFu) && SBCodepointIsValid(0u)); +static_assert(SBGeneralCategoryIsLetter(SBGeneralCategoryLO) && SBGeneralCategoryIsMark(SBGeneralCategoryMN)); +static_assert(SBGeneralCategoryIsNumber(SBGeneralCategoryND) && SBGeneralCategoryIsOther(SBGeneralCategoryCN)); +static_assert(SBUInt8InRange(5, 3, 7) && !SBUInt8InRange(2, 3, 7) && SBUInt32InRange(0x10940u, 0x10940u, 0x1095Fu)); +static_assert(std::is_same_v); + +// A constant-initialised table and a switch, as the enumerators allow. +struct Class { + std::uint8_t type; + bool strong; +}; +constexpr Class classes[] = { + {SBBidiTypeL, SBBidiTypeIsStrong(SBBidiTypeL)}, + {SBBidiTypeAN, SBBidiTypeIsStrong(SBBidiTypeAN)}, +}; +static_assert(classes[0].strong && !classes[1].strong); + +constexpr const char* name(std::uint8_t type) { + switch (type) { + case SBBidiTypeL: return "L"; + case SBBidiTypeR: return "R"; + case SBBidiTypeAL: return "AL"; + default: return "other"; + } +} +static_assert(std::string_view(name(SBBidiTypeAL)) == "AL"); + +int main() { + return macros::SHEENBIDI_VERSION_MAJOR == 3 && macros::SBLevelMax == 125 ? 0 : 1; +} diff --git a/tools/candidate_index.py b/tools/candidate_index.py new file mode 100644 index 0000000..68d202e --- /dev/null +++ b/tools/candidate_index.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""The compat namespace from mcpp-index's candidate branch, while the published +index does not yet carry compat.sheenbidi. + +compat.sheenbidi is added to mcpp-index by mcpplibs/mcpp-index#607. Until that +descriptor is on the index's main branch, a build of this repository cannot +resolve its one dependency from the published index. This tool then clones +the PR's branch, prepares the unchanged namespace views of its descriptors +(the index's own tests/prepare_mwrap_indices.py) and prints the checkout. + + --checkout-only print the candidate checkout (nothing once published), + for tools/test_template.py --index + --ci also append `[indices] compat = { path = }` to this + checkout's manifests (the root and examples/sheenbidi), in + CI's working copy only + +Once the published index has compat.sheenbidi, it does nothing. +""" +import argparse +from pathlib import Path +import subprocess +import sys +import urllib.error +import urllib.request + +INDEX = 'https://github.com/mcpplibs/mcpp-index' +BRANCH = 'compat-sheenbidi' +PUBLISHED = 'https://raw.githubusercontent.com/mcpplibs/mcpp-index/main/pkgs/c/compat.sheenbidi.lua' +MANIFESTS = ['mcpp.toml', 'examples/sheenbidi/mcpp.toml'] + + +def published() -> bool: + try: + with urllib.request.urlopen(PUBLISHED, timeout=60) as response: + return response.status == 200 + except urllib.error.HTTPError as error: + if error.code == 404: + return False + raise + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument('--checkout-only', action='store_true') + mode.add_argument('--ci', action='store_true') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + if published(): + print('compat.sheenbidi is published: the published index is used', file=sys.stderr) + return + checkout = root / 'target/candidate-index' + if not (checkout / 'index.toml').exists(): + subprocess.run(['git', 'clone', '--quiet', '--depth', '1', '--branch', BRANCH, INDEX + '.git', str(checkout)], + check=True, stdout=sys.stderr) + subprocess.run([sys.executable, str(checkout / 'tests/prepare_mwrap_indices.py')], cwd=checkout, check=True, + stdout=sys.stderr) + view = checkout / 'tests/mwrap/.indices/compat' + if not (view / 'pkgs/view/compat.sheenbidi.lua').exists(): + sys.exit(f'{INDEX} {BRANCH} has no compat.sheenbidi') + if args.ci: + for manifest in MANIFESTS: + with (root / manifest).open('a', encoding='utf-8', newline='\n') as out: + out.write('\n# CI only (tools/candidate_index.py): compat.sheenbidi from ' + BRANCH + '\n') + out.write('[indices]\ncompat = { path = "' + view.as_posix() + '" }\n') + print('candidate compat index for', manifest, file=sys.stderr) + print(checkout.as_posix()) + + +if __name__ == '__main__': + main() diff --git a/tools/check_exports.py b/tools/check_exports.py new file mode 100644 index 0000000..d4ebc9c --- /dev/null +++ b/tools/check_exports.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Audit the module surface against the pinned release's public SheenBidi headers. + +- src/sheenbidi.cppm, src/macros.cppm and tests/macros_mix_test.cpp are what + tools/gen_exports.py writes. +- Every type, struct tag and SB_PUBLIC function of the installed headers is + exported by `export using`, and nothing else is. +- Every enumerator of the headers' unnamed enums is a constant of its + typedef in src/macros.cppm. +- Every macro of the headers is mapped in src/macros.cppm (a constant or a + function), kept as a macro in with its + reason, or an include guard; and each mapping, enumerators included, is + checked against the raw header by tests/macros_mix_test.cpp. +""" +from pathlib import Path +import re +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from inventory import fetch, surface # noqa: E402 +from gen_exports import SIDE_HEADER, excluded # noqa: E402 + + +def mapped_names(text: str) -> set[str]: + text = text[text.index('export module'):] + names = set(re.findall(r'^inline constexpr auto&? (\w+) = c::value_\1;', text, flags=re.M)) + names |= set(re.findall(r'^constexpr auto \((\w+)\)\(', text, flags=re.M)) + return names + + +def main(): + root = Path(__file__).resolve().parents[1] + problems = [] + generated = subprocess.run([sys.executable, str(root / 'tools/gen_exports.py'), '--check'], capture_output=True, text=True) + if generated.returncode != 0: + problems.append(generated.stderr.strip() or generated.stdout.strip()) + sf = surface(fetch(root)) + module = (root / 'src/sheenbidi.cppm').read_text(encoding='utf-8') + exported = set(re.findall(r'^export using ::(\w+);', module, flags=re.M)) + declared = set(sf.types) | set(sf.tags) | set(sf.functions) + if declared - exported: + problems.append('src/sheenbidi.cppm lacks: ' + ', '.join(sorted(declared - exported))) + if exported - declared: + problems.append('src/sheenbidi.cppm exports what the headers do not declare: ' + ', '.join(sorted(exported - declared))) + + macros_text = (root / 'src/macros.cppm').read_text(encoding='utf-8') + enumerators = {m.group(2): m.group(1) # enumerator: its typedef + for m in re.finditer(r'^inline constexpr ::(\w+) (\w+) = ::\2;', macros_text, flags=re.M)} + if enumerators != sf.enumerators: + missing = sf.enumerators.keys() - enumerators.keys() + wrong = {n for n in sf.enumerators.keys() & enumerators.keys() if sf.enumerators[n] != enumerators[n]} + problems.append(f'enumerators: missing {sorted(missing)}, of the wrong type {sorted(wrong)}, ' + f'extra {sorted(enumerators.keys() - sf.enumerators.keys())}') + + mapped = mapped_names(macros_text) + macros = set(sf.macros) + guards = {m for m in macros if excluded(m)} + unaccounted = macros - mapped - SIDE_HEADER.keys() - guards + if unaccounted: + problems.append('macros neither mapped, side-header nor include guards: ' + ', '.join(sorted(unaccounted))) + both = SIDE_HEADER.keys() & mapped + if both: + problems.append('listed as side-header yet mapped: ' + ', '.join(sorted(both))) + stale = SIDE_HEADER.keys() - macros + if stale: + problems.append('listed but not in the headers: ' + ', '.join(sorted(stale))) + test = (root / 'tests/macros_mix_test.cpp').read_text(encoding='utf-8') + untested = {n for n in mapped | enumerators.keys() if f'm::{n}' not in test} + if untested: + problems.append('mapped but not checked against the header: ' + ', '.join(sorted(untested))) + side = (root / 'include/mwrap/sheenbidi/macros.hpp').read_text(encoding='utf-8') + if '#include ' not in side: + problems.append(' does not include ') + if problems: + sys.exit('\n'.join(problems)) + functions = sum(1 for m in sf.macros.values() if m.name in mapped and m.params is not None) + print(f'{len(sf.types)} types, {len(sf.tags)} struct tags and {len(sf.functions)} functions exported; ' + f'{len(enumerators)} enumerators of unnamed enums as constants of their typedefs; {len(macros)} macros: ' + f'{len(mapped) - functions} constants and {functions} functions mapped (each checked against the header), ' + f'{len(SIDE_HEADER)} kept as macros, {len(guards)} include guards') + + +if __name__ == '__main__': + main() diff --git a/tools/check_mwrap_release.py b/tools/check_mwrap_release.py new file mode 100644 index 0000000..9bedddb --- /dev/null +++ b/tools/check_mwrap_release.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Check the immutable mwrap release coordinate and bilingual entry pages.""" +import argparse +import hashlib +import json +from pathlib import Path +import re +import tomllib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--tag', help='release tag; omitted during PR checks') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + package = tomllib.loads((root / 'mcpp.toml').read_text(encoding="utf-8"))['package'] + assert package['namespace'] == 'mcpplibs.mwrap', package + assert re.fullmatch(r'[A-Za-z_][A-Za-z_0-9]*', package['name']), package['name'] + upstream = package['metadata']['mwrap']['upstream-version'] + version = package['version'] + assert version == upstream or re.fullmatch(re.escape(upstream) + r'\.[1-9][0-9]*', version), (version, upstream) + if args.tag: + assert args.tag == 'mwrap-' + version, (args.tag, version) + assert package['license'] == 'Apache-2.0', package['license'] + assert package['authors'] == ['mcpp-community'], package.get('authors') + license_text = (root / 'LICENSE').read_text(encoding='utf-8') + assert 'Apache License' in license_text and 'Version 2.0' in license_text + assert 'mcpp-community' in (root / 'NOTICE').read_text(encoding='utf-8') + documents = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8'))['documents'] + assert any(Path(d['path']).name.startswith('README') for d in documents) + assert any(Path(d['path']).name.startswith(('LICENSE', 'COPYING')) for d in documents) + deployed = {entry['from'] for entry in tomllib.loads((root / 'mcpp.toml').read_text(encoding='utf-8')).get('runtime', {}).get('deploy', [])} + assert {'LICENSE', 'NOTICE'} <= deployed + for document in documents: + path = Path(document['path']) + assert not path.is_absolute() and '..' not in path.parts and path.parts[0] == 'upstream' + assert hashlib.sha256((root / path).read_bytes()).hexdigest() == document['sha256'], path + assert document['source'] and path.as_posix() in deployed, path + identity = package['namespace'] + '.' + package['name'] + for name, other in [('README.md', 'README.zh.md'), ('README.zh.md', 'README.md')]: + text = (root / name).read_text(encoding="utf-8") + assert len(text.splitlines()) <= 150, name + assert '](' + other + ')' in '\n'.join(text.splitlines()[:12]), name + assert identity in text and version in text, name + assert text.count('```') % 2 == 0, name + print(identity + '@' + version) + + +if __name__ == '__main__': + main() diff --git a/tools/gen_exports.py b/tools/gen_exports.py new file mode 100644 index 0000000..3f813f3 --- /dev/null +++ b/tools/gen_exports.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Write the module sources and the generated macro test from the SheenBidi headers. + +- src/sheenbidi.cppm exports every type, struct tag and SB_PUBLIC function of + the installed headers by `export using` (the upstream globals, unchanged), + and re-exports the macros module. +- src/macros.cppm holds, in mcpplibs::mwrap::sheenbidi::macros under their + own names: + * the object-like macros (SBLevelInvalid, SBLevelMax, SBLevelDefaultLTR, + SBCodepointInvalid, SHEENBIDI_VERSION_* …) as `inline constexpr` of the + type of the macro's own expansion (the version string as a reference to + a constexpr char array); + * the function-like macros (SBBidiTypeIsStrong, SBCodepointIsValid, + SBGeneralCategoryIsLetter, SBUInt8InRange …) as constexpr function + templates whose body *is* the macro, expanded on the parameters; + * the enumerators of the headers' unnamed enums (SBBidiType*, + SBGeneralCategory*, SBScript*, SBStringEncoding*, SBTrue / SBFalse): + an unnamed enumeration is TU-local, so its enumerators cannot be + exported as they are. Each is a constant of the typedef the header + gives its values (`typedef SBUInt8 SBBidiType;` follows the enum). +- tests/macros_mix_test.cpp checks each mapping against the raw header in a + translation unit that includes it beside the import. + +With --check, nothing is written and a difference fails. +""" +import argparse +from pathlib import Path +import re +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from inventory import fetch, surface # noqa: E402 + +# Macros that stay macros, in (which is ). +SIDE_HEADER = { + 'SB_PUBLIC': 'an attribute of declarations (dllimport/dllexport under SB_CONFIG_DLL_*)', + 'SB_DEPRECATED': 'an attribute of declarations', + 'SB_EXTERN_C_BEGIN': 'opens an extern "C" block', + 'SB_EXTERN_C_END': 'closes an extern "C" block', +} + + +def excluded(name: str) -> str | None: + if re.fullmatch(r'_SB_PUBLIC_\w+_H|_SHEENBIDI_H', name): + return 'include guard' + return None + + +def classify(sf): + strings, constants, functions = [], [], [] + for macro in sf.macros.values(): + if macro.name in SIDE_HEADER or excluded(macro.name): + continue + if macro.params is not None: + functions.append(macro) + elif macro.body.startswith('"'): + strings.append(macro.name) + else: + constants.append(macro.name) + return strings, constants, functions + + +GUARD = '#include ' + +ROOT_HEADER = '''// Generated by tools/gen_exports.py from the SheenBidi 3.0.0 headers — do not +// edit by hand. +// +// mcpplibs.mwrap.sheenbidi — SheenBidi, the Unicode Bidirectional Algorithm +// (UAX #9, Unicode 17.0.0), as a C++ module. Every exported name is the +// upstream global of , unchanged: the types and struct +// tags, and every SB_PUBLIC function (SBAlgorithm, SBParagraph, SBLine, +// SBMirrorLocator, SBScriptLocator, SBCodepoint, SBAllocator, SBVersion). The +// root also re-exports the macro and enumerator mappings. The library is +// compat.sheenbidi, built from the same release. +module; +''' + GUARD + ''' + +export module mcpplibs.mwrap.sheenbidi; + +export import mcpplibs.mwrap.sheenbidi.macros; +''' + +MACROS_HEADER = '''// Generated by tools/gen_exports.py from the SheenBidi 3.0.0 headers — do not +// edit by hand. +// +// mcpplibs.mwrap.sheenbidi.macros — what declares that +// a named module cannot export as it is, under the original names in +// mcpplibs::mwrap::sheenbidi::macros: +// +// - object-like macros (SBLevelInvalid, SBLevelMax, SBLevelDefaultLTR, +// SBLevelDefaultRTL, SBCodepointInvalid, SBCodepointFaulty, +// SBCodepointMax, SB_TEXT_API_SUPPORTED, SHEENBIDI_VERSION_*) as `inline +// constexpr`, each of the type its macro expands to (SBLevelMax is an +// `int`, as 125 is; SBCodepointInvalid a `uint32_t`, as UINT32_MAX is), the +// version string as a reference to a constexpr char array; +// - function-like macros (SBUInt8InRange, SBBidiTypeIsStrong, +// SBCodepointIsValid, SBGeneralCategoryIsLetter …) as constexpr function +// templates. Each body is the macro itself, expanded on the parameters +// while the header's macros are defined (the declarator is parenthesised +// so the name is not expanded): the result has the macro's value and +// type for every argument type. Each argument is evaluated once, before +// the call; +// - the enumerators of the headers' unnamed enums (SBBidiType*, +// SBGeneralCategory*, SBScript*, SBStringEncoding*, SBTrue, SBFalse). An +// unnamed enumeration is TU-local, so they cannot be exported as they are: +// each is a constant of the typedef the header gives its values (the enum +// of SBBidiTypeL is followed by `typedef SBUInt8 SBBidiType;`). +// +// Attributes and the extern "C" brackets stay macros, in +// , included before the import. +// tools/check_exports.py accounts for every macro. +module; +''' + GUARD + ''' +''' + + +def render_root(sf) -> str: + out = [ROOT_HEADER] + out.append(f'// types ({len(sf.types)})') + out += [f'export using ::{n};' for n in sf.types] + out.append(f'\n// struct tags ({len(sf.tags)}; the opaque ones are declared, not defined)') + out += [f'export using ::{n};' for n in sf.tags] + out.append(f'\n// functions ({len(sf.functions)}; {", ".join(sf.deprecated)} is SB_DEPRECATED upstream)') + out += [f'export using ::{n};' for n in sf.functions] + return '\n'.join(out) + '\n' + + +def function(macro) -> list[str]: + tparams = ', '.join(f'class T{i}' for i in range(len(macro.params))) + params = ', '.join(f'T{i} {p}' for i, p in enumerate(macro.params)) + call = f'{macro.name}({", ".join(macro.params)})' + return [f'template <{tparams}>', f'constexpr auto ({macro.name})({params}) noexcept {{ return {call}; }}'] + + +def render_macros(sf) -> str: + strings, constants, functions = classify(sf) + out = [MACROS_HEADER] + out.append('export module mcpplibs.mwrap.sheenbidi.macros;\n') + out.append('// The header\'s values, taken while its macros are defined (module linkage).') + out.append('namespace mwrap_sheenbidi_capture {') + out += [f'inline constexpr auto value_{n} = {n};' for n in constants] + out += [f'inline constexpr char value_{n}[] = {n};' for n in strings] + out.append('} // namespace mwrap_sheenbidi_capture\n') + out.append('// ── computation macros, defined while the header\'s macros are ──────────────') + out.append('export namespace mcpplibs::mwrap::sheenbidi::macros {\n') + for macro in functions: + out.append(f'// #define {macro.name}({", ".join(macro.params)}) {macro.body}') + out += function(macro) + out.append('\n} // namespace mcpplibs::mwrap::sheenbidi::macros\n') + out.append('// Every mapped object-like name, so the definitions below are not rewritten.') + out += [f'#undef {n}' for n in constants + strings] + out.append('\nnamespace c = mwrap_sheenbidi_capture;\n') + out.append('export namespace mcpplibs::mwrap::sheenbidi::macros {\n') + out.append('// ── constants ───────────────────────────────────────────────────────────────') + out += [f'inline constexpr auto {n} = c::value_{n};' for n in constants] + out.append('\n// ── strings ─────────────────────────────────────────────────────────────────') + out += [f'inline constexpr auto& {n} = c::value_{n};' for n in strings] + by_type: dict[str, list[str]] = {} + for e, t in sf.enumerators.items(): + by_type.setdefault(t, []).append(e) + for t, names in by_type.items(): + out.append(f'\n// ── enumerators of the unnamed enum whose values are {t} ({len(names)}) ' + '─' * max(0, 20 - len(t))) + out += [f'inline constexpr ::{t} {n} = ::{n};' for n in names] + out.append('\n} // namespace mcpplibs::mwrap::sheenbidi::macros') + return '\n'.join(out) + '\n' + + +TEST_HEADER = '''// Generated by tools/gen_exports.py from the SheenBidi 3.0.0 headers — do not +// edit by hand. +// +// Every mapping against the raw header, in one translation unit that includes +// (that is, ) beside the +// import: +// +// - each constant has its macro's type and value (the macro is set aside +// with push_macro around the comparison: while it is defined it expands +// even inside `m::NAME`); +// - each enumerator constant has the value of the header's enumerator and +// the typedef the header gives it; +// - each function gives the macro's value and type for every argument of +// the sample sets (every 8-bit value, 16- and 32-bit boundaries, code +// points around the surrogates and the end of Unicode, ints); a +// function-like macro is not expanded when its name is followed by `)`, +// so `(m::NAME)(args)` calls the function beside the macro `NAME(args)`. +#include +#include + +import std; +import mcpplibs.mwrap.sheenbidi; + +namespace m = mcpplibs::mwrap::sheenbidi::macros; +''' + +TEST_SAMPLES = ''' +namespace { + +std::vector u8s() { // every 8-bit value: every bidi type, category, script + std::vector out; + for (int i = 0; i < 256; ++i) out.push_back(static_cast(i)); + return out; +} +std::vector u16s() { return {0, 1, 0x7f, 0xff, 0x100, 0xd7ff, 0xd800, 0xdbff, 0xdc00, 0xdfff, 0xe000, 0xfffd, 0xffff}; } +std::vector u32s() { + return {0u, 0x41u, 0x5d0u, 0xd7ffu, 0xd800u, 0xdbffu, 0xdc00u, 0xdfffu, 0xe000u, 0xfffdu, 0xffffu, + 0x10000u, 0x10940u, 0x10ffffu, 0x110000u, 0x7fffffffu, 0x80000000u, 0xffffffffu}; +} +std::vector ints() { return {0, 1, 2, 3, 4, 10, 14, 15, 23, 24, 125, 253, 254, 255, 256, -1, -2, 0xd800, 0x10ffff, 0x110000}; } + +// Calls `check` with every sample of every argument type. +template +void unary(F check) { + for (auto v : u8s()) check(v); + for (auto v : u16s()) check(v); + for (auto v : u32s()) check(v); + for (auto v : ints()) check(v); +} +template +void ternary(F check) { + auto over = [&](const auto& values) { + for (auto v : values) + for (auto s : values) + for (auto e : values) check(v, s, e); + }; + over(u16s()); + over(u32s()); + over(ints()); + over(std::vector{0, 1, 3, 4, 10, 11, 14, 15, 23, 24, 127, 128, 254, 255}); +} + +} // namespace +''' + + +def render_test(sf) -> str: + strings, constants, functions = classify(sf) + out = [TEST_HEADER] + out.append('// ── constants ───────────────────────────────────────────────────────────────') + for n in constants: + out.append(f'constexpr auto raw_{n} = {n};') + for n in strings: + out.append(f'using raw_type_{n} = decltype({n});') + out.append(f'constexpr std::string_view raw_{n} = {n};') + for n in constants: + out += [f'#pragma push_macro("{n}")', f'#undef {n}', + f'static_assert(std::is_same_v && m::{n} == raw_{n});', + f'#pragma pop_macro("{n}")'] + for n in strings: + out += [f'#pragma push_macro("{n}")', f'#undef {n}', + f'static_assert(std::is_same_v, std::remove_cvref_t>);', + f'static_assert(std::string_view(m::{n}) == raw_{n});', + f'#pragma pop_macro("{n}")'] + out.append('\n// ── enumerators ─────────────────────────────────────────────────────────────') + for n, t in sf.enumerators.items(): + out.append(f'static_assert(std::is_same_v && m::{n} == ::{n});') + out.append(TEST_SAMPLES) + out.append('// ── computation macros ──────────────────────────────────────────────────────') + for macro in functions: + params = ', '.join(f'auto {p}' for p in macro.params) + args = ', '.join(macro.params) + arity = {1: 'unary', 3: 'ternary'}[len(macro.params)] + out.append(f'TEST(MacroFunctions, {macro.name}) {{') + out.append(f' {arity}([]({params}) {{') + out += [f' static_assert(std::is_same_v);', + f' EXPECT_EQ({macro.name}({args}), (m::{macro.name})({args}));'] + out.append(' });') + out.append('}') + return '\n'.join(out) + '\n' + + +def outputs(root: Path) -> dict[Path, str]: + sf = surface(fetch(root)) + return { + root / 'src/sheenbidi.cppm': render_root(sf), + root / 'src/macros.cppm': render_macros(sf), + root / 'tests/macros_mix_test.cpp': render_test(sf), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--check', action='store_true') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + stale = [] + for path, text in outputs(root).items(): + # Text mode: a checkout with CRLF line ends (Windows) reads the same. + current = path.read_text(encoding='utf-8') if path.exists() else None + if current == text: + continue # unchanged: the timestamp is kept, nothing rebuilds + if args.check: + stale.append(path.relative_to(root).as_posix()) + else: + path.write_text(text, encoding='utf-8', newline='\n') + print('wrote', path.relative_to(root).as_posix()) + if stale: + sys.exit('not what tools/gen_exports.py writes: ' + ', '.join(stale)) + + +if __name__ == '__main__': + main() diff --git a/tools/inventory.py b/tools/inventory.py new file mode 100644 index 0000000..369d31f --- /dev/null +++ b/tools/inventory.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +"""The public surface of SheenBidi 3.0.0, from the pinned release. + +Shared by tools/gen_exports.py (which writes the module sources and the +generated macro test), tools/check_exports.py (which audits them) and +tools/run_conformance.py (which needs the release's Unicode test files). The +headers come from the upstream tag archive, downloaded once into +target/upstream and checked against upstream/documents.json: the same bytes +compat.sheenbidi builds. + +The public headers are the ones upstream installs (CMakeLists.txt, +SHEENBIDI_HEADERS; meson.build installs the same list). SBConfig.h, beside +them in Headers/, is not installed and no public header includes it: it is +the build's own switch file, and its two scratch-buffer sizes are internal. + +The headers are read as this build compiles them: compat.sheenbidi does not +define SB_CONFIG_EXPERIMENTAL_TEXT_API (upstream's default), so +SB_TEXT_API_SUPPORTED is 0 and every `#if SB_TEXT_API_SUPPORTED` block — the +experimental SBText, attribute and run-iterator API — declares nothing. +""" +from dataclasses import dataclass, field +import hashlib +import json +from pathlib import Path +import re +import sys +import tarfile +import urllib.request + +INSTALLED = [ # CMakeLists.txt: SHEENBIDI_HEADERS, in its order + 'SheenBidi.h', 'SBAlgorithm.h', 'SBAllocator.h', 'SBAttributeInfo.h', 'SBAttributeList.h', + 'SBAttributeRegistry.h', 'SBBase.h', 'SBBidiType.h', 'SBCodepoint.h', 'SBCodepointSequence.h', + 'SBGeneralCategory.h', 'SBLine.h', 'SBMirrorLocator.h', 'SBParagraph.h', 'SBRun.h', 'SBScript.h', + 'SBScriptLocator.h', 'SBText.h', 'SBTextConfig.h', 'SBTextIterators.h', 'SBTextType.h', 'SBVersion.h', +] +TEXT_API = '#if SB_TEXT_API_SUPPORTED' + + +def fetch(root: Path) -> Path: + pins = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8')) + cache = root / 'target/upstream' + cache.mkdir(parents=True, exist_ok=True) + archive = cache / ('SheenBidi-' + pins['upstream_version'] + '.tar.gz') + if not archive.exists() or hashlib.sha256(archive.read_bytes()).hexdigest() != pins['upstream_sha256']: + print('+ fetch', pins['upstream'], flush=True) + with urllib.request.urlopen(pins['upstream'], timeout=120) as response: + archive.write_bytes(response.read()) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + if digest != pins['upstream_sha256']: + sys.exit(f'upstream archive checksum {digest} != {pins["upstream_sha256"]}') + with tarfile.open(archive) as tar: + top = tar.getmembers()[0].name.split('/')[0] + if not (cache / top / 'Headers/SheenBidi/SheenBidi.h').exists(): + tar.extractall(cache, filter='data') + return cache / top + + +@dataclass +class Macro: + name: str + header: str + params: list[str] | None # None: object-like + body: str + + +@dataclass +class Surface: + types: list[str] = field(default_factory=list) # typedef names + tags: list[str] = field(default_factory=list) # struct tags (_SBRun, _SBLine …) + enumerators: dict[str, str] = field(default_factory=dict) # enumerator: the typedef the header gives its values + functions: list[str] = field(default_factory=list) + deprecated: list[str] = field(default_factory=list) + macros: dict[str, Macro] = field(default_factory=dict) + + +def strip(text: str) -> str: + text = re.sub(r'/\*.*?\*/', ' ', text, flags=re.S) + return re.sub(r'//[^\n]*', '', text) + + +def cut_text_api(text: str) -> str: + """The header without its `#if SB_TEXT_API_SUPPORTED` block (0 in this build).""" + start = text.find(TEXT_API) + if start < 0: + return text + depth, i = 0, start + for m in re.finditer(r'^[ \t]*#[ \t]*(if|ifdef|ifndef|endif)\b', text[start:], flags=re.M): + depth += 1 if m.group(1) != 'endif' else -1 + if depth == 0: + i = start + m.end() + break + else: + sys.exit('unterminated ' + TEXT_API) + return text[:start] + text[i:] + + +def surface(upstream: Path) -> Surface: + headers = upstream / 'Headers/SheenBidi' + s = Surface() + for name in INSTALLED: + raw = (headers / name).read_text(encoding='utf-8') + text = cut_text_api(strip(raw)) + for m in re.finditer(r'^[ \t]*#[ \t]*define[ \t]+(\w+)(\(([^)]*)\))?[ \t]*((?:[^\n]*\\\n)*[^\n]*)', text, flags=re.M): + params = [p.strip() for p in m.group(3).split(',')] if m.group(2) else None + body = ' '.join(m.group(4).replace('\\\n', ' ').split()) + s.macros.setdefault(m.group(1), Macro(m.group(1), name, params, body)) + if TEXT_API in raw and re.search(r'\bSB_PUBLIC\b|\btypedef\b', cut_text_api(strip(raw))): + sys.exit(f'{name}: declarations outside {TEXT_API}') + # Declarations only: the preprocessor's lines are not. + code = re.sub(r'^[ \t]*#(?:[^\n]*\\\n)*[^\n]*', '', text, flags=re.M) + code = re.sub(r'\bSB_EXTERN_C_(?:BEGIN|END)\b', ' ', code) + # Unnamed enums, each followed by the typedef the header gives its values + # (`enum { SBBidiTypeL = 0x01, … }; typedef SBUInt8 SBBidiType;`). + for m in re.finditer(r'\benum\s*\{(.*?)\}\s*;\s*typedef\s+\w+\s+(\w+)\s*;', code, flags=re.S): + for e in re.findall(r'\b([A-Za-z_]\w*)\s*(?:=[^,}]*)?(?:,|$)', m.group(1).strip()): + s.enumerators[e] = m.group(2) + if len(re.findall(r'\benum\b', code)) != len(re.findall(r'\benum\s*\{.*?\}\s*;\s*typedef', code, flags=re.S)): + sys.exit(f'{name}: an enum the inventory does not know (named, or without its typedef)') + code = re.sub(r'\benum\s*\{.*?\}\s*;', ';', code, flags=re.S) + # struct definitions: the tag, then the typedef name after the braces. + for m in re.finditer(r'\btypedef\s+struct\s+(\w+)\s*\{.*?\}\s*(\w+)\s*;', code, flags=re.S): + s.tags.append(m.group(1)) + s.types.append(m.group(2)) + code = re.sub(r'\btypedef\s+struct\s+\w+\s*\{.*?\}\s*\w+\s*;', ';', code, flags=re.S) + # Opaque references: `typedef const struct _SBLine *SBLineRef;`. + for m in re.finditer(r'\btypedef\s+(?:const\s+)?struct\s+(\w+)\s*\*\s*(\w+)\s*;', code): + s.tags.append(m.group(1)) + s.types.append(m.group(2)) + code = re.sub(r'\btypedef\s+(?:const\s+)?struct\s+\w+\s*\*\s*\w+\s*;', ';', code) + # Function pointer types: `typedef void *(*SBAllocatorAllocateBlockFunc)(…);`. + s.types += re.findall(r'\btypedef\b[^;(]*\(\s*\*\s*(\w+)\s*\)\s*\([^;]*\)\s*;', code) + code = re.sub(r'\btypedef\b[^;(]*\(\s*\*\s*\w+\s*\)\s*\([^;]*\)\s*;', ';', code) + # Plain typedefs: `typedef SBUInt8 SBLevel;`. + s.types += re.findall(r'\btypedef\s+[\w\s\*]+?\b(\w+)\s*;', code) + code = re.sub(r'\btypedef\b[^;]*;', ';', code) + # Functions: every one is SB_PUBLIC, some SB_DEPRECATED before it. + for m in re.finditer(r'(\bSB_DEPRECATED\s+)?\bSB_PUBLIC\b([^;{]*);', code): + f = re.search(r'\b(\w+)\s*\(', m.group(2)) + if not f: + sys.exit(f'{name}: SB_PUBLIC on something that is not a function: {m.group(2).strip()}') + s.functions.append(f.group(1)) + if m.group(1): + s.deprecated.append(f.group(1)) + rest = re.sub(r'(\bSB_DEPRECATED\s+)?\bSB_PUBLIC\b[^;{]*;', ';', code) + if re.search(r'\w', rest.replace(';', ' ')): + sys.exit(f'{name}: declarations the inventory does not know: {" ".join(rest.split())[:200]}') + s.types = sorted(set(s.types)) + s.tags = sorted(set(s.tags)) + s.functions = sorted(set(s.functions)) + s.deprecated = sorted(set(s.deprecated)) + return s + + +if __name__ == '__main__': + root = Path(__file__).resolve().parents[1] + sf = surface(fetch(root)) + print(f'{len(sf.types)} types, {len(sf.tags)} struct tags, {len(sf.enumerators)} enumerators, ' + f'{len(sf.functions)} functions ({len(sf.deprecated)} deprecated), {len(sf.macros)} macros') + print('types:', ' '.join(sf.types)) + print('tags:', ' '.join(sf.tags)) + print('enumerator types:', ' '.join(sorted(set(sf.enumerators.values())))) + print('functions:', ' '.join(sf.functions)) + print('deprecated:', ' '.join(sf.deprecated)) + print('object-like:', ' '.join(m.name for m in sf.macros.values() if m.params is None)) + print('function-like:', ' '.join(m.name for m in sf.macros.values() if m.params is not None)) diff --git a/tools/run_conformance.py b/tools/run_conformance.py new file mode 100644 index 0000000..8014b71 --- /dev/null +++ b/tools/run_conformance.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Run Unicode's bidi conformance tests on examples/sheenbidi, programs +written against import mcpplibs.mwrap.sheenbidi. + +The release tag archive is downloaded once into target/upstream and checked +against upstream/documents.json. Its Tools/Unicode/BidiTest.txt, +BidiCharacterTest.txt and BidiMirroring.txt are checked against the +checksums recorded there — those of the files Unicode publishes for 17.0.0 +(https://www.unicode.org/Public/17.0.0/ucd/) — and used unchanged: + +- BidiTest.txt: every data line under each paragraph direction its bitset + lists, the levels after L1 and the visual order; +- BidiCharacterTest.txt: every line — the paragraph level, the levels after + L1, the visual order, and the mirrored characters against BidiMirroring.txt. + +Nothing is excluded. Every test must pass, and the counts must be the +release's (a guard against reading nothing). +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from inventory import fetch # noqa: E402 + +# Unicode 17.0.0's counts. +BIDI_TESTS = 770241 +BIDI_CHARACTER_TESTS = 91707 + + +def build(example: Path, mcpp: str, toolchain: str | None) -> Path: + command = [mcpp, 'build', '--profile', 'release'] + (['--toolchain', toolchain] if toolchain else []) + print('+', ' '.join(command), flush=True) + subprocess.run(command, cwd=example, check=True) + suffix = '.exe' if os.name == 'nt' else '' + found = sorted((p for p in (example / 'target').rglob('BidiTest' + suffix) if p.parent.name == 'bin'), + key=lambda p: p.stat().st_mtime) + if not found: + sys.exit('examples/sheenbidi was not built') + return found[-1].parent + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--mcpp', default='mcpp') + parser.add_argument('--toolchain') + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + upstream = fetch(root) + pins = json.loads((root / 'upstream/documents.json').read_text(encoding='utf-8')) + for data in pins['conformance']: + digest = hashlib.sha256((upstream / data['path']).read_bytes()).hexdigest() + if digest != data['sha256']: + sys.exit(f'{data["path"]}: {digest} is not {data["source"]} ({data["sha256"]})') + print(f'ok {data["path"]} is {data["source"]}', flush=True) + bin_dir = build(root / 'examples/sheenbidi', args.mcpp, args.toolchain) + exe = lambda name: str(bin_dir / (name + ('.exe' if os.name == 'nt' else ''))) # noqa: E731 + unicode = upstream / 'Tools/Unicode' + + failed, counts = [], {} + for runner, files in (('BidiTest', ['BidiTest.txt']), + ('BidiCharacterTest', ['BidiCharacterTest.txt', 'BidiMirroring.txt'])): + run = subprocess.run([exe(runner)] + [str(unicode / f) for f in files], capture_output=True, text=True, + timeout=1800) + summary = re.search(runner + r': (\d+) tests, (\d+) errors', run.stdout) + counts[runner] = int(summary.group(1)) if summary else 0 + ok = run.returncode == 0 and summary is not None and summary.group(2) == '0' + print(f'{"ok " if ok else "FAIL"} {runner} {files[0]}: {summary.group(0) if summary else "no summary"}', + flush=True) + if not ok: + failed.append(files[0]) + print(run.stdout[-4000:], run.stderr[-4000:], flush=True) + + print(f'Unicode {pins["unicode_version"]}: BidiTest.txt {counts["BidiTest"]} tests; ' + f'BidiCharacterTest.txt {counts["BidiCharacterTest"]} tests; {len(failed)} failed') + if failed or counts['BidiTest'] != BIDI_TESTS or counts['BidiCharacterTest'] != BIDI_CHARACTER_TESTS: + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/test_template.py b/tools/test_template.py new file mode 100644 index 0000000..81ea805 --- /dev/null +++ b/tools/test_template.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Render an installed candidate's default mcpp template, then build and run it.""" +import argparse +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import tomllib + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mcpp", default="mcpp") + parser.add_argument("--namespace", help="index namespace for legacy manifests without package.namespace") + parser.add_argument("--toolchain", default="llvm@23.1.3") + parser.add_argument("--template", help="named template; omit for the package default") + parser.add_argument("--run-arg", action="append", default=[]) + parser.add_argument("--index", type=Path, help="candidate SDK/wrapper index for unpublished feature dependencies") + args = parser.parse_args() + root = Path(__file__).resolve().parents[1] + metadata = tomllib.loads((root / "mcpp.toml").read_text())["package"] + namespace = metadata.get("namespace", args.namespace) + assert namespace, "Supply the published index namespace for this legacy manifest" + name, version = metadata["name"], metadata["version"] + identity = namespace + "." + name + with tempfile.TemporaryDirectory(prefix=name + "-template-") as directory: + temporary = Path(directory) + home = temporary / "home" + package = home / "registry/data/xpkgs" / (namespace + "-x-" + name) / version / (name + "-" + version) + shutil.copytree(root, package, ignore=shutil.ignore_patterns( + ".git", "target", ".mcpp", ".xlings", ".candidate-index", "__pycache__", "mcpp.lock", "compile_commands.json", "mcpp-2026.*")) + index = home / "indices/mcpplibs/pkgs/mwrap" + index.mkdir(parents=True) + entry = '{url="https://example.invalid/candidate.tar.gz",sha256="fixture"}' + descriptor = ('package={spec="1",namespace=' + json.dumps(namespace) + ',name=' + json.dumps(name) + + ',description="Installed template fixture",type="package",mcpp="mcpp.toml",xpm={') + descriptor += ",".join(platform + "={[" + json.dumps(version) + "]=" + entry + "}" + for platform in ("linux", "macosx", "windows")) + "}}\n" + (index / (identity + ".lua")).write_text(descriptor) + env = dict(os.environ, MCPP_HOME=str(home)) + selector = identity + "@" + version + for command in ([args.mcpp, "new", "--list-templates", selector], + [args.mcpp, "new", "quickstart", "--template", selector + (":" + args.template if args.template else "")]): + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=temporary, env=env, check=True) + project = temporary / "quickstart" + manifest = project / "mcpp.toml" + text = manifest.read_text() + dependency = tomllib.loads(text)["dependencies"][identity] + options = {"version": dependency} if isinstance(dependency, str) else dependency + assert options["version"] == version, options + replacement = '{ path = ' + json.dumps(package.as_posix()) + if options.get("features"): + replacement += ', features = ' + json.dumps(options["features"]) + replacement += ' }' + lines = text.splitlines() + matches = [i for i, line in enumerate(lines) if line.startswith(json.dumps(identity) + " =")] + assert len(matches) == 1, "Expected exactly one qualified dependency from self.name" + lines[matches[0]] = json.dumps(identity) + " = " + replacement + manifest.write_text("\n".join(lines) + "\n") + if args.index: + candidate_index = args.index.resolve() + subprocess.run([os.sys.executable, str(candidate_index / "tests/prepare_mwrap_indices.py")], + cwd=candidate_index, check=True) + with manifest.open("a") as out: + out.write("\n[indices]\n") + for namespace in ("compat", "mcpplibs.mwrap"): + view = candidate_index / "tests/mwrap/.indices" / namespace + out.write(json.dumps(namespace) + ' = { path = ' + json.dumps(view.as_posix()) + ' }\n') + command = [args.mcpp, "build", "--toolchain", args.toolchain, "--strict"] + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + command = [args.mcpp, "run", "--toolchain", args.toolchain] + if args.run_arg: + command += ["--", *args.run_arg] + print("+", " ".join(command), flush=True) + subprocess.run(command, cwd=project, check=True) + + +if __name__ == "__main__": + main() diff --git a/upstream/documents.json b/upstream/documents.json new file mode 100644 index 0000000..09c0b83 --- /dev/null +++ b/upstream/documents.json @@ -0,0 +1,41 @@ +{ + "upstream": "https://github.com/Tehreer/SheenBidi/archive/refs/tags/v3.0.0.tar.gz", + "upstream_version": "3.0.0", + "upstream_sha256": "86c56014034739ba39a24c23eb00323b0bf6f737354f665786015fca842af786", + "unicode_version": "17.0.0", + "pruning": "Nothing of SheenBidi is vendored: the library is compat.sheenbidi 3.0.0, built from the same tag archive (Source/SheenBidi.c, its unity build, with the committed Unicode 17.0.0 tables of Source/Data). Its README and license (Apache-2.0) are retained here, with the Unicode License v3 of the Unicode Character Database its tables are generated from.", + "documents": [ + { + "path": "upstream/sheenbidi/README.md", + "source": "https://github.com/Tehreer/SheenBidi/blob/v3.0.0/README.md", + "sha256": "34376a693954902b6b587d29d008d07e8ac29e4e553e8319d2f1d5ab7e3ae1c8" + }, + { + "path": "upstream/sheenbidi/LICENSE", + "source": "https://github.com/Tehreer/SheenBidi/blob/v3.0.0/LICENSE", + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30" + }, + { + "path": "upstream/unicode/license.txt", + "source": "https://www.unicode.org/license.txt", + "sha256": "e7a93b009565cfce55919a381437ac4db883e9da2126fa28b91d12732bc53d96" + } + ], + "conformance": [ + { + "path": "Tools/Unicode/BidiTest.txt", + "source": "https://www.unicode.org/Public/17.0.0/ucd/BidiTest.txt", + "sha256": "888bdfc8090652272d1f859cdb00ae659e2dc6c26740be61ef1d03998a687620" + }, + { + "path": "Tools/Unicode/BidiCharacterTest.txt", + "source": "https://www.unicode.org/Public/17.0.0/ucd/BidiCharacterTest.txt", + "sha256": "a3e6e905ab5afbe318a96df5401d0372a04cd73ef139ab5e3cf0ae241c255488" + }, + { + "path": "Tools/Unicode/BidiMirroring.txt", + "source": "https://www.unicode.org/Public/17.0.0/ucd/BidiMirroring.txt", + "sha256": "a2f16fb873ab4fcdf3221cb1a8a85a134ddd6ed03603181823ff5206af3741ce" + } + ] +} diff --git a/upstream/sheenbidi/LICENSE b/upstream/sheenbidi/LICENSE new file mode 100755 index 0000000..d645695 --- /dev/null +++ b/upstream/sheenbidi/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/upstream/sheenbidi/README.md b/upstream/sheenbidi/README.md new file mode 100644 index 0000000..e4a246d --- /dev/null +++ b/upstream/sheenbidi/README.md @@ -0,0 +1,184 @@ +# SheenBidi + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![Linux CI](https://github.com/Tehreer/SheenBidi/actions/workflows/linux.yml/badge.svg)](https://github.com/Tehreer/SheenBidi/actions/workflows/linux.yml) +[![macOS CI](https://github.com/Tehreer/SheenBidi/actions/workflows/macos.yml/badge.svg)](https://github.com/Tehreer/SheenBidi/actions/workflows/macos.yml) +[![Windows CI](https://github.com/Tehreer/SheenBidi/actions/workflows/windows.yml/badge.svg)](https://github.com/Tehreer/SheenBidi/actions/workflows/windows.yml) +[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/k2vvegcdqsb9ld5a?svg=true)](https://ci.appveyor.com/project/mta452/sheenbidi) +[![Coverage Status](https://coveralls.io/repos/github/Tehreer/SheenBidi/badge.svg?branch=master)](https://coveralls.io/github/Tehreer/SheenBidi) + +SheenBidi is a lightweight, fast and stable implementation of the [Unicode Bidirectional Algorithm (UBA)](https://unicode.org/reports/tr9/). + +It is being used by multiple open source and commercial projects in different domains such as: +- **Game Engine:** + [Defold](https://github.com/defold/defold) +- **Games:** + [Watch Dogs: Legion](https://www.mobygames.com/game/152206/watch-dogs-legion/), + [SuperTuxKart](https://github.com/supertuxkart/stk-code), + [VVVVVV](https://github.com/TerryCavanagh/VVVVVV), + [DevilutionX](https://github.com/diasurgical/DevilutionX), + [Monaco 2](https://www.mobygames.com/game/240348/monaco-2/) +- **Multimedia:** + [SFML](https://github.com/SFML/SFML), + [JUCE](https://github.com/juce-framework/JUCE), + [Tracktion Engine](https://github.com/Tracktion/tracktion_engine), + [Rive Runtime](https://github.com/rive-app/rive-runtime) +- **Text:** + [Raqm](https://github.com/HOST-Oman/libraqm), + [Tehreer Android](https://github.com/Tehreer/Tehreer-Android), + [Tehreer Cocoa](https://github.com/Tehreer/Tehreer-Cocoa) +- **Mapping:** + [VTS Browser](https://github.com/melowntech/vts-browser-cpp) +- **Web:** + [Dropflow](https://github.com/chearon/dropflow), + [Itemizer](https://github.com/chearon/itemizer) + +*If you use SheenBidi in your project, I’d love to hear about it!* + +## Features +- **Object Based Design:** Facilitates modular and maintainable code. +- **Core Level Optimization:** Ensures high performance. +- **Thread Safe Architecture:** Suitable for multithreaded applications. +- **Lightweight API:** Simplifies integration. +- **Encoding Support:** UTF-8, UTF-16, and UTF-32. + +## API Overview + + +The above diagram provides a visual representation of the API applied to sample text. + +### SBCodepointSequence +Decodes a string buffer in a specified encoding into code points. + +### SBAlgorithm +Determines the bidirectional type of each code unit in the source string. It identifies paragraph boundaries as per rule [P1](https://www.unicode.org/reports/tr9/#P1) and allows creation of paragraph objects with explicit base levels or derived from rules [P2](https://www.unicode.org/reports/tr9/#P2)-[P3](https://www.unicode.org/reports/tr9/#P3). + +### SBParagraph +Represents a single paragraph processed with rules [X1](https://www.unicode.org/reports/tr9/#X1)-[I2](https://www.unicode.org/reports/tr9/#I2), providing resolved embedding levels for all code units. + +### SBLine +Represents a line of text processed with rules [L1](https://www.unicode.org/reports/tr9/#L1)-[L2](https://www.unicode.org/reports/tr9/#L2), offering reordered level runs instead of individual characters. + +### SBRun +Denotes a sequence of characters sharing the same embedding level. A run's direction is right-to-left if its embedding level is odd. + +### SBMirrorLocator +Identifies mirrored characters in a line as determined by rule [L4](https://www.unicode.org/reports/tr9/#L4). + +### SBScriptLocator +Assists in text shaping by locating script runs as specified in [UAX #24](https://www.unicode.org/reports/tr24/). + +## Dependency +SheenBidi relies solely on standard C library headers: `stddef.h`, `stdint.h`, `stdlib.h`, and `string.h`. + +## Compiling +SheenBidi can be compiled with any C compiler. + +### CMake +To build and install SheenBidi using CMake: + +```bash +cmake -S. -Bbuild-release -DCMAKE_BUILD_TYPE=Release +cmake --build build-release +sudo cmake --install build-release +``` + +For development and testing: + +```bash +cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Debug -DSB_CONFIG_UNITY=OFF +cmake --build build +ctest --test-dir build --output-on-failure +``` + +In other CMake projects, SheenBidi can be found via `find_package(SheenBidi)`, providing the target `SheenBidi::SheenBidi`. SheenBidi can also be used via `FetchContent`. + +### Meson +To build and install SheenBidi using Meson: + +```bash +meson setup builddir --buildtype=release +meson compile -C builddir +sudo meson install -C builddir +``` + +For testing: + +```bash +meson setup builddir -Dunity_mode=disabled +meson test -C builddir --print-errorlogs +``` + +## Example +A simple example in C11. + +```c +#include +#include +#include + +#include + +int main(int argc, const char * argv[]) { + /* Create code point sequence for a sample bidirectional text. */ + const char *bidiText = "یہ ایک )car( ہے۔"; + SBCodepointSequence codepointSequence = { SBStringEncodingUTF8, (void *)bidiText, strlen(bidiText) }; + + /* Extract the first bidirectional paragraph. */ + SBAlgorithmRef bidiAlgorithm = SBAlgorithmCreate(&codepointSequence); + SBParagraphRef firstParagraph = SBAlgorithmCreateParagraph(bidiAlgorithm, 0, INT32_MAX, SBLevelDefaultLTR); + SBUInteger paragraphLength = SBParagraphGetLength(firstParagraph); + + /* Create a line consisting of the whole paragraph and get its runs. */ + SBLineRef paragraphLine = SBParagraphCreateLine(firstParagraph, 0, paragraphLength); + SBUInteger runCount = SBLineGetRunCount(paragraphLine); + const SBRun *runArray = SBLineGetRunsPtr(paragraphLine); + + /* Log the details of each run in the line. */ + for (SBUInteger i = 0; i < runCount; i++) { + printf("Run Offset: %ld\n", (long)runArray[i].offset); + printf("Run Length: %ld\n", (long)runArray[i].length); + printf("Run Level: %ld\n\n", (long)runArray[i].level); + } + + /* Create a mirror locator and load the line in it. */ + SBMirrorLocatorRef mirrorLocator = SBMirrorLocatorCreate(); + SBMirrorLocatorLoadLine(mirrorLocator, paragraphLine, (void *)bidiText); + const SBMirrorAgent *mirrorAgent = SBMirrorLocatorGetAgent(mirrorLocator); + + /* Log the details of each mirror in the line. */ + while (SBMirrorLocatorMoveNext(mirrorLocator)) { + printf("Mirror Index: %ld\n", (long)mirrorAgent->index); + printf("Actual Code Point: %ld\n", (long)mirrorAgent->codepoint); + printf("Mirrored Code Point: %ld\n\n", (long)mirrorAgent->mirror); + } + + /* Release all objects. */ + SBMirrorLocatorRelease(mirrorLocator); + SBLineRelease(paragraphLine); + SBParagraphRelease(firstParagraph); + SBAlgorithmRelease(bidiAlgorithm); + + return 0; +} +``` + +## Support +If SheenBidi plays a role in your project, you’re welcome to [star the repository](https://github.com/Tehreer/SheenBidi/stargazers) or [contribute improvements](https://github.com/Tehreer/SheenBidi/pulls). Engagement from the community helps inform future development and ensures continued relevance. + +## License +``` +Copyright (C) 2014-2026 Muhammad Tayyab Akram + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +``` diff --git a/upstream/unicode/license.txt b/upstream/unicode/license.txt new file mode 100644 index 0000000..56da589 --- /dev/null +++ b/upstream/unicode/license.txt @@ -0,0 +1,39 @@ +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 1991-2026 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder.