From 7164503cd758e4dec412bd8686406384a1d7cf18 Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 14:08:23 +1000 Subject: [PATCH 1/5] feat(parser): add Utf8ClusterAssembler with grapheme cluster glue --- .../Epub/parsers/Utf8ClusterAssembler.cpp | 148 ++++++ lib/Epub/Epub/parsers/Utf8ClusterAssembler.h | 73 +++ test/CMakeLists.txt | 1 + test/parser_cluster/CMakeLists.txt | 17 + test/parser_cluster/ParserClusterTest.cpp | 466 ++++++++++++++++++ 5 files changed, 705 insertions(+) create mode 100644 lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp create mode 100644 lib/Epub/Epub/parsers/Utf8ClusterAssembler.h create mode 100644 test/parser_cluster/CMakeLists.txt create mode 100644 test/parser_cluster/ParserClusterTest.cpp diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp new file mode 100644 index 0000000000..02eaaae1ca --- /dev/null +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp @@ -0,0 +1,148 @@ +// lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp +// +// Host-safe UTF-8 decode + grapheme cluster assembly + WordJoin policy. See the header +// for the public contract and the task brief for the state-transition table. No heap, no +// Arduino/HAL deps — depends only on Utf8.h + WordJoin.h + EpdFontFamily.h. + +#include "Utf8ClusterAssembler.h" + +#include +#include + +namespace { + +// Expected total byte length of a UTF-8 sequence from its lead byte. Mirrors Utf8.cpp's +// utf8CodepointLen (kept local so this TU stays free of internal-linkage coupling). +uint8_t leadByteLen(uint8_t c) { + if (c < 0x80) return 1; // 0xxxxxxx + if ((c >> 5) == 0x6) return 2; // 110xxxxx + if ((c >> 4) == 0xE) return 3; // 1110xxxx + if ((c >> 3) == 0x1E) return 4; // 11110xxx + return 1; // invalid lead — treat as a 1-byte (replacement) cp +} + +// Classify a fully-decoded non-CJK codepoint into the caller-dispatch kind. +Utf8ClusterAssembler::NonCjkKind classifyNonCjk(uint32_t cp) { + using NonCjkKind = Utf8ClusterAssembler::NonCjkKind; + if (cp == 0x20 || cp == 0x09 || cp == 0x0A || cp == 0x0D) return NonCjkKind::Whitespace; + if (cp == 0x00A0 || cp == 0x202F) return NonCjkKind::Nbsp; + if (cp == 0xFEFF) return NonCjkKind::Feff; + return NonCjkKind::Latin; +} + +// Copy `state`'s staged base into `out`. Does NOT clear the staged base. +void fillFlushableFromState(const Utf8ClusterAssembler::State& state, Utf8ClusterAssembler::Flushable& out) { + memcpy(out.bytes, state.pendingCjkBase, state.pendingCjkBaseLen); + out.len = state.pendingCjkBaseLen; + out.join = state.pendingCjkBaseJoin; + out.fontStyle = state.fontStyleAtStage; + out.underline = state.underlineAtStage; +} + +} // namespace + +Utf8ClusterAssembler::ConsumeResult Utf8ClusterAssembler::tryConsumeCodepoint( + const char* s, int len, int& i, State& state, WordJoin& nextJoin, EpdFontFamily::Style callerFontStyle, + bool callerUnderline, Flushable& outFlushable, NonCjkKind& outNonCjkKind, uint32_t& outNonCjkCp, + uint8_t& outNonCjkLen) { + // --- Step 1/2: assemble a NUL-terminated scratch holding exactly one codepoint. --- + // utf8NextCodepoint stops on a NUL, so we MUST decode from this private scratch and never + // off s[] (which is not NUL-terminated). + char scratch[5]; + uint8_t scratchLen = 0; // bytes placed into scratch (full cp length) + uint8_t continuationBytes = 0; // bytes pulled from THIS call's s[] (drives i advance) + + if (state.pendingUtf8Len > 0) { + // Completing a codepoint split across callbacks. + const uint8_t expected = leadByteLen(state.pendingUtf8[0]); + memcpy(scratch, state.pendingUtf8, state.pendingUtf8Len); + scratchLen = state.pendingUtf8Len; + + while (scratchLen < expected && (i + continuationBytes) < len) { + scratch[scratchLen++] = s[i + continuationBytes]; + continuationBytes++; + } + + if (scratchLen < expected) { + // Still incomplete: stage what we pulled and signal NeedMore. i unchanged. + memcpy(state.pendingUtf8, scratch, scratchLen); + state.pendingUtf8Len = scratchLen; + return ConsumeResult::NeedMore; + } + state.pendingUtf8Len = 0; // carry-over consumed + } else { + if (i >= len) return ConsumeResult::NeedMore; // nothing to read (defensive) + const uint8_t expected = leadByteLen(static_cast(s[i])); + if (i + expected > len) { + // Partial sequence at the buffer tail — stage and signal NeedMore. i unchanged. + for (uint8_t b = 0; b < expected && (i + b) < len; ++b) state.pendingUtf8[b] = s[i + b]; + state.pendingUtf8Len = static_cast(len - i); + return ConsumeResult::NeedMore; + } + scratchLen = expected; + continuationBytes = expected; + memcpy(scratch, s + i, expected); + } + scratch[scratchLen] = '\0'; + + const unsigned char* p = reinterpret_cast(scratch); + const uint32_t cp = utf8NextCodepoint(&p); + const uint8_t cpLen = scratchLen; // full UTF-8 byte length of this codepoint + + // --- Step 3: classify — grapheme extender FIRST, then CJK breakable, then non-CJK. --- + + // Grapheme extender glued onto an existing staged base. + if (utf8IsGraphemeExtender(cp) && state.pendingCjkBaseLen > 0) { + // Step 4: overflow guard — if appending would exceed the staged buffer, force-flush the + // staged base and re-stage the extender as a NEW base with a reset join + fresh snapshot. + if (static_cast(state.pendingCjkBaseLen) + cpLen > sizeof(state.pendingCjkBase)) { + fillFlushableFromState(state, outFlushable); + memcpy(state.pendingCjkBase, scratch, cpLen); + state.pendingCjkBaseLen = cpLen; + state.pendingCjkBaseJoin = WordJoin::CjkBreak; // explicit reset, NOT the inherited join + state.fontStyleAtStage = callerFontStyle; + state.underlineAtStage = callerUnderline; + i += continuationBytes; + return ConsumeResult::EmittedFlushable; + } + memcpy(state.pendingCjkBase + state.pendingCjkBaseLen, scratch, cpLen); + state.pendingCjkBaseLen = static_cast(state.pendingCjkBaseLen + cpLen); + i += continuationBytes; + return ConsumeResult::StagedOnly; + } + + // CJK breakable base. + if (utf8IsCjkBreakable(cp)) { + const bool hadBase = state.pendingCjkBaseLen > 0; + if (hadBase) fillFlushableFromState(state, outFlushable); + // Stage the NEW base. + memcpy(state.pendingCjkBase, scratch, cpLen); + state.pendingCjkBaseLen = cpLen; + state.pendingCjkBaseJoin = nextJoin; + state.fontStyleAtStage = callerFontStyle; + state.underlineAtStage = callerUnderline; + nextJoin = WordJoin::CjkBreak; + i += continuationBytes; + return hadBase ? ConsumeResult::EmittedFlushable : ConsumeResult::StagedOnly; + } + + // Non-CJK: Latin / whitespace / NBSP / FEFF. Do NOT touch nextJoin — the caller sets it. + outNonCjkKind = classifyNonCjk(cp); + outNonCjkCp = cp; + outNonCjkLen = cpLen; + if (state.pendingCjkBaseLen > 0) { + fillFlushableFromState(state, outFlushable); + state.pendingCjkBaseLen = 0; // clear staged base + i += continuationBytes; + return ConsumeResult::EmittedAndNonCjk; + } + i += continuationBytes; + return ConsumeResult::NonCjkOnly; +} + +bool Utf8ClusterAssembler::flushPendingBase(State& state, Flushable& outFlushable) { + if (state.pendingCjkBaseLen == 0) return false; + fillFlushableFromState(state, outFlushable); + state.pendingCjkBaseLen = 0; + return true; +} diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h new file mode 100644 index 0000000000..e4fa612fd5 --- /dev/null +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h @@ -0,0 +1,73 @@ +// lib/Epub/Epub/parsers/Utf8ClusterAssembler.h +#pragma once + +#include + +#include "../WordJoin.h" // for WordJoin (enum-only, no layout deps) +#include // for EpdFontFamily::Style (used by State + Flushable + callerFontStyle) +#include // for utf8NextCodepoint + predicates + +class Utf8ClusterAssembler { + public: + // Classifies the non-CJK codepoint that the caller must dispatch on after an + // EmittedAndNonCjk / NonCjkOnly return. The four kinds map 1:1 to the four legacy + // branches in characterData (Latin buffer append / whitespace flush / NBSP synthesized + // token / FEFF discard). + enum class NonCjkKind : uint8_t { + Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer + Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space + Nbsp, // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue + Feff, // U+FEFF (BOM) — discard + }; + + enum class ConsumeResult : uint8_t { + NeedMore, // current codepoint incomplete (carry-over staged); i unchanged + StagedOnly, // codepoint consumed, staged into pendingCjkBase, no flushable emitted; i advanced + EmittedFlushable, // staged base flushed (caller emits it as a CJK token); + // current codepoint also staged/consumed; i advanced + NonCjkOnly, // no pending base existed (outFlushable INVALID — do not read). + // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. + EmittedAndNonCjk, // a pending base existed and was flushed: outFlushable filled. + // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. + }; + + // Cross-callback state — owned by the parser, threaded into each call. + // fontStyleAtStage / underlineAtStage are snapshotted at STAGE time (not flush time): a CJK + // base may be staged inside and flushed after closes; the flush must use the style + // in effect when the base was STAGED. + struct State { + uint8_t pendingUtf8[4] = {}; // leading bytes of a truncated codepoint + uint8_t pendingUtf8Len = 0; // 0..3 + char pendingCjkBase[16] = {}; // staged base + already-absorbed extenders + uint8_t pendingCjkBaseLen = 0; + WordJoin pendingCjkBaseJoin = WordJoin::Space; + EpdFontFamily::Style fontStyleAtStage = EpdFontFamily::REGULAR; + bool underlineAtStage = false; + }; + + // Payload emitted when a previously-staged CJK base must be flushed. + struct Flushable { + char bytes[16]; + uint8_t len; + WordJoin join; + EpdFontFamily::Style fontStyle; // snapshot at stage time + bool underline; // snapshot at stage time + }; + + // Consume up to one complete codepoint from s[i..len]. Returns the discriminant; out-params + // populated per the state-transition table. callerFontStyle / callerUnderline are the resolved + // style at THIS call; when a new base is staged they are snapshotted into State. + static ConsumeResult tryConsumeCodepoint( + const char* s, int len, int& i, + State& state, + WordJoin& nextJoin, + EpdFontFamily::Style callerFontStyle, + bool callerUnderline, + Flushable& outFlushable, // valid iff result in {EmittedFlushable, EmittedAndNonCjk} + NonCjkKind& outNonCjkKind, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} + uint32_t& outNonCjkCp, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} + uint8_t& outNonCjkLen); // total UTF-8 byte length of the non-CJK codepoint (1..4) + + // Force-flush any staged base. Returns true with outFlushable filled if a base was pending. + static bool flushPendingBase(State& state, Flushable& outFlushable); +}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2286f968ef..46d1651c7c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,3 +48,4 @@ add_subdirectory(font_resolver) add_subdirectory(font_boundary) add_subdirectory(parsed_text) add_subdirectory(utf8) +add_subdirectory(parser_cluster) diff --git a/test/parser_cluster/CMakeLists.txt b/test/parser_cluster/CMakeLists.txt new file mode 100644 index 0000000000..9972373f29 --- /dev/null +++ b/test/parser_cluster/CMakeLists.txt @@ -0,0 +1,17 @@ +# test/parser_cluster/CMakeLists.txt +add_executable(ParserClusterTest + ParserClusterTest.cpp + ${REPO_ROOT}/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp + ${REPO_ROOT}/lib/Utf8/Utf8.cpp +) +target_include_directories(ParserClusterTest PRIVATE + ${REPO_ROOT} + ${REPO_ROOT}/lib/Utf8 + ${REPO_ROOT}/lib/Epub + ${REPO_ROOT}/lib/Epub/Epub + ${REPO_ROOT}/lib/Epub/Epub/parsers + ${REPO_ROOT}/lib/EpdFont + ${REPO_ROOT}/test +) +target_link_libraries(ParserClusterTest PRIVATE crosspoint_test_common GTest::gtest_main) +gtest_discover_tests(ParserClusterTest) diff --git a/test/parser_cluster/ParserClusterTest.cpp b/test/parser_cluster/ParserClusterTest.cpp new file mode 100644 index 0000000000..0303ce712d --- /dev/null +++ b/test/parser_cluster/ParserClusterTest.cpp @@ -0,0 +1,466 @@ +// test/parser_cluster/ParserClusterTest.cpp +// +// Behavioural contract for Utf8ClusterAssembler — the host-safe state machine that +// performs UTF-8 decode + grapheme cluster assembly + WordJoin policy. The parser +// (Task 2.3c) consumes it; these 12 cases pin the contract that parser relies on. +// +// Helper `feed()` drives tryConsumeCodepoint over a byte buffer and collects every +// emitted Flushable, then drains any staged base with flushPendingBase. + +#include +#include + +#include +#include +#include +#include + +#include + +namespace { + +using ConsumeResult = Utf8ClusterAssembler::ConsumeResult; +using NonCjkKind = Utf8ClusterAssembler::NonCjkKind; +using Flushable = Utf8ClusterAssembler::Flushable; +using State = Utf8ClusterAssembler::State; + +// One observed call result, for assertions on call-by-call behaviour. +struct Step { + ConsumeResult result; + int iBefore; + int iAfter; + Flushable flushable; // valid iff result in {EmittedFlushable, EmittedAndNonCjk} + NonCjkKind nonCjkKind; + uint32_t nonCjkCp; + uint8_t nonCjkLen; +}; + +// Drive tryConsumeCodepoint across a whole buffer. callerFontStyle/callerUnderline are +// held constant for the whole feed (per-call snapshots are exercised directly in the +// style-snapshot tests). Stops looping when i no longer advances on a NeedMore so the +// caller can inspect the carry-over. +std::vector feed(const std::vector& bytes, State& state, WordJoin& nextJoin, + EpdFontFamily::Style callerFontStyle = EpdFontFamily::REGULAR, + bool callerUnderline = false) { + std::vector steps; + const char* s = reinterpret_cast(bytes.data()); + const int len = static_cast(bytes.size()); + int i = 0; + while (i < len) { + Step step; + step.iBefore = i; + step.flushable = Flushable{}; + step.nonCjkKind = NonCjkKind::Latin; + step.nonCjkCp = 0; + step.nonCjkLen = 0; + step.result = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, callerFontStyle, + callerUnderline, step.flushable, step.nonCjkKind, + step.nonCjkCp, step.nonCjkLen); + step.iAfter = i; + steps.push_back(step); + if (step.result == ConsumeResult::NeedMore) break; // carry-over staged; no more progress this buffer + } + return steps; +} + +uint32_t decodeOnly(const char* bytes, uint8_t len) { + char scratch[5] = {}; + memcpy(scratch, bytes, len); + const unsigned char* p = reinterpret_cast(scratch); + return utf8NextCodepoint(&p); +} + +// 1. NUL termination boundary: a 3-byte CJK cp where `len` excludes any trailing NUL. +// Classification fires only after the full cp is present; no read past s[len-1]. +TEST(ParserCluster, NulTerminationBoundary) { + // 日 U+65E5 = E6 97 A5, buffer holds exactly 3 bytes, no trailing NUL. + const std::vector buf = {0xE6, 0x97, 0xA5}; + State state; + WordJoin nextJoin = WordJoin::Space; + auto steps = feed(buf, state, nextJoin); + ASSERT_EQ(steps.size(), 1u); + EXPECT_EQ(steps[0].result, ConsumeResult::StagedOnly); + EXPECT_EQ(steps[0].iAfter, 3); // consumed all 3 bytes, none past the boundary + // The staged base is the full cp. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 3); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x65E5u); +} + +// 2. Carry-over across calls: split U+65E5 as {E6,97} then {A5}. +TEST(ParserCluster, CarryOverAcrossCalls) { + State state; + WordJoin nextJoin = WordJoin::Space; + + // Call 1: first 2 bytes only -> NeedMore, i unchanged, no flushable. + const std::vector part1 = {0xE6, 0x97}; + const char* s1 = reinterpret_cast(part1.data()); + int i1 = 0; + Flushable f1{}; + NonCjkKind k1 = NonCjkKind::Latin; + uint32_t cp1 = 0; + uint8_t l1 = 0; + auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 2, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, + k1, cp1, l1); + EXPECT_EQ(r1, ConsumeResult::NeedMore); + EXPECT_EQ(i1, 0); // i unchanged + EXPECT_EQ(state.pendingUtf8Len, 2); + + // Call 2: final byte -> stages the cluster. + const std::vector part2 = {0xA5}; + const char* s2 = reinterpret_cast(part2.data()); + int i2 = 0; + Flushable f2{}; + NonCjkKind k2 = NonCjkKind::Latin; + uint32_t cp2 = 0; + uint8_t l2 = 0; + auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, + k2, cp2, l2); + EXPECT_EQ(r2, ConsumeResult::StagedOnly); + EXPECT_EQ(i2, 1); + + // Drain emits one 3-byte cluster. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 3); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x65E5u); +} + +// 3. Single-callback NFD cluster: か (U+304B) + U+3099 in one call. +TEST(ParserCluster, SingleCallbackNfdCluster) { + // か U+304B = E3 81 8B, U+3099 = E3 82 99 + const std::vector buf = {0xE3, 0x81, 0x8B, 0xE3, 0x82, 0x99}; + State state; + WordJoin nextJoin = WordJoin::Space; + auto steps = feed(buf, state, nextJoin); + ASSERT_EQ(steps.size(), 2u); + EXPECT_EQ(steps[0].result, ConsumeResult::StagedOnly); // base staged + EXPECT_EQ(steps[1].result, ConsumeResult::StagedOnly); // extender absorbed, no flush + EXPECT_EQ(state.pendingCjkBaseLen, 6); // 3 + 3 bytes glued + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one 6-byte cluster spanning both codepoints +} + +// 4. Cross-callback NFD cluster: か in call 1 + U+3099 in call 2. +TEST(ParserCluster, CrossCallbackNfdCluster) { + State state; + WordJoin nextJoin = WordJoin::Space; + + const std::vector call1 = {0xE3, 0x81, 0x8B}; // か + auto s1 = feed(call1, state, nextJoin); + ASSERT_EQ(s1.size(), 1u); + EXPECT_EQ(s1[0].result, ConsumeResult::StagedOnly); + EXPECT_EQ(state.pendingCjkBaseLen, 3); // base survives the boundary + + const std::vector call2 = {0xE3, 0x82, 0x99}; // U+3099 + auto s2 = feed(call2, state, nextJoin); + ASSERT_EQ(s2.size(), 1u); + EXPECT_EQ(s2[0].result, ConsumeResult::StagedOnly); // extender glued onto carried base + EXPECT_EQ(state.pendingCjkBaseLen, 6); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one 6-byte cluster spanning both callbacks +} + +// 5. Variation Selector VS-1: 漢 (U+6F22) + U+FE00. +TEST(ParserCluster, VariationSelectorVs1) { + // 漢 = E6 BC A2, U+FE00 = EF B8 80 + const std::vector buf = {0xE6, 0xBC, 0xA2, 0xEF, 0xB8, 0x80}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one cluster of exactly 6 bytes +} + +// 6. Variation Selector Supplement: 漢 (3 bytes) + U+E0100 (4 bytes). +TEST(ParserCluster, VariationSelectorSupplement) { + // 漢 = E6 BC A2, U+E0100 = F3 A0 84 80 + const std::vector buf = {0xE6, 0xBC, 0xA2, 0xF3, 0xA0, 0x84, 0x80}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 7); // 3 + 4 bytes +} + +// 7. Halfwidth voicing: ｶ (U+FF76) + ﾞ (U+FF9E). +TEST(ParserCluster, HalfwidthVoicing) { + // ｶ U+FF76 = EF BD B6, ﾞ U+FF9E = EF BE 9E + const std::vector buf = {0xEF, 0xBD, 0xB6, 0xEF, 0xBE, 0x9E}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one 6-byte cluster +} + +// 8. Ideographic tone mark: CJK base + U+302A. Pins extender-before-breakable ordering: +// U+302A is BOTH a grapheme extender and CJK-breakable. If breakable were checked first +// this would be two clusters and this test would go red. +TEST(ParserCluster, IdeographicToneMarkGluesNotBreaks) { + // 日 = E6 97 A5, U+302A = E3 80 AA + const std::vector buf = {0xE6, 0x97, 0xA5, 0xE3, 0x80, 0xAA}; + State state; + WordJoin nextJoin = WordJoin::Space; + auto steps = feed(buf, state, nextJoin); + ASSERT_EQ(steps.size(), 2u); + EXPECT_EQ(steps[0].result, ConsumeResult::StagedOnly); // base + EXPECT_EQ(steps[1].result, ConsumeResult::StagedOnly); // tone mark glued, NOT a new base + EXPECT_EQ(state.pendingCjkBaseLen, 6); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // a single cluster, not two +} + +// 9. Overflow guard with join reset: base + enough extenders that the next append would +// exceed sizeof(pendingCjkBase) (16). The staged cluster force-flushes via +// EmittedFlushable, the overflowing extender becomes the new staged base, and the new +// base's join is reset to CjkBreak (NOT the inherited Glue/Space). +TEST(ParserCluster, OverflowGuardResetsJoin) { + // Base 漢 (3 bytes) then VS-1 extenders (3 bytes each: EF B8 80). + // 3 + 3 + 3 + 3 + 3 = 15 fits; the 5th extender (would make 18) overflows. + std::vector buf = {0xE6, 0xBC, 0xA2}; // 漢, 3 bytes + for (int n = 0; n < 5; ++n) buf.insert(buf.end(), {0xEF, 0xB8, 0x80}); // 5 × U+FE00 + + State state; + WordJoin nextJoin = WordJoin::Glue; // inherited join is Glue, to prove reset to CjkBreak + auto steps = feed(buf, state, nextJoin); + + // Find the force-flush step. + bool sawEmittedFlushable = false; + Flushable forced{}; + for (const auto& step : steps) { + if (step.result == ConsumeResult::EmittedFlushable) { + sawEmittedFlushable = true; + forced = step.flushable; + } + } + ASSERT_TRUE(sawEmittedFlushable); // (a) staged cluster force-flushed + EXPECT_EQ(forced.len, 15); // 漢 + 4 extenders = full 15-byte cluster + + // (b) the overflowing extender is now the new staged base. + EXPECT_EQ(state.pendingCjkBaseLen, 3); // the 5th U+FE00 alone + // (c) join reset to CjkBreak, NOT the inherited Glue. + EXPECT_EQ(state.pendingCjkBaseJoin, WordJoin::CjkBreak); +} + +// 10. EmittedAndNonCjk caller-append contract: 日 (U+65E5) then 'a' (U+0061). +// The caller MUST re-encode outNonCjkCp via utf8AppendCodepoint (NOT copy from s[]). +TEST(ParserCluster, EmittedAndNonCjkCallerAppendContract) { + // 日 = E6 97 A5, 'a' = 61 + const std::vector buf = {0xE6, 0x97, 0xA5, 0x61}; + State state; + WordJoin nextJoin = WordJoin::Space; + + const char* s = reinterpret_cast(buf.data()); + const int len = static_cast(buf.size()); + int i = 0; + + // Call A: stage 日 with underline=true, REGULAR. + Flushable fa{}; + NonCjkKind ka = NonCjkKind::Latin; + uint32_t cpa = 0; + uint8_t la = 0; + auto ra = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/true, fa, ka, cpa, la); + EXPECT_EQ(ra, ConsumeResult::StagedOnly); + EXPECT_EQ(i, 3); + + // Call B: consume 'a' with underline=false -> force flush 日. + Flushable fb{}; + NonCjkKind kb = NonCjkKind::Latin; + uint32_t cpb = 0; + uint8_t lb = 0; + auto rb = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/false, fb, kb, cpb, lb); + EXPECT_EQ(rb, ConsumeResult::EmittedAndNonCjk); // (a) + EXPECT_EQ(fb.len, 3); // (b) + EXPECT_EQ(decodeOnly(fb.bytes, fb.len), 0x65E5u); + EXPECT_EQ(kb, NonCjkKind::Latin); // (c) + EXPECT_EQ(cpb, 0x0061u); // (d) + EXPECT_EQ(lb, 1); // (e) + EXPECT_EQ(i, 4); // (f) advanced past 'a' + EXPECT_EQ(state.pendingCjkBaseLen, 0); // (g) state cleared + // (h)(i) stage-time snapshot survives, NOT call-B's false/REGULAR. + EXPECT_TRUE(fb.underline); + EXPECT_EQ(fb.fontStyle, EpdFontFamily::REGULAR); + + // The caller MUST re-encode outNonCjkCp via utf8AppendCodepoint (NOT copy s[i_prev..i-1]). + std::string reencoded; + utf8AppendCodepoint(cpb, reencoded); + ASSERT_EQ(reencoded.size(), 1u); + EXPECT_EQ(static_cast(reencoded[0]), 0x61); + + // NonCjkOnly sub-case: ' ' alone with no preceding base. + { + const std::vector sp = {0x20}; + State st2; + WordJoin nj2 = WordJoin::Space; + const char* ss = reinterpret_cast(sp.data()); + int j = 0; + Flushable fx{}; + NonCjkKind kx = NonCjkKind::Latin; + uint32_t cpx = 0; + uint8_t lx = 0; + auto rx = Utf8ClusterAssembler::tryConsumeCodepoint(ss, 1, j, st2, nj2, EpdFontFamily::REGULAR, false, fx, kx, + cpx, lx); + EXPECT_EQ(rx, ConsumeResult::NonCjkOnly); + EXPECT_EQ(kx, NonCjkKind::Whitespace); // do NOT read fx (outFlushable INVALID) + EXPECT_EQ(j, 1); + } + + // Other NonCjkKind variants after a staged base: 日 + ' '/NBSP/FEFF. + auto classifyAfterBase = [](const std::vector& tail) -> NonCjkKind { + std::vector b = {0xE6, 0x97, 0xA5}; // 日 + b.insert(b.end(), tail.begin(), tail.end()); + State st; + WordJoin nj = WordJoin::Space; + const char* ss = reinterpret_cast(b.data()); + const int ln = static_cast(b.size()); + int j = 0; + NonCjkKind seen = NonCjkKind::Latin; + while (j < ln) { + Flushable fl{}; + NonCjkKind kk = NonCjkKind::Latin; + uint32_t cc = 0; + uint8_t ll = 0; + auto r = Utf8ClusterAssembler::tryConsumeCodepoint(ss, ln, j, st, nj, EpdFontFamily::REGULAR, false, fl, kk, + cc, ll); + if (r == ConsumeResult::EmittedAndNonCjk || r == ConsumeResult::NonCjkOnly) { + seen = kk; + break; + } + } + return seen; + }; + EXPECT_EQ(classifyAfterBase({0x20}), NonCjkKind::Whitespace); // ' ' + EXPECT_EQ(classifyAfterBase({0xC2, 0xA0}), NonCjkKind::Nbsp); // NBSP U+00A0 + EXPECT_EQ(classifyAfterBase({0xEF, 0xBB, 0xBF}), NonCjkKind::Feff); // FEFF +} + +// 11. Split-Latin codepoint across callbacks: é (U+00E9 = C3 A9) split {C3} then {A9}. +TEST(ParserCluster, SplitLatinCodepointAcrossCallbacks) { + State state; + WordJoin nextJoin = WordJoin::Space; + + // Call 1: {C3} only -> NeedMore, carry-over staged, i unchanged. + const std::vector part1 = {0xC3}; + const char* s1 = reinterpret_cast(part1.data()); + int i1 = 0; + Flushable f1{}; + NonCjkKind k1 = NonCjkKind::Latin; + uint32_t cp1 = 0; + uint8_t l1 = 0; + auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 1, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, + k1, cp1, l1); + EXPECT_EQ(r1, ConsumeResult::NeedMore); + EXPECT_EQ(i1, 0); + EXPECT_EQ(state.pendingUtf8Len, 1); + + // Call 2: {A9} -> NonCjkOnly, Latin, cp 0x00E9, len 2. + const std::vector part2 = {0xA9}; + const char* s2 = reinterpret_cast(part2.data()); + int i2 = 0; + Flushable f2{}; + NonCjkKind k2 = NonCjkKind::Latin; + uint32_t cp2 = 0; + uint8_t l2 = 0; + auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, + k2, cp2, l2); + EXPECT_EQ(r2, ConsumeResult::NonCjkOnly); + EXPECT_EQ(k2, NonCjkKind::Latin); + EXPECT_EQ(cp2, 0x00E9u); + EXPECT_EQ(l2, 2); // FULL byte length, even though i advanced by only 1 continuation byte + EXPECT_EQ(i2, 1); + + // Lossless re-encode even though the two bytes never appeared together in one callback. + std::string scratch; + utf8AppendCodepoint(cp2, scratch); + ASSERT_EQ(scratch.size(), 2u); + EXPECT_EQ(static_cast(scratch[0]), 0xC3); + EXPECT_EQ(static_cast(scratch[1]), 0xA9); +} + +// 12. Style snapshot at stage time, NOT flush time. Proxies 日本. +TEST(ParserCluster, StyleSnapshotAtStageTimeNotFlushTime) { + // 日 = E6 97 A5, 本 U+672C = E6 9C AC + const std::vector buf = {0xE6, 0x97, 0xA5, 0xE6, 0x9C, 0xAC}; + State state; + WordJoin nextJoin = WordJoin::Space; + + const char* s = reinterpret_cast(buf.data()); + const int len = static_cast(buf.size()); + int i = 0; + + // Call A: 日 under REGULAR + underline=true. + Flushable fa{}; + NonCjkKind ka = NonCjkKind::Latin; + uint32_t cpa = 0; + uint8_t la = 0; + auto ra = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/true, fa, ka, cpa, la); + EXPECT_EQ(ra, ConsumeResult::StagedOnly); + EXPECT_EQ(state.pendingCjkBaseLen, 3); + EXPECT_TRUE(state.underlineAtStage); + EXPECT_EQ(state.fontStyleAtStage, EpdFontFamily::REGULAR); + + // Call B: 本 under REGULAR + underline=false -> flush 日 with STAGE-time snapshot. + Flushable fb{}; + NonCjkKind kb = NonCjkKind::Latin; + uint32_t cpb = 0; + uint8_t lb = 0; + auto rb = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/false, fb, kb, cpb, lb); + EXPECT_EQ(rb, ConsumeResult::EmittedFlushable); + EXPECT_EQ(fb.len, 3); + EXPECT_EQ(decodeOnly(fb.bytes, fb.len), 0x65E5u); // 日 + EXPECT_TRUE(fb.underline); // stage-time true, NOT call-B false + EXPECT_EQ(fb.fontStyle, EpdFontFamily::REGULAR); + EXPECT_EQ(state.pendingCjkBaseLen, 3); // 本 now staged + EXPECT_FALSE(state.underlineAtStage); // 本 staged under underline=false + + // Drain: 本 flushes with underline=false. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x672Cu); // 本 + EXPECT_FALSE(drained.underline); + + // Symmetric sub-test: 日本 — 日 staged under underline=false, 本 under true. + { + State st; + WordJoin nj = WordJoin::Space; + int j = 0; + + Flushable g1{}; + NonCjkKind gk1 = NonCjkKind::Latin; + uint32_t gc1 = 0; + uint8_t gl1 = 0; + auto gr1 = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, j, st, nj, EpdFontFamily::REGULAR, + /*callerUnderline=*/false, g1, gk1, gc1, gl1); + EXPECT_EQ(gr1, ConsumeResult::StagedOnly); // 日 staged under underline=false + + Flushable g2{}; + NonCjkKind gk2 = NonCjkKind::Latin; + uint32_t gc2 = 0; + uint8_t gl2 = 0; + auto gr2 = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, j, st, nj, EpdFontFamily::REGULAR, + /*callerUnderline=*/true, g2, gk2, gc2, gl2); + EXPECT_EQ(gr2, ConsumeResult::EmittedFlushable); + EXPECT_FALSE(g2.underline); // 日's stage-time underline=false + EXPECT_TRUE(st.underlineAtStage); // 本 now staged under underline=true + } +} + +} // namespace From cd84c32304ae6808c6ef564778dd60df4c1d0206 Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 14:17:47 +1000 Subject: [PATCH 2/5] feat(parser): emit CJK as breakable words via Utf8ClusterAssembler --- .../Epub/parsers/ChapterHtmlSlimParser.cpp | 337 +++++++++--------- lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h | 10 + test/parsed_text/ParsedTextLayoutTest.cpp | 27 ++ 3 files changed, 196 insertions(+), 178 deletions(-) diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index ba80f9e1d5..d0a13d2380 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -9,6 +9,8 @@ #include #include +#include +#include #include #include @@ -159,9 +161,10 @@ void ChapterHtmlSlimParser::flushPendingAnchor() { pendingAnchorId.clear(); } -// flush the contents of partWordBuffer to currentTextBlock -void ChapterHtmlSlimParser::flushPartWordBuffer() { - // Determine font style from depth-based tracking and CSS effective style +// Resolve the current font style from depth-based tracking and CSS effective style. +// Verbatim extraction of flushPartWordBuffer's former style block (including underline), so the +// Latin flush path stays byte-identical and the assembler can stage this same snapshot at stage time. +EpdFontFamily::Style ChapterHtmlSlimParser::currentFontStyle() const { const bool isBold = boldUntilDepth < depth || effectiveBold; const bool isItalic = italicUntilDepth < depth || effectiveItalic; const bool isUnderline = underlineUntilDepth < depth || effectiveUnderline; @@ -182,16 +185,92 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() { } else if (effectiveSub) { fontStyle = static_cast(fontStyle | EpdFontFamily::SUB); } + return fontStyle; +} - // flush the buffer +// flush the contents of partWordBuffer to currentTextBlock +void ChapterHtmlSlimParser::flushPartWordBuffer() { partWordBuffer[partWordBufferIndex] = '\0'; - currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextJoin); + currentTextBlock->addWord(partWordBuffer, currentFontStyle(), false, nextJoin); partWordBufferIndex = 0; nextJoin = WordJoin::Space; } +// Drain any pending text — staged CJK base (via the assembler) and/or the Latin partWordBuffer — +// into currentTextBlock. Returns which kind flushed LAST so inline-close callers can decide the +// next join: Latin re-glues (Glue), CJK keeps CjkBreak (stays wrappable). Idempotent when nothing +// is staged. +ChapterHtmlSlimParser::FlushedKind ChapterHtmlSlimParser::flushPendingText() { + if (!currentTextBlock) return FlushedKind::None; + bool flushedCjk = false; + Utf8ClusterAssembler::Flushable f; + if (Utf8ClusterAssembler::flushPendingBase(clusterState, f)) { + // Use the stage-time snapshot (f.fontStyle / f.underline), NOT currentFontStyle()/ + // effectiveUnderline — the style may have changed since the base was staged. + currentTextBlock->addWord(std::string(f.bytes, f.len), f.fontStyle, f.underline, f.join); + flushedCjk = true; + } + if (partWordBufferIndex > 0) { + flushPartWordBuffer(); + return FlushedKind::Latin; // Latin flushed last → Glue is safe for inline-close callers + } + return flushedCjk ? FlushedKind::Cjk : FlushedKind::None; +} + +void ChapterHtmlSlimParser::emitCjkToken(const Utf8ClusterAssembler::Flushable& f) { + if (!currentTextBlock) return; + currentTextBlock->addWord(std::string(f.bytes, f.len), f.fontStyle, f.underline, f.join); +} + +void ChapterHtmlSlimParser::dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind, uint32_t cp) { + switch (kind) { + case Utf8ClusterAssembler::NonCjkKind::Latin: { + // Re-encode from cp (NOT from s[]) — the codepoint may have straddled a callback boundary. + std::string scratch; + utf8AppendCodepoint(cp, scratch); + // Append the codepoint ATOMICALLY: flush first if it wouldn't fit, so a multi-byte cp is + // never split across the MAX_WORD_SIZE boundary (no orphaned continuation bytes). + if (partWordBufferIndex + static_cast(scratch.size()) >= MAX_WORD_SIZE) { + flushPartWordBuffer(); + } + for (char b : scratch) partWordBuffer[partWordBufferIndex++] = b; + break; + } + case Utf8ClusterAssembler::NonCjkKind::Whitespace: + flushPendingText(); + nextJoin = WordJoin::Space; + break; + case Utf8ClusterAssembler::NonCjkKind::Nbsp: + flushPendingText(); + currentTextBlock->addWord(" ", currentFontStyle(), /*underline=*/false, WordJoin::Glue); + nextJoin = WordJoin::Glue; + break; + case Utf8ClusterAssembler::NonCjkKind::Feff: + break; // discard BOM / ZWNBSP + default: + assert(false); + std::abort(); + } +} + +// Split currentTextBlock into pages when it grows past ~750 words, freeing a lot of memory. +// Runs after each CJK emit AND once at the end of characterData to bound Latin-heavy callbacks too. +void ChapterHtmlSlimParser::splitLongBlockIfNeeded() { + if (!currentTextBlock || currentTextBlock->size() <= 750) return; + LOG_DBG("EHP", "Text block too long, splitting into multiple pages"); + const int horizontalInset = currentTextBlock->getBlockStyle().totalHorizontalInset(); + const uint16_t effectiveWidth = (horizontalInset < viewportWidth) + ? static_cast(viewportWidth - horizontalInset) + : viewportWidth; + currentTextBlock->layoutAndExtractLines( + renderer, fontId, effectiveWidth, + [this](const std::shared_ptr& tb) { addLineToPage(tb); }, false); +} + // start a new text block if needed void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { + flushPendingText(); // Round-7 M9: drain pending CJK base / Latin into the previous block + // before opening the new one. Idempotent if nothing staged. nextJoin = WordJoin::Space; // New block = new paragraph, no continuation if (currentTextBlock) { // already have a text block running and it is empty - just reuse it @@ -218,9 +297,7 @@ void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { } void ChapterHtmlSlimParser::emitHorizontalRule(const BlockStyle& blockStyle) { - if (partWordBufferIndex > 0) { - flushPartWordBuffer(); - } + flushPendingText(); if (currentTextBlock) { const BlockStyle parentBlockStyle = currentTextBlock->getBlockStyle(); @@ -369,13 +446,12 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* if (strcmp(name, "table") == 0) { // skip nested tables if (self->tableDepth > 0) { + self->flushPendingText(); // Round-11: preserve outer-paragraph text before the table opens self->tableDepth += 1; return; } - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); self->tableDepth += 1; self->tableRowIndex = 0; self->tableColIndex = 0; @@ -391,9 +467,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } if (self->tableDepth == 1 && (strcmp(name, "td") == 0 || strcmp(name, "th") == 0)) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); self->tableColIndex += 1; auto tableCellBlockStyle = BlockStyle(); @@ -417,9 +491,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->inlineStyleStack.push_back(headerStyle); self->updateEffectiveInlineStyle(); self->characterData(userData, headerText.c_str(), static_cast(headerText.length())); - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); self->nextJoin = WordJoin::Space; self->inlineStyleStack.pop_back(); self->updateEffectiveInlineStyle(); @@ -597,9 +669,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } // Flush any pending text block so it appears before the image - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); if (self->currentTextBlock && !self->currentTextBlock->isEmpty()) { const BlockStyle parentBlockStyle = self->currentTextBlock->getBlockStyle(); self->startNewTextBlock(parentBlockStyle); @@ -741,9 +811,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* if (isInternalLink) { // Flush buffer before style change - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->insideFootnoteLink = true; self->footnoteLinkDepth = self->depth; @@ -806,10 +876,8 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->updateEffectiveInlineStyle(); } else if (matches(name, BLOCK_TAGS, std::size(BLOCK_TAGS))) { if (strcmp(name, "br") == 0) { - if (self->partWordBufferIndex > 0) { - // flush word preceding
to currentTextBlock before calling startNewTextBlock - self->flushPartWordBuffer(); - } + // flush text preceding
to currentTextBlock before calling startNewTextBlock + self->flushPendingText(); self->startNewTextBlock(self->blockStyleStack.back().withoutBottom()); } else { self->currentCssStyle = cssStyle; @@ -825,9 +893,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } } else if (matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS))) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->underlineUntilDepth = std::min(self->underlineUntilDepth, self->depth); // Push inline style entry for underline tag @@ -848,9 +916,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->updateEffectiveInlineStyle(); } else if (matches(name, BOLD_TAGS, std::size(BOLD_TAGS))) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->boldUntilDepth = std::min(self->boldUntilDepth, self->depth); // Push inline style entry for bold tag @@ -871,9 +939,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->updateEffectiveInlineStyle(); } else if (matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS))) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->italicUntilDepth = std::min(self->italicUntilDepth, self->depth); // Push inline style entry for italic tag @@ -893,9 +961,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->inlineStyleStack.push_back(entry); self->updateEffectiveInlineStyle(); } else if (strcmp(name, "sup") == 0 || strcmp(name, "sub") == 0) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } StyleStackEntry entry; entry.depth = self->depth; @@ -913,9 +981,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* if (cssStyle.hasFontWeight() || cssStyle.hasFontStyle() || cssStyle.hasTextDecoration() || cssStyle.hasDirection() || cssStyle.hasVerticalAlign()) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } StyleStackEntry entry; entry.depth = self->depth; // Track depth for matching pop @@ -991,129 +1059,42 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char self->currentFootnote.number[self->currentFootnoteLinkTextLen] = '\0'; } - for (int i = 0; i < len; i++) { - if (isWhitespace(s[i])) { - // Currently looking at whitespace, if there's anything in the partWordBuffer, flush it - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } - // Whitespace is a real word boundary — reset continuation state - self->nextJoin = WordJoin::Space; - // Skip the whitespace char - continue; - } - - // Detect U+00A0 (non-breaking space, UTF-8: 0xC2 0xA0) or - // U+202F (narrow no-break space, UTF-8: 0xE2 0x80 0xAF). - // - // Both are rendered as a visible space but must never allow a line break around them. - // We split the no-break space into its own word token and link the surrounding words - // with continuation flags so the layout engine treats them as an indivisible group. - // - // Example: "200 Quadratkilometer" or "200 Quadratkilometer" - // Input bytes: "200\xC2\xA0Quadratkilometer" (or 0xE2 0x80 0xAF for U+202F) - // Tokens produced: - // [0] "200" continues=false - // [1] " " continues=true (attaches to "200", no gap) - // [2] "Quadratkilometer" continues=true (attaches to " ", no gap) - // - // The continuation flags prevent the line-breaker from inserting a line break - // between "200" and "Quadratkilometer". However, "Quadratkilometer" is now a - // standalone word for hyphenation purposes, so Liang patterns can produce - // "200 Quadrat-" / "kilometer" instead of the unusable "200" / "Quadratkilometer". - if (static_cast(s[i]) == 0xC2 && i + 1 < len && static_cast(s[i + 1]) == 0xA0) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } - - self->partWordBuffer[0] = ' '; - self->partWordBuffer[1] = '\0'; - self->partWordBufferIndex = 1; - self->nextJoin = WordJoin::Glue; // Attach space to previous word (no break). - self->flushPartWordBuffer(); - - self->nextJoin = WordJoin::Glue; // Next real word attaches to this space (no break). - - i++; // Skip the second byte (0xA0) - continue; - } - - // U+202F (narrow no-break space) — identical logic to U+00A0 above. - if (static_cast(s[i]) == 0xE2 && i + 2 < len && static_cast(s[i + 1]) == 0x80 && - static_cast(s[i + 2]) == 0xAF) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } - - self->partWordBuffer[0] = ' '; - self->partWordBuffer[1] = '\0'; - self->partWordBufferIndex = 1; - self->nextJoin = WordJoin::Glue; - self->flushPartWordBuffer(); - - self->nextJoin = WordJoin::Glue; - - i += 2; // Skip the remaining two bytes (0x80 0xAF) - continue; - } - - // Skip Zero Width No-Break Space / BOM (U+FEFF) = 0xEF 0xBB 0xBF - const XML_Char FEFF_BYTE_1 = static_cast(0xEF); - const XML_Char FEFF_BYTE_2 = static_cast(0xBB); - const XML_Char FEFF_BYTE_3 = static_cast(0xBF); - - if (s[i] == FEFF_BYTE_1) { - // Check if the next two bytes complete the 3-byte sequence - if ((i + 2 < len) && (s[i + 1] == FEFF_BYTE_2) && (s[i + 2] == FEFF_BYTE_3)) { - // Sequence 0xEF 0xBB 0xBF found! - i += 2; // Skip the next two bytes - continue; // Move to the next iteration - } + // Drive the cluster assembler one codepoint at a time. It stages CJK bases into clusterState + // (so a CJK base + its NFD extenders becomes a single, breakable word) and classifies non-CJK + // codepoints (Latin / whitespace / NBSP / FEFF) for the legacy branches below. CJK runs now break + // at the viewport edge instead of the old fixed MAX_WORD_SIZE word cut. + for (int i = 0; i < len;) { + Utf8ClusterAssembler::Flushable f; + Utf8ClusterAssembler::NonCjkKind kind; + uint32_t cp = 0; + uint8_t cpLen = 0; + const Utf8ClusterAssembler::ConsumeResult r = Utf8ClusterAssembler::tryConsumeCodepoint( + s, len, i, self->clusterState, self->nextJoin, self->currentFontStyle(), self->effectiveUnderline, f, kind, cp, + cpLen); + + switch (r) { + case Utf8ClusterAssembler::ConsumeResult::NeedMore: + return; // codepoint split across callbacks — resume next callback + case Utf8ClusterAssembler::ConsumeResult::StagedOnly: + break; // keep looping + case Utf8ClusterAssembler::ConsumeResult::EmittedFlushable: + self->emitCjkToken(f); + self->splitLongBlockIfNeeded(); + break; + case Utf8ClusterAssembler::ConsumeResult::NonCjkOnly: + self->dispatchNonCjk(kind, cp); + break; + case Utf8ClusterAssembler::ConsumeResult::EmittedAndNonCjk: + self->emitCjkToken(f); + self->splitLongBlockIfNeeded(); + self->dispatchNonCjk(kind, cp); + break; } - - // If we're about to run out of space, then cut the word off and start a new one. - // For CJK text (no spaces), this is the primary word-breaking mechanism. - // We must avoid splitting multi-byte UTF-8 sequences across word boundaries, - // otherwise the trailing bytes become orphaned continuation bytes that the - // decoder can't interpret. - if (self->partWordBufferIndex >= MAX_WORD_SIZE) { - int safeLen = utf8SafeTruncateBuffer(self->partWordBuffer, self->partWordBufferIndex); - - if (safeLen < self->partWordBufferIndex && safeLen > 0) { - // Incomplete UTF-8 sequence at the end — save it before flushing - int overflow = self->partWordBufferIndex - safeLen; - char saved[4]; - for (int j = 0; j < overflow; j++) { - saved[j] = self->partWordBuffer[safeLen + j]; - } - self->partWordBufferIndex = safeLen; - self->flushPartWordBuffer(); - for (int j = 0; j < overflow; j++) { - self->partWordBuffer[j] = saved[j]; - } - self->partWordBufferIndex = overflow; - } else { - self->flushPartWordBuffer(); - } - } - - self->partWordBuffer[self->partWordBufferIndex++] = s[i]; } - // If we have > 750 words buffered up, perform the layout and consume out all but the last line - // There should be enough here to build out 1-2 full pages and doing this will free up a lot of - // memory. - // Spotted when reading Intermezzo, there are some really long text blocks in there. - if (self->currentTextBlock->size() > 750) { - LOG_DBG("EHP", "Text block too long, splitting into multiple pages"); - const int horizontalInset = self->currentTextBlock->getBlockStyle().totalHorizontalInset(); - const uint16_t effectiveWidth = (horizontalInset < self->viewportWidth) - ? static_cast(self->viewportWidth - horizontalInset) - : self->viewportWidth; - self->currentTextBlock->layoutAndExtractLines( - self->renderer, self->fontId, effectiveWidth, - [self](const std::shared_ptr& textBlock) { self->addLineToPage(textBlock); }, false); - } + // Bound Latin-heavy callbacks too: split once after draining the chunk (CJK emits already split + // per-token above). See splitLongBlockIfNeeded for the rationale. + self->splitLongBlockIfNeeded(); } void XMLCALL ChapterHtmlSlimParser::defaultHandlerExpand(void* userData, const XML_Char* s, const int len) { @@ -1151,27 +1132,27 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n if (self->tableDepth > 1 && strcmp(name, "table") == 0) { // get rid of all text inside the nested table self->partWordBufferIndex = 0; + self->clusterState = Utf8ClusterAssembler::State{}; // drop staged CJK too (NOT a flush) self->tableDepth -= 1; LOG_DBG("EHP", "nested table detected, get rid of its content"); return; } - // Flush buffer with current style BEFORE any style changes - if (self->partWordBufferIndex > 0) { - // Flush if style will change OR if we're closing a block/structural element - const bool isInlineTag = !headerOrBlockTag && !tableStructuralTag && - !matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) && self->depth != 1; - const bool shouldFlush = styleWillChange || headerOrBlockTag || matches(name, BOLD_TAGS, std::size(BOLD_TAGS)) || - matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS)) || - matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS)) || tableStructuralTag || - matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) || self->depth == 1; - - if (shouldFlush) { - self->flushPartWordBuffer(); - // If closing an inline element, the next word fragment continues the same visual word - if (isInlineTag) { - self->nextJoin = WordJoin::Glue; - } + // Flush buffer with current style BEFORE any style changes. + // Flush if style will change OR if we're closing a block/structural element. + const bool isInlineTag = !headerOrBlockTag && !tableStructuralTag && + !matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) && self->depth != 1; + const bool shouldFlush = styleWillChange || headerOrBlockTag || matches(name, BOLD_TAGS, std::size(BOLD_TAGS)) || + matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS)) || + matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS)) || tableStructuralTag || + matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) || self->depth == 1; + + if (shouldFlush) { + const FlushedKind kind = self->flushPendingText(); + // If closing an inline element, a Latin fragment continues the same visual word; a CJK base + // must stay on CjkBreak so it remains wrappable across the boundary. + if (isInlineTag && kind == FlushedKind::Latin) { + self->nextJoin = WordJoin::Glue; } } diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h index cc9874ab04..309b25ab31 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h @@ -14,6 +14,7 @@ #include "Epub/blocks/TextBlock.h" #include "Epub/css/CssParser.h" #include "Epub/css/CssStyle.h" +#include "Utf8ClusterAssembler.h" class Page; class GfxRenderer; @@ -37,6 +38,10 @@ class ChapterHtmlSlimParser { char partWordBuffer[MAX_WORD_SIZE + 1] = {}; int partWordBufferIndex = 0; WordJoin nextJoin = WordJoin::Space; // how the next flushed word joins the previous (inline element boundary) + // Kind of text drained by flushPendingText(): inline-close callers re-glue Latin (Glue) but must + // leave a CJK base on CjkBreak so it stays wrappable across the boundary. + enum class FlushedKind : uint8_t { None, Latin, Cjk }; + Utf8ClusterAssembler::State clusterState; // cross-callback CJK base / truncated-codepoint staging std::unique_ptr currentTextBlock = nullptr; std::unique_ptr currentPage = nullptr; int16_t currentPageNextY = 0; @@ -100,6 +105,11 @@ class ChapterHtmlSlimParser { void startNewTextBlock(const BlockStyle& blockStyle); void flushPendingAnchor(); void flushPartWordBuffer(); + EpdFontFamily::Style currentFontStyle() const; + FlushedKind flushPendingText(); + void emitCjkToken(const Utf8ClusterAssembler::Flushable& f); + void dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind, uint32_t cp); + void splitLongBlockIfNeeded(); void makePages(); static void applyDirectionToEntry(StyleStackEntry& entry, const CssStyle& css); void emitHorizontalRule(const BlockStyle& blockStyle); diff --git a/test/parsed_text/ParsedTextLayoutTest.cpp b/test/parsed_text/ParsedTextLayoutTest.cpp index d47cca58a0..b20eba539b 100644 --- a/test/parsed_text/ParsedTextLayoutTest.cpp +++ b/test/parsed_text/ParsedTextLayoutTest.cpp @@ -337,3 +337,30 @@ TEST(ParsedTextLayout, FocusReadingPreservesUnderlineOnCjkCluster) { EXPECT_EQ(static_cast(styles[0]) & static_cast(EpdFontFamily::BOLD), 0) << "focus-bypass must still skip focus-bolding of a single CJK cluster"; } + +// CJK across an inline-style boundary MUST keep CjkBreak (stays wrappable). +TEST(ParsedTextLayout, ParserSequenceCjkAcrossInlineStyleStaysBreakable) { + ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false, + /*focusReadingEnabled=*/false, leftAligned()); + text.addWord("ab", EpdFontFamily::REGULAR); // join=Space + text.addWord("\xE6\x97\xA5", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 日 + text.addWord("\xE6\x9C\xAC", EpdFontFamily::ITALIC, false, WordJoin::CjkBreak); // 本 (italic via ) + text.addWord("\xE8\xAA\x9E", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 語 + auto lines = linesOf(text, 3 * kCell); + ASSERT_EQ(lines.size(), 2u); + EXPECT_EQ(lines[0], "ab\xE6\x97\xA5"); // ab日 + EXPECT_EQ(lines[1], "\xE6\x9C\xAC\xE8\xAA\x9E"); // 本語 +} + +// Latin inline-close MUST keep Glue (preserves quickly). +TEST(ParsedTextLayout, ParserSequenceLatinAcrossInlineStyleStaysGlued) { + ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false, + /*focusReadingEnabled=*/false, leftAligned()); + text.addWord("a", EpdFontFamily::REGULAR); // join=Space + text.addWord("quick", EpdFontFamily::REGULAR); // join=Space + text.addWord("ly", EpdFontFamily::ITALIC, false, WordJoin::Glue); // Glue — inline close + auto lines = linesOf(text, 7 * kCell); + ASSERT_EQ(lines.size(), 2u); + EXPECT_EQ(lines[0], "a"); + EXPECT_EQ(lines[1], "quickly"); +} From 309cfcbabd2c36dedbefdbd28237da812e5f11d7 Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 14:22:54 +1000 Subject: [PATCH 3/5] chore: bump SECTION_FILE_VERSION + full verification --- docs/file-formats.md | 7 ++++--- lib/Epub/Epub/Section.cpp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/file-formats.md b/docs/file-formats.md index 0b505fda04..3121796fdf 100644 --- a/docs/file-formats.md +++ b/docs/file-formats.md @@ -90,13 +90,13 @@ if (parsedSize != fileSize) { ## `section.bin` -### Version 25 +### Version 27 Each file in `sections/*.bin` stores one laid-out spine section. The header is also the cache-busting key: if any layout-affecting setting differs from the current reader settings, the section is discarded and rebuilt. -Version 25 includes: +Version 27 includes: - cache-busting fields for paragraph alignment, hyphenation, embedded CSS, image rendering mode, and Focus Reading @@ -105,6 +105,7 @@ Version 25 includes: - paragraph and list-item LUTs used by KOReader sync page refinement - optional per-word Focus Reading split metadata - per-page footnote entries +- CJK (Japanese) paragraphs wrap at viewport width via per-character breakable tokens (v27) ImHex pattern: @@ -113,7 +114,7 @@ import std.mem; import std.string; import std.core; -#define EXPECTED_VERSION 25 +#define EXPECTED_VERSION 27 #define MAX_STRING_LENGTH 65535 #define FOOTNOTE_NUMBER_LEN 32 #define FOOTNOTE_HREF_LEN 96 diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index c13da52672..3a268c0d84 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -10,7 +10,7 @@ #include "parsers/ChapterHtmlSlimParser.h" namespace { -constexpr uint8_t SECTION_FILE_VERSION = 26; +constexpr uint8_t SECTION_FILE_VERSION = 27; // was 26: CJK line-breaking changes layout output constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(uint8_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) + sizeof(bool) + sizeof(uint8_t) + sizeof(bool) + sizeof(uint32_t) + sizeof(uint32_t) + From 1e13f0b9d84d32a58368fc9b87332f27b81c68ea Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 14:49:59 +1000 Subject: [PATCH 4/5] =?UTF-8?q?fix(parser):=20preserve=20word=20order=20at?= =?UTF-8?q?=20Latin=E2=86=92CJK=20boundary;=20correct=20stale=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp | 15 ++++++++++++++- lib/Epub/Epub/parsers/Utf8ClusterAssembler.h | 6 +++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index d0a13d2380..6b536af60d 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -1068,6 +1068,7 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char Utf8ClusterAssembler::NonCjkKind kind; uint32_t cp = 0; uint8_t cpLen = 0; + const WordJoin joinBefore = self->nextJoin; const Utf8ClusterAssembler::ConsumeResult r = Utf8ClusterAssembler::tryConsumeCodepoint( s, len, i, self->clusterState, self->nextJoin, self->currentFontStyle(), self->effectiveUnderline, f, kind, cp, cpLen); @@ -1076,7 +1077,19 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char case Utf8ClusterAssembler::ConsumeResult::NeedMore: return; // codepoint split across callbacks — resume next callback case Utf8ClusterAssembler::ConsumeResult::StagedOnly: - break; // keep looping + // A CJK base was just staged. If Latin text was pending in partWordBuffer (a Latin→CJK + // boundary with no separating whitespace, e.g. "PDF版" / "5月"), it must be emitted BEFORE + // the staged base so word order is preserved. The base wrongly inherited the Latin run's + // join when it was staged; the Latin owns that join, and the base should break from the + // Latin (CjkBreak — they abut with no space, breakable). (Invariant: partWordBuffer and a + // staged base are never both non-empty except transiently here, so this drains cleanly.) + if (self->partWordBufferIndex > 0) { + self->nextJoin = joinBefore; // restore the Latin run's own join + self->flushPartWordBuffer(); // emit the Latin token IN ORDER + self->clusterState.pendingCjkBaseJoin = WordJoin::CjkBreak; // staged base breaks from the Latin + self->nextJoin = WordJoin::CjkBreak; // CJK run continues: successor breaks too + } + break; case Utf8ClusterAssembler::ConsumeResult::EmittedFlushable: self->emitCjkToken(f); self->splitLongBlockIfNeeded(); diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h index e4fa612fd5..342f632b64 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h @@ -10,9 +10,9 @@ class Utf8ClusterAssembler { public: // Classifies the non-CJK codepoint that the caller must dispatch on after an - // EmittedAndNonCjk / NonCjkOnly return. The four kinds map 1:1 to the four legacy - // branches in characterData (Latin buffer append / whitespace flush / NBSP synthesized - // token / FEFF discard). + // EmittedAndNonCjk / NonCjkOnly return. The four kinds map 1:1 to the four branches + // in ChapterHtmlSlimParser::dispatchNonCjk (Latin buffer append / whitespace flush / + // NBSP synthesized token / FEFF discard). enum class NonCjkKind : uint8_t { Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space From 4c94f9276526ea0cc983945fc156433e1d74c6da Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 15:02:01 +1000 Subject: [PATCH 5/5] style: apply clang-format (CI v21) --- .../Epub/parsers/ChapterHtmlSlimParser.cpp | 20 +++---- .../Epub/parsers/Utf8ClusterAssembler.cpp | 2 +- lib/Epub/Epub/parsers/Utf8ClusterAssembler.h | 54 ++++++++--------- test/parsed_text/ParsedTextLayoutTest.cpp | 18 +++--- test/parser_cluster/ParserClusterTest.cpp | 60 +++++++++---------- 5 files changed, 74 insertions(+), 80 deletions(-) diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index 6b536af60d..942fbccd88 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -259,12 +259,10 @@ void ChapterHtmlSlimParser::splitLongBlockIfNeeded() { if (!currentTextBlock || currentTextBlock->size() <= 750) return; LOG_DBG("EHP", "Text block too long, splitting into multiple pages"); const int horizontalInset = currentTextBlock->getBlockStyle().totalHorizontalInset(); - const uint16_t effectiveWidth = (horizontalInset < viewportWidth) - ? static_cast(viewportWidth - horizontalInset) - : viewportWidth; + const uint16_t effectiveWidth = + (horizontalInset < viewportWidth) ? static_cast(viewportWidth - horizontalInset) : viewportWidth; currentTextBlock->layoutAndExtractLines( - renderer, fontId, effectiveWidth, - [this](const std::shared_ptr& tb) { addLineToPage(tb); }, false); + renderer, fontId, effectiveWidth, [this](const std::shared_ptr& tb) { addLineToPage(tb); }, false); } // start a new text block if needed @@ -1084,10 +1082,10 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char // Latin (CjkBreak — they abut with no space, breakable). (Invariant: partWordBuffer and a // staged base are never both non-empty except transiently here, so this drains cleanly.) if (self->partWordBufferIndex > 0) { - self->nextJoin = joinBefore; // restore the Latin run's own join - self->flushPartWordBuffer(); // emit the Latin token IN ORDER - self->clusterState.pendingCjkBaseJoin = WordJoin::CjkBreak; // staged base breaks from the Latin - self->nextJoin = WordJoin::CjkBreak; // CJK run continues: successor breaks too + self->nextJoin = joinBefore; // restore the Latin run's own join + self->flushPartWordBuffer(); // emit the Latin token IN ORDER + self->clusterState.pendingCjkBaseJoin = WordJoin::CjkBreak; // staged base breaks from the Latin + self->nextJoin = WordJoin::CjkBreak; // CJK run continues: successor breaks too } break; case Utf8ClusterAssembler::ConsumeResult::EmittedFlushable: @@ -1153,8 +1151,8 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n // Flush buffer with current style BEFORE any style changes. // Flush if style will change OR if we're closing a block/structural element. - const bool isInlineTag = !headerOrBlockTag && !tableStructuralTag && - !matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) && self->depth != 1; + const bool isInlineTag = + !headerOrBlockTag && !tableStructuralTag && !matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) && self->depth != 1; const bool shouldFlush = styleWillChange || headerOrBlockTag || matches(name, BOLD_TAGS, std::size(BOLD_TAGS)) || matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS)) || matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS)) || tableStructuralTag || diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp index 02eaaae1ca..20b3fbf1c0 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp @@ -49,7 +49,7 @@ Utf8ClusterAssembler::ConsumeResult Utf8ClusterAssembler::tryConsumeCodepoint( // utf8NextCodepoint stops on a NUL, so we MUST decode from this private scratch and never // off s[] (which is not NUL-terminated). char scratch[5]; - uint8_t scratchLen = 0; // bytes placed into scratch (full cp length) + uint8_t scratchLen = 0; // bytes placed into scratch (full cp length) uint8_t continuationBytes = 0; // bytes pulled from THIS call's s[] (drives i advance) if (state.pendingUtf8Len > 0) { diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h index 342f632b64..d311721a27 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h @@ -1,11 +1,12 @@ // lib/Epub/Epub/parsers/Utf8ClusterAssembler.h #pragma once +#include // for EpdFontFamily::Style (used by State + Flushable + callerFontStyle) +#include // for utf8NextCodepoint + predicates + #include -#include "../WordJoin.h" // for WordJoin (enum-only, no layout deps) -#include // for EpdFontFamily::Style (used by State + Flushable + callerFontStyle) -#include // for utf8NextCodepoint + predicates +#include "../WordJoin.h" // for WordJoin (enum-only, no layout deps) class Utf8ClusterAssembler { public: @@ -14,21 +15,21 @@ class Utf8ClusterAssembler { // in ChapterHtmlSlimParser::dispatchNonCjk (Latin buffer append / whitespace flush / // NBSP synthesized token / FEFF discard). enum class NonCjkKind : uint8_t { - Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer - Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space - Nbsp, // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue - Feff, // U+FEFF (BOM) — discard + Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer + Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space + Nbsp, // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue + Feff, // U+FEFF (BOM) — discard }; enum class ConsumeResult : uint8_t { - NeedMore, // current codepoint incomplete (carry-over staged); i unchanged - StagedOnly, // codepoint consumed, staged into pendingCjkBase, no flushable emitted; i advanced - EmittedFlushable, // staged base flushed (caller emits it as a CJK token); - // current codepoint also staged/consumed; i advanced - NonCjkOnly, // no pending base existed (outFlushable INVALID — do not read). - // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. - EmittedAndNonCjk, // a pending base existed and was flushed: outFlushable filled. - // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. + NeedMore, // current codepoint incomplete (carry-over staged); i unchanged + StagedOnly, // codepoint consumed, staged into pendingCjkBase, no flushable emitted; i advanced + EmittedFlushable, // staged base flushed (caller emits it as a CJK token); + // current codepoint also staged/consumed; i advanced + NonCjkOnly, // no pending base existed (outFlushable INVALID — do not read). + // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. + EmittedAndNonCjk, // a pending base existed and was flushed: outFlushable filled. + // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. }; // Cross-callback state — owned by the parser, threaded into each call. @@ -36,9 +37,9 @@ class Utf8ClusterAssembler { // base may be staged inside and flushed after closes; the flush must use the style // in effect when the base was STAGED. struct State { - uint8_t pendingUtf8[4] = {}; // leading bytes of a truncated codepoint - uint8_t pendingUtf8Len = 0; // 0..3 - char pendingCjkBase[16] = {}; // staged base + already-absorbed extenders + uint8_t pendingUtf8[4] = {}; // leading bytes of a truncated codepoint + uint8_t pendingUtf8Len = 0; // 0..3 + char pendingCjkBase[16] = {}; // staged base + already-absorbed extenders uint8_t pendingCjkBaseLen = 0; WordJoin pendingCjkBaseJoin = WordJoin::Space; EpdFontFamily::Style fontStyleAtStage = EpdFontFamily::REGULAR; @@ -50,23 +51,20 @@ class Utf8ClusterAssembler { char bytes[16]; uint8_t len; WordJoin join; - EpdFontFamily::Style fontStyle; // snapshot at stage time - bool underline; // snapshot at stage time + EpdFontFamily::Style fontStyle; // snapshot at stage time + bool underline; // snapshot at stage time }; // Consume up to one complete codepoint from s[i..len]. Returns the discriminant; out-params // populated per the state-transition table. callerFontStyle / callerUnderline are the resolved // style at THIS call; when a new base is staged they are snapshotted into State. static ConsumeResult tryConsumeCodepoint( - const char* s, int len, int& i, - State& state, - WordJoin& nextJoin, - EpdFontFamily::Style callerFontStyle, + const char* s, int len, int& i, State& state, WordJoin& nextJoin, EpdFontFamily::Style callerFontStyle, bool callerUnderline, - Flushable& outFlushable, // valid iff result in {EmittedFlushable, EmittedAndNonCjk} - NonCjkKind& outNonCjkKind, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} - uint32_t& outNonCjkCp, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} - uint8_t& outNonCjkLen); // total UTF-8 byte length of the non-CJK codepoint (1..4) + Flushable& outFlushable, // valid iff result in {EmittedFlushable, EmittedAndNonCjk} + NonCjkKind& outNonCjkKind, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} + uint32_t& outNonCjkCp, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} + uint8_t& outNonCjkLen); // total UTF-8 byte length of the non-CJK codepoint (1..4) // Force-flush any staged base. Returns true with outFlushable filled if a base was pending. static bool flushPendingBase(State& state, Flushable& outFlushable); diff --git a/test/parsed_text/ParsedTextLayoutTest.cpp b/test/parsed_text/ParsedTextLayoutTest.cpp index b20eba539b..193348976a 100644 --- a/test/parsed_text/ParsedTextLayoutTest.cpp +++ b/test/parsed_text/ParsedTextLayoutTest.cpp @@ -342,23 +342,23 @@ TEST(ParsedTextLayout, FocusReadingPreservesUnderlineOnCjkCluster) { TEST(ParsedTextLayout, ParserSequenceCjkAcrossInlineStyleStaysBreakable) { ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false, /*focusReadingEnabled=*/false, leftAligned()); - text.addWord("ab", EpdFontFamily::REGULAR); // join=Space - text.addWord("\xE6\x97\xA5", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 日 - text.addWord("\xE6\x9C\xAC", EpdFontFamily::ITALIC, false, WordJoin::CjkBreak); // 本 (italic via ) - text.addWord("\xE8\xAA\x9E", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 語 + text.addWord("ab", EpdFontFamily::REGULAR); // join=Space + text.addWord("\xE6\x97\xA5", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 日 + text.addWord("\xE6\x9C\xAC", EpdFontFamily::ITALIC, false, WordJoin::CjkBreak); // 本 (italic via ) + text.addWord("\xE8\xAA\x9E", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 語 auto lines = linesOf(text, 3 * kCell); ASSERT_EQ(lines.size(), 2u); - EXPECT_EQ(lines[0], "ab\xE6\x97\xA5"); // ab日 - EXPECT_EQ(lines[1], "\xE6\x9C\xAC\xE8\xAA\x9E"); // 本語 + EXPECT_EQ(lines[0], "ab\xE6\x97\xA5"); // ab日 + EXPECT_EQ(lines[1], "\xE6\x9C\xAC\xE8\xAA\x9E"); // 本語 } // Latin inline-close MUST keep Glue (preserves quickly). TEST(ParsedTextLayout, ParserSequenceLatinAcrossInlineStyleStaysGlued) { ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false, /*focusReadingEnabled=*/false, leftAligned()); - text.addWord("a", EpdFontFamily::REGULAR); // join=Space - text.addWord("quick", EpdFontFamily::REGULAR); // join=Space - text.addWord("ly", EpdFontFamily::ITALIC, false, WordJoin::Glue); // Glue — inline close + text.addWord("a", EpdFontFamily::REGULAR); // join=Space + text.addWord("quick", EpdFontFamily::REGULAR); // join=Space + text.addWord("ly", EpdFontFamily::ITALIC, false, WordJoin::Glue); // Glue — inline close auto lines = linesOf(text, 7 * kCell); ASSERT_EQ(lines.size(), 2u); EXPECT_EQ(lines[0], "a"); diff --git a/test/parser_cluster/ParserClusterTest.cpp b/test/parser_cluster/ParserClusterTest.cpp index 0303ce712d..ab2aae4904 100644 --- a/test/parser_cluster/ParserClusterTest.cpp +++ b/test/parser_cluster/ParserClusterTest.cpp @@ -9,14 +9,13 @@ #include #include +#include #include #include #include #include -#include - namespace { using ConsumeResult = Utf8ClusterAssembler::ConsumeResult; @@ -40,8 +39,7 @@ struct Step { // style-snapshot tests). Stops looping when i no longer advances on a NeedMore so the // caller can inspect the carry-over. std::vector feed(const std::vector& bytes, State& state, WordJoin& nextJoin, - EpdFontFamily::Style callerFontStyle = EpdFontFamily::REGULAR, - bool callerUnderline = false) { + EpdFontFamily::Style callerFontStyle = EpdFontFamily::REGULAR, bool callerUnderline = false) { std::vector steps; const char* s = reinterpret_cast(bytes.data()); const int len = static_cast(bytes.size()); @@ -53,9 +51,9 @@ std::vector feed(const std::vector& bytes, State& state, WordJoin step.nonCjkKind = NonCjkKind::Latin; step.nonCjkCp = 0; step.nonCjkLen = 0; - step.result = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, callerFontStyle, - callerUnderline, step.flushable, step.nonCjkKind, - step.nonCjkCp, step.nonCjkLen); + step.result = + Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, callerFontStyle, callerUnderline, + step.flushable, step.nonCjkKind, step.nonCjkCp, step.nonCjkLen); step.iAfter = i; steps.push_back(step); if (step.result == ConsumeResult::NeedMore) break; // carry-over staged; no more progress this buffer @@ -101,8 +99,8 @@ TEST(ParserCluster, CarryOverAcrossCalls) { NonCjkKind k1 = NonCjkKind::Latin; uint32_t cp1 = 0; uint8_t l1 = 0; - auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 2, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, - k1, cp1, l1); + auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 2, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, k1, + cp1, l1); EXPECT_EQ(r1, ConsumeResult::NeedMore); EXPECT_EQ(i1, 0); // i unchanged EXPECT_EQ(state.pendingUtf8Len, 2); @@ -115,8 +113,8 @@ TEST(ParserCluster, CarryOverAcrossCalls) { NonCjkKind k2 = NonCjkKind::Latin; uint32_t cp2 = 0; uint8_t l2 = 0; - auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, - k2, cp2, l2); + auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, k2, + cp2, l2); EXPECT_EQ(r2, ConsumeResult::StagedOnly); EXPECT_EQ(i2, 1); @@ -231,7 +229,7 @@ TEST(ParserCluster, IdeographicToneMarkGluesNotBreaks) { TEST(ParserCluster, OverflowGuardResetsJoin) { // Base 漢 (3 bytes) then VS-1 extenders (3 bytes each: EF B8 80). // 3 + 3 + 3 + 3 + 3 = 15 fits; the 5th extender (would make 18) overflows. - std::vector buf = {0xE6, 0xBC, 0xA2}; // 漢, 3 bytes + std::vector buf = {0xE6, 0xBC, 0xA2}; // 漢, 3 bytes for (int n = 0; n < 5; ++n) buf.insert(buf.end(), {0xEF, 0xB8, 0x80}); // 5 × U+FE00 State state; @@ -286,12 +284,12 @@ TEST(ParserCluster, EmittedAndNonCjkCallerAppendContract) { auto rb = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, /*callerUnderline=*/false, fb, kb, cpb, lb); EXPECT_EQ(rb, ConsumeResult::EmittedAndNonCjk); // (a) - EXPECT_EQ(fb.len, 3); // (b) + EXPECT_EQ(fb.len, 3); // (b) EXPECT_EQ(decodeOnly(fb.bytes, fb.len), 0x65E5u); - EXPECT_EQ(kb, NonCjkKind::Latin); // (c) - EXPECT_EQ(cpb, 0x0061u); // (d) - EXPECT_EQ(lb, 1); // (e) - EXPECT_EQ(i, 4); // (f) advanced past 'a' + EXPECT_EQ(kb, NonCjkKind::Latin); // (c) + EXPECT_EQ(cpb, 0x0061u); // (d) + EXPECT_EQ(lb, 1); // (e) + EXPECT_EQ(i, 4); // (f) advanced past 'a' EXPECT_EQ(state.pendingCjkBaseLen, 0); // (g) state cleared // (h)(i) stage-time snapshot survives, NOT call-B's false/REGULAR. EXPECT_TRUE(fb.underline); @@ -314,8 +312,8 @@ TEST(ParserCluster, EmittedAndNonCjkCallerAppendContract) { NonCjkKind kx = NonCjkKind::Latin; uint32_t cpx = 0; uint8_t lx = 0; - auto rx = Utf8ClusterAssembler::tryConsumeCodepoint(ss, 1, j, st2, nj2, EpdFontFamily::REGULAR, false, fx, kx, - cpx, lx); + auto rx = + Utf8ClusterAssembler::tryConsumeCodepoint(ss, 1, j, st2, nj2, EpdFontFamily::REGULAR, false, fx, kx, cpx, lx); EXPECT_EQ(rx, ConsumeResult::NonCjkOnly); EXPECT_EQ(kx, NonCjkKind::Whitespace); // do NOT read fx (outFlushable INVALID) EXPECT_EQ(j, 1); @@ -336,8 +334,8 @@ TEST(ParserCluster, EmittedAndNonCjkCallerAppendContract) { NonCjkKind kk = NonCjkKind::Latin; uint32_t cc = 0; uint8_t ll = 0; - auto r = Utf8ClusterAssembler::tryConsumeCodepoint(ss, ln, j, st, nj, EpdFontFamily::REGULAR, false, fl, kk, - cc, ll); + auto r = + Utf8ClusterAssembler::tryConsumeCodepoint(ss, ln, j, st, nj, EpdFontFamily::REGULAR, false, fl, kk, cc, ll); if (r == ConsumeResult::EmittedAndNonCjk || r == ConsumeResult::NonCjkOnly) { seen = kk; break; @@ -345,9 +343,9 @@ TEST(ParserCluster, EmittedAndNonCjkCallerAppendContract) { } return seen; }; - EXPECT_EQ(classifyAfterBase({0x20}), NonCjkKind::Whitespace); // ' ' - EXPECT_EQ(classifyAfterBase({0xC2, 0xA0}), NonCjkKind::Nbsp); // NBSP U+00A0 - EXPECT_EQ(classifyAfterBase({0xEF, 0xBB, 0xBF}), NonCjkKind::Feff); // FEFF + EXPECT_EQ(classifyAfterBase({0x20}), NonCjkKind::Whitespace); // ' ' + EXPECT_EQ(classifyAfterBase({0xC2, 0xA0}), NonCjkKind::Nbsp); // NBSP U+00A0 + EXPECT_EQ(classifyAfterBase({0xEF, 0xBB, 0xBF}), NonCjkKind::Feff); // FEFF } // 11. Split-Latin codepoint across callbacks: é (U+00E9 = C3 A9) split {C3} then {A9}. @@ -363,8 +361,8 @@ TEST(ParserCluster, SplitLatinCodepointAcrossCallbacks) { NonCjkKind k1 = NonCjkKind::Latin; uint32_t cp1 = 0; uint8_t l1 = 0; - auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 1, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, - k1, cp1, l1); + auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 1, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, k1, + cp1, l1); EXPECT_EQ(r1, ConsumeResult::NeedMore); EXPECT_EQ(i1, 0); EXPECT_EQ(state.pendingUtf8Len, 1); @@ -377,8 +375,8 @@ TEST(ParserCluster, SplitLatinCodepointAcrossCallbacks) { NonCjkKind k2 = NonCjkKind::Latin; uint32_t cp2 = 0; uint8_t l2 = 0; - auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, - k2, cp2, l2); + auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, k2, + cp2, l2); EXPECT_EQ(r2, ConsumeResult::NonCjkOnly); EXPECT_EQ(k2, NonCjkKind::Latin); EXPECT_EQ(cp2, 0x00E9u); @@ -426,7 +424,7 @@ TEST(ParserCluster, StyleSnapshotAtStageTimeNotFlushTime) { EXPECT_EQ(rb, ConsumeResult::EmittedFlushable); EXPECT_EQ(fb.len, 3); EXPECT_EQ(decodeOnly(fb.bytes, fb.len), 0x65E5u); // 日 - EXPECT_TRUE(fb.underline); // stage-time true, NOT call-B false + EXPECT_TRUE(fb.underline); // stage-time true, NOT call-B false EXPECT_EQ(fb.fontStyle, EpdFontFamily::REGULAR); EXPECT_EQ(state.pendingCjkBaseLen, 3); // 本 now staged EXPECT_FALSE(state.underlineAtStage); // 本 staged under underline=false @@ -458,8 +456,8 @@ TEST(ParserCluster, StyleSnapshotAtStageTimeNotFlushTime) { auto gr2 = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, j, st, nj, EpdFontFamily::REGULAR, /*callerUnderline=*/true, g2, gk2, gc2, gl2); EXPECT_EQ(gr2, ConsumeResult::EmittedFlushable); - EXPECT_FALSE(g2.underline); // 日's stage-time underline=false - EXPECT_TRUE(st.underlineAtStage); // 本 now staged under underline=true + EXPECT_FALSE(g2.underline); // 日's stage-time underline=false + EXPECT_TRUE(st.underlineAtStage); // 本 now staged under underline=true } }