From cf1b711216543bcd62b108cba842c554bcd60607 Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 16:41:43 +1000 Subject: [PATCH 1/4] fix(epub): preserve U+FEFF no-break between CJK characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 日本 previously wrapped identically to 日本 — the ZWNBSP no-break intent was silently dropped because the trailing CJK base inherited the run's default CjkBreak join. FEFF (ZWNBSP) produces no token, so its only effect is an intrinsic no-break: Utf8ClusterAssembler now sets nextJoin = Glue across it, so the next CJK base stages with Glue instead of CjkBreak. The parser's Feff dispatch keeps discarding the codepoint and no longer needs to touch the join. Add a host regression test (ParserCluster.FeffIsNoBreakBetweenCjk): 日本 stays glued (Glue) while plain 日本 stays breakable (CjkBreak). Red before the fix, green after. --- .../Epub/parsers/ChapterHtmlSlimParser.cpp | 4 ++- .../Epub/parsers/Utf8ClusterAssembler.cpp | 6 ++++- lib/Epub/Epub/parsers/Utf8ClusterAssembler.h | 3 ++- test/parser_cluster/ParserClusterTest.cpp | 27 +++++++++++++++++++ 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index 942fbccd88..d098f7d4d7 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -246,7 +246,9 @@ void ChapterHtmlSlimParser::dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind nextJoin = WordJoin::Glue; break; case Utf8ClusterAssembler::NonCjkKind::Feff: - break; // discard BOM / ZWNBSP + // Discard the BOM / ZWNBSP codepoint. Its no-break (Glue) join is already applied by + // Utf8ClusterAssembler (FEFF is an intrinsic no-break), so do NOT touch nextJoin here. + break; default: assert(false); std::abort(); diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp index 20b3fbf1c0..279863e091 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp @@ -126,10 +126,14 @@ Utf8ClusterAssembler::ConsumeResult Utf8ClusterAssembler::tryConsumeCodepoint( return hadBase ? ConsumeResult::EmittedFlushable : ConsumeResult::StagedOnly; } - // Non-CJK: Latin / whitespace / NBSP / FEFF. Do NOT touch nextJoin — the caller sets it. + // Non-CJK: Latin / whitespace / NBSP / FEFF. The caller sets nextJoin for Latin/whitespace/ + // NBSP (those couple the join to a parser-side flush/emit action). FEFF (ZWNBSP) is the lone + // exception: it produces no token, so its only effect is an intrinsic no-break — glue across + // it here so the next CJK base stages with Glue instead of the run's default CjkBreak. outNonCjkKind = classifyNonCjk(cp); outNonCjkCp = cp; outNonCjkLen = cpLen; + if (outNonCjkKind == NonCjkKind::Feff) nextJoin = WordJoin::Glue; if (state.pendingCjkBaseLen > 0) { fillFlushableFromState(state, outFlushable); state.pendingCjkBaseLen = 0; // clear staged base diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h index d311721a27..16ea8dd91a 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h @@ -18,7 +18,8 @@ class Utf8ClusterAssembler { Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space Nbsp, // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue - Feff, // U+FEFF (BOM) — discard + Feff, // U+FEFF (BOM / ZWNBSP) — caller discards the codepoint; the assembler has already + // set nextJoin = Glue so the no-break intent survives (next CJK base glues, not breaks) }; enum class ConsumeResult : uint8_t { diff --git a/test/parser_cluster/ParserClusterTest.cpp b/test/parser_cluster/ParserClusterTest.cpp index ab2aae4904..80e655394b 100644 --- a/test/parser_cluster/ParserClusterTest.cpp +++ b/test/parser_cluster/ParserClusterTest.cpp @@ -461,4 +461,31 @@ TEST(ParserCluster, StyleSnapshotAtStageTimeNotFlushTime) { } } +// 13. FEFF (ZWNBSP) is a no-break: 日本 must NOT break before 本, while plain +// 日本 DOES. The ZWNBSP carries an intrinsic no-break intent, so the assembler must set +// nextJoin = Glue across it — the trailing base then stages with Glue, not CjkBreak. +TEST(ParserCluster, FeffIsNoBreakBetweenCjk) { + // 日 = E6 97 A5, U+FEFF = EF BB BF, 本 = E6 9C AC + const std::vector withFeff = {0xE6, 0x97, 0xA5, 0xEF, 0xBB, 0xBF, 0xE6, 0x9C, 0xAC}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(withFeff, state, nextJoin); + + // The trailing base 本 is still staged; drain it and inspect its join. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x672Cu); // 本 + EXPECT_EQ(drained.join, WordJoin::Glue); // no break across the ZWNBSP + + // Control: plain 日本 (no FEFF) keeps the ordinary CJK run join — breakable before 本. + const std::vector plain = {0xE6, 0x97, 0xA5, 0xE6, 0x9C, 0xAC}; + State st2; + WordJoin nj2 = WordJoin::Space; + feed(plain, st2, nj2); + Flushable d2; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(st2, d2)); + EXPECT_EQ(decodeOnly(d2.bytes, d2.len), 0x672Cu); // 本 + EXPECT_EQ(d2.join, WordJoin::CjkBreak); // ordinary CJK run: may break here +} + } // namespace From b74759f20f15d309252ad170d73867fdaeb5d3a6 Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 16:41:50 +1000 Subject: [PATCH 2/4] fix(font): enforce setFallback static-glyph precondition on primaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setFallback asserted only the fallback's glyphMissHandler == nullptr, but resolveGlyph's pointer-identity miss detection runs on the selected primary font too (any of the four, per style) — not just the fallback. Assert every non-null primary symmetrically so the already-documented precondition is structurally enforced on both sides. Harmless today (the UI primaries that receive a fallback are static built-in fonts); this is a footgun guard for a future ring-buffer-backed primary. --- lib/EpdFont/EpdFontFamily.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/EpdFont/EpdFontFamily.h b/lib/EpdFont/EpdFontFamily.h index 0a40cf67c4..3af1d158d4 100644 --- a/lib/EpdFont/EpdFontFamily.h +++ b/lib/EpdFont/EpdFontFamily.h @@ -68,7 +68,14 @@ class EpdFontFamily { /// constraint instead of leaving it to documentation. void setFallback(const EpdFont* f) { if (f) { - assert(f->data->glyphMissHandler == nullptr && + // resolveGlyph runs its pointer-identity miss detection on BOTH the selected primary + // (any of the four, per style) AND the fallback, so EVERY non-null primary must satisfy + // the same static-glyph-storage precondition as the fallback — not just the fallback. + assert((regular == nullptr || regular->data->glyphMissHandler == nullptr) && + (bold == nullptr || bold->data->glyphMissHandler == nullptr) && + (italic == nullptr || italic->data->glyphMissHandler == nullptr) && + (boldItalic == nullptr || boldItalic->data->glyphMissHandler == nullptr) && + f->data->glyphMissHandler == nullptr && "setFallback: ring-buffer-backed fonts (glyphMissHandler != nullptr) " "are unsafe with the resolver's pointer-identity miss detection. " "If you need SD-backed fallback, use an SD-safe lookup that does not " From 811d787be3c94a8ab0dc3a347da285f94cabda5f Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 16:56:15 +1000 Subject: [PATCH 3/4] fix(epub): scope FEFF no-break to a breakable CJK adjacency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The prior commit set nextJoin = Glue for every FEFF, which clobbered a preceding Space (日本): the space exists only as the join, so glueing across the FEFF silently swallowed it (and forbade the break). Guard the upgrade on nextJoin == CjkBreak, so FEFF only converts a breakable CJK adjacency into a no-break and leaves a real space — or an already-set Glue — untouched. Add ParserCluster.FeffDoesNotSwallowPrecedingSpace, driven from the post-space state (nextJoin = Space) the parser hands the assembler. Red before the guard, green after; revert-check confirmed non-vacuous. --- .../Epub/parsers/Utf8ClusterAssembler.cpp | 6 ++++-- test/parser_cluster/ParserClusterTest.cpp | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp index 279863e091..8b595a0d81 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp @@ -129,11 +129,13 @@ Utf8ClusterAssembler::ConsumeResult Utf8ClusterAssembler::tryConsumeCodepoint( // Non-CJK: Latin / whitespace / NBSP / FEFF. The caller sets nextJoin for Latin/whitespace/ // NBSP (those couple the join to a parser-side flush/emit action). FEFF (ZWNBSP) is the lone // exception: it produces no token, so its only effect is an intrinsic no-break — glue across - // it here so the next CJK base stages with Glue instead of the run's default CjkBreak. + // it here so the next CJK base stages with Glue instead of the run's default CjkBreak. Only + // upgrade a breakable CJK adjacency (CjkBreak): a preceding Space is a real space carried as + // the join, and an existing Glue is already no-break, so leave both untouched. outNonCjkKind = classifyNonCjk(cp); outNonCjkCp = cp; outNonCjkLen = cpLen; - if (outNonCjkKind == NonCjkKind::Feff) nextJoin = WordJoin::Glue; + if (outNonCjkKind == NonCjkKind::Feff && nextJoin == WordJoin::CjkBreak) nextJoin = WordJoin::Glue; if (state.pendingCjkBaseLen > 0) { fillFlushableFromState(state, outFlushable); state.pendingCjkBaseLen = 0; // clear staged base diff --git a/test/parser_cluster/ParserClusterTest.cpp b/test/parser_cluster/ParserClusterTest.cpp index 80e655394b..394b3b3f9c 100644 --- a/test/parser_cluster/ParserClusterTest.cpp +++ b/test/parser_cluster/ParserClusterTest.cpp @@ -488,4 +488,23 @@ TEST(ParserCluster, FeffIsNoBreakBetweenCjk) { EXPECT_EQ(d2.join, WordJoin::CjkBreak); // ordinary CJK run: may break here } +// 14. FEFF's no-break only upgrades a breakable CJK adjacency (CjkBreak); it must NOT clobber a +// preceding space. In 日本 the parser's dispatchNonCjk(Whitespace) sets +// nextJoin = Space before the FEFF codepoint is consumed (the space exists only as that join, +// never as a token), so we enter FEFF with nextJoin = Space — feed 本 from that state. +// Glueing across the FEFF here would silently swallow the space, so the trailing base must +// keep Space (and stay breakable), NOT become Glue. +TEST(ParserCluster, FeffDoesNotSwallowPrecedingSpace) { + // U+FEFF = EF BB BF, 本 = E6 9C AC. nextJoin = Space models the just-processed space. + const std::vector buf = {0xEF, 0xBB, 0xBF, 0xE6, 0x9C, 0xAC}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x672Cu); // 本 + EXPECT_EQ(drained.join, WordJoin::Space); // space survives the ZWNBSP +} + } // namespace From 83767bf2ae269ebc89597f61e6f0b9caa7392e83 Mon Sep 17 00:00:00 2001 From: Mitsuki Fukunaga Date: Thu, 18 Jun 2026 17:00:29 +1000 Subject: [PATCH 4/4] =?UTF-8?q?docs(epub):=20reflect=20conditional=20FEFF?= =?UTF-8?q?=E2=86=92Glue=20upgrade=20in=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard added in the prior commit makes the FEFF→Glue upgrade conditional (only when the adjacency is breakable, nextJoin == CjkBreak), but the NonCjkKind::Feff doc and the parser's Feff-case comment still read as unconditional. Correct both to state the assembler upgrades only a breakable adjacency and leaves a real Space or existing Glue untouched. --- lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp | 5 +++-- lib/Epub/Epub/parsers/Utf8ClusterAssembler.h | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index d098f7d4d7..4a353073dd 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -246,8 +246,9 @@ void ChapterHtmlSlimParser::dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind nextJoin = WordJoin::Glue; break; case Utf8ClusterAssembler::NonCjkKind::Feff: - // Discard the BOM / ZWNBSP codepoint. Its no-break (Glue) join is already applied by - // Utf8ClusterAssembler (FEFF is an intrinsic no-break), so do NOT touch nextJoin here. + // Discard the BOM / ZWNBSP codepoint. If nextJoin was CjkBreak the assembler has already + // upgraded it to Glue (intrinsic no-break); a real Space or existing Glue is left untouched. + // Either way, do NOT touch nextJoin here. break; default: assert(false); diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h index 16ea8dd91a..e05702a53f 100644 --- a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h @@ -18,8 +18,9 @@ class Utf8ClusterAssembler { Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space Nbsp, // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue - Feff, // U+FEFF (BOM / ZWNBSP) — caller discards the codepoint; the assembler has already - // set nextJoin = Glue so the no-break intent survives (next CJK base glues, not breaks) + Feff, // U+FEFF (BOM / ZWNBSP) — caller discards the codepoint; the assembler upgrades a + // breakable adjacency (nextJoin CjkBreak→Glue) so the no-break intent survives, but + // leaves a real Space (or an existing Glue) untouched }; enum class ConsumeResult : uint8_t {