diff --git a/docs/file-formats.md b/docs/file-formats.md index 0b505fda04..3121796fdf 100644 --- a/docs/file-formats.md +++ b/docs/file-formats.md @@ -90,13 +90,13 @@ if (parsedSize != fileSize) { ## `section.bin` -### Version 25 +### Version 27 Each file in `sections/*.bin` stores one laid-out spine section. The header is also the cache-busting key: if any layout-affecting setting differs from the current reader settings, the section is discarded and rebuilt. -Version 25 includes: +Version 27 includes: - cache-busting fields for paragraph alignment, hyphenation, embedded CSS, image rendering mode, and Focus Reading @@ -105,6 +105,7 @@ Version 25 includes: - paragraph and list-item LUTs used by KOReader sync page refinement - optional per-word Focus Reading split metadata - per-page footnote entries +- CJK (Japanese) paragraphs wrap at viewport width via per-character breakable tokens (v27) ImHex pattern: @@ -113,7 +114,7 @@ import std.mem; import std.string; import std.core; -#define EXPECTED_VERSION 25 +#define EXPECTED_VERSION 27 #define MAX_STRING_LENGTH 65535 #define FOOTNOTE_NUMBER_LEN 32 #define FOOTNOTE_HREF_LEN 96 diff --git a/lib/Epub/Epub/Section.cpp b/lib/Epub/Epub/Section.cpp index c13da52672..3a268c0d84 100644 --- a/lib/Epub/Epub/Section.cpp +++ b/lib/Epub/Epub/Section.cpp @@ -10,7 +10,7 @@ #include "parsers/ChapterHtmlSlimParser.h" namespace { -constexpr uint8_t SECTION_FILE_VERSION = 26; +constexpr uint8_t SECTION_FILE_VERSION = 27; // was 26: CJK line-breaking changes layout output constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(uint8_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) + sizeof(bool) + sizeof(uint8_t) + sizeof(bool) + sizeof(uint32_t) + sizeof(uint32_t) + diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp index ba80f9e1d5..942fbccd88 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp @@ -9,6 +9,8 @@ #include #include +#include +#include #include #include @@ -159,9 +161,10 @@ void ChapterHtmlSlimParser::flushPendingAnchor() { pendingAnchorId.clear(); } -// flush the contents of partWordBuffer to currentTextBlock -void ChapterHtmlSlimParser::flushPartWordBuffer() { - // Determine font style from depth-based tracking and CSS effective style +// Resolve the current font style from depth-based tracking and CSS effective style. +// Verbatim extraction of flushPartWordBuffer's former style block (including underline), so the +// Latin flush path stays byte-identical and the assembler can stage this same snapshot at stage time. +EpdFontFamily::Style ChapterHtmlSlimParser::currentFontStyle() const { const bool isBold = boldUntilDepth < depth || effectiveBold; const bool isItalic = italicUntilDepth < depth || effectiveItalic; const bool isUnderline = underlineUntilDepth < depth || effectiveUnderline; @@ -182,16 +185,90 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() { } else if (effectiveSub) { fontStyle = static_cast(fontStyle | EpdFontFamily::SUB); } + return fontStyle; +} - // flush the buffer +// flush the contents of partWordBuffer to currentTextBlock +void ChapterHtmlSlimParser::flushPartWordBuffer() { partWordBuffer[partWordBufferIndex] = '\0'; - currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextJoin); + currentTextBlock->addWord(partWordBuffer, currentFontStyle(), false, nextJoin); partWordBufferIndex = 0; nextJoin = WordJoin::Space; } +// Drain any pending text — staged CJK base (via the assembler) and/or the Latin partWordBuffer — +// into currentTextBlock. Returns which kind flushed LAST so inline-close callers can decide the +// next join: Latin re-glues (Glue), CJK keeps CjkBreak (stays wrappable). Idempotent when nothing +// is staged. +ChapterHtmlSlimParser::FlushedKind ChapterHtmlSlimParser::flushPendingText() { + if (!currentTextBlock) return FlushedKind::None; + bool flushedCjk = false; + Utf8ClusterAssembler::Flushable f; + if (Utf8ClusterAssembler::flushPendingBase(clusterState, f)) { + // Use the stage-time snapshot (f.fontStyle / f.underline), NOT currentFontStyle()/ + // effectiveUnderline — the style may have changed since the base was staged. + currentTextBlock->addWord(std::string(f.bytes, f.len), f.fontStyle, f.underline, f.join); + flushedCjk = true; + } + if (partWordBufferIndex > 0) { + flushPartWordBuffer(); + return FlushedKind::Latin; // Latin flushed last → Glue is safe for inline-close callers + } + return flushedCjk ? FlushedKind::Cjk : FlushedKind::None; +} + +void ChapterHtmlSlimParser::emitCjkToken(const Utf8ClusterAssembler::Flushable& f) { + if (!currentTextBlock) return; + currentTextBlock->addWord(std::string(f.bytes, f.len), f.fontStyle, f.underline, f.join); +} + +void ChapterHtmlSlimParser::dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind, uint32_t cp) { + switch (kind) { + case Utf8ClusterAssembler::NonCjkKind::Latin: { + // Re-encode from cp (NOT from s[]) — the codepoint may have straddled a callback boundary. + std::string scratch; + utf8AppendCodepoint(cp, scratch); + // Append the codepoint ATOMICALLY: flush first if it wouldn't fit, so a multi-byte cp is + // never split across the MAX_WORD_SIZE boundary (no orphaned continuation bytes). + if (partWordBufferIndex + static_cast(scratch.size()) >= MAX_WORD_SIZE) { + flushPartWordBuffer(); + } + for (char b : scratch) partWordBuffer[partWordBufferIndex++] = b; + break; + } + case Utf8ClusterAssembler::NonCjkKind::Whitespace: + flushPendingText(); + nextJoin = WordJoin::Space; + break; + case Utf8ClusterAssembler::NonCjkKind::Nbsp: + flushPendingText(); + currentTextBlock->addWord(" ", currentFontStyle(), /*underline=*/false, WordJoin::Glue); + nextJoin = WordJoin::Glue; + break; + case Utf8ClusterAssembler::NonCjkKind::Feff: + break; // discard BOM / ZWNBSP + default: + assert(false); + std::abort(); + } +} + +// Split currentTextBlock into pages when it grows past ~750 words, freeing a lot of memory. +// Runs after each CJK emit AND once at the end of characterData to bound Latin-heavy callbacks too. +void ChapterHtmlSlimParser::splitLongBlockIfNeeded() { + if (!currentTextBlock || currentTextBlock->size() <= 750) return; + LOG_DBG("EHP", "Text block too long, splitting into multiple pages"); + const int horizontalInset = currentTextBlock->getBlockStyle().totalHorizontalInset(); + const uint16_t effectiveWidth = + (horizontalInset < viewportWidth) ? static_cast(viewportWidth - horizontalInset) : viewportWidth; + currentTextBlock->layoutAndExtractLines( + renderer, fontId, effectiveWidth, [this](const std::shared_ptr& tb) { addLineToPage(tb); }, false); +} + // start a new text block if needed void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { + flushPendingText(); // Round-7 M9: drain pending CJK base / Latin into the previous block + // before opening the new one. Idempotent if nothing staged. nextJoin = WordJoin::Space; // New block = new paragraph, no continuation if (currentTextBlock) { // already have a text block running and it is empty - just reuse it @@ -218,9 +295,7 @@ void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) { } void ChapterHtmlSlimParser::emitHorizontalRule(const BlockStyle& blockStyle) { - if (partWordBufferIndex > 0) { - flushPartWordBuffer(); - } + flushPendingText(); if (currentTextBlock) { const BlockStyle parentBlockStyle = currentTextBlock->getBlockStyle(); @@ -369,13 +444,12 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* if (strcmp(name, "table") == 0) { // skip nested tables if (self->tableDepth > 0) { + self->flushPendingText(); // Round-11: preserve outer-paragraph text before the table opens self->tableDepth += 1; return; } - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); self->tableDepth += 1; self->tableRowIndex = 0; self->tableColIndex = 0; @@ -391,9 +465,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } if (self->tableDepth == 1 && (strcmp(name, "td") == 0 || strcmp(name, "th") == 0)) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); self->tableColIndex += 1; auto tableCellBlockStyle = BlockStyle(); @@ -417,9 +489,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->inlineStyleStack.push_back(headerStyle); self->updateEffectiveInlineStyle(); self->characterData(userData, headerText.c_str(), static_cast(headerText.length())); - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); self->nextJoin = WordJoin::Space; self->inlineStyleStack.pop_back(); self->updateEffectiveInlineStyle(); @@ -597,9 +667,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } // Flush any pending text block so it appears before the image - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } + self->flushPendingText(); if (self->currentTextBlock && !self->currentTextBlock->isEmpty()) { const BlockStyle parentBlockStyle = self->currentTextBlock->getBlockStyle(); self->startNewTextBlock(parentBlockStyle); @@ -741,9 +809,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* if (isInternalLink) { // Flush buffer before style change - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->insideFootnoteLink = true; self->footnoteLinkDepth = self->depth; @@ -806,10 +874,8 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->updateEffectiveInlineStyle(); } else if (matches(name, BLOCK_TAGS, std::size(BLOCK_TAGS))) { if (strcmp(name, "br") == 0) { - if (self->partWordBufferIndex > 0) { - // flush word preceding
to currentTextBlock before calling startNewTextBlock - self->flushPartWordBuffer(); - } + // flush text preceding
to currentTextBlock before calling startNewTextBlock + self->flushPendingText(); self->startNewTextBlock(self->blockStyleStack.back().withoutBottom()); } else { self->currentCssStyle = cssStyle; @@ -825,9 +891,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* } } else if (matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS))) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->underlineUntilDepth = std::min(self->underlineUntilDepth, self->depth); // Push inline style entry for underline tag @@ -848,9 +914,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->updateEffectiveInlineStyle(); } else if (matches(name, BOLD_TAGS, std::size(BOLD_TAGS))) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->boldUntilDepth = std::min(self->boldUntilDepth, self->depth); // Push inline style entry for bold tag @@ -871,9 +937,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->updateEffectiveInlineStyle(); } else if (matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS))) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } self->italicUntilDepth = std::min(self->italicUntilDepth, self->depth); // Push inline style entry for italic tag @@ -893,9 +959,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* self->inlineStyleStack.push_back(entry); self->updateEffectiveInlineStyle(); } else if (strcmp(name, "sup") == 0 || strcmp(name, "sub") == 0) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } StyleStackEntry entry; entry.depth = self->depth; @@ -913,9 +979,9 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char* if (cssStyle.hasFontWeight() || cssStyle.hasFontStyle() || cssStyle.hasTextDecoration() || cssStyle.hasDirection() || cssStyle.hasVerticalAlign()) { // Flush buffer before style change so preceding text gets current style - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - self->nextJoin = WordJoin::Glue; + { + const FlushedKind kind = self->flushPendingText(); + if (kind == FlushedKind::Latin) self->nextJoin = WordJoin::Glue; } StyleStackEntry entry; entry.depth = self->depth; // Track depth for matching pop @@ -991,129 +1057,55 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char self->currentFootnote.number[self->currentFootnoteLinkTextLen] = '\0'; } - for (int i = 0; i < len; i++) { - if (isWhitespace(s[i])) { - // Currently looking at whitespace, if there's anything in the partWordBuffer, flush it - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } - // Whitespace is a real word boundary — reset continuation state - self->nextJoin = WordJoin::Space; - // Skip the whitespace char - continue; - } - - // Detect U+00A0 (non-breaking space, UTF-8: 0xC2 0xA0) or - // U+202F (narrow no-break space, UTF-8: 0xE2 0x80 0xAF). - // - // Both are rendered as a visible space but must never allow a line break around them. - // We split the no-break space into its own word token and link the surrounding words - // with continuation flags so the layout engine treats them as an indivisible group. - // - // Example: "200 Quadratkilometer" or "200 Quadratkilometer" - // Input bytes: "200\xC2\xA0Quadratkilometer" (or 0xE2 0x80 0xAF for U+202F) - // Tokens produced: - // [0] "200" continues=false - // [1] " " continues=true (attaches to "200", no gap) - // [2] "Quadratkilometer" continues=true (attaches to " ", no gap) - // - // The continuation flags prevent the line-breaker from inserting a line break - // between "200" and "Quadratkilometer". However, "Quadratkilometer" is now a - // standalone word for hyphenation purposes, so Liang patterns can produce - // "200 Quadrat-" / "kilometer" instead of the unusable "200" / "Quadratkilometer". - if (static_cast(s[i]) == 0xC2 && i + 1 < len && static_cast(s[i + 1]) == 0xA0) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } - - self->partWordBuffer[0] = ' '; - self->partWordBuffer[1] = '\0'; - self->partWordBufferIndex = 1; - self->nextJoin = WordJoin::Glue; // Attach space to previous word (no break). - self->flushPartWordBuffer(); - - self->nextJoin = WordJoin::Glue; // Next real word attaches to this space (no break). - - i++; // Skip the second byte (0xA0) - continue; - } - - // U+202F (narrow no-break space) — identical logic to U+00A0 above. - if (static_cast(s[i]) == 0xE2 && i + 2 < len && static_cast(s[i + 1]) == 0x80 && - static_cast(s[i + 2]) == 0xAF) { - if (self->partWordBufferIndex > 0) { - self->flushPartWordBuffer(); - } - - self->partWordBuffer[0] = ' '; - self->partWordBuffer[1] = '\0'; - self->partWordBufferIndex = 1; - self->nextJoin = WordJoin::Glue; - self->flushPartWordBuffer(); - - self->nextJoin = WordJoin::Glue; - - i += 2; // Skip the remaining two bytes (0x80 0xAF) - continue; - } - - // Skip Zero Width No-Break Space / BOM (U+FEFF) = 0xEF 0xBB 0xBF - const XML_Char FEFF_BYTE_1 = static_cast(0xEF); - const XML_Char FEFF_BYTE_2 = static_cast(0xBB); - const XML_Char FEFF_BYTE_3 = static_cast(0xBF); - - if (s[i] == FEFF_BYTE_1) { - // Check if the next two bytes complete the 3-byte sequence - if ((i + 2 < len) && (s[i + 1] == FEFF_BYTE_2) && (s[i + 2] == FEFF_BYTE_3)) { - // Sequence 0xEF 0xBB 0xBF found! - i += 2; // Skip the next two bytes - continue; // Move to the next iteration - } - } - - // If we're about to run out of space, then cut the word off and start a new one. - // For CJK text (no spaces), this is the primary word-breaking mechanism. - // We must avoid splitting multi-byte UTF-8 sequences across word boundaries, - // otherwise the trailing bytes become orphaned continuation bytes that the - // decoder can't interpret. - if (self->partWordBufferIndex >= MAX_WORD_SIZE) { - int safeLen = utf8SafeTruncateBuffer(self->partWordBuffer, self->partWordBufferIndex); - - if (safeLen < self->partWordBufferIndex && safeLen > 0) { - // Incomplete UTF-8 sequence at the end — save it before flushing - int overflow = self->partWordBufferIndex - safeLen; - char saved[4]; - for (int j = 0; j < overflow; j++) { - saved[j] = self->partWordBuffer[safeLen + j]; - } - self->partWordBufferIndex = safeLen; - self->flushPartWordBuffer(); - for (int j = 0; j < overflow; j++) { - self->partWordBuffer[j] = saved[j]; + // Drive the cluster assembler one codepoint at a time. It stages CJK bases into clusterState + // (so a CJK base + its NFD extenders becomes a single, breakable word) and classifies non-CJK + // codepoints (Latin / whitespace / NBSP / FEFF) for the legacy branches below. CJK runs now break + // at the viewport edge instead of the old fixed MAX_WORD_SIZE word cut. + for (int i = 0; i < len;) { + Utf8ClusterAssembler::Flushable f; + Utf8ClusterAssembler::NonCjkKind kind; + uint32_t cp = 0; + uint8_t cpLen = 0; + const WordJoin joinBefore = self->nextJoin; + const Utf8ClusterAssembler::ConsumeResult r = Utf8ClusterAssembler::tryConsumeCodepoint( + s, len, i, self->clusterState, self->nextJoin, self->currentFontStyle(), self->effectiveUnderline, f, kind, cp, + cpLen); + + switch (r) { + case Utf8ClusterAssembler::ConsumeResult::NeedMore: + return; // codepoint split across callbacks — resume next callback + case Utf8ClusterAssembler::ConsumeResult::StagedOnly: + // A CJK base was just staged. If Latin text was pending in partWordBuffer (a Latin→CJK + // boundary with no separating whitespace, e.g. "PDF版" / "5月"), it must be emitted BEFORE + // the staged base so word order is preserved. The base wrongly inherited the Latin run's + // join when it was staged; the Latin owns that join, and the base should break from the + // Latin (CjkBreak — they abut with no space, breakable). (Invariant: partWordBuffer and a + // staged base are never both non-empty except transiently here, so this drains cleanly.) + if (self->partWordBufferIndex > 0) { + self->nextJoin = joinBefore; // restore the Latin run's own join + self->flushPartWordBuffer(); // emit the Latin token IN ORDER + self->clusterState.pendingCjkBaseJoin = WordJoin::CjkBreak; // staged base breaks from the Latin + self->nextJoin = WordJoin::CjkBreak; // CJK run continues: successor breaks too } - self->partWordBufferIndex = overflow; - } else { - self->flushPartWordBuffer(); - } + break; + case Utf8ClusterAssembler::ConsumeResult::EmittedFlushable: + self->emitCjkToken(f); + self->splitLongBlockIfNeeded(); + break; + case Utf8ClusterAssembler::ConsumeResult::NonCjkOnly: + self->dispatchNonCjk(kind, cp); + break; + case Utf8ClusterAssembler::ConsumeResult::EmittedAndNonCjk: + self->emitCjkToken(f); + self->splitLongBlockIfNeeded(); + self->dispatchNonCjk(kind, cp); + break; } - - self->partWordBuffer[self->partWordBufferIndex++] = s[i]; } - // If we have > 750 words buffered up, perform the layout and consume out all but the last line - // There should be enough here to build out 1-2 full pages and doing this will free up a lot of - // memory. - // Spotted when reading Intermezzo, there are some really long text blocks in there. - if (self->currentTextBlock->size() > 750) { - LOG_DBG("EHP", "Text block too long, splitting into multiple pages"); - const int horizontalInset = self->currentTextBlock->getBlockStyle().totalHorizontalInset(); - const uint16_t effectiveWidth = (horizontalInset < self->viewportWidth) - ? static_cast(self->viewportWidth - horizontalInset) - : self->viewportWidth; - self->currentTextBlock->layoutAndExtractLines( - self->renderer, self->fontId, effectiveWidth, - [self](const std::shared_ptr& textBlock) { self->addLineToPage(textBlock); }, false); - } + // Bound Latin-heavy callbacks too: split once after draining the chunk (CJK emits already split + // per-token above). See splitLongBlockIfNeeded for the rationale. + self->splitLongBlockIfNeeded(); } void XMLCALL ChapterHtmlSlimParser::defaultHandlerExpand(void* userData, const XML_Char* s, const int len) { @@ -1151,27 +1143,27 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n if (self->tableDepth > 1 && strcmp(name, "table") == 0) { // get rid of all text inside the nested table self->partWordBufferIndex = 0; + self->clusterState = Utf8ClusterAssembler::State{}; // drop staged CJK too (NOT a flush) self->tableDepth -= 1; LOG_DBG("EHP", "nested table detected, get rid of its content"); return; } - // Flush buffer with current style BEFORE any style changes - if (self->partWordBufferIndex > 0) { - // Flush if style will change OR if we're closing a block/structural element - const bool isInlineTag = !headerOrBlockTag && !tableStructuralTag && - !matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) && self->depth != 1; - const bool shouldFlush = styleWillChange || headerOrBlockTag || matches(name, BOLD_TAGS, std::size(BOLD_TAGS)) || - matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS)) || - matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS)) || tableStructuralTag || - matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) || self->depth == 1; - - if (shouldFlush) { - self->flushPartWordBuffer(); - // If closing an inline element, the next word fragment continues the same visual word - if (isInlineTag) { - self->nextJoin = WordJoin::Glue; - } + // Flush buffer with current style BEFORE any style changes. + // Flush if style will change OR if we're closing a block/structural element. + const bool isInlineTag = + !headerOrBlockTag && !tableStructuralTag && !matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) && self->depth != 1; + const bool shouldFlush = styleWillChange || headerOrBlockTag || matches(name, BOLD_TAGS, std::size(BOLD_TAGS)) || + matches(name, ITALIC_TAGS, std::size(ITALIC_TAGS)) || + matches(name, UNDERLINE_TAGS, std::size(UNDERLINE_TAGS)) || tableStructuralTag || + matches(name, IMAGE_TAGS, std::size(IMAGE_TAGS)) || self->depth == 1; + + if (shouldFlush) { + const FlushedKind kind = self->flushPendingText(); + // If closing an inline element, a Latin fragment continues the same visual word; a CJK base + // must stay on CjkBreak so it remains wrappable across the boundary. + if (isInlineTag && kind == FlushedKind::Latin) { + self->nextJoin = WordJoin::Glue; } } diff --git a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h index cc9874ab04..309b25ab31 100644 --- a/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h +++ b/lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h @@ -14,6 +14,7 @@ #include "Epub/blocks/TextBlock.h" #include "Epub/css/CssParser.h" #include "Epub/css/CssStyle.h" +#include "Utf8ClusterAssembler.h" class Page; class GfxRenderer; @@ -37,6 +38,10 @@ class ChapterHtmlSlimParser { char partWordBuffer[MAX_WORD_SIZE + 1] = {}; int partWordBufferIndex = 0; WordJoin nextJoin = WordJoin::Space; // how the next flushed word joins the previous (inline element boundary) + // Kind of text drained by flushPendingText(): inline-close callers re-glue Latin (Glue) but must + // leave a CJK base on CjkBreak so it stays wrappable across the boundary. + enum class FlushedKind : uint8_t { None, Latin, Cjk }; + Utf8ClusterAssembler::State clusterState; // cross-callback CJK base / truncated-codepoint staging std::unique_ptr currentTextBlock = nullptr; std::unique_ptr currentPage = nullptr; int16_t currentPageNextY = 0; @@ -100,6 +105,11 @@ class ChapterHtmlSlimParser { void startNewTextBlock(const BlockStyle& blockStyle); void flushPendingAnchor(); void flushPartWordBuffer(); + EpdFontFamily::Style currentFontStyle() const; + FlushedKind flushPendingText(); + void emitCjkToken(const Utf8ClusterAssembler::Flushable& f); + void dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind, uint32_t cp); + void splitLongBlockIfNeeded(); void makePages(); static void applyDirectionToEntry(StyleStackEntry& entry, const CssStyle& css); void emitHorizontalRule(const BlockStyle& blockStyle); diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp new file mode 100644 index 0000000000..20b3fbf1c0 --- /dev/null +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp @@ -0,0 +1,148 @@ +// lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp +// +// Host-safe UTF-8 decode + grapheme cluster assembly + WordJoin policy. See the header +// for the public contract and the task brief for the state-transition table. No heap, no +// Arduino/HAL deps — depends only on Utf8.h + WordJoin.h + EpdFontFamily.h. + +#include "Utf8ClusterAssembler.h" + +#include +#include + +namespace { + +// Expected total byte length of a UTF-8 sequence from its lead byte. Mirrors Utf8.cpp's +// utf8CodepointLen (kept local so this TU stays free of internal-linkage coupling). +uint8_t leadByteLen(uint8_t c) { + if (c < 0x80) return 1; // 0xxxxxxx + if ((c >> 5) == 0x6) return 2; // 110xxxxx + if ((c >> 4) == 0xE) return 3; // 1110xxxx + if ((c >> 3) == 0x1E) return 4; // 11110xxx + return 1; // invalid lead — treat as a 1-byte (replacement) cp +} + +// Classify a fully-decoded non-CJK codepoint into the caller-dispatch kind. +Utf8ClusterAssembler::NonCjkKind classifyNonCjk(uint32_t cp) { + using NonCjkKind = Utf8ClusterAssembler::NonCjkKind; + if (cp == 0x20 || cp == 0x09 || cp == 0x0A || cp == 0x0D) return NonCjkKind::Whitespace; + if (cp == 0x00A0 || cp == 0x202F) return NonCjkKind::Nbsp; + if (cp == 0xFEFF) return NonCjkKind::Feff; + return NonCjkKind::Latin; +} + +// Copy `state`'s staged base into `out`. Does NOT clear the staged base. +void fillFlushableFromState(const Utf8ClusterAssembler::State& state, Utf8ClusterAssembler::Flushable& out) { + memcpy(out.bytes, state.pendingCjkBase, state.pendingCjkBaseLen); + out.len = state.pendingCjkBaseLen; + out.join = state.pendingCjkBaseJoin; + out.fontStyle = state.fontStyleAtStage; + out.underline = state.underlineAtStage; +} + +} // namespace + +Utf8ClusterAssembler::ConsumeResult Utf8ClusterAssembler::tryConsumeCodepoint( + const char* s, int len, int& i, State& state, WordJoin& nextJoin, EpdFontFamily::Style callerFontStyle, + bool callerUnderline, Flushable& outFlushable, NonCjkKind& outNonCjkKind, uint32_t& outNonCjkCp, + uint8_t& outNonCjkLen) { + // --- Step 1/2: assemble a NUL-terminated scratch holding exactly one codepoint. --- + // utf8NextCodepoint stops on a NUL, so we MUST decode from this private scratch and never + // off s[] (which is not NUL-terminated). + char scratch[5]; + uint8_t scratchLen = 0; // bytes placed into scratch (full cp length) + uint8_t continuationBytes = 0; // bytes pulled from THIS call's s[] (drives i advance) + + if (state.pendingUtf8Len > 0) { + // Completing a codepoint split across callbacks. + const uint8_t expected = leadByteLen(state.pendingUtf8[0]); + memcpy(scratch, state.pendingUtf8, state.pendingUtf8Len); + scratchLen = state.pendingUtf8Len; + + while (scratchLen < expected && (i + continuationBytes) < len) { + scratch[scratchLen++] = s[i + continuationBytes]; + continuationBytes++; + } + + if (scratchLen < expected) { + // Still incomplete: stage what we pulled and signal NeedMore. i unchanged. + memcpy(state.pendingUtf8, scratch, scratchLen); + state.pendingUtf8Len = scratchLen; + return ConsumeResult::NeedMore; + } + state.pendingUtf8Len = 0; // carry-over consumed + } else { + if (i >= len) return ConsumeResult::NeedMore; // nothing to read (defensive) + const uint8_t expected = leadByteLen(static_cast(s[i])); + if (i + expected > len) { + // Partial sequence at the buffer tail — stage and signal NeedMore. i unchanged. + for (uint8_t b = 0; b < expected && (i + b) < len; ++b) state.pendingUtf8[b] = s[i + b]; + state.pendingUtf8Len = static_cast(len - i); + return ConsumeResult::NeedMore; + } + scratchLen = expected; + continuationBytes = expected; + memcpy(scratch, s + i, expected); + } + scratch[scratchLen] = '\0'; + + const unsigned char* p = reinterpret_cast(scratch); + const uint32_t cp = utf8NextCodepoint(&p); + const uint8_t cpLen = scratchLen; // full UTF-8 byte length of this codepoint + + // --- Step 3: classify — grapheme extender FIRST, then CJK breakable, then non-CJK. --- + + // Grapheme extender glued onto an existing staged base. + if (utf8IsGraphemeExtender(cp) && state.pendingCjkBaseLen > 0) { + // Step 4: overflow guard — if appending would exceed the staged buffer, force-flush the + // staged base and re-stage the extender as a NEW base with a reset join + fresh snapshot. + if (static_cast(state.pendingCjkBaseLen) + cpLen > sizeof(state.pendingCjkBase)) { + fillFlushableFromState(state, outFlushable); + memcpy(state.pendingCjkBase, scratch, cpLen); + state.pendingCjkBaseLen = cpLen; + state.pendingCjkBaseJoin = WordJoin::CjkBreak; // explicit reset, NOT the inherited join + state.fontStyleAtStage = callerFontStyle; + state.underlineAtStage = callerUnderline; + i += continuationBytes; + return ConsumeResult::EmittedFlushable; + } + memcpy(state.pendingCjkBase + state.pendingCjkBaseLen, scratch, cpLen); + state.pendingCjkBaseLen = static_cast(state.pendingCjkBaseLen + cpLen); + i += continuationBytes; + return ConsumeResult::StagedOnly; + } + + // CJK breakable base. + if (utf8IsCjkBreakable(cp)) { + const bool hadBase = state.pendingCjkBaseLen > 0; + if (hadBase) fillFlushableFromState(state, outFlushable); + // Stage the NEW base. + memcpy(state.pendingCjkBase, scratch, cpLen); + state.pendingCjkBaseLen = cpLen; + state.pendingCjkBaseJoin = nextJoin; + state.fontStyleAtStage = callerFontStyle; + state.underlineAtStage = callerUnderline; + nextJoin = WordJoin::CjkBreak; + i += continuationBytes; + return hadBase ? ConsumeResult::EmittedFlushable : ConsumeResult::StagedOnly; + } + + // Non-CJK: Latin / whitespace / NBSP / FEFF. Do NOT touch nextJoin — the caller sets it. + outNonCjkKind = classifyNonCjk(cp); + outNonCjkCp = cp; + outNonCjkLen = cpLen; + if (state.pendingCjkBaseLen > 0) { + fillFlushableFromState(state, outFlushable); + state.pendingCjkBaseLen = 0; // clear staged base + i += continuationBytes; + return ConsumeResult::EmittedAndNonCjk; + } + i += continuationBytes; + return ConsumeResult::NonCjkOnly; +} + +bool Utf8ClusterAssembler::flushPendingBase(State& state, Flushable& outFlushable) { + if (state.pendingCjkBaseLen == 0) return false; + fillFlushableFromState(state, outFlushable); + state.pendingCjkBaseLen = 0; + return true; +} diff --git a/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h new file mode 100644 index 0000000000..d311721a27 --- /dev/null +++ b/lib/Epub/Epub/parsers/Utf8ClusterAssembler.h @@ -0,0 +1,71 @@ +// lib/Epub/Epub/parsers/Utf8ClusterAssembler.h +#pragma once + +#include // for EpdFontFamily::Style (used by State + Flushable + callerFontStyle) +#include // for utf8NextCodepoint + predicates + +#include + +#include "../WordJoin.h" // for WordJoin (enum-only, no layout deps) + +class Utf8ClusterAssembler { + public: + // Classifies the non-CJK codepoint that the caller must dispatch on after an + // EmittedAndNonCjk / NonCjkOnly return. The four kinds map 1:1 to the four branches + // in ChapterHtmlSlimParser::dispatchNonCjk (Latin buffer append / whitespace flush / + // NBSP synthesized token / FEFF discard). + enum class NonCjkKind : uint8_t { + Latin, // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer + Whitespace, // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space + Nbsp, // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue + Feff, // U+FEFF (BOM) — discard + }; + + enum class ConsumeResult : uint8_t { + NeedMore, // current codepoint incomplete (carry-over staged); i unchanged + StagedOnly, // codepoint consumed, staged into pendingCjkBase, no flushable emitted; i advanced + EmittedFlushable, // staged base flushed (caller emits it as a CJK token); + // current codepoint also staged/consumed; i advanced + NonCjkOnly, // no pending base existed (outFlushable INVALID — do not read). + // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. + EmittedAndNonCjk, // a pending base existed and was flushed: outFlushable filled. + // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced. + }; + + // Cross-callback state — owned by the parser, threaded into each call. + // fontStyleAtStage / underlineAtStage are snapshotted at STAGE time (not flush time): a CJK + // base may be staged inside and flushed after closes; the flush must use the style + // in effect when the base was STAGED. + struct State { + uint8_t pendingUtf8[4] = {}; // leading bytes of a truncated codepoint + uint8_t pendingUtf8Len = 0; // 0..3 + char pendingCjkBase[16] = {}; // staged base + already-absorbed extenders + uint8_t pendingCjkBaseLen = 0; + WordJoin pendingCjkBaseJoin = WordJoin::Space; + EpdFontFamily::Style fontStyleAtStage = EpdFontFamily::REGULAR; + bool underlineAtStage = false; + }; + + // Payload emitted when a previously-staged CJK base must be flushed. + struct Flushable { + char bytes[16]; + uint8_t len; + WordJoin join; + EpdFontFamily::Style fontStyle; // snapshot at stage time + bool underline; // snapshot at stage time + }; + + // Consume up to one complete codepoint from s[i..len]. Returns the discriminant; out-params + // populated per the state-transition table. callerFontStyle / callerUnderline are the resolved + // style at THIS call; when a new base is staged they are snapshotted into State. + static ConsumeResult tryConsumeCodepoint( + const char* s, int len, int& i, State& state, WordJoin& nextJoin, EpdFontFamily::Style callerFontStyle, + bool callerUnderline, + Flushable& outFlushable, // valid iff result in {EmittedFlushable, EmittedAndNonCjk} + NonCjkKind& outNonCjkKind, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} + uint32_t& outNonCjkCp, // valid iff result in {NonCjkOnly, EmittedAndNonCjk} + uint8_t& outNonCjkLen); // total UTF-8 byte length of the non-CJK codepoint (1..4) + + // Force-flush any staged base. Returns true with outFlushable filled if a base was pending. + static bool flushPendingBase(State& state, Flushable& outFlushable); +}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2286f968ef..46d1651c7c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -48,3 +48,4 @@ add_subdirectory(font_resolver) add_subdirectory(font_boundary) add_subdirectory(parsed_text) add_subdirectory(utf8) +add_subdirectory(parser_cluster) diff --git a/test/parsed_text/ParsedTextLayoutTest.cpp b/test/parsed_text/ParsedTextLayoutTest.cpp index d47cca58a0..193348976a 100644 --- a/test/parsed_text/ParsedTextLayoutTest.cpp +++ b/test/parsed_text/ParsedTextLayoutTest.cpp @@ -337,3 +337,30 @@ TEST(ParsedTextLayout, FocusReadingPreservesUnderlineOnCjkCluster) { EXPECT_EQ(static_cast(styles[0]) & static_cast(EpdFontFamily::BOLD), 0) << "focus-bypass must still skip focus-bolding of a single CJK cluster"; } + +// CJK across an inline-style boundary MUST keep CjkBreak (stays wrappable). +TEST(ParsedTextLayout, ParserSequenceCjkAcrossInlineStyleStaysBreakable) { + ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false, + /*focusReadingEnabled=*/false, leftAligned()); + text.addWord("ab", EpdFontFamily::REGULAR); // join=Space + text.addWord("\xE6\x97\xA5", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 日 + text.addWord("\xE6\x9C\xAC", EpdFontFamily::ITALIC, false, WordJoin::CjkBreak); // 本 (italic via ) + text.addWord("\xE8\xAA\x9E", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak); // 語 + auto lines = linesOf(text, 3 * kCell); + ASSERT_EQ(lines.size(), 2u); + EXPECT_EQ(lines[0], "ab\xE6\x97\xA5"); // ab日 + EXPECT_EQ(lines[1], "\xE6\x9C\xAC\xE8\xAA\x9E"); // 本語 +} + +// Latin inline-close MUST keep Glue (preserves quickly). +TEST(ParsedTextLayout, ParserSequenceLatinAcrossInlineStyleStaysGlued) { + ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false, + /*focusReadingEnabled=*/false, leftAligned()); + text.addWord("a", EpdFontFamily::REGULAR); // join=Space + text.addWord("quick", EpdFontFamily::REGULAR); // join=Space + text.addWord("ly", EpdFontFamily::ITALIC, false, WordJoin::Glue); // Glue — inline close + auto lines = linesOf(text, 7 * kCell); + ASSERT_EQ(lines.size(), 2u); + EXPECT_EQ(lines[0], "a"); + EXPECT_EQ(lines[1], "quickly"); +} diff --git a/test/parser_cluster/CMakeLists.txt b/test/parser_cluster/CMakeLists.txt new file mode 100644 index 0000000000..9972373f29 --- /dev/null +++ b/test/parser_cluster/CMakeLists.txt @@ -0,0 +1,17 @@ +# test/parser_cluster/CMakeLists.txt +add_executable(ParserClusterTest + ParserClusterTest.cpp + ${REPO_ROOT}/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp + ${REPO_ROOT}/lib/Utf8/Utf8.cpp +) +target_include_directories(ParserClusterTest PRIVATE + ${REPO_ROOT} + ${REPO_ROOT}/lib/Utf8 + ${REPO_ROOT}/lib/Epub + ${REPO_ROOT}/lib/Epub/Epub + ${REPO_ROOT}/lib/Epub/Epub/parsers + ${REPO_ROOT}/lib/EpdFont + ${REPO_ROOT}/test +) +target_link_libraries(ParserClusterTest PRIVATE crosspoint_test_common GTest::gtest_main) +gtest_discover_tests(ParserClusterTest) diff --git a/test/parser_cluster/ParserClusterTest.cpp b/test/parser_cluster/ParserClusterTest.cpp new file mode 100644 index 0000000000..ab2aae4904 --- /dev/null +++ b/test/parser_cluster/ParserClusterTest.cpp @@ -0,0 +1,464 @@ +// test/parser_cluster/ParserClusterTest.cpp +// +// Behavioural contract for Utf8ClusterAssembler — the host-safe state machine that +// performs UTF-8 decode + grapheme cluster assembly + WordJoin policy. The parser +// (Task 2.3c) consumes it; these 12 cases pin the contract that parser relies on. +// +// Helper `feed()` drives tryConsumeCodepoint over a byte buffer and collects every +// emitted Flushable, then drains any staged base with flushPendingBase. + +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +using ConsumeResult = Utf8ClusterAssembler::ConsumeResult; +using NonCjkKind = Utf8ClusterAssembler::NonCjkKind; +using Flushable = Utf8ClusterAssembler::Flushable; +using State = Utf8ClusterAssembler::State; + +// One observed call result, for assertions on call-by-call behaviour. +struct Step { + ConsumeResult result; + int iBefore; + int iAfter; + Flushable flushable; // valid iff result in {EmittedFlushable, EmittedAndNonCjk} + NonCjkKind nonCjkKind; + uint32_t nonCjkCp; + uint8_t nonCjkLen; +}; + +// Drive tryConsumeCodepoint across a whole buffer. callerFontStyle/callerUnderline are +// held constant for the whole feed (per-call snapshots are exercised directly in the +// style-snapshot tests). Stops looping when i no longer advances on a NeedMore so the +// caller can inspect the carry-over. +std::vector feed(const std::vector& bytes, State& state, WordJoin& nextJoin, + EpdFontFamily::Style callerFontStyle = EpdFontFamily::REGULAR, bool callerUnderline = false) { + std::vector steps; + const char* s = reinterpret_cast(bytes.data()); + const int len = static_cast(bytes.size()); + int i = 0; + while (i < len) { + Step step; + step.iBefore = i; + step.flushable = Flushable{}; + step.nonCjkKind = NonCjkKind::Latin; + step.nonCjkCp = 0; + step.nonCjkLen = 0; + step.result = + Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, callerFontStyle, callerUnderline, + step.flushable, step.nonCjkKind, step.nonCjkCp, step.nonCjkLen); + step.iAfter = i; + steps.push_back(step); + if (step.result == ConsumeResult::NeedMore) break; // carry-over staged; no more progress this buffer + } + return steps; +} + +uint32_t decodeOnly(const char* bytes, uint8_t len) { + char scratch[5] = {}; + memcpy(scratch, bytes, len); + const unsigned char* p = reinterpret_cast(scratch); + return utf8NextCodepoint(&p); +} + +// 1. NUL termination boundary: a 3-byte CJK cp where `len` excludes any trailing NUL. +// Classification fires only after the full cp is present; no read past s[len-1]. +TEST(ParserCluster, NulTerminationBoundary) { + // 日 U+65E5 = E6 97 A5, buffer holds exactly 3 bytes, no trailing NUL. + const std::vector buf = {0xE6, 0x97, 0xA5}; + State state; + WordJoin nextJoin = WordJoin::Space; + auto steps = feed(buf, state, nextJoin); + ASSERT_EQ(steps.size(), 1u); + EXPECT_EQ(steps[0].result, ConsumeResult::StagedOnly); + EXPECT_EQ(steps[0].iAfter, 3); // consumed all 3 bytes, none past the boundary + // The staged base is the full cp. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 3); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x65E5u); +} + +// 2. Carry-over across calls: split U+65E5 as {E6,97} then {A5}. +TEST(ParserCluster, CarryOverAcrossCalls) { + State state; + WordJoin nextJoin = WordJoin::Space; + + // Call 1: first 2 bytes only -> NeedMore, i unchanged, no flushable. + const std::vector part1 = {0xE6, 0x97}; + const char* s1 = reinterpret_cast(part1.data()); + int i1 = 0; + Flushable f1{}; + NonCjkKind k1 = NonCjkKind::Latin; + uint32_t cp1 = 0; + uint8_t l1 = 0; + auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 2, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, k1, + cp1, l1); + EXPECT_EQ(r1, ConsumeResult::NeedMore); + EXPECT_EQ(i1, 0); // i unchanged + EXPECT_EQ(state.pendingUtf8Len, 2); + + // Call 2: final byte -> stages the cluster. + const std::vector part2 = {0xA5}; + const char* s2 = reinterpret_cast(part2.data()); + int i2 = 0; + Flushable f2{}; + NonCjkKind k2 = NonCjkKind::Latin; + uint32_t cp2 = 0; + uint8_t l2 = 0; + auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, k2, + cp2, l2); + EXPECT_EQ(r2, ConsumeResult::StagedOnly); + EXPECT_EQ(i2, 1); + + // Drain emits one 3-byte cluster. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 3); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x65E5u); +} + +// 3. Single-callback NFD cluster: か (U+304B) + U+3099 in one call. +TEST(ParserCluster, SingleCallbackNfdCluster) { + // か U+304B = E3 81 8B, U+3099 = E3 82 99 + const std::vector buf = {0xE3, 0x81, 0x8B, 0xE3, 0x82, 0x99}; + State state; + WordJoin nextJoin = WordJoin::Space; + auto steps = feed(buf, state, nextJoin); + ASSERT_EQ(steps.size(), 2u); + EXPECT_EQ(steps[0].result, ConsumeResult::StagedOnly); // base staged + EXPECT_EQ(steps[1].result, ConsumeResult::StagedOnly); // extender absorbed, no flush + EXPECT_EQ(state.pendingCjkBaseLen, 6); // 3 + 3 bytes glued + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one 6-byte cluster spanning both codepoints +} + +// 4. Cross-callback NFD cluster: か in call 1 + U+3099 in call 2. +TEST(ParserCluster, CrossCallbackNfdCluster) { + State state; + WordJoin nextJoin = WordJoin::Space; + + const std::vector call1 = {0xE3, 0x81, 0x8B}; // か + auto s1 = feed(call1, state, nextJoin); + ASSERT_EQ(s1.size(), 1u); + EXPECT_EQ(s1[0].result, ConsumeResult::StagedOnly); + EXPECT_EQ(state.pendingCjkBaseLen, 3); // base survives the boundary + + const std::vector call2 = {0xE3, 0x82, 0x99}; // U+3099 + auto s2 = feed(call2, state, nextJoin); + ASSERT_EQ(s2.size(), 1u); + EXPECT_EQ(s2[0].result, ConsumeResult::StagedOnly); // extender glued onto carried base + EXPECT_EQ(state.pendingCjkBaseLen, 6); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one 6-byte cluster spanning both callbacks +} + +// 5. Variation Selector VS-1: 漢 (U+6F22) + U+FE00. +TEST(ParserCluster, VariationSelectorVs1) { + // 漢 = E6 BC A2, U+FE00 = EF B8 80 + const std::vector buf = {0xE6, 0xBC, 0xA2, 0xEF, 0xB8, 0x80}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one cluster of exactly 6 bytes +} + +// 6. Variation Selector Supplement: 漢 (3 bytes) + U+E0100 (4 bytes). +TEST(ParserCluster, VariationSelectorSupplement) { + // 漢 = E6 BC A2, U+E0100 = F3 A0 84 80 + const std::vector buf = {0xE6, 0xBC, 0xA2, 0xF3, 0xA0, 0x84, 0x80}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 7); // 3 + 4 bytes +} + +// 7. Halfwidth voicing: ｶ (U+FF76) + ﾞ (U+FF9E). +TEST(ParserCluster, HalfwidthVoicing) { + // ｶ U+FF76 = EF BD B6, ﾞ U+FF9E = EF BE 9E + const std::vector buf = {0xEF, 0xBD, 0xB6, 0xEF, 0xBE, 0x9E}; + State state; + WordJoin nextJoin = WordJoin::Space; + feed(buf, state, nextJoin); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // one 6-byte cluster +} + +// 8. Ideographic tone mark: CJK base + U+302A. Pins extender-before-breakable ordering: +// U+302A is BOTH a grapheme extender and CJK-breakable. If breakable were checked first +// this would be two clusters and this test would go red. +TEST(ParserCluster, IdeographicToneMarkGluesNotBreaks) { + // 日 = E6 97 A5, U+302A = E3 80 AA + const std::vector buf = {0xE6, 0x97, 0xA5, 0xE3, 0x80, 0xAA}; + State state; + WordJoin nextJoin = WordJoin::Space; + auto steps = feed(buf, state, nextJoin); + ASSERT_EQ(steps.size(), 2u); + EXPECT_EQ(steps[0].result, ConsumeResult::StagedOnly); // base + EXPECT_EQ(steps[1].result, ConsumeResult::StagedOnly); // tone mark glued, NOT a new base + EXPECT_EQ(state.pendingCjkBaseLen, 6); + + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(drained.len, 6); // a single cluster, not two +} + +// 9. Overflow guard with join reset: base + enough extenders that the next append would +// exceed sizeof(pendingCjkBase) (16). The staged cluster force-flushes via +// EmittedFlushable, the overflowing extender becomes the new staged base, and the new +// base's join is reset to CjkBreak (NOT the inherited Glue/Space). +TEST(ParserCluster, OverflowGuardResetsJoin) { + // Base 漢 (3 bytes) then VS-1 extenders (3 bytes each: EF B8 80). + // 3 + 3 + 3 + 3 + 3 = 15 fits; the 5th extender (would make 18) overflows. + std::vector buf = {0xE6, 0xBC, 0xA2}; // 漢, 3 bytes + for (int n = 0; n < 5; ++n) buf.insert(buf.end(), {0xEF, 0xB8, 0x80}); // 5 × U+FE00 + + State state; + WordJoin nextJoin = WordJoin::Glue; // inherited join is Glue, to prove reset to CjkBreak + auto steps = feed(buf, state, nextJoin); + + // Find the force-flush step. + bool sawEmittedFlushable = false; + Flushable forced{}; + for (const auto& step : steps) { + if (step.result == ConsumeResult::EmittedFlushable) { + sawEmittedFlushable = true; + forced = step.flushable; + } + } + ASSERT_TRUE(sawEmittedFlushable); // (a) staged cluster force-flushed + EXPECT_EQ(forced.len, 15); // 漢 + 4 extenders = full 15-byte cluster + + // (b) the overflowing extender is now the new staged base. + EXPECT_EQ(state.pendingCjkBaseLen, 3); // the 5th U+FE00 alone + // (c) join reset to CjkBreak, NOT the inherited Glue. + EXPECT_EQ(state.pendingCjkBaseJoin, WordJoin::CjkBreak); +} + +// 10. EmittedAndNonCjk caller-append contract: 日 (U+65E5) then 'a' (U+0061). +// The caller MUST re-encode outNonCjkCp via utf8AppendCodepoint (NOT copy from s[]). +TEST(ParserCluster, EmittedAndNonCjkCallerAppendContract) { + // 日 = E6 97 A5, 'a' = 61 + const std::vector buf = {0xE6, 0x97, 0xA5, 0x61}; + State state; + WordJoin nextJoin = WordJoin::Space; + + const char* s = reinterpret_cast(buf.data()); + const int len = static_cast(buf.size()); + int i = 0; + + // Call A: stage 日 with underline=true, REGULAR. + Flushable fa{}; + NonCjkKind ka = NonCjkKind::Latin; + uint32_t cpa = 0; + uint8_t la = 0; + auto ra = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/true, fa, ka, cpa, la); + EXPECT_EQ(ra, ConsumeResult::StagedOnly); + EXPECT_EQ(i, 3); + + // Call B: consume 'a' with underline=false -> force flush 日. + Flushable fb{}; + NonCjkKind kb = NonCjkKind::Latin; + uint32_t cpb = 0; + uint8_t lb = 0; + auto rb = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/false, fb, kb, cpb, lb); + EXPECT_EQ(rb, ConsumeResult::EmittedAndNonCjk); // (a) + EXPECT_EQ(fb.len, 3); // (b) + EXPECT_EQ(decodeOnly(fb.bytes, fb.len), 0x65E5u); + EXPECT_EQ(kb, NonCjkKind::Latin); // (c) + EXPECT_EQ(cpb, 0x0061u); // (d) + EXPECT_EQ(lb, 1); // (e) + EXPECT_EQ(i, 4); // (f) advanced past 'a' + EXPECT_EQ(state.pendingCjkBaseLen, 0); // (g) state cleared + // (h)(i) stage-time snapshot survives, NOT call-B's false/REGULAR. + EXPECT_TRUE(fb.underline); + EXPECT_EQ(fb.fontStyle, EpdFontFamily::REGULAR); + + // The caller MUST re-encode outNonCjkCp via utf8AppendCodepoint (NOT copy s[i_prev..i-1]). + std::string reencoded; + utf8AppendCodepoint(cpb, reencoded); + ASSERT_EQ(reencoded.size(), 1u); + EXPECT_EQ(static_cast(reencoded[0]), 0x61); + + // NonCjkOnly sub-case: ' ' alone with no preceding base. + { + const std::vector sp = {0x20}; + State st2; + WordJoin nj2 = WordJoin::Space; + const char* ss = reinterpret_cast(sp.data()); + int j = 0; + Flushable fx{}; + NonCjkKind kx = NonCjkKind::Latin; + uint32_t cpx = 0; + uint8_t lx = 0; + auto rx = + Utf8ClusterAssembler::tryConsumeCodepoint(ss, 1, j, st2, nj2, EpdFontFamily::REGULAR, false, fx, kx, cpx, lx); + EXPECT_EQ(rx, ConsumeResult::NonCjkOnly); + EXPECT_EQ(kx, NonCjkKind::Whitespace); // do NOT read fx (outFlushable INVALID) + EXPECT_EQ(j, 1); + } + + // Other NonCjkKind variants after a staged base: 日 + ' '/NBSP/FEFF. + auto classifyAfterBase = [](const std::vector& tail) -> NonCjkKind { + std::vector b = {0xE6, 0x97, 0xA5}; // 日 + b.insert(b.end(), tail.begin(), tail.end()); + State st; + WordJoin nj = WordJoin::Space; + const char* ss = reinterpret_cast(b.data()); + const int ln = static_cast(b.size()); + int j = 0; + NonCjkKind seen = NonCjkKind::Latin; + while (j < ln) { + Flushable fl{}; + NonCjkKind kk = NonCjkKind::Latin; + uint32_t cc = 0; + uint8_t ll = 0; + auto r = + Utf8ClusterAssembler::tryConsumeCodepoint(ss, ln, j, st, nj, EpdFontFamily::REGULAR, false, fl, kk, cc, ll); + if (r == ConsumeResult::EmittedAndNonCjk || r == ConsumeResult::NonCjkOnly) { + seen = kk; + break; + } + } + return seen; + }; + EXPECT_EQ(classifyAfterBase({0x20}), NonCjkKind::Whitespace); // ' ' + EXPECT_EQ(classifyAfterBase({0xC2, 0xA0}), NonCjkKind::Nbsp); // NBSP U+00A0 + EXPECT_EQ(classifyAfterBase({0xEF, 0xBB, 0xBF}), NonCjkKind::Feff); // FEFF +} + +// 11. Split-Latin codepoint across callbacks: é (U+00E9 = C3 A9) split {C3} then {A9}. +TEST(ParserCluster, SplitLatinCodepointAcrossCallbacks) { + State state; + WordJoin nextJoin = WordJoin::Space; + + // Call 1: {C3} only -> NeedMore, carry-over staged, i unchanged. + const std::vector part1 = {0xC3}; + const char* s1 = reinterpret_cast(part1.data()); + int i1 = 0; + Flushable f1{}; + NonCjkKind k1 = NonCjkKind::Latin; + uint32_t cp1 = 0; + uint8_t l1 = 0; + auto r1 = Utf8ClusterAssembler::tryConsumeCodepoint(s1, 1, i1, state, nextJoin, EpdFontFamily::REGULAR, false, f1, k1, + cp1, l1); + EXPECT_EQ(r1, ConsumeResult::NeedMore); + EXPECT_EQ(i1, 0); + EXPECT_EQ(state.pendingUtf8Len, 1); + + // Call 2: {A9} -> NonCjkOnly, Latin, cp 0x00E9, len 2. + const std::vector part2 = {0xA9}; + const char* s2 = reinterpret_cast(part2.data()); + int i2 = 0; + Flushable f2{}; + NonCjkKind k2 = NonCjkKind::Latin; + uint32_t cp2 = 0; + uint8_t l2 = 0; + auto r2 = Utf8ClusterAssembler::tryConsumeCodepoint(s2, 1, i2, state, nextJoin, EpdFontFamily::REGULAR, false, f2, k2, + cp2, l2); + EXPECT_EQ(r2, ConsumeResult::NonCjkOnly); + EXPECT_EQ(k2, NonCjkKind::Latin); + EXPECT_EQ(cp2, 0x00E9u); + EXPECT_EQ(l2, 2); // FULL byte length, even though i advanced by only 1 continuation byte + EXPECT_EQ(i2, 1); + + // Lossless re-encode even though the two bytes never appeared together in one callback. + std::string scratch; + utf8AppendCodepoint(cp2, scratch); + ASSERT_EQ(scratch.size(), 2u); + EXPECT_EQ(static_cast(scratch[0]), 0xC3); + EXPECT_EQ(static_cast(scratch[1]), 0xA9); +} + +// 12. Style snapshot at stage time, NOT flush time. Proxies 日本. +TEST(ParserCluster, StyleSnapshotAtStageTimeNotFlushTime) { + // 日 = E6 97 A5, 本 U+672C = E6 9C AC + const std::vector buf = {0xE6, 0x97, 0xA5, 0xE6, 0x9C, 0xAC}; + State state; + WordJoin nextJoin = WordJoin::Space; + + const char* s = reinterpret_cast(buf.data()); + const int len = static_cast(buf.size()); + int i = 0; + + // Call A: 日 under REGULAR + underline=true. + Flushable fa{}; + NonCjkKind ka = NonCjkKind::Latin; + uint32_t cpa = 0; + uint8_t la = 0; + auto ra = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/true, fa, ka, cpa, la); + EXPECT_EQ(ra, ConsumeResult::StagedOnly); + EXPECT_EQ(state.pendingCjkBaseLen, 3); + EXPECT_TRUE(state.underlineAtStage); + EXPECT_EQ(state.fontStyleAtStage, EpdFontFamily::REGULAR); + + // Call B: 本 under REGULAR + underline=false -> flush 日 with STAGE-time snapshot. + Flushable fb{}; + NonCjkKind kb = NonCjkKind::Latin; + uint32_t cpb = 0; + uint8_t lb = 0; + auto rb = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, i, state, nextJoin, EpdFontFamily::REGULAR, + /*callerUnderline=*/false, fb, kb, cpb, lb); + EXPECT_EQ(rb, ConsumeResult::EmittedFlushable); + EXPECT_EQ(fb.len, 3); + EXPECT_EQ(decodeOnly(fb.bytes, fb.len), 0x65E5u); // 日 + EXPECT_TRUE(fb.underline); // stage-time true, NOT call-B false + EXPECT_EQ(fb.fontStyle, EpdFontFamily::REGULAR); + EXPECT_EQ(state.pendingCjkBaseLen, 3); // 本 now staged + EXPECT_FALSE(state.underlineAtStage); // 本 staged under underline=false + + // Drain: 本 flushes with underline=false. + Flushable drained; + ASSERT_TRUE(Utf8ClusterAssembler::flushPendingBase(state, drained)); + EXPECT_EQ(decodeOnly(drained.bytes, drained.len), 0x672Cu); // 本 + EXPECT_FALSE(drained.underline); + + // Symmetric sub-test: 日本 — 日 staged under underline=false, 本 under true. + { + State st; + WordJoin nj = WordJoin::Space; + int j = 0; + + Flushable g1{}; + NonCjkKind gk1 = NonCjkKind::Latin; + uint32_t gc1 = 0; + uint8_t gl1 = 0; + auto gr1 = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, j, st, nj, EpdFontFamily::REGULAR, + /*callerUnderline=*/false, g1, gk1, gc1, gl1); + EXPECT_EQ(gr1, ConsumeResult::StagedOnly); // 日 staged under underline=false + + Flushable g2{}; + NonCjkKind gk2 = NonCjkKind::Latin; + uint32_t gc2 = 0; + uint8_t gl2 = 0; + auto gr2 = Utf8ClusterAssembler::tryConsumeCodepoint(s, len, j, st, nj, EpdFontFamily::REGULAR, + /*callerUnderline=*/true, g2, gk2, gc2, gl2); + EXPECT_EQ(gr2, ConsumeResult::EmittedFlushable); + EXPECT_FALSE(g2.underline); // 日's stage-time underline=false + EXPECT_TRUE(st.underlineAtStage); // 本 now staged under underline=true + } +} + +} // namespace