feat(epub): Japanese CJK line-breaking PR F — Utf8ClusterAssembler + viewport-width CJK wrapping

docs/file-formats.md

@@ -90,13 +90,13 @@ if (parsedSize != fileSize) {
 
 ## `section.bin`
 
-### Version 25
+### Version 27
 
 Each file in `sections/*.bin` stores one laid-out spine section. The header is
 also the cache-busting key: if any layout-affecting setting differs from the
 current reader settings, the section is discarded and rebuilt.
 
-Version 25 includes:
+Version 27 includes:
 
 - cache-busting fields for paragraph alignment, hyphenation, embedded CSS,
   image rendering mode, and Focus Reading
@@ -105,6 +105,7 @@ Version 25 includes:
 - paragraph and list-item LUTs used by KOReader sync page refinement
 - optional per-word Focus Reading split metadata
 - per-page footnote entries
+- CJK (Japanese) paragraphs wrap at viewport width via per-character breakable tokens (v27)
 
 ImHex pattern:
 
@@ -113,7 +114,7 @@ import std.mem;
 import std.string;
 import std.core;
 
-#define EXPECTED_VERSION 25
+#define EXPECTED_VERSION 27
 #define MAX_STRING_LENGTH 65535
 #define FOOTNOTE_NUMBER_LEN 32
 #define FOOTNOTE_HREF_LEN 96

lib/Epub/Epub/Section.cpp

@@ -10,7 +10,7 @@
 #include "parsers/ChapterHtmlSlimParser.h"
 
 namespace {
-constexpr uint8_t SECTION_FILE_VERSION = 26;
+constexpr uint8_t SECTION_FILE_VERSION = 27;  // was 26: CJK line-breaking changes layout output
 constexpr uint32_t HEADER_SIZE = sizeof(uint8_t) + sizeof(int) + sizeof(float) + sizeof(bool) + sizeof(uint8_t) +
                                  sizeof(uint16_t) + sizeof(uint16_t) + sizeof(uint16_t) + sizeof(bool) + sizeof(bool) +
                                  sizeof(uint8_t) + sizeof(bool) + sizeof(uint32_t) + sizeof(uint32_t) +

lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h

@@ -14,6 +14,7 @@
 #include "Epub/blocks/TextBlock.h"
 #include "Epub/css/CssParser.h"
 #include "Epub/css/CssStyle.h"
+#include "Utf8ClusterAssembler.h"
 
 class Page;
 class GfxRenderer;
@@ -37,6 +38,10 @@ class ChapterHtmlSlimParser {
   char partWordBuffer[MAX_WORD_SIZE + 1] = {};
   int partWordBufferIndex = 0;
   WordJoin nextJoin = WordJoin::Space;  // how the next flushed word joins the previous (inline element boundary)
+  // Kind of text drained by flushPendingText(): inline-close callers re-glue Latin (Glue) but must
+  // leave a CJK base on CjkBreak so it stays wrappable across the boundary.
+  enum class FlushedKind : uint8_t { None, Latin, Cjk };
+  Utf8ClusterAssembler::State clusterState;  // cross-callback CJK base / truncated-codepoint staging
   std::unique_ptr<ParsedText> currentTextBlock = nullptr;
   std::unique_ptr<Page> currentPage = nullptr;
   int16_t currentPageNextY = 0;
@@ -100,6 +105,11 @@ class ChapterHtmlSlimParser {
   void startNewTextBlock(const BlockStyle& blockStyle);
   void flushPendingAnchor();
   void flushPartWordBuffer();
+  EpdFontFamily::Style currentFontStyle() const;
+  FlushedKind flushPendingText();
+  void emitCjkToken(const Utf8ClusterAssembler::Flushable& f);
+  void dispatchNonCjk(Utf8ClusterAssembler::NonCjkKind kind, uint32_t cp);
+  void splitLongBlockIfNeeded();
   void makePages();
   static void applyDirectionToEntry(StyleStackEntry& entry, const CssStyle& css);
   void emitHorizontalRule(const BlockStyle& blockStyle);

lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp

@@ -0,0 +1,148 @@
+// lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp
+//
+// Host-safe UTF-8 decode + grapheme cluster assembly + WordJoin policy. See the header
+// for the public contract and the task brief for the state-transition table. No heap, no
+// Arduino/HAL deps — depends only on Utf8.h + WordJoin.h + EpdFontFamily.h.
+
+#include "Utf8ClusterAssembler.h"
+
+#include <cstddef>
+#include <cstring>
+
+namespace {
+
+// Expected total byte length of a UTF-8 sequence from its lead byte. Mirrors Utf8.cpp's
+// utf8CodepointLen (kept local so this TU stays free of internal-linkage coupling).
+uint8_t leadByteLen(uint8_t c) {
+  if (c < 0x80) return 1;          // 0xxxxxxx
+  if ((c >> 5) == 0x6) return 2;   // 110xxxxx
+  if ((c >> 4) == 0xE) return 3;   // 1110xxxx
+  if ((c >> 3) == 0x1E) return 4;  // 11110xxx
+  return 1;                        // invalid lead — treat as a 1-byte (replacement) cp
+}
+
+// Classify a fully-decoded non-CJK codepoint into the caller-dispatch kind.
+Utf8ClusterAssembler::NonCjkKind classifyNonCjk(uint32_t cp) {
+  using NonCjkKind = Utf8ClusterAssembler::NonCjkKind;
+  if (cp == 0x20 || cp == 0x09 || cp == 0x0A || cp == 0x0D) return NonCjkKind::Whitespace;
+  if (cp == 0x00A0 || cp == 0x202F) return NonCjkKind::Nbsp;
+  if (cp == 0xFEFF) return NonCjkKind::Feff;
+  return NonCjkKind::Latin;
+}
+
+// Copy `state`'s staged base into `out`. Does NOT clear the staged base.
+void fillFlushableFromState(const Utf8ClusterAssembler::State& state, Utf8ClusterAssembler::Flushable& out) {
+  memcpy(out.bytes, state.pendingCjkBase, state.pendingCjkBaseLen);
+  out.len = state.pendingCjkBaseLen;
+  out.join = state.pendingCjkBaseJoin;
+  out.fontStyle = state.fontStyleAtStage;
+  out.underline = state.underlineAtStage;
+}
+
+}  // namespace
+
+Utf8ClusterAssembler::ConsumeResult Utf8ClusterAssembler::tryConsumeCodepoint(
+    const char* s, int len, int& i, State& state, WordJoin& nextJoin, EpdFontFamily::Style callerFontStyle,
+    bool callerUnderline, Flushable& outFlushable, NonCjkKind& outNonCjkKind, uint32_t& outNonCjkCp,
+    uint8_t& outNonCjkLen) {
+  // --- Step 1/2: assemble a NUL-terminated scratch holding exactly one codepoint. ---
+  // utf8NextCodepoint stops on a NUL, so we MUST decode from this private scratch and never
+  // off s[] (which is not NUL-terminated).
+  char scratch[5];
+  uint8_t scratchLen = 0;         // bytes placed into scratch (full cp length)
+  uint8_t continuationBytes = 0;  // bytes pulled from THIS call's s[] (drives i advance)
+
+  if (state.pendingUtf8Len > 0) {
+    // Completing a codepoint split across callbacks.
+    const uint8_t expected = leadByteLen(state.pendingUtf8[0]);
+    memcpy(scratch, state.pendingUtf8, state.pendingUtf8Len);
+    scratchLen = state.pendingUtf8Len;
+
+    while (scratchLen < expected && (i + continuationBytes) < len) {
+      scratch[scratchLen++] = s[i + continuationBytes];
+      continuationBytes++;
+    }
+
+    if (scratchLen < expected) {
+      // Still incomplete: stage what we pulled and signal NeedMore. i unchanged.
+      memcpy(state.pendingUtf8, scratch, scratchLen);
+      state.pendingUtf8Len = scratchLen;
+      return ConsumeResult::NeedMore;
+    }
+    state.pendingUtf8Len = 0;  // carry-over consumed
+  } else {
+    if (i >= len) return ConsumeResult::NeedMore;  // nothing to read (defensive)
+    const uint8_t expected = leadByteLen(static_cast<uint8_t>(s[i]));
+    if (i + expected > len) {
+      // Partial sequence at the buffer tail — stage and signal NeedMore. i unchanged.
+      for (uint8_t b = 0; b < expected && (i + b) < len; ++b) state.pendingUtf8[b] = s[i + b];
+      state.pendingUtf8Len = static_cast<uint8_t>(len - i);
+      return ConsumeResult::NeedMore;
+    }
+    scratchLen = expected;
+    continuationBytes = expected;
+    memcpy(scratch, s + i, expected);
+  }
+  scratch[scratchLen] = '\0';
+
+  const unsigned char* p = reinterpret_cast<const unsigned char*>(scratch);
+  const uint32_t cp = utf8NextCodepoint(&p);
+  const uint8_t cpLen = scratchLen;  // full UTF-8 byte length of this codepoint
+
+  // --- Step 3: classify — grapheme extender FIRST, then CJK breakable, then non-CJK. ---
+
+  // Grapheme extender glued onto an existing staged base.
+  if (utf8IsGraphemeExtender(cp) && state.pendingCjkBaseLen > 0) {
+    // Step 4: overflow guard — if appending would exceed the staged buffer, force-flush the
+    // staged base and re-stage the extender as a NEW base with a reset join + fresh snapshot.
+    if (static_cast<size_t>(state.pendingCjkBaseLen) + cpLen > sizeof(state.pendingCjkBase)) {
+      fillFlushableFromState(state, outFlushable);
+      memcpy(state.pendingCjkBase, scratch, cpLen);
+      state.pendingCjkBaseLen = cpLen;
+      state.pendingCjkBaseJoin = WordJoin::CjkBreak;  // explicit reset, NOT the inherited join
+      state.fontStyleAtStage = callerFontStyle;
+      state.underlineAtStage = callerUnderline;
+      i += continuationBytes;
+      return ConsumeResult::EmittedFlushable;
+    }
+    memcpy(state.pendingCjkBase + state.pendingCjkBaseLen, scratch, cpLen);
+    state.pendingCjkBaseLen = static_cast<uint8_t>(state.pendingCjkBaseLen + cpLen);
+    i += continuationBytes;
+    return ConsumeResult::StagedOnly;
+  }
+
+  // CJK breakable base.
+  if (utf8IsCjkBreakable(cp)) {
+    const bool hadBase = state.pendingCjkBaseLen > 0;
+    if (hadBase) fillFlushableFromState(state, outFlushable);
+    // Stage the NEW base.
+    memcpy(state.pendingCjkBase, scratch, cpLen);
+    state.pendingCjkBaseLen = cpLen;
+    state.pendingCjkBaseJoin = nextJoin;
+    state.fontStyleAtStage = callerFontStyle;
+    state.underlineAtStage = callerUnderline;
+    nextJoin = WordJoin::CjkBreak;
+    i += continuationBytes;
+    return hadBase ? ConsumeResult::EmittedFlushable : ConsumeResult::StagedOnly;
+  }
+
+  // Non-CJK: Latin / whitespace / NBSP / FEFF. Do NOT touch nextJoin — the caller sets it.
+  outNonCjkKind = classifyNonCjk(cp);
+  outNonCjkCp = cp;
+  outNonCjkLen = cpLen;
+  if (state.pendingCjkBaseLen > 0) {
+    fillFlushableFromState(state, outFlushable);
+    state.pendingCjkBaseLen = 0;  // clear staged base
+    i += continuationBytes;
+    return ConsumeResult::EmittedAndNonCjk;
+  }
+  i += continuationBytes;
+  return ConsumeResult::NonCjkOnly;
+}
+
+bool Utf8ClusterAssembler::flushPendingBase(State& state, Flushable& outFlushable) {
+  if (state.pendingCjkBaseLen == 0) return false;
+  fillFlushableFromState(state, outFlushable);
+  state.pendingCjkBaseLen = 0;
+  return true;
+}

lib/Epub/Epub/parsers/Utf8ClusterAssembler.h

@@ -0,0 +1,71 @@
+// lib/Epub/Epub/parsers/Utf8ClusterAssembler.h
+#pragma once
+
+#include <EpdFontFamily.h>  // for EpdFontFamily::Style (used by State + Flushable + callerFontStyle)
+#include <Utf8.h>           // for utf8NextCodepoint + predicates
+
+#include <cstdint>
+
+#include "../WordJoin.h"  // for WordJoin (enum-only, no layout deps)
+
+class Utf8ClusterAssembler {
+ public:
+  // Classifies the non-CJK codepoint that the caller must dispatch on after an
+  // EmittedAndNonCjk / NonCjkOnly return. The four kinds map 1:1 to the four branches
+  // in ChapterHtmlSlimParser::dispatchNonCjk (Latin buffer append / whitespace flush /
+  // NBSP synthesized token / FEFF discard).
+  enum class NonCjkKind : uint8_t {
+    Latin,       // ordinary printable codepoint — caller appends raw UTF-8 bytes to Latin buffer
+    Whitespace,  // U+0020 / U+0009 / U+000A / U+000D — flush Latin, set nextJoin = Space
+    Nbsp,        // U+00A0 / U+202F — flush, emit synthesized " " token with Glue, then nextJoin = Glue
+    Feff,        // U+FEFF (BOM) — discard
+  };
+
+  enum class ConsumeResult : uint8_t {
+    NeedMore,          // current codepoint incomplete (carry-over staged); i unchanged
+    StagedOnly,        // codepoint consumed, staged into pendingCjkBase, no flushable emitted; i advanced
+    EmittedFlushable,  // staged base flushed (caller emits it as a CJK token);
+                       // current codepoint also staged/consumed; i advanced
+    NonCjkOnly,        // no pending base existed (outFlushable INVALID — do not read).
+                       // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced.
+    EmittedAndNonCjk,  // a pending base existed and was flushed: outFlushable filled.
+                       // outNonCjkKind + outNonCjkCp + outNonCjkLen describe the non-CJK cp; i advanced.
+  };
+
+  // Cross-callback state — owned by the parser, threaded into each call.
+  // fontStyleAtStage / underlineAtStage are snapshotted at STAGE time (not flush time): a CJK
+  // base may be staged inside <u> and flushed after </u> closes; the flush must use the style
+  // in effect when the base was STAGED.
+  struct State {
+    uint8_t pendingUtf8[4] = {};   // leading bytes of a truncated codepoint
+    uint8_t pendingUtf8Len = 0;    // 0..3
+    char pendingCjkBase[16] = {};  // staged base + already-absorbed extenders
+    uint8_t pendingCjkBaseLen = 0;
+    WordJoin pendingCjkBaseJoin = WordJoin::Space;
+    EpdFontFamily::Style fontStyleAtStage = EpdFontFamily::REGULAR;
+    bool underlineAtStage = false;
+  };
+
+  // Payload emitted when a previously-staged CJK base must be flushed.
+  struct Flushable {
+    char bytes[16];
+    uint8_t len;
+    WordJoin join;
+    EpdFontFamily::Style fontStyle;  // snapshot at stage time
+    bool underline;                  // snapshot at stage time
+  };
+
+  // Consume up to one complete codepoint from s[i..len]. Returns the discriminant; out-params
+  // populated per the state-transition table. callerFontStyle / callerUnderline are the resolved
+  // style at THIS call; when a new base is staged they are snapshotted into State.
+  static ConsumeResult tryConsumeCodepoint(
+      const char* s, int len, int& i, State& state, WordJoin& nextJoin, EpdFontFamily::Style callerFontStyle,
+      bool callerUnderline,
+      Flushable& outFlushable,    // valid iff result in {EmittedFlushable, EmittedAndNonCjk}
+      NonCjkKind& outNonCjkKind,  // valid iff result in {NonCjkOnly, EmittedAndNonCjk}
+      uint32_t& outNonCjkCp,      // valid iff result in {NonCjkOnly, EmittedAndNonCjk}
+      uint8_t& outNonCjkLen);     // total UTF-8 byte length of the non-CJK codepoint (1..4)
+
+  // Force-flush any staged base. Returns true with outFlushable filled if a base was pending.
+  static bool flushPendingBase(State& state, Flushable& outFlushable);
+};

test/CMakeLists.txt

@@ -48,3 +48,4 @@ add_subdirectory(font_resolver)
 add_subdirectory(font_boundary)
 add_subdirectory(parsed_text)
 add_subdirectory(utf8)
+add_subdirectory(parser_cluster)

test/parsed_text/ParsedTextLayoutTest.cpp

@@ -337,3 +337,30 @@ TEST(ParsedTextLayout, FocusReadingPreservesUnderlineOnCjkCluster) {
   EXPECT_EQ(static_cast<int>(styles[0]) & static_cast<int>(EpdFontFamily::BOLD), 0)
       << "focus-bypass must still skip focus-bolding of a single CJK cluster";
 }
+
+// CJK across an inline-style boundary MUST keep CjkBreak (stays wrappable).
+TEST(ParsedTextLayout, ParserSequenceCjkAcrossInlineStyleStaysBreakable) {
+  ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false,
+                  /*focusReadingEnabled=*/false, leftAligned());
+  text.addWord("ab", EpdFontFamily::REGULAR);                                       // join=Space
+  text.addWord("\xE6\x97\xA5", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak);  // 日
+  text.addWord("\xE6\x9C\xAC", EpdFontFamily::ITALIC, false, WordJoin::CjkBreak);   // 本 (italic via <em>)
+  text.addWord("\xE8\xAA\x9E", EpdFontFamily::REGULAR, false, WordJoin::CjkBreak);  // 語
+  auto lines = linesOf(text, 3 * kCell);
+  ASSERT_EQ(lines.size(), 2u);
+  EXPECT_EQ(lines[0], "ab\xE6\x97\xA5");            // ab日
+  EXPECT_EQ(lines[1], "\xE6\x9C\xAC\xE8\xAA\x9E");  // 本語
+}
+
+// Latin inline-close MUST keep Glue (preserves quickly).
+TEST(ParsedTextLayout, ParserSequenceLatinAcrossInlineStyleStaysGlued) {
+  ParsedText text(/*extraParagraphSpacing=*/false, /*hyphenationEnabled=*/false,
+                  /*focusReadingEnabled=*/false, leftAligned());
+  text.addWord("a", EpdFontFamily::REGULAR);                         // join=Space
+  text.addWord("quick", EpdFontFamily::REGULAR);                     // join=Space
+  text.addWord("ly", EpdFontFamily::ITALIC, false, WordJoin::Glue);  // Glue — inline close
+  auto lines = linesOf(text, 7 * kCell);
+  ASSERT_EQ(lines.size(), 2u);
+  EXPECT_EQ(lines[0], "a");
+  EXPECT_EQ(lines[1], "quickly");
+}

test/parser_cluster/CMakeLists.txt

@@ -0,0 +1,17 @@
+# test/parser_cluster/CMakeLists.txt
+add_executable(ParserClusterTest
+  ParserClusterTest.cpp
+  ${REPO_ROOT}/lib/Epub/Epub/parsers/Utf8ClusterAssembler.cpp
+  ${REPO_ROOT}/lib/Utf8/Utf8.cpp
+)
+target_include_directories(ParserClusterTest PRIVATE
+  ${REPO_ROOT}
+  ${REPO_ROOT}/lib/Utf8
+  ${REPO_ROOT}/lib/Epub
+  ${REPO_ROOT}/lib/Epub/Epub
+  ${REPO_ROOT}/lib/Epub/Epub/parsers
+  ${REPO_ROOT}/lib/EpdFont
+  ${REPO_ROOT}/test
+)
+target_link_libraries(ParserClusterTest PRIVATE crosspoint_test_common GTest::gtest_main)
+gtest_discover_tests(ParserClusterTest)