refactor(epub): Japanese CJK line-breaking PR E — WordJoin setup

lib/Epub/Epub/ParsedText.h

@@ -7,6 +7,7 @@
 #include <string>
 #include <vector>
 
+#include "WordJoin.h"
 #include "blocks/BlockStyle.h"
 #include "blocks/TextBlock.h"
 
@@ -15,7 +16,7 @@ class ITextMetrics;
 class ParsedText {
   std::vector<std::string> words;
   std::vector<EpdFontFamily::Style> wordStyles;
-  std::vector<bool> wordContinues;      // true = word attaches to previous (no space before it)
+  std::vector<WordJoin> wordJoins;      // how each word joins the previous (space / glue / cjk-break)
   std::vector<bool> wordIsFocusSuffix;  // true = token is the regular tail of a focus bold-prefix split
   BlockStyle blockStyle;
   bool extraParagraphSpacing;
@@ -26,19 +27,19 @@ class ParsedText {
   std::vector<std::string> reorderedWordsScratch;
   std::vector<EpdFontFamily::Style> reorderedStylesScratch;
   std::vector<uint16_t> reorderedWidthsScratch;
-  std::vector<bool> reorderedContinuesScratch;
+  std::vector<WordJoin> reorderedJoinsScratch;
   std::vector<bool> reorderedFocusSuffixScratch;
   std::vector<uint16_t> visualOrderScratch;
 
   int resolveFirstLineIndent(bool isFirstLine, const ITextMetrics& renderer, int fontId) const;
   std::vector<size_t> computeLineBreaks(const ITextMetrics& renderer, int fontId, int pageWidth,
-                                        std::vector<uint16_t>& wordWidths, std::vector<bool>& continuesVec);
+                                        std::vector<uint16_t>& wordWidths, std::vector<WordJoin>& joinsVec);
   std::vector<size_t> computeHyphenatedLineBreaks(const ITextMetrics& renderer, int fontId, int pageWidth,
-                                                  std::vector<uint16_t>& wordWidths, std::vector<bool>& continuesVec);
+                                                  std::vector<uint16_t>& wordWidths, std::vector<WordJoin>& joinsVec);
   bool hyphenateWordAtIndex(size_t wordIndex, int availableWidth, const ITextMetrics& renderer, int fontId,
                             std::vector<uint16_t>& wordWidths, bool allowFallbackBreaks);
   void extractLine(size_t breakIndex, int pageWidth, const std::vector<uint16_t>& wordWidths,
-                   const std::vector<bool>& continuesVec, const std::vector<size_t>& lineBreakIndices,
+                   const std::vector<WordJoin>& joinsVec, const std::vector<size_t>& lineBreakIndices,
                    const std::function<void(std::shared_ptr<TextBlock>)>& processLine, const ITextMetrics& renderer,
                    int fontId);
   std::vector<uint16_t> calculateWordWidths(const ITextMetrics& renderer, int fontId);
@@ -54,7 +55,8 @@ class ParsedText {
         hasRtlWord(false) {}
   ~ParsedText() = default;
 
-  void addWord(std::string word, EpdFontFamily::Style fontStyle, bool underline = false, bool attachToPrevious = false);
+  void addWord(std::string word, EpdFontFamily::Style fontStyle, bool underline = false,
+               WordJoin join = WordJoin::Space);
   void setBlockStyle(const BlockStyle& blockStyle) { this->blockStyle = blockStyle; }
   BlockStyle& getBlockStyle() { return blockStyle; }
   size_t size() const { return words.size(); }

lib/Epub/Epub/WordJoin.h

@@ -0,0 +1,14 @@
+// lib/Epub/Epub/WordJoin.h
+#pragma once
+
+#include <cstdint>
+
+// How the current word joins the previous one in a line:
+//   Space    — a space precedes this word; the layout may break here.
+//   Glue     — no space and no break (e.g. NBSP, hyphenation remainder bridge).
+//   CjkBreak — no space, but the layout MAY break here (the only state CJK needs).
+// Predicates are free `constexpr` so call sites read naturally.
+enum class WordJoin : uint8_t { Space, Glue, CjkBreak };
+
+constexpr bool needsSpaceBefore(WordJoin j) { return j == WordJoin::Space; }
+constexpr bool canBreakBefore(WordJoin j) { return j != WordJoin::Glue; }

lib/Epub/Epub/parsers/ChapterHtmlSlimParser.cpp

@@ -185,14 +185,14 @@ void ChapterHtmlSlimParser::flushPartWordBuffer() {
 
   // flush the buffer
   partWordBuffer[partWordBufferIndex] = '\0';
-  currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextWordContinues);
+  currentTextBlock->addWord(partWordBuffer, fontStyle, false, nextJoin);
   partWordBufferIndex = 0;
-  nextWordContinues = false;
+  nextJoin = WordJoin::Space;
 }
 
 // start a new text block if needed
 void ChapterHtmlSlimParser::startNewTextBlock(const BlockStyle& blockStyle) {
-  nextWordContinues = false;  // New block = new paragraph, no continuation
+  nextJoin = WordJoin::Space;  // New block = new paragraph, no continuation
   if (currentTextBlock) {
     // already have a text block running and it is empty - just reuse it
     if (currentTextBlock->isEmpty()) {
@@ -420,7 +420,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
     if (self->partWordBufferIndex > 0) {
       self->flushPartWordBuffer();
     }
-    self->nextWordContinues = false;
+    self->nextJoin = WordJoin::Space;
     self->inlineStyleStack.pop_back();
     self->updateEffectiveInlineStyle();
 
@@ -743,7 +743,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
       // Flush buffer before style change
       if (self->partWordBufferIndex > 0) {
         self->flushPartWordBuffer();
-        self->nextWordContinues = true;
+        self->nextJoin = WordJoin::Glue;
       }
       self->insideFootnoteLink = true;
       self->footnoteLinkDepth = self->depth;
@@ -827,7 +827,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
     // Flush buffer before style change so preceding text gets current style
     if (self->partWordBufferIndex > 0) {
       self->flushPartWordBuffer();
-      self->nextWordContinues = true;
+      self->nextJoin = WordJoin::Glue;
     }
     self->underlineUntilDepth = std::min(self->underlineUntilDepth, self->depth);
     // Push inline style entry for underline tag
@@ -850,7 +850,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
     // Flush buffer before style change so preceding text gets current style
     if (self->partWordBufferIndex > 0) {
       self->flushPartWordBuffer();
-      self->nextWordContinues = true;
+      self->nextJoin = WordJoin::Glue;
     }
     self->boldUntilDepth = std::min(self->boldUntilDepth, self->depth);
     // Push inline style entry for bold tag
@@ -873,7 +873,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
     // Flush buffer before style change so preceding text gets current style
     if (self->partWordBufferIndex > 0) {
       self->flushPartWordBuffer();
-      self->nextWordContinues = true;
+      self->nextJoin = WordJoin::Glue;
     }
     self->italicUntilDepth = std::min(self->italicUntilDepth, self->depth);
     // Push inline style entry for italic tag
@@ -895,7 +895,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
   } else if (strcmp(name, "sup") == 0 || strcmp(name, "sub") == 0) {
     if (self->partWordBufferIndex > 0) {
       self->flushPartWordBuffer();
-      self->nextWordContinues = true;
+      self->nextJoin = WordJoin::Glue;
     }
     StyleStackEntry entry;
     entry.depth = self->depth;
@@ -915,7 +915,7 @@ void XMLCALL ChapterHtmlSlimParser::startElement(void* userData, const XML_Char*
       // Flush buffer before style change so preceding text gets current style
       if (self->partWordBufferIndex > 0) {
         self->flushPartWordBuffer();
-        self->nextWordContinues = true;
+        self->nextJoin = WordJoin::Glue;
       }
       StyleStackEntry entry;
       entry.depth = self->depth;  // Track depth for matching pop
@@ -998,7 +998,7 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
         self->flushPartWordBuffer();
       }
       // Whitespace is a real word boundary — reset continuation state
-      self->nextWordContinues = false;
+      self->nextJoin = WordJoin::Space;
       // Skip the whitespace char
       continue;
     }
@@ -1029,10 +1029,10 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
       self->partWordBuffer[0] = ' ';
       self->partWordBuffer[1] = '\0';
       self->partWordBufferIndex = 1;
-      self->nextWordContinues = true;  // Attach space to previous word (no break).
+      self->nextJoin = WordJoin::Glue;  // Attach space to previous word (no break).
       self->flushPartWordBuffer();
 
-      self->nextWordContinues = true;  // Next real word attaches to this space (no break).
+      self->nextJoin = WordJoin::Glue;  // Next real word attaches to this space (no break).
 
       i++;  // Skip the second byte (0xA0)
       continue;
@@ -1048,10 +1048,10 @@ void XMLCALL ChapterHtmlSlimParser::characterData(void* userData, const XML_Char
       self->partWordBuffer[0] = ' ';
       self->partWordBuffer[1] = '\0';
       self->partWordBufferIndex = 1;
-      self->nextWordContinues = true;
+      self->nextJoin = WordJoin::Glue;
       self->flushPartWordBuffer();
 
-      self->nextWordContinues = true;
+      self->nextJoin = WordJoin::Glue;
 
       i += 2;  // Skip the remaining two bytes (0x80 0xAF)
       continue;
@@ -1170,7 +1170,7 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n
       self->flushPartWordBuffer();
       // If closing an inline element, the next word fragment continues the same visual word
       if (isInlineTag) {
-        self->nextWordContinues = true;
+        self->nextJoin = WordJoin::Glue;
       }
     }
   }
@@ -1198,18 +1198,18 @@ void XMLCALL ChapterHtmlSlimParser::endElement(void* userData, const XML_Char* n
   }
 
   if (self->tableDepth == 1 && (strcmp(name, "td") == 0 || strcmp(name, "th") == 0)) {
-    self->nextWordContinues = false;
+    self->nextJoin = WordJoin::Space;
   }
 
   if (self->tableDepth == 1 && (strcmp(name, "tr") == 0)) {
-    self->nextWordContinues = false;
+    self->nextJoin = WordJoin::Space;
   }
 
   if (self->tableDepth == 1 && strcmp(name, "table") == 0) {
     self->tableDepth -= 1;
     self->tableRowIndex = 0;
     self->tableColIndex = 0;
-    self->nextWordContinues = false;
+    self->nextJoin = WordJoin::Space;
   }
 
   // Leaving bold tag

lib/Epub/Epub/parsers/ChapterHtmlSlimParser.h

@@ -36,7 +36,7 @@ class ChapterHtmlSlimParser {
   // leave one char at end for null pointer
   char partWordBuffer[MAX_WORD_SIZE + 1] = {};
   int partWordBufferIndex = 0;
-  bool nextWordContinues = false;  // true when next flushed word attaches to previous (inline element boundary)
+  WordJoin nextJoin = WordJoin::Space;  // how the next flushed word joins the previous (inline element boundary)
   std::unique_ptr<ParsedText> currentTextBlock = nullptr;
   std::unique_ptr<Page> currentPage = nullptr;
   int16_t currentPageNextY = 0;

lib/Utf8/Utf8.h

@@ -32,7 +32,19 @@ inline bool utf8IsCjkBreakable(const uint32_t cp) {
          || (cp >= 0xFF01 && cp <= 0xFF60)     // Fullwidth Latin / Punctuation
          || (cp >= 0xFF65 && cp <= 0xFFEF)     // Halfwidth Katakana / Hangul
          || (cp >= 0x20000 && cp <= 0x2A6DF)   // CJK Extension B
-         || (cp >= 0x2A700 && cp <= 0x2B73F);  // CJK Extension C
+         || (cp >= 0x2A700 && cp <= 0x2B73F)   // CJK Extension C
+         || (cp >= 0x1B000 && cp <= 0x1B0FF)   // Kana Supplement
+         || (cp >= 0x1B100 && cp <= 0x1B12F)   // Kana Extended-A
+         || (cp >= 0x2B740 && cp <= 0x2B81F)   // CJK Extension D
+         || (cp >= 0x2B820 && cp <= 0x2CEAF)   // CJK Extension E
+         || (cp >= 0x2CEB0 && cp <= 0x2EBEF)   // CJK Extension F
+         || (cp >= 0x2F800 && cp <= 0x2FA1F)   // CJK Compatibility Ideographs Supplement
+         || (cp >= 0x30000 && cp <= 0x3134F)   // CJK Extension G
+         || (cp >= 0x1AFF0 && cp <= 0x1AFFF)   // Kana Extended-B
+         || (cp >= 0x1B130 && cp <= 0x1B16F)   // Small Kana Extension
+         || (cp >= 0x2EBF0 && cp <= 0x2EE5F)   // CJK Extension I
+         || (cp >= 0x31350 && cp <= 0x323AF)   // CJK Extension H
+         || (cp >= 0x323B0 && cp <= 0x3347F);  // CJK Extension J
 }
 
 // Returns true for Unicode combining diacritical marks that should not advance the cursor.
@@ -42,3 +54,14 @@ inline bool utf8IsCombiningMark(const uint32_t cp) {
          || (cp >= 0x20D0 && cp <= 0x20FF)   // Combining Diacritical Marks for Symbols
          || (cp >= 0xFE20 && cp <= 0xFE2F);  // Combining Half Marks
 }
+
+// Returns true for grapheme extenders that must stay glued to a preceding base
+// codepoint in layout (combining marks, kana voicing marks, ideographic tone
+// marks, variation selectors).
+inline bool utf8IsGraphemeExtender(uint32_t cp) {
+  return utf8IsCombiningMark(cp) || (cp >= 0x3099 && cp <= 0x309A)  // Combining katakana-hiragana voicing
+         || (cp >= 0x302A && cp <= 0x302F)                          // Ideographic tone marks
+         || (cp >= 0xFF9E && cp <= 0xFF9F)                          // Halfwidth voicing marks
+         || (cp >= 0xFE00 && cp <= 0xFE0F)                          // Variation Selectors VS-1..VS-16
+         || (cp >= 0xE0100 && cp <= 0xE01EF);                       // Variation Selectors Supplement
+}

test/CMakeLists.txt

@@ -47,3 +47,4 @@ add_subdirectory(cjk_font_parity)
 add_subdirectory(font_resolver)
 add_subdirectory(font_boundary)
 add_subdirectory(parsed_text)
+add_subdirectory(utf8)