fix(chat): stream English answers sentence by sentence

examples/loopx-chat-agent-smoke.py

@@ -18,7 +18,7 @@
 if str(REPO_ROOT) not in sys.path:
     sys.path.insert(0, str(REPO_ROOT))
 
-from loopx.chat import VisibleResponseStreamFilter  # noqa: E402
+from loopx.chat import CHAT_REVIEW_OPEN_TAG, VisibleResponseStreamFilter, redact_local_paths  # noqa: E402
 from loopx.chat_agent import (  # noqa: E402
     CodexChatAgentError,
     CodexChatAgentSession,
@@ -247,7 +247,46 @@ def get(self, *, timeout: float) -> dict[str, object]:
     assert chinese_early, "Chinese prose must stream without whitespace or sentence punctuation"
     assert chinese_early + chinese_filter.finish() == long_chinese
 
+    english_filter = VisibleResponseStreamFilter()
+    english_chunks = ["Version 1.2 is ready", ".", " See example.com for notes! Next,", " run the check? Done"]
+    english_visible = [english_filter.feed(chunk) for chunk in english_chunks]
+    assert english_visible[0] == "", "a decimal or version point must not end a sentence"
+    assert english_visible[1] == "", "sentence punctuation waits for the following whitespace"
+    assert english_visible[2] == "Version 1.2 is ready. See example.com for notes! ", english_visible
+    assert english_visible[3] == "Next, run the check? ", english_visible
+    assert "".join(english_visible) + english_filter.finish() == "".join(english_chunks)
+
+    # An early sentence end must not hold back a long tail in the same chunk:
+    # the tail still streams through the length fallback before the provider
+    # sends anything else.
+    tail_filter = VisibleResponseStreamFilter()
+    tail_chunk = "Ready. " + "word " * 100
+    tail_visible = tail_filter.feed(tail_chunk)
+    assert tail_visible.startswith("Ready. word "), tail_visible[:40]
+    assert len(tail_chunk) - len(tail_visible) < 160, len(tail_visible)
+    assert tail_visible + tail_filter.finish() == tail_chunk
+
     protected_path = "/home/example/project"
+    tail_path_filter = VisibleResponseStreamFilter(protected_paths=[protected_path])
+    tail_path_text = "Ready. " + "word " * 40 + f"see {protected_path}/notes.txt " + "word " * 40
+    tail_path_chunks = [tail_path_text, CHAT_REVIEW_OPEN_TAG + '{"hidden": true}']
+    tail_path_early = tail_path_filter.feed(tail_path_chunks[0])
+    assert len(tail_path_text) - len(tail_path_early) < 160, len(tail_path_early)
+    tail_path_visible = tail_path_early + tail_path_filter.feed(tail_path_chunks[1]) + tail_path_filter.finish()
+    assert protected_path not in tail_path_visible, tail_path_visible
+    assert "hidden" not in tail_path_visible, tail_path_visible
+    assert tail_path_visible == redact_local_paths(tail_path_text, protected_paths=[protected_path]), tail_path_visible
+
+    english_path_filter = VisibleResponseStreamFilter(protected_paths=[protected_path])
+    english_path_text = f"The report is in {protected_path}/notes.txt. Review it next. "
+    english_path_visible = "".join(
+        english_path_filter.feed(english_path_text[index : index + 7])
+        for index in range(0, len(english_path_text), 7)
+    ) + english_path_filter.finish()
+    assert protected_path not in english_path_visible, english_path_visible
+    # Splitting at sentences must not change what redaction produces.
+    assert english_path_visible == redact_local_paths(english_path_text, protected_paths=[protected_path]), english_path_visible
+
     path_filter = VisibleResponseStreamFilter(protected_paths=[protected_path])
     path_text = ("前" * 150) + protected_path + "/secret.txt 后续内容"
     path_early = "".join(

loopx/chat.py

@@ -106,6 +106,10 @@ class VisibleResponseStreamFilter:
     """Stream safe operator text while withholding the structured review envelope."""
 
     _FLUSH_BOUNDARIES = {"\n", "。", "！", "？"}
+    # Latin sentence punctuation ends a sentence only before whitespace, so
+    # decimals, versions, file names and URLs never split. The split lands on
+    # whitespace, which the length fallback below already treats as safe.
+    _SPACED_SENTENCE_ENDINGS = {".", "!", "?"}
     _MAX_PENDING_CHARS = 160
 
     def __init__(self, *, protected_paths: Iterable[Path | str] = ()) -> None:
@@ -114,36 +118,50 @@ def __init__(self, *, protected_paths: Iterable[Path | str] = ()) -> None:
         self.visible_pending = ""
         self.envelope_started = False
 
-    def _accept_visible(self, text: str, *, final: bool) -> str:
-        self.visible_pending += text
-        if final:
-            ready = self.visible_pending
-            self.visible_pending = ""
-            return redact_local_paths(ready, protected_paths=self.protected_paths)
+    def _next_boundary(self, pending: str) -> int:
         boundary = -1
-        search_limit = min(len(self.visible_pending), self._MAX_PENDING_CHARS)
-        for index, character in enumerate(self.visible_pending[:search_limit]):
+        search_limit = min(len(pending), self._MAX_PENDING_CHARS)
+        for index, character in enumerate(pending[:search_limit]):
             if character in self._FLUSH_BOUNDARIES:
                 boundary = index + 1
-        if boundary < 0 and len(self.visible_pending) >= self._MAX_PENDING_CHARS:
-            prefix = self.visible_pending[: self._MAX_PENDING_CHARS + 1]
+            elif (
+                character in self._SPACED_SENTENCE_ENDINGS
+                and pending[index + 1 : index + 2] in {" ", "\t"}
+            ):
+                boundary = index + 2
+        if boundary < 0 and len(pending) >= self._MAX_PENDING_CHARS:
+            prefix = pending[: self._MAX_PENDING_CHARS + 1]
             whitespace = max(prefix.rfind(" "), prefix.rfind("\t"))
             if whitespace >= 0:
                 boundary = whitespace + 1
             elif _ABSOLUTE_LOCAL_PATH.search(prefix) is None:
                 boundary = self._MAX_PENDING_CHARS
             else:
                 for index, character in enumerate(
-                    self.visible_pending[self._MAX_PENDING_CHARS :],
+                    pending[self._MAX_PENDING_CHARS :],
                     start=self._MAX_PENDING_CHARS,
                 ):
                     if character in " \t\r\n`'\"<>":
                         boundary = index + 1
                         break
-        if boundary < 0:
+        return boundary
+
+    def _accept_visible(self, text: str, *, final: bool) -> str:
+        self.visible_pending += text
+        if final:
+            ready = self.visible_pending
+            self.visible_pending = ""
+            return redact_local_paths(ready, protected_paths=self.protected_paths)
+        # One chunk can hold several safe boundaries. Keep cutting until none
+        # is left, so an early sentence never holds back a long tail that the
+        # length fallback would otherwise release.
+        ready_length = 0
+        while (boundary := self._next_boundary(self.visible_pending[ready_length:])) > 0:
+            ready_length += boundary
+        if not ready_length:
             return ""
-        ready = self.visible_pending[:boundary]
-        self.visible_pending = self.visible_pending[boundary:]
+        ready = self.visible_pending[:ready_length]
+        self.visible_pending = self.visible_pending[ready_length:]
         return redact_local_paths(ready, protected_paths=self.protected_paths)
 
     def feed(self, chunk: str) -> str: