diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index fd9d0d827..f3ab2f688 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -1280,7 +1280,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } if bool(args.get("sessionScope", False)): params["sessionId"] = self._session_id - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.search", params, ) @@ -1298,7 +1298,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) method = methods.get(kind) if method is None: return json.dumps({"error": f"unknown memory kind: {kind}"}) - item = self._bridge.request( + item = self._bridge_request_with_retry( method, {"id": item_id, "namespace": self._runtime_namespace()} ) if not item: @@ -1343,7 +1343,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } ) if tool_name == "memos_timeline": - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.timeline", { "episodeId": args.get("episodeId", self._episode_id), @@ -1358,12 +1358,12 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) params = {"limit": limit, "namespace": self._runtime_namespace()} if args.get("status"): params["status"] = args["status"] - return json.dumps(self._bridge.request("skill.list", params)) + return json.dumps(self._bridge_request_with_retry("skill.list", params)) if tool_name == "memos_environment": query = (args.get("query") or "").strip() limit = self._int_arg(args, "limit", 5, 1, 30) if not query: - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.list_world_models", {"limit": limit, "offset": 0, "namespace": self._runtime_namespace()}, ) @@ -1379,7 +1379,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) "queried": False, } ) - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.search", { "agent": "hermes", @@ -1412,7 +1412,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) skill_id = (args.get("id") or "").strip() if not skill_id: return json.dumps({"error": "missing id"}) - skill = self._bridge.request( + skill = self._bridge_request_with_retry( "skill.get", { "id": skill_id, @@ -1724,6 +1724,85 @@ def _is_transport_closed(self, err: Exception) -> bool: msg = str(err).lower() return "broken pipe" in msg or "bridge closed" in msg or "transport_closed" in msg + def _describe_keepalive_error(self, err: Exception) -> str: + """Human-readable tag for keepalive log lines.""" + if isinstance(err, BridgeError): + return f"{err.code} ({err.message})" + return f"{err.__class__.__name__} ({err})" + + def _should_reconnect_after_keepalive_failure(self, err: Exception) -> bool: + """Return True when a keepalive failure indicates a broken bridge. + + Reconnect on: + + 1. `transport_closed` — existing behaviour, still covers broken + pipes. + 2. `BridgeError("timeout", …)` — the exact code raised when the + bridge stops answering. This is the reported symptom of + issue #1722: the Node subprocess is technically alive but + the event loop is wedged, so every `core.health` call hits + the 10 s timeout. + 3. Any exception observed while the bridge subprocess is + already dead (`Popen.poll()` returned non-None). Belt-and- + suspenders for hangs that don't raise a transport error yet + still map to a subprocess we can no longer talk to. + + A live subprocess raising a generic (non-transport) error must + NOT trigger a reconnect — otherwise transient parse noise would + create a reconnect storm. + """ + if self._is_transport_closed(err): + return True + if isinstance(err, BridgeError) and err.code == "timeout": + return True + bridge = self._bridge + if bridge is not None: + proc = getattr(bridge, "_proc", None) + poll = getattr(proc, "poll", None) + if callable(poll): + try: + exit_code = poll() + except Exception: + exit_code = None + if exit_code is not None: + return True + return False + + def _bridge_request_with_retry( + self, + method: str, + params: dict[str, Any] | None = None, + *, + timeout: float | None = None, + ) -> dict[str, Any]: + """Read-path helper: request → on transport_closed, reconnect + retry once. + + Mirrors the existing ``sync_turn`` reconnect+retry pattern so + every read-path memory tool (`memos_search`, `memos_get`, + `memos_timeline`, `memos_environment`, `memos_skill_list`, + `memos_skill_get`) recovers when the bridge dies mid-session + instead of surfacing the raw `[transport_closed]` string to the + model. Only one retry is attempted — a second failure surfaces + verbatim so the caller can see what went wrong. + """ + assert self._bridge is not None + kwargs: dict[str, Any] = {} + if timeout is not None: + kwargs["timeout"] = timeout + try: + return self._bridge.request(method, params, **kwargs) or {} + except Exception as err: + if not self._is_transport_closed(err): + raise + logger.warning( + "MemOS: bridge transport closed during %s; reconnecting and retrying once — %s", + method, + err, + ) + self._reconnect_bridge(self._session_id, timeout=30.0) + assert self._bridge is not None + return self._bridge.request(method, params, **kwargs) or {} + def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> None: # Don't reconnect if we're shutting down if self._bridge_keepalive_stop.is_set(): @@ -1798,8 +1877,11 @@ def _run() -> None: assert self._bridge is not None self._bridge.request("core.health", {}, timeout=10.0) except Exception as err: - if self._is_transport_closed(err): - logger.info("MemOS: bridge keepalive reconnecting after transport close") + if self._should_reconnect_after_keepalive_failure(err): + logger.info( + "MemOS: bridge keepalive reconnecting after %s", + self._describe_keepalive_error(err), + ) with contextlib.suppress(Exception): self._reconnect_bridge(self._session_id, timeout=10.0) else: diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 5b986ec66..d6ed4da15 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -172,6 +172,17 @@ def request( ) -> dict[str, Any]: if self._closed: raise BridgeError("transport_closed", "bridge client is closed") + # Fast-fail: if the subprocess has already exited we must not + # write to a broken pipe. OS-level pipe buffering would happily + # accept the write, then the caller would park on `waiter.wait` + # for the full per-request timeout even though the bridge is + # already dead. Detect that state up front. + exit_code = self._proc.poll() + if exit_code is not None: + raise BridgeError( + "transport_closed", + f"bridge subprocess exited (code={exit_code})", + ) with self._lock: rpc_id = self._next_id self._next_id += 1 @@ -274,88 +285,115 @@ def close(self) -> None: except subprocess.TimeoutExpired: logger.error("MemOS: bridge process %d could not be killed", pid) - # 5. Clean up pending requests + # 5. Clean up pending requests — shared helper with `_read_loop`. + self._abort_pending("bridge closed") + + # ─── Internals ── + + def _read_loop(self) -> None: + assert self._proc.stdout is not None + try: + for line in self._proc.stdout: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + logger.debug("bridge: malformed line: %r", line[:120]) + continue + if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg): + self._resolve(msg) + continue + if msg.get("method") == "events.notify": + for cb in list(self._events): + try: + cb(msg.get("params") or {}) + except Exception: + logger.debug("event listener threw", exc_info=True) + continue + if msg.get("method") == "logs.forward": + for cb in list(self._logs): + try: + cb(msg.get("params") or {}) + except Exception: + logger.debug("log listener threw", exc_info=True) + continue + # Reverse-direction request: the bridge is asking the + # adapter to do something (e.g. run a fallback LLM call + # via `host.llm.complete`). Dispatch to the registered + # handler and write the response back synchronously. + method = msg.get("method") + rpc_id = msg.get("id") + if ( + isinstance(method, str) + and rpc_id is not None + and "result" not in msg + and "error" not in msg + ): + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue + params = msg.get("params") or {} + if not isinstance(params, dict): + params = {} + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) + continue + except Exception: + # Any read-side exception (subprocess crash surfaced as OSError, + # stdout iteration errors) is terminal for the reader thread. + # We fall through to the finally block so pending waiters are + # released instead of parking on their per-request timeouts. + logger.debug("bridge: reader loop terminated by exception", exc_info=True) + finally: + # Normal EOF path (bridge subprocess exited cleanly) reaches + # here without exception. Either way we must wake pending + # waiters with a transport_closed error — otherwise every + # in-flight caller sleeps for the full 30 s wall clock and + # then reports a misleading "[timeout] …" (issue #1722). + self._abort_pending("bridge subprocess exited") + + def _abort_pending(self, reason: str) -> None: + """Mark the client closed and wake every parked JSON-RPC waiter. + + Called from both ``_read_loop``'s finally block (subprocess + died) and ``close`` (client shutting down). Idempotent — + subsequent calls with the same client are no-ops because + ``_pending`` is cleared and ``_closed`` is latched. + """ + with self._host_handlers_cv: + self._closed = True + self._host_handlers_cv.notify_all() with self._lock: for entry in list(self._pending.values()): entry["error"] = { "code": -32000, - "message": "bridge closed", + "message": reason, "data": {"code": "transport_closed"}, } entry["event"].set() self._pending.clear() - # ─── Internals ── - - def _read_loop(self) -> None: - assert self._proc.stdout is not None - for line in self._proc.stdout: - line = line.strip() - if not line: - continue - try: - msg = json.loads(line) - except json.JSONDecodeError: - logger.debug("bridge: malformed line: %r", line[:120]) - continue - if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg): - self._resolve(msg) - continue - if msg.get("method") == "events.notify": - for cb in list(self._events): - try: - cb(msg.get("params") or {}) - except Exception: - logger.debug("event listener threw", exc_info=True) - continue - if msg.get("method") == "logs.forward": - for cb in list(self._logs): - try: - cb(msg.get("params") or {}) - except Exception: - logger.debug("log listener threw", exc_info=True) - continue - # Reverse-direction request: the bridge is asking the - # adapter to do something (e.g. run a fallback LLM call - # via `host.llm.complete`). Dispatch to the registered - # handler and write the response back synchronously. - method = msg.get("method") - rpc_id = msg.get("id") - if ( - isinstance(method, str) - and rpc_id is not None - and "result" not in msg - and "error" not in msg - ): - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) - continue - params = msg.get("params") or {} - if not isinstance(params, dict): - params = {} - try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) - continue - def _host_handler_for( self, method: str, diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index b47d5660e..3a6ad7b87 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -20,6 +20,7 @@ import unittest from pathlib import Path +from typing import Any from unittest.mock import patch @@ -49,6 +50,8 @@ def __init__(self, *_args, **_kwargs) -> None: self._stdin_lines: list[str] = [] self.stdout = _ServerStream() self.stderr = io.StringIO() + # None = alive; set to an int to simulate an exited subprocess. + self._returncode: int | None = None # Patch the write path so writes accumulate in `_stdin_lines` # and the server can peek at incoming requests. @@ -61,13 +64,16 @@ def _write(s: str) -> int: self.stdin.write = _write # type: ignore[assignment] - # The client just needs wait/kill to exist; they are no-ops here. + # The client just needs wait/kill/poll to exist; they are no-ops here. def wait(self, timeout: float | None = None) -> int: - return 0 + return self._returncode if self._returncode is not None else 0 def kill(self) -> None: pass + def poll(self) -> int | None: + return self._returncode + class _ServerStream(io.StringIO): """Script bridge responses as if coming from the Node subprocess.""" @@ -295,6 +301,56 @@ def test_close_is_idempotent(self) -> None: client.close() client.close() # second call must not raise + def test_reader_exit_marks_pending_as_transport_closed(self) -> None: + """R1: reader-thread EOF wakes every parked waiter with transport_closed.""" + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + + # Issue a request whose response is never scripted; the waiter + # will park on the event. + thread_result: dict[str, Any] = {} + + def _issue() -> None: + try: + client.request("never_answered", {"foo": "bar"}, timeout=10.0) + except Exception as err: + thread_result["error"] = err + + import threading as _th + + t = _th.Thread(target=_issue) + t.start() + # Wait until the request is registered as pending. + deadline = time.monotonic() + 1.0 + while time.monotonic() < deadline and not client._pending: + time.sleep(0.01) + self.assertTrue(client._pending, "request should be pending") + + # Simulate bridge subprocess dying — reader hits EOF. + self._fake.stdout._done = True + self._fake.stdout._event.set() + + t.join(timeout=2.0) + self.assertFalse(t.is_alive(), "waiter should be released within 2s") + err = thread_result.get("error") + self.assertIsInstance(err, BridgeError) + self.assertEqual(err.code, "transport_closed") # type: ignore[union-attr] + # After the reader exits, pending must be empty. + self.assertEqual(client._pending, {}) + + def test_request_fast_fails_when_subprocess_already_dead(self) -> None: + """R2: `poll()` non-None short-circuits before touching stdin.""" + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + # Simulate OOM kill on Linux (SIGKILL -> exit code 137). + self._fake._returncode = 137 + stdin_lines_before = list(self._fake._stdin_lines) + with self.assertRaises(BridgeError) as ctx: + client.request("core.health", {}) + self.assertEqual(ctx.exception.code, "transport_closed") + # No new writes should have gone to stdin. + self.assertEqual(self._fake._stdin_lines, stdin_lines_before) + def test_stdio_bridge_starts_without_viewer_by_default(self) -> None: client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") assert self._fake is not None @@ -627,6 +683,117 @@ def request(self, method, params=None, **_kwargs): self.assertIn("局势", retry_payload["agentText"]) self.assertEqual(retry_payload["toolCalls"][0]["name"], "search_files") + def test_keepalive_reconnects_on_health_timeout(self) -> None: + """R3: `BridgeError('timeout', …)` must trigger a keepalive reconnect.""" + p = self._provider_mod.MemTensorProvider() + + class LiveBridge: + def __init__(self) -> None: + self._proc = type("P", (), {"poll": staticmethod(lambda: None)})() + + p._bridge = LiveBridge() + err = BridgeError("timeout", "core.health did not respond within 10.0s") + self.assertTrue(p._should_reconnect_after_keepalive_failure(err)) + + def test_keepalive_reconnects_when_subprocess_is_dead(self) -> None: + """R3: any error surfaces from a dead subprocess must reconnect.""" + p = self._provider_mod.MemTensorProvider() + + class DeadBridge: + def __init__(self) -> None: + self._proc = type("P", (), {"poll": staticmethod(lambda: 137)})() + + p._bridge = DeadBridge() + err = RuntimeError("boom — subprocess is gone") + self.assertTrue(p._should_reconnect_after_keepalive_failure(err)) + + def test_keepalive_does_not_reconnect_on_transient_generic_error(self) -> None: + """R3: live subprocess + generic error must NOT trigger a reconnect.""" + p = self._provider_mod.MemTensorProvider() + + class LiveBridge: + def __init__(self) -> None: + self._proc = type("P", (), {"poll": staticmethod(lambda: None)})() + + p._bridge = LiveBridge() + err = BridgeError("internal", "transient parse noise") + self.assertFalse(p._should_reconnect_after_keepalive_failure(err)) + + def test_handle_tool_call_retries_memos_search_after_transport_closed(self) -> None: + """R4: read path reconnects and retries once on transport_closed.""" + + class FlakyBridge: + def __init__(self) -> None: + self.calls: list[tuple[str, dict]] = [] + self._first = True + + def request(self, method: str, params: dict | None = None, **_kwargs): + self.calls.append((method, params or {})) + if self._first: + self._first = False + raise BridgeError("transport_closed", "[Errno 32] Broken pipe") + if method == "memory.search": + return { + "hits": [ + { + "tier": 2, + "refKind": "trace", + "refId": "tr-99", + "score": 0.42, + "snippet": "recovered hit", + } + ] + } + return {} + + flaky = FlakyBridge() + p = self._provider_mod.MemTensorProvider() + p._bridge = flaky + p._session_id = "sess-recover" + p._episode_id = "ep-recover" + + reconnect_calls: list[str] = [] + + def _fake_reconnect(session_id: str = "", *, timeout: float = 30.0) -> None: + reconnect_calls.append(session_id) + # The reconnect swaps _bridge; keep the same fake so subsequent + # calls resolve. The important behavioural invariant is that + # _reconnect_bridge was invoked before the retry. + p._bridge = flaky + + p._reconnect_bridge = _fake_reconnect # type: ignore[method-assign] + + result = json.loads(p.handle_tool_call("memos_search", {"query": "what did I do earlier?"})) + self.assertEqual(reconnect_calls, ["sess-recover"]) + self.assertEqual(result["hits"][0]["refId"], "tr-99") + # Exactly two bridge invocations: the failing one + the retry. + methods = [m for m, _ in flaky.calls] + self.assertEqual(methods, ["memory.search", "memory.search"]) + + def test_handle_tool_call_surfaces_second_transport_failure(self) -> None: + """R4: if the retry also fails the tool response carries the error text.""" + + class DeadBridge: + def __init__(self) -> None: + self.calls: list[str] = [] + + def request(self, method: str, params: dict | None = None, **_kwargs): + self.calls.append(method) + raise BridgeError("transport_closed", "still down after reconnect") + + dead = DeadBridge() + p = self._provider_mod.MemTensorProvider() + p._bridge = dead + p._session_id = "sess-perm-dead" + + p._reconnect_bridge = lambda session_id="", *, timeout=30.0: None # type: ignore[method-assign] + + parsed = json.loads(p.handle_tool_call("memos_search", {"query": "x"})) + self.assertIn("error", parsed) + self.assertIn("still down after reconnect", parsed["error"]) + # Exactly two invocations: initial + one retry. + self.assertEqual(dead.calls, ["memory.search", "memory.search"]) + def test_get_config_schema_describes_known_fields(self) -> None: p = self._provider_mod.MemTensorProvider() schema = p.get_config_schema()