diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py index fd9d0d827..6d1a8fed5 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py @@ -1280,7 +1280,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } if bool(args.get("sessionScope", False)): params["sessionId"] = self._session_id - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.search", params, ) @@ -1298,7 +1298,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) method = methods.get(kind) if method is None: return json.dumps({"error": f"unknown memory kind: {kind}"}) - item = self._bridge.request( + item = self._bridge_request_with_retry( method, {"id": item_id, "namespace": self._runtime_namespace()} ) if not item: @@ -1343,7 +1343,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) } ) if tool_name == "memos_timeline": - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.timeline", { "episodeId": args.get("episodeId", self._episode_id), @@ -1358,12 +1358,12 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) params = {"limit": limit, "namespace": self._runtime_namespace()} if args.get("status"): params["status"] = args["status"] - return json.dumps(self._bridge.request("skill.list", params)) + return json.dumps(self._bridge_request_with_retry("skill.list", params)) if tool_name == "memos_environment": query = (args.get("query") or "").strip() limit = self._int_arg(args, "limit", 5, 1, 30) if not query: - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.list_world_models", {"limit": limit, "offset": 0, "namespace": self._runtime_namespace()}, ) @@ -1379,7 +1379,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) "queried": False, } ) - resp = self._bridge.request( + resp = self._bridge_request_with_retry( "memory.search", { "agent": "hermes", @@ -1412,7 +1412,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any) skill_id = (args.get("id") or "").strip() if not skill_id: return json.dumps({"error": "missing id"}) - skill = self._bridge.request( + skill = self._bridge_request_with_retry( "skill.get", { "id": skill_id, @@ -1785,6 +1785,38 @@ def _ensure_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> bool self._bridge = None return False + def _bridge_request_with_retry( + self, + method: str, + params: dict[str, Any], + *, + timeout: float | None = None, + ) -> dict[str, Any]: + """Call the bridge with a single reconnect+retry on `transport_closed`. + + Mirrors the retry pattern `sync_turn` already uses so read-path + memory tools (`memos_search`, `memos_get`, …) also recover from a + stale bridge instead of surfacing a bare error on the first + attempt. Any other error is re-raised for the caller to handle. + See issue #1722. + """ + assert self._bridge is not None + call_kwargs: dict[str, Any] = {"timeout": timeout} if timeout is not None else {} + try: + return self._bridge.request(method, params, **call_kwargs) + except Exception as err: + if not self._is_transport_closed(err): + raise + logger.warning( + "MemOS: bridge transport closed during %s; reconnecting and retrying once — %s", + method, + err, + ) + # One reconnect, one retry, then surface whatever comes back. + self._reconnect_bridge(self._session_id, timeout=30.0) + assert self._bridge is not None + return self._bridge.request(method, params, **call_kwargs) + def _start_bridge_keepalive(self) -> None: if self._bridge_keepalive_thread and self._bridge_keepalive_thread.is_alive(): return @@ -1798,8 +1830,11 @@ def _run() -> None: assert self._bridge is not None self._bridge.request("core.health", {}, timeout=10.0) except Exception as err: - if self._is_transport_closed(err): - logger.info("MemOS: bridge keepalive reconnecting after transport close") + if self._should_reconnect_after_keepalive_failure(err): + logger.info( + "MemOS: bridge keepalive reconnecting after health failure — %s", + err, + ) with contextlib.suppress(Exception): self._reconnect_bridge(self._session_id, timeout=10.0) else: @@ -1812,6 +1847,35 @@ def _run() -> None: ) self._bridge_keepalive_thread.start() + def _should_reconnect_after_keepalive_failure(self, err: Exception) -> bool: + """Decide whether a `core.health` failure warrants a reconnect. + + Historically we only reconnected on `transport_closed`. Issue + #1722 showed that a hung bridge surfaces as `BridgeError("timeout", + …)` while the pipe is still nominally open, so the client never + recovered. Reconnect for: + + - the historical `transport_closed` case, + - `BridgeError("timeout", …)` (the health probe timing out means + the bridge is not answering anything), + - any error observed while the subprocess is already dead + (`Popen.poll()` returns a non-`None` exit code). + """ + if self._is_transport_closed(err): + return True + if isinstance(err, BridgeError) and err.code == "timeout": + return True + bridge = self._bridge + if bridge is not None: + proc = getattr(bridge, "_proc", None) + poll = getattr(proc, "poll", None) + try: + if callable(poll) and poll() is not None: + return True + except Exception: # pragma: no cover — defensive + pass + return False + def _turn_start(self, query: str, *, session_id: str = "") -> str: assert self._bridge is not None host_runtime = self._host_runtime_context() diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py index 5b986ec66..2ab6c8af1 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py @@ -172,6 +172,19 @@ def request( ) -> dict[str, Any]: if self._closed: raise BridgeError("transport_closed", "bridge client is closed") + # Fast-fail path: if the subprocess has already exited, do NOT + # write to a dead pipe (the OS may buffer the write for a while + # and the caller would then park for the full timeout waiting for + # a reader thread that has already returned). Surface + # `transport_closed` immediately so upstream retry / reconnect + # logic can kick in. See issue #1722. + poll = getattr(self._proc, "poll", None) + exit_code = poll() if callable(poll) else None + if exit_code is not None: + raise BridgeError( + "transport_closed", + f"bridge subprocess exited (code={exit_code})", + ) with self._lock: rpc_id = self._next_id self._next_id += 1 @@ -239,11 +252,22 @@ def register_host_handler( def close(self) -> None: if self._closed: + # Even if the reader thread already flipped `_closed` (subprocess + # exit → `_abort_pending`), the caller's explicit close should + # still tear down the subprocess so nothing lingers. + self._teardown_process() return with self._host_handlers_cv: self._closed = True self._host_handlers_cv.notify_all() + self._teardown_process() + + # Clean up any request that was still pending when close() was + # called explicitly (e.g. shutdown before the subprocess exited). + self._abort_pending("bridge closed") + + def _teardown_process(self) -> None: pid = self.pid # 1. Close stdin (triggers bridge's graceful exit) @@ -274,87 +298,110 @@ def close(self) -> None: except subprocess.TimeoutExpired: logger.error("MemOS: bridge process %d could not be killed", pid) - # 5. Clean up pending requests - with self._lock: - for entry in list(self._pending.values()): - entry["error"] = { - "code": -32000, - "message": "bridge closed", - "data": {"code": "transport_closed"}, - } - entry["event"].set() - self._pending.clear() - # ─── Internals ── def _read_loop(self) -> None: assert self._proc.stdout is not None - for line in self._proc.stdout: - line = line.strip() - if not line: - continue - try: - msg = json.loads(line) - except json.JSONDecodeError: - logger.debug("bridge: malformed line: %r", line[:120]) - continue - if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg): - self._resolve(msg) - continue - if msg.get("method") == "events.notify": - for cb in list(self._events): - try: - cb(msg.get("params") or {}) - except Exception: - logger.debug("event listener threw", exc_info=True) - continue - if msg.get("method") == "logs.forward": - for cb in list(self._logs): - try: - cb(msg.get("params") or {}) - except Exception: - logger.debug("log listener threw", exc_info=True) - continue - # Reverse-direction request: the bridge is asking the - # adapter to do something (e.g. run a fallback LLM call - # via `host.llm.complete`). Dispatch to the registered - # handler and write the response back synchronously. - method = msg.get("method") - rpc_id = msg.get("id") - if ( - isinstance(method, str) - and rpc_id is not None - and "result" not in msg - and "error" not in msg - ): - handler = self._host_handler_for(method) - if handler is None: - self._send_response( - rpc_id, - error={ - "code": -32601, - "message": f"method not found: {method}", - "data": {"code": "unknown_method"}, - }, - ) + exit_reason = "bridge subprocess exited" + try: + for line in self._proc.stdout: + line = line.strip() + if not line: continue - params = msg.get("params") or {} - if not isinstance(params, dict): - params = {} try: - result = handler(params) - self._send_response(rpc_id, result=result) - except Exception as err: - logger.warning("host handler %s failed: %s", method, err) - self._send_response( - rpc_id, - error={ - "code": -32000, - "message": str(err) or err.__class__.__name__, - "data": {"code": "host_handler_failed"}, - }, - ) - continue + msg = json.loads(line) + except json.JSONDecodeError: + logger.debug("bridge: malformed line: %r", line[:120]) + continue + if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg): + self._resolve(msg) + continue + if msg.get("method") == "events.notify": + for cb in list(self._events): + try: + cb(msg.get("params") or {}) + except Exception: + logger.debug("event listener threw", exc_info=True) + continue + if msg.get("method") == "logs.forward": + for cb in list(self._logs): + try: + cb(msg.get("params") or {}) + except Exception: + logger.debug("log listener threw", exc_info=True) + continue + # Reverse-direction request: the bridge is asking the + # adapter to do something (e.g. run a fallback LLM call + # via `host.llm.complete`). Dispatch to the registered + # handler and write the response back synchronously. + method = msg.get("method") + rpc_id = msg.get("id") + if ( + isinstance(method, str) + and rpc_id is not None + and "result" not in msg + and "error" not in msg + ): + handler = self._host_handler_for(method) + if handler is None: + self._send_response( + rpc_id, + error={ + "code": -32601, + "message": f"method not found: {method}", + "data": {"code": "unknown_method"}, + }, + ) + continue + params = msg.get("params") or {} + if not isinstance(params, dict): + params = {} + try: + result = handler(params) + self._send_response(rpc_id, result=result) + except Exception as err: + logger.warning("host handler %s failed: %s", method, err) + self._send_response( + rpc_id, + error={ + "code": -32000, + "message": str(err) or err.__class__.__name__, + "data": {"code": "host_handler_failed"}, + }, + ) + continue + except Exception as err: + exit_reason = f"bridge read loop crashed: {err}" + logger.debug("bridge: read loop raised %s", err, exc_info=True) + finally: + # Whether we exited cleanly (stdout EOF because the subprocess + # died) or via an exception, every JSON-RPC waiter parked in + # `request()` MUST be woken with a `transport_closed` error — + # otherwise callers park for their full timeout waiting for a + # response that will never arrive. Issue #1722. + self._abort_pending(exit_reason) + + def _abort_pending(self, reason: str) -> None: + """Wake every parked JSON-RPC waiter with a `transport_closed` error. + + Called from the reader thread's `finally` block (subprocess exit + or stream error) so callers don't have to wait the full per- + request timeout to learn that the bridge is gone. Also idempotent + with `close()` — the second run just finds `self._pending` empty. + """ + with self._host_handlers_cv: + self._closed = True + self._host_handlers_cv.notify_all() + with self._lock: + pending = list(self._pending.values()) + self._pending.clear() + for entry in pending: + entry["error"] = { + "code": -32000, + "message": reason, + "data": {"code": "transport_closed"}, + } + entry["event"].set() def _host_handler_for( self, diff --git a/apps/memos-local-plugin/tests/python/test_bridge_client.py b/apps/memos-local-plugin/tests/python/test_bridge_client.py index b47d5660e..8d78866e8 100644 --- a/apps/memos-local-plugin/tests/python/test_bridge_client.py +++ b/apps/memos-local-plugin/tests/python/test_bridge_client.py @@ -20,6 +20,7 @@ import unittest from pathlib import Path +from typing import Any from unittest.mock import patch @@ -45,6 +46,7 @@ class FakePopen: def __init__(self, *_args, **_kwargs) -> None: self.cmd = list(_args[0]) if _args else [] self.pid = 12345 + self._exit_code: int | None = None self.stdin = io.StringIO() self._stdin_lines: list[str] = [] self.stdout = _ServerStream() @@ -55,18 +57,33 @@ def __init__(self, *_args, **_kwargs) -> None: orig_write = self.stdin.write def _write(s: str) -> int: + # Real kernels happily accept writes to a not-yet-flushed pipe + # even after the subprocess exits (data goes into the OS buffer + # and only errors when the buffer fills). Mimic that so the + # fast-fail path in `request()` has to rely on `poll()` — not + # on a fortuitous write error — to detect the dead subprocess. self._stdin_lines.append(s) - self.stdout.on_request(s) + if self._exit_code is None: + self.stdout.on_request(s) return orig_write(s) self.stdin.write = _write # type: ignore[assignment] - # The client just needs wait/kill to exist; they are no-ops here. + # The client just needs wait/kill/poll to exist; they are no-ops here. def wait(self, timeout: float | None = None) -> int: - return 0 + return self._exit_code if self._exit_code is not None else 0 def kill(self) -> None: - pass + self._exit_code = -9 + + def poll(self) -> int | None: + return self._exit_code + + def simulate_exit(self, code: int = 0) -> None: + """Simulate the bridge subprocess dying: close stdout + set exit code.""" + self._exit_code = code + self.stdout._done = True + self.stdout._event.set() class _ServerStream(io.StringIO): @@ -343,6 +360,82 @@ def _wait_for_client_write(self, predicate, timeout: float = 2.0) -> dict: time.sleep(0.01) self.fail("timed out waiting for client write") + def test_reader_exit_marks_pending_as_transport_closed(self) -> None: + """Pending waiters must fail fast when the reader thread exits. + + Reproduces the #1722 symptom: bridge subprocess dies silently, + the reader thread returns without notifying anyone, and callers + park for the full 30 s timeout. After the fix, pending requests + surface `BridgeError("transport_closed", …)` within a second. + """ + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + + result: dict[str, Any] = {} + + def _issue() -> None: + try: + client.request("memory.search", {"query": "x"}, timeout=10.0) + except BridgeError as err: + result["error"] = err + except Exception as err: + result["error"] = err + + # Drain the server-side auto-response for memory.search so the + # request stays pending when the subprocess dies. + self._fake.stdout._queue.clear() + original_on_request = self._fake.stdout.on_request + + def _swallow(raw: str) -> None: + try: + req = json.loads(raw) + except json.JSONDecodeError: + return + if req.get("method") == "memory.search": + return # do NOT respond, mimic a hung bridge + original_on_request(raw) + + self._fake.stdout.on_request = _swallow # type: ignore[assignment] + + worker = threading.Thread(target=_issue, daemon=True) + worker.start() + time.sleep(0.1) # let the request enqueue + + # Simulate bridge subprocess exit (stdout EOF + poll returns code). + self._fake.simulate_exit(1) + + worker.join(timeout=2.0) + self.assertFalse(worker.is_alive(), "request did not fail fast on subprocess exit") + self.assertIsInstance(result.get("error"), BridgeError) + self.assertEqual(result["error"].code, "transport_closed") + client.close() + + def test_request_fast_fails_when_subprocess_already_dead(self) -> None: + """A dead subprocess must not accept new writes; short-circuit at request().""" + client = MemosBridgeClient(bridge_path="/tmp/bridge.cts") + assert self._fake is not None + + # Give the bridge time to become "healthy" once so subsequent writes + # would ordinarily succeed, then kill the subprocess. + self._fake.simulate_exit(1) + # Reader thread should have already marked us closed via _abort_pending. + # Even a caller that hasn't observed that yet must get transport_closed + # from the fast-fail poll() check inside request(). + time.sleep(0.1) + + # Fresh call must NOT append to _stdin_lines — poll() shortcut wins. + stdin_before = list(self._fake._stdin_lines) + with self.assertRaises(BridgeError) as ctx: + client.request("core.health", timeout=1.0) + self.assertEqual(ctx.exception.code, "transport_closed") + stdin_after = list(self._fake._stdin_lines) + self.assertEqual( + stdin_before, + stdin_after, + "request() wrote to stdin of a dead subprocess", + ) + client.close() + class MemTensorProviderTests(unittest.TestCase): """Exercise `MemTensorProvider` against a mocked bridge.""" @@ -627,6 +720,141 @@ def request(self, method, params=None, **_kwargs): self.assertIn("局势", retry_payload["agentText"]) self.assertEqual(retry_payload["toolCalls"][0]["name"], "search_files") + def test_handle_tool_call_retries_memos_search_after_transport_closed(self) -> None: + """Read-path tools reconnect + retry once after a stale bridge pipe. + + Guards issue #1722: after the bridge subprocess dies the first + call raises `transport_closed`, but the adapter reconnects and + the retried call succeeds so the user still sees hits. + """ + + class BrokenSearchBridge: + def __init__(self) -> None: + self.closed = False + + def close(self) -> None: + self.closed = True + + def request(self, method, params=None, **_kwargs): + if method == "memory.search": + raise BridgeError("transport_closed", "[Errno 32] Broken pipe") + return {} + + class HealthyBridge: + def __init__(self) -> None: + self.calls: list[tuple[str, dict]] = [] + + def register_host_handler(self, *_args, **_kwargs) -> None: + return None + + def request(self, method, params=None, **_kwargs): + self.calls.append((method, params or {})) + if method == "session.open": + return {"sessionId": (params or {}).get("sessionId", "sess")} + if method == "memory.search": + return {"hits": [{"snippet": "recovered hit"}]} + return {} + + broken = BrokenSearchBridge() + replacement = HealthyBridge() + p = self._provider_mod.MemTensorProvider() + p._bridge = broken + p._session_id = "sess_tui_long_running" + p._episode_id = "" + + with patch("memos_provider.MemosBridgeClient", return_value=replacement): + raw = p.handle_tool_call("memos_search", {"query": "what is in memory?"}) + + parsed = json.loads(raw) + self.assertNotIn("error", parsed, f"expected recovery, got: {parsed}") + self.assertEqual(parsed["hits"][0]["snippet"], "recovered hit") + self.assertTrue(broken.closed, "stale bridge was not closed before retry") + methods = [m for m, _ in replacement.calls] + self.assertIn("session.open", methods) + self.assertEqual(methods[-1], "memory.search") + + def test_handle_tool_call_surfaces_second_transport_failure(self) -> None: + """If the retried call also fails, surface the error verbatim (no infinite loop).""" + + class BrokenSearchBridge: + def close(self) -> None: + pass + + def request(self, method, params=None, **_kwargs): + if method == "memory.search": + raise BridgeError("transport_closed", "still down") + return {} + + class StillBrokenBridge: + def register_host_handler(self, *_args, **_kwargs) -> None: + return None + + def request(self, method, params=None, **_kwargs): + if method == "session.open": + return {"sessionId": (params or {}).get("sessionId", "sess")} + if method == "memory.search": + raise BridgeError("transport_closed", "still down after reconnect") + return {} + + p = self._provider_mod.MemTensorProvider() + p._bridge = BrokenSearchBridge() + p._session_id = "sess" + + with patch("memos_provider.MemosBridgeClient", return_value=StillBrokenBridge()): + raw = p.handle_tool_call("memos_search", {"query": "anything"}) + parsed = json.loads(raw) + self.assertIn("error", parsed) + self.assertIn("still down", parsed["error"]) + + def test_keepalive_reconnects_on_health_timeout(self) -> None: + """A `core.health` timeout must trigger a reconnect (not just DEBUG log). + + Issue #1722: keepalive previously only reconnected on + `transport_closed`. When the bridge subprocess hangs and every + request times out, keepalive should still detect this and rebuild + the client. + """ + p = self._provider_mod.MemTensorProvider() + + class HangingBridge: + _proc = type("_P", (), {"poll": staticmethod(lambda: None)})() + + def request(self, method, params=None, **_kwargs): + raise BridgeError("timeout", "core.health did not respond within 10.0s") + + p._bridge = HangingBridge() + self.assertTrue( + p._should_reconnect_after_keepalive_failure( + BridgeError("timeout", "core.health did not respond within 10.0s"), + ) + ) + + def test_keepalive_reconnects_when_subprocess_is_dead(self) -> None: + """A dead subprocess (poll() != None) triggers reconnect even on generic errors.""" + p = self._provider_mod.MemTensorProvider() + + class DeadBridge: + _proc = type("_P", (), {"poll": staticmethod(lambda: 1)})() + + p._bridge = DeadBridge() + # Even a non-transport, non-timeout error must count as reconnect-worthy + # when the subprocess is already gone. + self.assertTrue( + p._should_reconnect_after_keepalive_failure(RuntimeError("unexpected")), + ) + + def test_keepalive_does_not_reconnect_on_transient_generic_error(self) -> None: + """Generic errors on a live subprocess must NOT trigger reconnect.""" + p = self._provider_mod.MemTensorProvider() + + class LiveBridge: + _proc = type("_P", (), {"poll": staticmethod(lambda: None)})() + + p._bridge = LiveBridge() + self.assertFalse( + p._should_reconnect_after_keepalive_failure(ValueError("transient parse error")), + ) + def test_get_config_schema_describes_known_fields(self) -> None: p = self._provider_mod.MemTensorProvider() schema = p.get_config_schema()