Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 73 additions & 9 deletions apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1280,7 +1280,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
}
if bool(args.get("sessionScope", False)):
params["sessionId"] = self._session_id
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.search",
params,
)
Expand All @@ -1298,7 +1298,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
method = methods.get(kind)
if method is None:
return json.dumps({"error": f"unknown memory kind: {kind}"})
item = self._bridge.request(
item = self._bridge_request_with_retry(
method, {"id": item_id, "namespace": self._runtime_namespace()}
)
if not item:
Expand Down Expand Up @@ -1343,7 +1343,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
}
)
if tool_name == "memos_timeline":
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.timeline",
{
"episodeId": args.get("episodeId", self._episode_id),
Expand All @@ -1358,12 +1358,12 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
params = {"limit": limit, "namespace": self._runtime_namespace()}
if args.get("status"):
params["status"] = args["status"]
return json.dumps(self._bridge.request("skill.list", params))
return json.dumps(self._bridge_request_with_retry("skill.list", params))
if tool_name == "memos_environment":
query = (args.get("query") or "").strip()
limit = self._int_arg(args, "limit", 5, 1, 30)
if not query:
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.list_world_models",
{"limit": limit, "offset": 0, "namespace": self._runtime_namespace()},
)
Expand All @@ -1379,7 +1379,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
"queried": False,
}
)
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.search",
{
"agent": "hermes",
Expand Down Expand Up @@ -1412,7 +1412,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
skill_id = (args.get("id") or "").strip()
if not skill_id:
return json.dumps({"error": "missing id"})
skill = self._bridge.request(
skill = self._bridge_request_with_retry(
"skill.get",
{
"id": skill_id,
Expand Down Expand Up @@ -1785,6 +1785,38 @@ def _ensure_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> bool
self._bridge = None
return False

def _bridge_request_with_retry(
self,
method: str,
params: dict[str, Any],
*,
timeout: float | None = None,
) -> dict[str, Any]:
"""Call the bridge with a single reconnect+retry on `transport_closed`.

Mirrors the retry pattern `sync_turn` already uses so read-path
memory tools (`memos_search`, `memos_get`, …) also recover from a
stale bridge instead of surfacing a bare error on the first
attempt. Any other error is re-raised for the caller to handle.
See issue #1722.
"""
assert self._bridge is not None
call_kwargs: dict[str, Any] = {"timeout": timeout} if timeout is not None else {}
try:
return self._bridge.request(method, params, **call_kwargs)
except Exception as err:
if not self._is_transport_closed(err):
raise
logger.warning(
"MemOS: bridge transport closed during %s; reconnecting and retrying once — %s",
method,
err,
)
# One reconnect, one retry, then surface whatever comes back.
self._reconnect_bridge(self._session_id, timeout=30.0)
assert self._bridge is not None
return self._bridge.request(method, params, **call_kwargs)

def _start_bridge_keepalive(self) -> None:
if self._bridge_keepalive_thread and self._bridge_keepalive_thread.is_alive():
return
Expand All @@ -1798,8 +1830,11 @@ def _run() -> None:
assert self._bridge is not None
self._bridge.request("core.health", {}, timeout=10.0)
except Exception as err:
if self._is_transport_closed(err):
logger.info("MemOS: bridge keepalive reconnecting after transport close")
if self._should_reconnect_after_keepalive_failure(err):
logger.info(
"MemOS: bridge keepalive reconnecting after health failure — %s",
err,
)
with contextlib.suppress(Exception):
self._reconnect_bridge(self._session_id, timeout=10.0)
else:
Expand All @@ -1812,6 +1847,35 @@ def _run() -> None:
)
self._bridge_keepalive_thread.start()

def _should_reconnect_after_keepalive_failure(self, err: Exception) -> bool:
"""Decide whether a `core.health` failure warrants a reconnect.

Historically we only reconnected on `transport_closed`. Issue
#1722 showed that a hung bridge surfaces as `BridgeError("timeout",
…)` while the pipe is still nominally open, so the client never
recovered. Reconnect for:

- the historical `transport_closed` case,
- `BridgeError("timeout", …)` (the health probe timing out means
the bridge is not answering anything),
- any error observed while the subprocess is already dead
(`Popen.poll()` returns a non-`None` exit code).
"""
if self._is_transport_closed(err):
return True
if isinstance(err, BridgeError) and err.code == "timeout":
return True
bridge = self._bridge
if bridge is not None:
proc = getattr(bridge, "_proc", None)
poll = getattr(proc, "poll", None)
try:
if callable(poll) and poll() is not None:
return True
except Exception: # pragma: no cover — defensive
pass
return False

def _turn_start(self, query: str, *, session_id: str = "") -> str:
assert self._bridge is not None
host_runtime = self._host_runtime_context()
Expand Down
197 changes: 122 additions & 75 deletions apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,19 @@ def request(
) -> dict[str, Any]:
if self._closed:
raise BridgeError("transport_closed", "bridge client is closed")
# Fast-fail path: if the subprocess has already exited, do NOT
# write to a dead pipe (the OS may buffer the write for a while
# and the caller would then park for the full timeout waiting for
# a reader thread that has already returned). Surface
# `transport_closed` immediately so upstream retry / reconnect
# logic can kick in. See issue #1722.
poll = getattr(self._proc, "poll", None)
exit_code = poll() if callable(poll) else None
if exit_code is not None:
raise BridgeError(
"transport_closed",
f"bridge subprocess exited (code={exit_code})",
)
with self._lock:
rpc_id = self._next_id
self._next_id += 1
Expand Down Expand Up @@ -239,11 +252,22 @@ def register_host_handler(

def close(self) -> None:
if self._closed:
# Even if the reader thread already flipped `_closed` (subprocess
# exit → `_abort_pending`), the caller's explicit close should
# still tear down the subprocess so nothing lingers.
self._teardown_process()
return
with self._host_handlers_cv:
self._closed = True
self._host_handlers_cv.notify_all()

self._teardown_process()

# Clean up any request that was still pending when close() was
# called explicitly (e.g. shutdown before the subprocess exited).
self._abort_pending("bridge closed")

def _teardown_process(self) -> None:
pid = self.pid

# 1. Close stdin (triggers bridge's graceful exit)
Expand Down Expand Up @@ -274,87 +298,110 @@ def close(self) -> None:
except subprocess.TimeoutExpired:
logger.error("MemOS: bridge process %d could not be killed", pid)

# 5. Clean up pending requests
with self._lock:
for entry in list(self._pending.values()):
entry["error"] = {
"code": -32000,
"message": "bridge closed",
"data": {"code": "transport_closed"},
}
entry["event"].set()
self._pending.clear()

# ─── Internals ──

def _read_loop(self) -> None:
assert self._proc.stdout is not None
for line in self._proc.stdout:
line = line.strip()
if not line:
continue
try:
msg = json.loads(line)
except json.JSONDecodeError:
logger.debug("bridge: malformed line: %r", line[:120])
continue
if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg):
self._resolve(msg)
continue
if msg.get("method") == "events.notify":
for cb in list(self._events):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("event listener threw", exc_info=True)
continue
if msg.get("method") == "logs.forward":
for cb in list(self._logs):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("log listener threw", exc_info=True)
continue
# Reverse-direction request: the bridge is asking the
# adapter to do something (e.g. run a fallback LLM call
# via `host.llm.complete`). Dispatch to the registered
# handler and write the response back synchronously.
method = msg.get("method")
rpc_id = msg.get("id")
if (
isinstance(method, str)
and rpc_id is not None
and "result" not in msg
and "error" not in msg
):
handler = self._host_handler_for(method)
if handler is None:
self._send_response(
rpc_id,
error={
"code": -32601,
"message": f"method not found: {method}",
"data": {"code": "unknown_method"},
},
)
exit_reason = "bridge subprocess exited"
try:
for line in self._proc.stdout:
line = line.strip()
if not line:
continue
params = msg.get("params") or {}
if not isinstance(params, dict):
params = {}
try:
result = handler(params)
self._send_response(rpc_id, result=result)
except Exception as err:
logger.warning("host handler %s failed: %s", method, err)
self._send_response(
rpc_id,
error={
"code": -32000,
"message": str(err) or err.__class__.__name__,
"data": {"code": "host_handler_failed"},
},
)
continue
msg = json.loads(line)
except json.JSONDecodeError:
logger.debug("bridge: malformed line: %r", line[:120])
continue
if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg):
self._resolve(msg)
continue
if msg.get("method") == "events.notify":
for cb in list(self._events):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("event listener threw", exc_info=True)
continue
if msg.get("method") == "logs.forward":
for cb in list(self._logs):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("log listener threw", exc_info=True)
continue
# Reverse-direction request: the bridge is asking the
# adapter to do something (e.g. run a fallback LLM call
# via `host.llm.complete`). Dispatch to the registered
# handler and write the response back synchronously.
method = msg.get("method")
rpc_id = msg.get("id")
if (
isinstance(method, str)
and rpc_id is not None
and "result" not in msg
and "error" not in msg
):
handler = self._host_handler_for(method)
if handler is None:
self._send_response(
rpc_id,
error={
"code": -32601,
"message": f"method not found: {method}",
"data": {"code": "unknown_method"},
},
)
continue
params = msg.get("params") or {}
if not isinstance(params, dict):
params = {}
try:
result = handler(params)
self._send_response(rpc_id, result=result)
except Exception as err:
logger.warning("host handler %s failed: %s", method, err)
self._send_response(
rpc_id,
error={
"code": -32000,
"message": str(err) or err.__class__.__name__,
"data": {"code": "host_handler_failed"},
},
)
continue
except Exception as err:
exit_reason = f"bridge read loop crashed: {err}"
logger.debug("bridge: read loop raised %s", err, exc_info=True)
finally:
# Whether we exited cleanly (stdout EOF because the subprocess
# died) or via an exception, every JSON-RPC waiter parked in
# `request()` MUST be woken with a `transport_closed` error —
# otherwise callers park for their full timeout waiting for a
# response that will never arrive. Issue #1722.
self._abort_pending(exit_reason)

def _abort_pending(self, reason: str) -> None:
"""Wake every parked JSON-RPC waiter with a `transport_closed` error.

Called from the reader thread's `finally` block (subprocess exit
or stream error) so callers don't have to wait the full per-
request timeout to learn that the bridge is gone. Also idempotent
with `close()` — the second run just finds `self._pending` empty.
"""
with self._host_handlers_cv:
self._closed = True
self._host_handlers_cv.notify_all()
with self._lock:
pending = list(self._pending.values())
self._pending.clear()
for entry in pending:
entry["error"] = {
"code": -32000,
"message": reason,
"data": {"code": "transport_closed"},
}
entry["event"].set()

def _host_handler_for(
self,
Expand Down
Loading
Loading