Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 91 additions & 9 deletions apps/memos-local-plugin/adapters/hermes/memos_provider/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1280,7 +1280,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
}
if bool(args.get("sessionScope", False)):
params["sessionId"] = self._session_id
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.search",
params,
)
Expand All @@ -1298,7 +1298,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
method = methods.get(kind)
if method is None:
return json.dumps({"error": f"unknown memory kind: {kind}"})
item = self._bridge.request(
item = self._bridge_request_with_retry(
method, {"id": item_id, "namespace": self._runtime_namespace()}
)
if not item:
Expand Down Expand Up @@ -1343,7 +1343,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
}
)
if tool_name == "memos_timeline":
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.timeline",
{
"episodeId": args.get("episodeId", self._episode_id),
Expand All @@ -1358,12 +1358,12 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
params = {"limit": limit, "namespace": self._runtime_namespace()}
if args.get("status"):
params["status"] = args["status"]
return json.dumps(self._bridge.request("skill.list", params))
return json.dumps(self._bridge_request_with_retry("skill.list", params))
if tool_name == "memos_environment":
query = (args.get("query") or "").strip()
limit = self._int_arg(args, "limit", 5, 1, 30)
if not query:
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.list_world_models",
{"limit": limit, "offset": 0, "namespace": self._runtime_namespace()},
)
Expand All @@ -1379,7 +1379,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
"queried": False,
}
)
resp = self._bridge.request(
resp = self._bridge_request_with_retry(
"memory.search",
{
"agent": "hermes",
Expand Down Expand Up @@ -1412,7 +1412,7 @@ def handle_tool_call(self, tool_name: str, args: dict[str, Any], **_kwargs: Any)
skill_id = (args.get("id") or "").strip()
if not skill_id:
return json.dumps({"error": "missing id"})
skill = self._bridge.request(
skill = self._bridge_request_with_retry(
"skill.get",
{
"id": skill_id,
Expand Down Expand Up @@ -1724,6 +1724,85 @@ def _is_transport_closed(self, err: Exception) -> bool:
msg = str(err).lower()
return "broken pipe" in msg or "bridge closed" in msg or "transport_closed" in msg

def _describe_keepalive_error(self, err: Exception) -> str:
"""Human-readable tag for keepalive log lines."""
if isinstance(err, BridgeError):
return f"{err.code} ({err.message})"
return f"{err.__class__.__name__} ({err})"

def _should_reconnect_after_keepalive_failure(self, err: Exception) -> bool:
"""Return True when a keepalive failure indicates a broken bridge.

Reconnect on:

1. `transport_closed` — existing behaviour, still covers broken
pipes.
2. `BridgeError("timeout", …)` — the exact code raised when the
bridge stops answering. This is the reported symptom of
issue #1722: the Node subprocess is technically alive but
the event loop is wedged, so every `core.health` call hits
the 10 s timeout.
3. Any exception observed while the bridge subprocess is
already dead (`Popen.poll()` returned non-None). Belt-and-
suspenders for hangs that don't raise a transport error yet
still map to a subprocess we can no longer talk to.

A live subprocess raising a generic (non-transport) error must
NOT trigger a reconnect — otherwise transient parse noise would
create a reconnect storm.
"""
if self._is_transport_closed(err):
return True
if isinstance(err, BridgeError) and err.code == "timeout":
return True
bridge = self._bridge
if bridge is not None:
proc = getattr(bridge, "_proc", None)
poll = getattr(proc, "poll", None)
if callable(poll):
try:
exit_code = poll()
except Exception:
exit_code = None
if exit_code is not None:
return True
return False

def _bridge_request_with_retry(
self,
method: str,
params: dict[str, Any] | None = None,
*,
timeout: float | None = None,
) -> dict[str, Any]:
"""Read-path helper: request → on transport_closed, reconnect + retry once.

Mirrors the existing ``sync_turn`` reconnect+retry pattern so
every read-path memory tool (`memos_search`, `memos_get`,
`memos_timeline`, `memos_environment`, `memos_skill_list`,
`memos_skill_get`) recovers when the bridge dies mid-session
instead of surfacing the raw `[transport_closed]` string to the
model. Only one retry is attempted — a second failure surfaces
verbatim so the caller can see what went wrong.
"""
assert self._bridge is not None
kwargs: dict[str, Any] = {}
if timeout is not None:
kwargs["timeout"] = timeout
try:
return self._bridge.request(method, params, **kwargs) or {}
except Exception as err:
if not self._is_transport_closed(err):
raise
logger.warning(
"MemOS: bridge transport closed during %s; reconnecting and retrying once — %s",
method,
err,
)
self._reconnect_bridge(self._session_id, timeout=30.0)
assert self._bridge is not None
return self._bridge.request(method, params, **kwargs) or {}

def _reconnect_bridge(self, session_id: str = "", *, timeout: float = 30.0) -> None:
# Don't reconnect if we're shutting down
if self._bridge_keepalive_stop.is_set():
Expand Down Expand Up @@ -1798,8 +1877,11 @@ def _run() -> None:
assert self._bridge is not None
self._bridge.request("core.health", {}, timeout=10.0)
except Exception as err:
if self._is_transport_closed(err):
logger.info("MemOS: bridge keepalive reconnecting after transport close")
if self._should_reconnect_after_keepalive_failure(err):
logger.info(
"MemOS: bridge keepalive reconnecting after %s",
self._describe_keepalive_error(err),
)
with contextlib.suppress(Exception):
self._reconnect_bridge(self._session_id, timeout=10.0)
else:
Expand Down
184 changes: 111 additions & 73 deletions apps/memos-local-plugin/adapters/hermes/memos_provider/bridge_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,17 @@ def request(
) -> dict[str, Any]:
if self._closed:
raise BridgeError("transport_closed", "bridge client is closed")
# Fast-fail: if the subprocess has already exited we must not
# write to a broken pipe. OS-level pipe buffering would happily
# accept the write, then the caller would park on `waiter.wait`
# for the full per-request timeout even though the bridge is
# already dead. Detect that state up front.
exit_code = self._proc.poll()
if exit_code is not None:
raise BridgeError(
"transport_closed",
f"bridge subprocess exited (code={exit_code})",
)
with self._lock:
rpc_id = self._next_id
self._next_id += 1
Expand Down Expand Up @@ -274,88 +285,115 @@ def close(self) -> None:
except subprocess.TimeoutExpired:
logger.error("MemOS: bridge process %d could not be killed", pid)

# 5. Clean up pending requests
# 5. Clean up pending requests — shared helper with `_read_loop`.
self._abort_pending("bridge closed")

# ─── Internals ──

def _read_loop(self) -> None:
assert self._proc.stdout is not None
try:
for line in self._proc.stdout:
line = line.strip()
if not line:
continue
try:
msg = json.loads(line)
except json.JSONDecodeError:
logger.debug("bridge: malformed line: %r", line[:120])
continue
if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg):
self._resolve(msg)
continue
if msg.get("method") == "events.notify":
for cb in list(self._events):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("event listener threw", exc_info=True)
continue
if msg.get("method") == "logs.forward":
for cb in list(self._logs):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("log listener threw", exc_info=True)
continue
# Reverse-direction request: the bridge is asking the
# adapter to do something (e.g. run a fallback LLM call
# via `host.llm.complete`). Dispatch to the registered
# handler and write the response back synchronously.
method = msg.get("method")
rpc_id = msg.get("id")
if (
isinstance(method, str)
and rpc_id is not None
and "result" not in msg
and "error" not in msg
):
handler = self._host_handler_for(method)
if handler is None:
self._send_response(
rpc_id,
error={
"code": -32601,
"message": f"method not found: {method}",
"data": {"code": "unknown_method"},
},
)
continue
params = msg.get("params") or {}
if not isinstance(params, dict):
params = {}
try:
result = handler(params)
self._send_response(rpc_id, result=result)
except Exception as err:
logger.warning("host handler %s failed: %s", method, err)
self._send_response(
rpc_id,
error={
"code": -32000,
"message": str(err) or err.__class__.__name__,
"data": {"code": "host_handler_failed"},
},
)
continue
except Exception:
# Any read-side exception (subprocess crash surfaced as OSError,
# stdout iteration errors) is terminal for the reader thread.
# We fall through to the finally block so pending waiters are
# released instead of parking on their per-request timeouts.
logger.debug("bridge: reader loop terminated by exception", exc_info=True)
finally:
# Normal EOF path (bridge subprocess exited cleanly) reaches
# here without exception. Either way we must wake pending
# waiters with a transport_closed error — otherwise every
# in-flight caller sleeps for the full 30 s wall clock and
# then reports a misleading "[timeout] …" (issue #1722).
self._abort_pending("bridge subprocess exited")

def _abort_pending(self, reason: str) -> None:
"""Mark the client closed and wake every parked JSON-RPC waiter.

Called from both ``_read_loop``'s finally block (subprocess
died) and ``close`` (client shutting down). Idempotent —
subsequent calls with the same client are no-ops because
``_pending`` is cleared and ``_closed`` is latched.
"""
with self._host_handlers_cv:
self._closed = True
self._host_handlers_cv.notify_all()
with self._lock:
for entry in list(self._pending.values()):
entry["error"] = {
"code": -32000,
"message": "bridge closed",
"message": reason,
"data": {"code": "transport_closed"},
}
entry["event"].set()
self._pending.clear()

# ─── Internals ──

def _read_loop(self) -> None:
assert self._proc.stdout is not None
for line in self._proc.stdout:
line = line.strip()
if not line:
continue
try:
msg = json.loads(line)
except json.JSONDecodeError:
logger.debug("bridge: malformed line: %r", line[:120])
continue
if "id" in msg and msg["id"] is not None and ("result" in msg or "error" in msg):
self._resolve(msg)
continue
if msg.get("method") == "events.notify":
for cb in list(self._events):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("event listener threw", exc_info=True)
continue
if msg.get("method") == "logs.forward":
for cb in list(self._logs):
try:
cb(msg.get("params") or {})
except Exception:
logger.debug("log listener threw", exc_info=True)
continue
# Reverse-direction request: the bridge is asking the
# adapter to do something (e.g. run a fallback LLM call
# via `host.llm.complete`). Dispatch to the registered
# handler and write the response back synchronously.
method = msg.get("method")
rpc_id = msg.get("id")
if (
isinstance(method, str)
and rpc_id is not None
and "result" not in msg
and "error" not in msg
):
handler = self._host_handler_for(method)
if handler is None:
self._send_response(
rpc_id,
error={
"code": -32601,
"message": f"method not found: {method}",
"data": {"code": "unknown_method"},
},
)
continue
params = msg.get("params") or {}
if not isinstance(params, dict):
params = {}
try:
result = handler(params)
self._send_response(rpc_id, result=result)
except Exception as err:
logger.warning("host handler %s failed: %s", method, err)
self._send_response(
rpc_id,
error={
"code": -32000,
"message": str(err) or err.__class__.__name__,
"data": {"code": "host_handler_failed"},
},
)
continue

def _host_handler_for(
self,
method: str,
Expand Down
Loading
Loading