From c9c401bfc32339338e79b14e1f9ce42760903cb0 Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Mon, 12 Oct 2026 17:25:50 +0800 Subject: [PATCH 1/2] fix(plugin): defer rescue-spawn while the managed bridge unit is active (#2467) During a systemd restart window (~8-10s) :18800 probes "free" while the supervised daemon is mid-start. ensure_viewer_daemon treated that as spawn clearance, so a session host rescue-spawned its own --daemon, won first bind, and the memos-bridge.service unit crash-looped forever on "port busy" while the rogue answered /api/v1/health green (413 restarts/24h observed in #2467). Killing the rogue did not help: the host keepalive/reconnect path re-entered ensure_viewer_daemon within ~3s (shared_bridge_runtime.reconnect runs before_spawn on every respawn). Gate the rescue-spawn path on `systemctl is-active` for MEMOS_BRIDGE_SYSTEMD_UNIT (default memos-bridge.service): while the unit is active/activating, wait up to 20s for the managed daemon to take the port (report the viewer up when it does) instead of racing it; spawn again only once the unit leaves active/activating. Any failure of the check (no systemctl, no systemd bus, timeout) degrades to today's unconditional rescue behavior, so macOS/Windows/containers and manually managed daemons are unaffected. Unit-side yield (bind-retry backoff for a daemon losing first-bind to a mid-start unit) is left as follow-up work. --- .../hermes/memos_provider/daemon_manager.py | 115 +++++++++ .../tests/python/test_daemon_manager.py | 225 ++++++++++++++++++ 2 files changed, 340 insertions(+) create mode 100644 apps/memos-local-plugin/tests/python/test_daemon_manager.py diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py index d7e6b6e8d..bdaff3009 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py @@ -56,6 +56,17 @@ VIEWER_PROBE_TTL_SEC = 30.0 VIEWER_START_LOCK_TIMEOUT_SEC = 20.0 VIEWER_START_LOCK_STALE_SEC = 60.0 +# Issue #2467: during a systemd restart window the managed bridge daemon is +# mid-start while :18800 still probes "free" (nothing is bound yet). A session +# host that rescue-spawns its own `--daemon` then wins first bind, the unit +# crash-loops forever on "port busy", and the rogue answers +# `/api/v1/health` — monitoring stays green while the supervised process is +# gone. When a supervisor-managed bridge unit is active or activating, wait +# for it instead of racing it. +MANAGED_BRIDGE_UNIT_ENV = "MEMOS_BRIDGE_SYSTEMD_UNIT" +DEFAULT_MANAGED_BRIDGE_UNIT = "memos-bridge.service" +MANAGED_UNIT_GRACE_SEC = 20.0 +MANAGED_UNIT_POLL_SEC = 1.0 # Bound how long a cached `_bridge_ok` answer is trusted. Without this, a # single transient `_node_available()` failure during gateway startup # (subprocess race, `.env` loaded after the first probe) would pin the @@ -356,6 +367,99 @@ def _probe_json_url(url: str) -> LoopbackProbeResult: return "occupied" +def _managed_bridge_unit() -> str | None: + """Return the supervisor-managed bridge unit to defer to, if any. + + ``MEMOS_BRIDGE_SYSTEMD_UNIT`` renames the unit for deployments that + don't use the default name; setting it empty disables the check. + """ + raw = os.environ.get(MANAGED_BRIDGE_UNIT_ENV, DEFAULT_MANAGED_BRIDGE_UNIT) + return raw.strip() or None + + +def _systemd_unit_state(unit: str) -> str | None: + """Return ``systemctl is-active ``, or None when undetectable. + + Any failure — no systemctl binary, no systemd bus, timeout — degrades to + None so hosts without systemd keep the unconditional rescue-spawn + behavior. The state is printed on stdout even for non-zero exit codes + ("inactive"/"failed" exit 3), so only the output matters. + """ + systemctl = shutil.which("systemctl") + if not systemctl: + return None + try: + completed = subprocess.run( + [systemctl, "is-active", unit], + capture_output=True, + text=True, + timeout=2.0, + ) + except Exception: + return None + return (completed.stdout or "").strip().lower() or None + + +def _managed_daemon_gate() -> Literal["spawn", "running", "refuse"]: + """Decide whether the rescue-spawn path may start a ``--daemon``. + + ``spawn``: no managed-unit signal (systemd absent, unit inactive or + failed), or the unit just left active/activating — rescue exactly as + before. + ``running``: the managed daemon took :18800 while we waited — the + caller must report the viewer as up instead of spawning. + ``refuse``: the unit is still active/activating after the grace window + — spawning a competing daemon would start the #2467 port war, so fail + closed like the occupied/unknown paths. + """ + unit = _managed_bridge_unit() + if not unit: + return "spawn" + state = _systemd_unit_state(unit) + if state not in ("active", "activating"): + return "spawn" + logger.info( + "MemOS: managed bridge unit %s is %s while port %d probes free — " + "waiting for the managed daemon instead of rescue-spawning", + unit, + state, + HERMES_VIEWER_PORT, + ) + deadline = time.time() + MANAGED_UNIT_GRACE_SEC + while True: + status = _probe_viewer() + if status == "running_memos": + return "running" + if status == "occupied": + logger.warning( + "MemOS: port %d was claimed by a non-MemOS service while " + "waiting for managed unit %s", + HERMES_VIEWER_PORT, + unit, + ) + return "refuse" + state = _systemd_unit_state(unit) + if state not in ("active", "activating"): + logger.info( + "MemOS: managed bridge unit %s is no longer %s — proceeding " + "with rescue-spawn", + unit, + state or "active/activating", + ) + return "spawn" + if time.time() >= deadline: + logger.warning( + "MemOS: managed bridge unit %s still %s after %.0fs and port " + "%d still free; refusing to spawn a competing daemon", + unit, + state or "unknown", + MANAGED_UNIT_GRACE_SEC, + HERMES_VIEWER_PORT, + ) + return "refuse" + time.sleep(MANAGED_UNIT_POLL_SEC) + + def ensure_viewer_daemon( *, probe_only: bool = False, @@ -429,6 +533,17 @@ def ensure_viewer_daemon( "memory capture will continue without the web panel", ) return False + # #2467: a "free" probe cannot distinguish "no daemon" from "the + # supervisor-managed daemon is mid-restart". Defer to the managed + # unit before spawning a competing --daemon. + gate = _managed_daemon_gate() + if gate == "running": + _viewer_status = "running_memos" + _viewer_last_probe_at = time.time() + logger.info("MemOS: managed bridge daemon is serving port %d", HERMES_VIEWER_PORT) + return True + if gate == "refuse": + return False if not ensure_bridge_running(): return False diff --git a/apps/memos-local-plugin/tests/python/test_daemon_manager.py b/apps/memos-local-plugin/tests/python/test_daemon_manager.py new file mode 100644 index 000000000..666307bfb --- /dev/null +++ b/apps/memos-local-plugin/tests/python/test_daemon_manager.py @@ -0,0 +1,225 @@ +"""Unit tests for the daemon_manager managed-daemon gate (issue #2467). + +During a systemd restart window the managed bridge daemon is mid-start +while :18800 probes ``free``; a session host that rescue-spawns its own +``--daemon`` then wins first bind and the unit crash-loops forever while +the rogue answers ``/api/v1/health``. The gate under test defers to a +supervisor-managed unit (``systemctl is-active``) instead of racing it. + +These tests stub the probe/subprocess layers — no real Node bridge and no +systemd required. + +Run: + python3 -m unittest tests.python.test_daemon_manager +""" + +from __future__ import annotations + +import contextlib +import os +import subprocess +import sys +import unittest + +from pathlib import Path +from unittest.mock import patch + + +_ADAPTER_ROOT = Path(__file__).resolve().parent.parent.parent / "adapters" / "hermes" +_PLUGIN_DIR = _ADAPTER_ROOT / "memos_provider" +for _p in (_ADAPTER_ROOT, _PLUGIN_DIR): + if str(_p) not in sys.path: + sys.path.insert(0, str(_p)) + +import daemon_manager as daemon_manager_mod # noqa: E402 + + +UNIT = "memos-bridge.service" + + +class FakeDaemon: + returncode = None + + def poll(self): + return None + + +@contextlib.contextmanager +def acquired_lock(_runtime_home=None): + yield True + + +class ManagedDaemonGateTests(unittest.TestCase): + """Rescue-spawn must yield to an active/activating managed unit (#2467).""" + + def tearDown(self) -> None: + daemon_manager_mod._viewer_status = None + daemon_manager_mod._viewer_last_probe_at = 0.0 + daemon_manager_mod._viewer_process = None + + def test_activating_managed_unit_blocks_rescue_spawn(self) -> None: + """Unit mid-restart + port probes free: no rogue ``--daemon``.""" + with ( + patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), + patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object( + daemon_manager_mod, "_systemd_unit_state", return_value="activating", create=True + ), + patch.object(daemon_manager_mod, "MANAGED_UNIT_GRACE_SEC", 0.05, create=True), + patch.object(daemon_manager_mod, "MANAGED_UNIT_POLL_SEC", 0.01, create=True), + patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), + patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), + patch.object(daemon_manager_mod.subprocess, "Popen") as popen, + ): + self.assertFalse(daemon_manager_mod.ensure_viewer_daemon()) + popen.assert_not_called() + + def test_active_managed_unit_blocks_rescue_spawn(self) -> None: + """Unit active but :18800 not yet bound (pipeline warming up): no spawn.""" + with ( + patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), + patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object(daemon_manager_mod, "_systemd_unit_state", return_value="active", create=True), + patch.object(daemon_manager_mod, "MANAGED_UNIT_GRACE_SEC", 0.05, create=True), + patch.object(daemon_manager_mod, "MANAGED_UNIT_POLL_SEC", 0.01, create=True), + patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), + patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), + patch.object(daemon_manager_mod.subprocess, "Popen") as popen, + ): + self.assertFalse(daemon_manager_mod.ensure_viewer_daemon()) + popen.assert_not_called() + + def test_managed_daemon_coming_up_is_waited_for_not_raced(self) -> None: + """The managed daemon binds during the grace window: report up, no spawn.""" + with ( + patch.object( + daemon_manager_mod, + "_probe_viewer", + side_effect=["free", "free", "running_memos"], + ), + patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object( + daemon_manager_mod, + "_systemd_unit_state", + side_effect=["activating", "activating"], + create=True, + ), + patch.object(daemon_manager_mod, "MANAGED_UNIT_GRACE_SEC", 5.0, create=True), + patch.object(daemon_manager_mod, "MANAGED_UNIT_POLL_SEC", 0.01, create=True), + patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), + patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), + patch.object(daemon_manager_mod.subprocess, "Popen") as popen, + ): + self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) + popen.assert_not_called() + self.assertEqual(daemon_manager_mod._viewer_status, "running_memos") + + def test_managed_unit_failure_releases_rescue_spawn(self) -> None: + """Unit left active/activating (failed/stopped): rescue-spawn proceeds.""" + with ( + patch.object( + daemon_manager_mod, + "_probe_viewer", + side_effect=["free", "free", "free", "running_memos"], + ), + patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object( + daemon_manager_mod, + "_systemd_unit_state", + side_effect=["activating", "failed"], + create=True, + ), + patch.object(daemon_manager_mod, "MANAGED_UNIT_GRACE_SEC", 5.0, create=True), + patch.object(daemon_manager_mod, "MANAGED_UNIT_POLL_SEC", 0.01, create=True), + patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), + patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), + patch.object(daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon()) as popen, + ): + self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) + popen.assert_called_once() + + def test_no_managed_unit_keeps_unconditional_rescue_spawn(self) -> None: + """No supervisor signal (systemd absent / unit disabled): today's behavior.""" + with ( + patch.object( + daemon_manager_mod, + "_probe_viewer", + side_effect=["free", "free", "running_memos"], + ), + patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=None, create=True), + patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), + patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), + patch.object(daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon()) as popen, + ): + self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) + popen.assert_called_once() + + def test_inactive_managed_unit_still_allows_rescue_spawn(self) -> None: + """Unit exists but is stopped: the host is the only way to serve :18800.""" + with ( + patch.object( + daemon_manager_mod, + "_probe_viewer", + side_effect=["free", "free", "running_memos"], + ), + patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object(daemon_manager_mod, "_systemd_unit_state", return_value="inactive", create=True), + patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), + patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), + patch.object(daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon()) as popen, + ): + self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) + popen.assert_called_once() + + +class ManagedBridgeUnitConfigTests(unittest.TestCase): + def test_default_unit_is_memos_bridge_service(self) -> None: + env = {k: v for k, v in os.environ.items() if k != "MEMOS_BRIDGE_SYSTEMD_UNIT"} + with patch.dict(os.environ, env, clear=True): + self.assertEqual(daemon_manager_mod._managed_bridge_unit(), UNIT) + + def test_env_renames_the_unit(self) -> None: + with patch.dict(os.environ, {"MEMOS_BRIDGE_SYSTEMD_UNIT": "my-bridge.service"}): + self.assertEqual(daemon_manager_mod._managed_bridge_unit(), "my-bridge.service") + + def test_empty_env_disables_the_gate(self) -> None: + with patch.dict(os.environ, {"MEMOS_BRIDGE_SYSTEMD_UNIT": " "}): + self.assertIsNone(daemon_manager_mod._managed_bridge_unit()) + + +class SystemdUnitStateTests(unittest.TestCase): + def test_degrades_to_none_without_systemctl(self) -> None: + with patch.object(daemon_manager_mod.shutil, "which", return_value=None) as which: + self.assertIsNone(daemon_manager_mod._systemd_unit_state(UNIT)) + which.assert_called_once_with("systemctl") + + def test_reads_stdout_regardless_of_exit_code(self) -> None: + completed = subprocess.CompletedProcess( + args=[], returncode=3, stdout="activating\n", stderr="" + ) + with ( + patch.object(daemon_manager_mod.shutil, "which", return_value="/usr/bin/systemctl"), + patch.object(daemon_manager_mod.subprocess, "run", return_value=completed) as run, + ): + self.assertEqual(daemon_manager_mod._systemd_unit_state(UNIT), "activating") + run.assert_called_once_with( + ["/usr/bin/systemctl", "is-active", UNIT], + capture_output=True, + text=True, + timeout=2.0, + ) + + def test_degrades_to_none_when_systemctl_fails(self) -> None: + with ( + patch.object(daemon_manager_mod.shutil, "which", return_value="/usr/bin/systemctl"), + patch.object( + daemon_manager_mod.subprocess, + "run", + side_effect=subprocess.TimeoutExpired(cmd="systemctl", timeout=2.0), + ), + ): + self.assertIsNone(daemon_manager_mod._systemd_unit_state(UNIT)) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() From 6f042035938e667c4529416eb8e07b84d6033ed8 Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Mon, 12 Oct 2026 17:35:21 +0800 Subject: [PATCH 2/2] style(plugin): ruff-format the #2467 gate and its tests Acceptance requires the repo-enforced `ruff format --check` (.github/workflows/python-tests.yml:95-96) to pass. No semantic change: one gate log string joined onto a single line, patch.object call wrapping normalized in the new test file. Verified: uvx ruff@0.11.8 format --check . -> 636 files already formatted; ruff check --no-fix . -> All checks passed; python test suite still 144/144 OK. --- .../hermes/memos_provider/daemon_manager.py | 3 +- .../tests/python/test_daemon_manager.py | 44 ++++++++++++++----- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py b/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py index bdaff3009..8acd721f6 100644 --- a/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py +++ b/apps/memos-local-plugin/adapters/hermes/memos_provider/daemon_manager.py @@ -441,8 +441,7 @@ def _managed_daemon_gate() -> Literal["spawn", "running", "refuse"]: state = _systemd_unit_state(unit) if state not in ("active", "activating"): logger.info( - "MemOS: managed bridge unit %s is no longer %s — proceeding " - "with rescue-spawn", + "MemOS: managed bridge unit %s is no longer %s — proceeding with rescue-spawn", unit, state or "active/activating", ) diff --git a/apps/memos-local-plugin/tests/python/test_daemon_manager.py b/apps/memos-local-plugin/tests/python/test_daemon_manager.py index 666307bfb..4571560f9 100644 --- a/apps/memos-local-plugin/tests/python/test_daemon_manager.py +++ b/apps/memos-local-plugin/tests/python/test_daemon_manager.py @@ -61,7 +61,9 @@ def test_activating_managed_unit_blocks_rescue_spawn(self) -> None: """Unit mid-restart + port probes free: no rogue ``--daemon``.""" with ( patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), - patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object( + daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True + ), patch.object( daemon_manager_mod, "_systemd_unit_state", return_value="activating", create=True ), @@ -78,8 +80,12 @@ def test_active_managed_unit_blocks_rescue_spawn(self) -> None: """Unit active but :18800 not yet bound (pipeline warming up): no spawn.""" with ( patch.object(daemon_manager_mod, "_probe_viewer", return_value="free"), - patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), - patch.object(daemon_manager_mod, "_systemd_unit_state", return_value="active", create=True), + patch.object( + daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True + ), + patch.object( + daemon_manager_mod, "_systemd_unit_state", return_value="active", create=True + ), patch.object(daemon_manager_mod, "MANAGED_UNIT_GRACE_SEC", 0.05, create=True), patch.object(daemon_manager_mod, "MANAGED_UNIT_POLL_SEC", 0.01, create=True), patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), @@ -97,7 +103,9 @@ def test_managed_daemon_coming_up_is_waited_for_not_raced(self) -> None: "_probe_viewer", side_effect=["free", "free", "running_memos"], ), - patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object( + daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True + ), patch.object( daemon_manager_mod, "_systemd_unit_state", @@ -122,7 +130,9 @@ def test_managed_unit_failure_releases_rescue_spawn(self) -> None: "_probe_viewer", side_effect=["free", "free", "free", "running_memos"], ), - patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), + patch.object( + daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True + ), patch.object( daemon_manager_mod, "_systemd_unit_state", @@ -133,7 +143,9 @@ def test_managed_unit_failure_releases_rescue_spawn(self) -> None: patch.object(daemon_manager_mod, "MANAGED_UNIT_POLL_SEC", 0.01, create=True), patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), - patch.object(daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon()) as popen, + patch.object( + daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon() + ) as popen, ): self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) popen.assert_called_once() @@ -146,10 +158,14 @@ def test_no_managed_unit_keeps_unconditional_rescue_spawn(self) -> None: "_probe_viewer", side_effect=["free", "free", "running_memos"], ), - patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=None, create=True), + patch.object( + daemon_manager_mod, "_managed_bridge_unit", return_value=None, create=True + ), patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), - patch.object(daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon()) as popen, + patch.object( + daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon() + ) as popen, ): self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) popen.assert_called_once() @@ -162,11 +178,17 @@ def test_inactive_managed_unit_still_allows_rescue_spawn(self) -> None: "_probe_viewer", side_effect=["free", "free", "running_memos"], ), - patch.object(daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True), - patch.object(daemon_manager_mod, "_systemd_unit_state", return_value="inactive", create=True), + patch.object( + daemon_manager_mod, "_managed_bridge_unit", return_value=UNIT, create=True + ), + patch.object( + daemon_manager_mod, "_systemd_unit_state", return_value="inactive", create=True + ), patch.object(daemon_manager_mod, "_viewer_start_lock", acquired_lock), patch.object(daemon_manager_mod, "ensure_bridge_running", return_value=True), - patch.object(daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon()) as popen, + patch.object( + daemon_manager_mod.subprocess, "Popen", return_value=FakeDaemon() + ) as popen, ): self.assertTrue(daemon_manager_mod.ensure_viewer_daemon()) popen.assert_called_once()